supports localized title

This commit is contained in:
Your Name 2024-07-13 00:16:47 -04:00 committed by Henri Dickson
parent 9b13e8028e
commit 65098a137d
151 changed files with 16260 additions and 12811 deletions

View file

@ -52,7 +52,6 @@ jobs:
NEODB_SITE_NAME: test NEODB_SITE_NAME: test
NEODB_SITE_DOMAIN: test.domain NEODB_SITE_DOMAIN: test.domain
NEODB_SECRET_KEY: test NEODB_SECRET_KEY: test
NEODB_LANGUAGE: zh-hans
run: | run: |
python manage.py compilemessages -i .venv -l zh_Hans python manage.py compilemessages -i .venv -l zh_Hans
python manage.py test python manage.py test

View file

@ -46,8 +46,8 @@ env = environ.FileAwareEnv(
NEODB_SITE_LINKS=(dict, {}), NEODB_SITE_LINKS=(dict, {}),
# Alternative domains # Alternative domains
NEODB_ALTERNATIVE_DOMAINS=(list, []), NEODB_ALTERNATIVE_DOMAINS=(list, []),
# Default language # Preferred languages in catalog
NEODB_LANGUAGE=(str, "zh-hans"), NEODB_PREFERRED_LANGUAGES=(list, ["en", "zh"]), # , "ja", "ko", "de", "fr", "es"
# Invite only mode # Invite only mode
# when True: user will not be able to register unless with invite token # when True: user will not be able to register unless with invite token
# (generated by `neodb-manage invite --create`) # (generated by `neodb-manage invite --create`)
@ -408,14 +408,38 @@ if SLACK_TOKEN:
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md" MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md"
LANGUAGE_CODE = env("NEODB_LANGUAGE", default="zh-hans") # type: ignore SUPPORTED_UI_LANGUAGES = {
LOCALE_PATHS = [os.path.join(BASE_DIR, "locale")] "en": _("English"),
LANGUAGES = ( "zh-hans": _("Simplified Chinese"),
("en", _("English")), "zh-hant": _("Traditional Chinese"),
("zh-hans", _("Simplified Chinese")), }
("zh-hant", _("Traditional Chinese")),
LANGUAGES = SUPPORTED_UI_LANGUAGES.items()
def _init_language_settings(preferred_lanugages_env):
default_language = None
preferred_lanugages = []
for pl in preferred_lanugages_env:
lang = pl.strip().lower()
if not default_language:
if lang in SUPPORTED_UI_LANGUAGES:
default_language = lang
elif lang == "zh":
default_language = "zh-hans"
if lang.startswith("zh-"):
lang = "zh"
if lang not in preferred_lanugages:
preferred_lanugages.append(lang)
return default_language or "en", preferred_lanugages or ["en"]
LANGUAGE_CODE, PREFERRED_LANGUAGES = _init_language_settings(
env("NEODB_PREFERRED_LANGUAGES")
) )
LOCALE_PATHS = [os.path.join(BASE_DIR, "locale")]
TIME_ZONE = env("NEODB_TIMEZONE", default="Asia/Shanghai") # type: ignore TIME_ZONE = env("NEODB_TIMEZONE", default="Asia/Shanghai") # type: ignore
USE_I18N = True USE_I18N = True

View file

@ -17,7 +17,6 @@ work data seems asymmetric (a book links to a work, but may not listed in that w
""" """
from os.path import exists
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from django.core.validators import MaxValueValidator, MinValueValidator from django.core.validators import MaxValueValidator, MinValueValidator
@ -37,6 +36,7 @@ from catalog.common import (
PrimaryLookupIdDescriptor, PrimaryLookupIdDescriptor,
jsondata, jsondata,
) )
from catalog.common.models import SCRIPT_CHOICES
from .utils import * from .utils import *
@ -90,6 +90,7 @@ class Edition(Item):
"pages", "pages",
"price", "price",
"brief", "brief",
"localized_description",
"contents", "contents",
] ]
subtitle = jsondata.CharField( subtitle = jsondata.CharField(
@ -113,7 +114,12 @@ class Edition(Item):
default=list, default=list,
) )
language = jsondata.CharField( language = jsondata.CharField(
_("language"), null=True, blank=True, default=None, max_length=500 _("language"),
null=False,
blank=True,
default=None,
max_length=500,
choices=SCRIPT_CHOICES,
) )
pub_house = jsondata.CharField( pub_house = jsondata.CharField(
_("publishing house"), null=True, blank=False, default=None, max_length=500 _("publishing house"), null=True, blank=False, default=None, max_length=500

View file

@ -13,12 +13,16 @@ from django.db import connection, models
from django.db.models import QuerySet, Value from django.db.models import QuerySet, Value
from django.template.defaultfilters import default from django.template.defaultfilters import default
from django.utils import timezone from django.utils import timezone
from django.utils.translation import get_language
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from loguru import logger from loguru import logger
from ninja import Field, Schema from ninja import Field, Schema
from polymorphic.models import PolymorphicModel from polymorphic.models import PolymorphicModel
from catalog.common import jsondata from catalog.common import jsondata
from common.models import LANGUAGE_CHOICES, LOCALE_CHOICES, SCRIPT_CHOICES
from common.models.lang import get_current_locales
from common.models.misc import uniq
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
@ -258,9 +262,16 @@ class BaseSchema(Schema):
external_resources: list[ExternalResourceSchema] | None external_resources: list[ExternalResourceSchema] | None
class LocalizedTitleSchema(Schema):
lang: str
text: str
class ItemInSchema(Schema): class ItemInSchema(Schema):
title: str title: str
brief: str brief: str
localized_title: list[LocalizedTitleSchema] = []
localized_description: list[LocalizedTitleSchema] = []
cover_image_url: str | None cover_image_url: str | None
rating: float | None rating: float | None
rating_count: int | None rating_count: int | None
@ -270,6 +281,63 @@ class ItemSchema(BaseSchema, ItemInSchema):
pass pass
def get_locale_choices_for_jsonform(choices):
"""return list for jsonform schema"""
return [{"title": v, "value": k} for k, v in choices]
LOCALE_CHOICES_JSONFORM = get_locale_choices_for_jsonform(LOCALE_CHOICES)
LANGUAGE_CHOICES_JSONFORM = get_locale_choices_for_jsonform(LANGUAGE_CHOICES)
LOCALIZED_TITLE_SCHEMA = {
"type": "list",
"items": {
"type": "dict",
"keys": {
"lang": {
"type": "string",
"title": _("language"),
"choices": LOCALE_CHOICES_JSONFORM,
},
"text": {"type": "string", "title": _("content")},
},
"required": ["lang", "s"],
},
"uniqueItems": True,
}
LOCALIZED_DESCRIPTION_SCHEMA = {
"type": "list",
"items": {
"type": "dict",
"keys": {
"lang": {
"type": "string",
"title": _("language"),
"choices": LOCALE_CHOICES_JSONFORM,
},
"text": {"type": "string", "title": _("content"), "widget": "textarea"},
},
"required": ["lang", "s"],
},
"uniqueItems": True,
}
def LanguageListField():
return jsondata.ArrayField(
verbose_name=_("language"),
base_field=models.CharField(blank=True, default="", max_length=100),
null=True,
blank=True,
default=list,
# schema={
# "type": "list",
# "items": {"type": "string", "choices": LANGUAGE_CHOICES_JSONFORM},
# },
)
class Item(PolymorphicModel): class Item(PolymorphicModel):
if TYPE_CHECKING: if TYPE_CHECKING:
external_resources: QuerySet["ExternalResource"] external_resources: QuerySet["ExternalResource"]
@ -308,6 +376,22 @@ class Item(PolymorphicModel):
related_name="merged_from_items", related_name="merged_from_items",
) )
localized_title = jsondata.JSONField(
verbose_name=_("title"),
null=False,
blank=True,
default=list,
schema=LOCALIZED_TITLE_SCHEMA,
)
localized_description = jsondata.JSONField(
verbose_name=_("description"),
null=False,
blank=True,
default=list,
schema=LOCALIZED_DESCRIPTION_SCHEMA,
)
class Meta: class Meta:
index_together = [ index_together = [
[ [
@ -494,12 +578,52 @@ class Item(PolymorphicModel):
def class_name(self) -> str: def class_name(self) -> str:
return self.__class__.__name__.lower() return self.__class__.__name__.lower()
@property def get_localized_title(self) -> str | None:
def display_title(self) -> str: if self.localized_title:
return self.title locales = get_current_locales()
for loc in locales:
v = next(
filter(lambda t: t["lang"] == loc, self.localized_title), {}
).get("text")
if v:
return v
def get_localized_description(self) -> str | None:
if self.localized_description:
locales = get_current_locales()
for loc in locales:
v = next(
filter(lambda t: t["lang"] == loc, self.localized_description), {}
).get("text")
if v:
return v
@property @property
def display_description(self): def display_title(self) -> str:
return (
self.get_localized_title()
or self.title
or (
self.orig_title # type:ignore
if hasattr(self, "orig_title")
else ""
)
) or (self.localized_title[0]["text"] if self.localized_title else "")
@property
def display_description(self) -> str:
return (
self.get_localized_description()
or self.brief
or (
self.localized_description[0]["text"]
if self.localized_description
else ""
)
)
@property
def brief_description(self):
return self.brief[:155] return self.brief[:155]
@classmethod @classmethod
@ -547,7 +671,13 @@ class Item(PolymorphicModel):
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", "title",
"brief", "brief",
"localized_title",
"localized_description",
] # list of metadata keys to copy from resource to item ] # list of metadata keys to copy from resource to item
METADATA_MERGE_LIST = [
"localized_title",
"localized_description",
]
@classmethod @classmethod
def copy_metadata(cls, metadata: dict[str, Any]) -> dict[str, Any]: def copy_metadata(cls, metadata: dict[str, Any]) -> dict[str, Any]:
@ -568,19 +698,26 @@ class Item(PolymorphicModel):
else None else None
) )
def merge_data_from_external_resource(
self, p: "ExternalResource", ignore_existing_content: bool = False
):
for k in self.METADATA_COPY_LIST:
v = p.metadata.get(k)
if v:
if not getattr(self, k) or ignore_existing_content:
setattr(self, k, v)
elif k in self.METADATA_MERGE_LIST:
setattr(self, k, uniq((v or []) + getattr(self, k, [])))
if p.cover and (not self.has_cover() or ignore_existing_content):
self.cover = p.cover
def merge_data_from_external_resources(self, ignore_existing_content: bool = False): def merge_data_from_external_resources(self, ignore_existing_content: bool = False):
"""Subclass may override this""" """Subclass may override this"""
lookup_ids = [] lookup_ids = []
for p in self.external_resources.all(): for p in self.external_resources.all():
lookup_ids.append((p.id_type, p.id_value)) lookup_ids.append((p.id_type, p.id_value))
lookup_ids += p.other_lookup_ids.items() lookup_ids += p.other_lookup_ids.items()
for k in self.METADATA_COPY_LIST: self.merge_data_from_external_resource(p, ignore_existing_content)
if p.metadata.get(k) and (
not getattr(self, k) or ignore_existing_content
):
setattr(self, k, p.metadata.get(k))
if p.cover and (not self.has_cover() or ignore_existing_content):
self.cover = p.cover
self.update_lookup_ids(list(set(lookup_ids))) self.update_lookup_ids(list(set(lookup_ids)))
def update_linked_items_from_external_resource(self, resource: "ExternalResource"): def update_linked_items_from_external_resource(self, resource: "ExternalResource"):

View file

@ -349,8 +349,12 @@ def crawl_related_resources_task(resource_pk):
if not site and w.get("url"): if not site and w.get("url"):
site = SiteManager.get_site_by_url(w["url"]) site = SiteManager.get_site_by_url(w["url"])
if site: if site:
site.get_resource_ready(ignore_existing_content=False, auto_link=True) res = site.get_resource_ready(
ignore_existing_content=False, auto_link=True
)
item = site.get_item() item = site.get_item()
if item and res and w in resource.prematched_resources:
item.merge_data_from_external_resource(res)
if item: if item:
logger.info(f"crawled {w} {item}") logger.info(f"crawled {w} {item}")
else: else:

5
catalog/common/tests.py Normal file
View file

@ -0,0 +1,5 @@
from django.test import TestCase as DjangoTestCase
class TestCase(DjangoTestCase):
databases = "__all__"

View file

@ -3,6 +3,7 @@ from django.utils.translation import gettext_lazy as _
from catalog.models import * from catalog.models import *
from common.forms import PreviewImageInput from common.forms import PreviewImageInput
from common.models import DEFAULT_CATALOG_LANGUAGE, detect_language, uniq
CatalogForms = {} CatalogForms = {}
@ -39,6 +40,73 @@ def _EditForm(item_model):
"cover": PreviewImageInput(), "cover": PreviewImageInput(),
} }
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.migrate_initial()
# {'id': 547, 'primary_lookup_id_type': 'imdb', 'primary_lookup_id_value': 'tt0056923', 'cover': <ImageFieldFile: item/tmdb_movie/2024/01/12/10973d2b-1d20-4e37-8c3c-ecc89e671a80.jpg>, 'orig_title': 'Charade', 'other_title': [], 'director': ['Stanley Donen'], 'playwright': ['Peter Stone'], 'actor': ['Cary Grant', 'Audrey Hepburn', 'Walter Matthau', 'James Coburn', 'George Kennedy', 'Dominique Minot', 'Ned Glass', 'Jacques Marin', 'Paul Bonifas', 'Thomas Chelimsky', 'Marc Arian', 'Claudine Berg', 'Marcel Bernier', 'Albert Daumergue', 'Raoul Delfosse', 'Stanley Donen', 'Jean Gold', 'Chantal Goya', 'Clément Harari', 'Monte Landis', 'Bernard Musson', 'Antonio Passalia', 'Jacques Préboist', 'Peter Stone', 'Michel Thomass', 'Roger Trapp', 'Louis Viret'], 'genre': ['喜剧', '悬疑', '爱情'], 'showtime': [{'time': '1963-12-05', 'region': ''}], 'site': '', 'area': [], 'language': ['English', 'Français', 'Deutsch', 'Italiano'], 'year': 1963, 'duration': '', 'localized_title': [], 'localized_description': []}
def migrate_initial(self):
if self.initial and self.instance:
if (
"localized_title" in self.Meta.fields
and not self.initial["localized_title"]
):
titles = []
if self.instance.title:
titles.append(
{
"lang": detect_language(self.instance.title),
"text": self.instance.title,
}
)
if (
hasattr(self.instance, "orig_title")
and self.instance.orig_title
):
titles.append(
{
"lang": detect_language(self.instance.orig_title),
"text": self.instance.orig_title,
}
)
if (
hasattr(self.instance, "other_title")
and self.instance.other_title
):
for t in self.instance.other_title:
titles.append(
{
"lang": detect_language(t),
"text": self.instance.orig_title,
}
)
if not titles:
titles = [
{"lang": DEFAULT_CATALOG_LANGUAGE, "text": "<no title>"}
]
self.initial["localized_title"] = uniq(titles) # type:ignore
if (
"localized_description" in self.Meta.fields
and not self.initial["localized_description"]
):
if self.instance.brief:
d = {
"lang": detect_language(self.instance.brief),
"text": self.instance.brief,
}
else:
d = {
"lang": self.initial["localized_title"][0]["lang"],
"text": "",
}
self.initial["localized_description"] = [d] # type:ignore
# if (
# "language" in self.Meta.fields
# and self.initial["language"]
# ):
# if isinstance(self.initial["language"], str):
def clean(self): def clean(self):
data = super().clean() or {} data = super().clean() or {}
t, v = self.Meta.model.lookup_id_cleanup( t, v = self.Meta.model.lookup_id_cleanup(

View file

@ -39,9 +39,11 @@ class Game(Item):
douban_game = PrimaryLookupIdDescriptor(IdType.DoubanGame) douban_game = PrimaryLookupIdDescriptor(IdType.DoubanGame)
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"brief", # "brief",
"other_title", # "other_title",
"localized_title",
"localized_description",
"designer", "designer",
"artist", "artist",
"developer", "developer",

View file

@ -5,6 +5,8 @@ from catalog.models import *
class IGDBTestCase(TestCase): class IGDBTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id_type = IdType.IGDB t_id_type = IdType.IGDB
t_id_value = "portal-2" t_id_value = "portal-2"
@ -42,7 +44,9 @@ class IGDBTestCase(TestCase):
) )
self.assertIsInstance(site.resource.item, Game) self.assertIsInstance(site.resource.item, Game)
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IGDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IGDB)
self.assertEqual(site.resource.item.genre, ["Role-playing (RPG)", "Adventure"]) self.assertEqual(
site.resource.item.genre, ["Puzzle", "Role-playing (RPG)", "Adventure"]
)
self.assertEqual( self.assertEqual(
site.resource.item.primary_lookup_id_value, site.resource.item.primary_lookup_id_value,
"the-legend-of-zelda-breath-of-the-wild", "the-legend-of-zelda-breath-of-the-wild",
@ -50,6 +54,8 @@ class IGDBTestCase(TestCase):
class SteamTestCase(TestCase): class SteamTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id_type = IdType.Steam t_id_type = IdType.Steam
t_id_value = "620" t_id_value = "620"
@ -70,10 +76,7 @@ class SteamTestCase(TestCase):
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "Portal 2") self.assertEqual(site.resource.metadata["title"], "Portal 2")
self.assertEqual( self.assertEqual(site.resource.metadata["brief"][:6], "Sequel")
site.resource.metadata["brief"],
"“终身测试计划”现已升级,您可以为您自己或您的好友设计合作谜题!",
)
self.assertIsInstance(site.resource.item, Game) self.assertIsInstance(site.resource.item, Game)
self.assertEqual(site.resource.item.steam, "620") self.assertEqual(site.resource.item.steam, "620")
self.assertEqual( self.assertEqual(
@ -82,6 +85,8 @@ class SteamTestCase(TestCase):
class DoubanGameTestCase(TestCase): class DoubanGameTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id_type = IdType.DoubanGame t_id_type = IdType.DoubanGame
t_id_value = "10734307" t_id_value = "10734307"
@ -100,16 +105,16 @@ class DoubanGameTestCase(TestCase):
self.assertEqual(site.ready, False) self.assertEqual(site.ready, False)
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "传送门2 Portal 2")
self.assertIsInstance(site.resource.item, Game) self.assertIsInstance(site.resource.item, Game)
self.assertEqual(site.resource.item.display_title, "Portal 2")
self.assertEqual(site.resource.item.douban_game, "10734307") self.assertEqual(site.resource.item.douban_game, "10734307")
self.assertEqual( self.assertEqual(site.resource.item.genre, ["第一人称射击", "益智"])
site.resource.item.genre, ["第一人称射击", "益智", "射击", "动作"]
)
self.assertEqual(site.resource.item.other_title, []) self.assertEqual(site.resource.item.other_title, [])
class BangumiGameTestCase(TestCase): class BangumiGameTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_parse(self): def test_parse(self):
t_id_type = IdType.Bangumi t_id_type = IdType.Bangumi
@ -124,6 +129,8 @@ class BangumiGameTestCase(TestCase):
class BoardGameGeekTestCase(TestCase): class BoardGameGeekTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_scrape(self): def test_scrape(self):
t_url = "https://boardgamegeek.com/boardgame/167791" t_url = "https://boardgamegeek.com/boardgame/167791"
@ -134,15 +141,17 @@ class BoardGameGeekTestCase(TestCase):
self.assertEqual(site.ready, False) self.assertEqual(site.ready, False)
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "Terraforming Mars")
self.assertIsInstance(site.resource.item, Game) self.assertIsInstance(site.resource.item, Game)
self.assertEqual(site.resource.item.display_title, "Terraforming Mars")
self.assertEqual(site.resource.item.platform, ["Boardgame"]) self.assertEqual(site.resource.item.platform, ["Boardgame"])
self.assertEqual(site.resource.item.genre[0], "Economic") self.assertEqual(site.resource.item.genre[0], "Economic")
self.assertEqual(site.resource.item.other_title[0], "殖民火星") # self.assertEqual(site.resource.item.other_title[0], "殖民火星")
self.assertEqual(site.resource.item.designer, ["Jacob Fryxelius"]) self.assertEqual(site.resource.item.designer, ["Jacob Fryxelius"])
class MultiGameSitesTestCase(TestCase): class MultiGameSitesTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_games(self): def test_games(self):
url1 = "https://www.igdb.com/games/portal-2" url1 = "https://www.igdb.com/games/portal-2"

View file

@ -13,6 +13,7 @@ from catalog.common import (
PrimaryLookupIdDescriptor, PrimaryLookupIdDescriptor,
jsondata, jsondata,
) )
from catalog.common.models import LanguageListField
class MovieInSchema(ItemInSchema): class MovieInSchema(ItemInSchema):
@ -43,9 +44,10 @@ class Movie(Item):
douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie) douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie)
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"localized_title",
"orig_title", "orig_title",
"other_title", # "other_title",
"director", "director",
"playwright", "playwright",
"actor", "actor",
@ -59,7 +61,8 @@ class Movie(Item):
# "season_number", # "season_number",
# "episodes", # "episodes",
# "single_episode_length", # "single_episode_length",
"brief", "localized_description",
# "brief",
] ]
orig_title = jsondata.CharField( orig_title = jsondata.CharField(
verbose_name=_("original title"), blank=True, default="", max_length=500 verbose_name=_("original title"), blank=True, default="", max_length=500
@ -141,17 +144,7 @@ class Movie(Item):
blank=True, blank=True,
default=list, default=list,
) )
language = jsondata.ArrayField( language = LanguageListField()
verbose_name=_("language"),
base_field=models.CharField(
blank=True,
default="",
max_length=100,
),
null=True,
blank=True,
default=list,
)
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True) year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
duration = jsondata.CharField( duration = jsondata.CharField(
verbose_name=_("length"), blank=True, default="", max_length=200 verbose_name=_("length"), blank=True, default="", max_length=200

View file

@ -4,6 +4,8 @@ from catalog.common import *
class DoubanMovieTestCase(TestCase): class DoubanMovieTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id = "3541415" t_id = "3541415"
t_url = "https://movie.douban.com/subject/3541415/" t_url = "https://movie.douban.com/subject/3541415/"
@ -28,6 +30,8 @@ class DoubanMovieTestCase(TestCase):
class TMDBMovieTestCase(TestCase): class TMDBMovieTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id = "293767" t_id = "293767"
t_url = ( t_url = (
@ -49,13 +53,17 @@ class TMDBMovieTestCase(TestCase):
self.assertEqual(site.ready, False) self.assertEqual(site.ready, False)
self.assertEqual(site.id_value, "293767") self.assertEqual(site.id_value, "293767")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.resource.metadata["title"], "比利·林恩的中场战事") self.assertEqual(
site.resource.metadata["title"], "Billy Lynn's Long Halftime Walk"
)
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.__class__.__name__, "Movie") self.assertEqual(site.resource.item.__class__.__name__, "Movie")
self.assertEqual(site.resource.item.imdb, "tt2513074") self.assertEqual(site.resource.item.imdb, "tt2513074")
class IMDBMovieTestCase(TestCase): class IMDBMovieTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id = "tt1375666" t_id = "tt1375666"
t_url = "https://www.imdb.com/title/tt1375666/" t_url = "https://www.imdb.com/title/tt1375666/"
@ -75,24 +83,28 @@ class IMDBMovieTestCase(TestCase):
self.assertEqual(site.ready, False) self.assertEqual(site.ready, False)
self.assertEqual(site.id_value, "tt1375666") self.assertEqual(site.id_value, "tt1375666")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.resource.metadata["title"], "盗梦空间") self.assertEqual(site.resource.metadata["title"], "Inception")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.imdb, "tt1375666") self.assertEqual(site.resource.item.imdb, "tt1375666")
class BangumiMovieTestCase(TestCase): class BangumiMovieTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_scrape(self): def test_scrape(self):
url = "https://bgm.tv/subject/237" url = "https://bgm.tv/subject/237"
site = SiteManager.get_site_by_url(url) site = SiteManager.get_site_by_url(url)
self.assertEqual(site.id_value, "237") self.assertEqual(site.id_value, "237")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.resource.metadata["title"], "攻壳机动队") self.assertEqual(site.resource.item.display_title, "GHOST IN THE SHELL")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.imdb, "tt0113568") self.assertEqual(site.resource.item.imdb, "tt0113568")
class MultiMovieSitesTestCase(TestCase): class MultiMovieSitesTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_movies(self): def test_movies(self):
url1 = "https://www.themoviedb.org/movie/27205-inception" url1 = "https://www.themoviedb.org/movie/27205-inception"

View file

@ -41,12 +41,14 @@ class Album(Item):
douban_music = PrimaryLookupIdDescriptor(IdType.DoubanMusic) douban_music = PrimaryLookupIdDescriptor(IdType.DoubanMusic)
spotify_album = PrimaryLookupIdDescriptor(IdType.Spotify_Album) spotify_album = PrimaryLookupIdDescriptor(IdType.Spotify_Album)
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"other_title", # "other_title",
"localized_title",
"artist", "artist",
"company", "company",
"track_list", "track_list",
"brief", # "brief",
"localized_description",
"album_type", "album_type",
"media", "media",
"disc_count", "disc_count",

View file

@ -69,7 +69,7 @@ class DoubanMusicTestCase(TestCase):
self.assertIsInstance(site.resource.item, Album) self.assertIsInstance(site.resource.item, Album)
self.assertEqual(site.resource.item.barcode, "0077774644020") self.assertEqual(site.resource.item.barcode, "0077774644020")
self.assertEqual(site.resource.item.genre, ["摇滚"]) self.assertEqual(site.resource.item.genre, ["摇滚"])
self.assertEqual(site.resource.item.other_title, ["橡胶灵魂"]) self.assertEqual(len(site.resource.item.localized_title), 2)
class MultiMusicSitesTestCase(TestCase): class MultiMusicSitesTestCase(TestCase):

View file

@ -14,6 +14,7 @@ from catalog.common import (
ItemType, ItemType,
jsondata, jsondata,
) )
from catalog.common.models import LanguageListField
from catalog.common.utils import DEFAULT_ITEM_COVER from catalog.common.utils import DEFAULT_ITEM_COVER
@ -124,13 +125,7 @@ class Performance(Item):
blank=False, blank=False,
default=list, default=list,
) )
language = jsondata.ArrayField( language = LanguageListField()
verbose_name=_("language"),
base_field=models.CharField(blank=False, default="", max_length=200),
null=False,
blank=True,
default=list,
)
director = jsondata.ArrayField( director = jsondata.ArrayField(
verbose_name=_("director"), verbose_name=_("director"),
base_field=models.CharField(blank=False, default="", max_length=500), base_field=models.CharField(blank=False, default="", max_length=500),
@ -211,10 +206,12 @@ class Performance(Item):
verbose_name=_("website"), max_length=1000, null=True, blank=True verbose_name=_("website"), max_length=1000, null=True, blank=True
) )
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"brief", # "brief",
"localized_title",
"localized_description",
"orig_title", "orig_title",
"other_title", # "other_title",
"genre", "genre",
"language", "language",
"opening_date", "opening_date",
@ -266,13 +263,7 @@ class PerformanceProduction(Item):
blank=True, blank=True,
default=list, default=list,
) )
language = jsondata.ArrayField( language = LanguageListField()
verbose_name=_("language"),
base_field=models.CharField(blank=False, default="", max_length=200),
null=False,
blank=True,
default=list,
)
director = jsondata.ArrayField( director = jsondata.ArrayField(
verbose_name=_("director"), verbose_name=_("director"),
base_field=models.CharField(blank=False, default="", max_length=500), base_field=models.CharField(blank=False, default="", max_length=500),
@ -353,10 +344,12 @@ class PerformanceProduction(Item):
verbose_name=_("website"), max_length=1000, null=True, blank=True verbose_name=_("website"), max_length=1000, null=True, blank=True
) )
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", "localized_title",
"brief", "localized_description",
# "title",
# "brief",
"orig_title", "orig_title",
"other_title", # "other_title",
"language", "language",
"opening_date", "opening_date",
"closing_date", "closing_date",
@ -389,7 +382,9 @@ class PerformanceProduction(Item):
@property @property
def display_title(self): def display_title(self):
return f"{self.show.title if self.show else ''} {self.title}" return (
f"{self.show.display_title if self.show else ''} {super().display_title}"
)
@property @property
def cover_image_url(self) -> str | None: def cover_image_url(self) -> str | None:

View file

@ -5,6 +5,8 @@ from catalog.common.sites import crawl_related_resources_task
class DoubanDramaTestCase(TestCase): class DoubanDramaTestCase(TestCase):
databases = "__all__"
def setUp(self): def setUp(self):
pass pass
@ -29,10 +31,10 @@ class DoubanDramaTestCase(TestCase):
site = SiteManager.get_site_by_url(t_url) site = SiteManager.get_site_by_url(t_url)
resource = site.get_resource_ready() resource = site.get_resource_ready()
item = site.get_item() item = site.get_item()
self.assertEqual(item.title, "不眠之人·拿破仑")
self.assertEqual( self.assertEqual(
item.other_title, ["眠らない男・ナポレオン ―愛と栄光の涯(はて)に―"] item.display_title, "眠らない男・ナポレオン ―愛と栄光の涯(はて)に―"
) )
self.assertEqual(len(item.localized_title), 2)
self.assertEqual(item.genre, ["音乐剧"]) self.assertEqual(item.genre, ["音乐剧"])
self.assertEqual(item.troupe, ["宝塚歌剧团"]) self.assertEqual(item.troupe, ["宝塚歌剧团"])
self.assertEqual(item.composer, ["ジェラール・プレスギュルヴィック"]) self.assertEqual(item.composer, ["ジェラール・プレスギュルヴィック"])
@ -41,7 +43,7 @@ class DoubanDramaTestCase(TestCase):
site = SiteManager.get_site_by_url(t_url) site = SiteManager.get_site_by_url(t_url)
resource = site.get_resource_ready() resource = site.get_resource_ready()
item = site.get_item() item = site.get_item()
self.assertEqual(item.title, "相声说垮鬼子们") self.assertEqual(item.display_title, "相聲說垮鬼子們")
self.assertEqual(item.opening_date, "1997-05") self.assertEqual(item.opening_date, "1997-05")
self.assertEqual(item.location, ["臺北新舞臺"]) self.assertEqual(item.location, ["臺北新舞臺"])
@ -54,7 +56,8 @@ class DoubanDramaTestCase(TestCase):
if item is None: if item is None:
raise ValueError() raise ValueError()
self.assertEqual(item.orig_title, "Iphigenie auf Tauris") self.assertEqual(item.orig_title, "Iphigenie auf Tauris")
self.assertEqual(sorted(item.other_title), ["死而复生的伊菲格尼"]) print(item.localized_title)
self.assertEqual(len(item.localized_title), 3)
self.assertEqual(item.opening_date, "1974-04-21") self.assertEqual(item.opening_date, "1974-04-21")
self.assertEqual(item.choreographer, ["Pina Bausch"]) self.assertEqual(item.choreographer, ["Pina Bausch"])
@ -68,9 +71,9 @@ class DoubanDramaTestCase(TestCase):
item = site.get_item() item = site.get_item()
if item is None: if item is None:
raise ValueError() raise ValueError()
self.assertEqual(item.title, "红花侠") self.assertEqual(item.display_title, "THE SCARLET PIMPERNEL")
self.assertEqual(sorted(item.other_title), ["THE SCARLET PIMPERNEL"]) self.assertEqual(len(item.localized_title), 3)
self.assertEqual(len(item.brief), 545) self.assertEqual(len(item.display_description), 545)
self.assertEqual(item.genre, ["音乐剧"]) self.assertEqual(item.genre, ["音乐剧"])
# self.assertEqual( # self.assertEqual(
# item.version, ["08星组公演版", "10年月組公演版", "17年星組公演版", "ュージカル2017年版"] # item.version, ["08星组公演版", "10年月組公演版", "17年星組公演版", "ュージカル2017年版"]
@ -80,7 +83,7 @@ class DoubanDramaTestCase(TestCase):
item.playwright, ["小池修一郎", "Baroness Orczy原作", "小池 修一郎"] item.playwright, ["小池修一郎", "Baroness Orczy原作", "小池 修一郎"]
) )
self.assertEqual( self.assertEqual(
item.actor, sorted(item.actor, key=lambda a: a["name"]),
[ [
{"name": "安蘭けい", "role": ""}, {"name": "安蘭けい", "role": ""},
{"name": "柚希礼音", "role": ""}, {"name": "柚希礼音", "role": ""},
@ -110,7 +113,10 @@ class DoubanDramaTestCase(TestCase):
self.assertEqual(productions[2].closing_date, "2017-03-17") self.assertEqual(productions[2].closing_date, "2017-03-17")
self.assertEqual(productions[3].opening_date, "2017-11-13") self.assertEqual(productions[3].opening_date, "2017-11-13")
self.assertEqual(productions[3].closing_date, None) self.assertEqual(productions[3].closing_date, None)
self.assertEqual(productions[3].title, "ミュージカル2017年") self.assertEqual(
productions[3].display_title,
"THE SCARLET PIMPERNEL ミュージカル2017年",
)
self.assertEqual(len(productions[3].actor), 6) self.assertEqual(len(productions[3].actor), 6)
self.assertEqual(productions[3].language, ["日语"]) self.assertEqual(productions[3].language, ["日语"])
self.assertEqual(productions[3].opening_date, "2017-11-13") self.assertEqual(productions[3].opening_date, "2017-11-13")

View file

@ -15,6 +15,7 @@ from catalog.common import (
PrimaryLookupIdDescriptor, PrimaryLookupIdDescriptor,
jsondata, jsondata,
) )
from catalog.common.models import LanguageListField
class PodcastInSchema(ItemInSchema): class PodcastInSchema(ItemInSchema):
@ -44,6 +45,8 @@ class Podcast(Item):
default=list, default=list,
) )
language = LanguageListField()
hosts = jsondata.ArrayField( hosts = jsondata.ArrayField(
verbose_name=_("host"), verbose_name=_("host"),
base_field=models.CharField(blank=True, default="", max_length=200), base_field=models.CharField(blank=True, default="", max_length=200),
@ -55,8 +58,11 @@ class Podcast(Item):
) )
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"brief", # "brief",
"localized_title",
"localized_description",
"language",
"hosts", "hosts",
"genre", "genre",
"official_site", "official_site",

View file

@ -91,7 +91,7 @@ class PodcastRSSFeedTestCase(TestCase):
metadata["official_site"], "http://www.bbc.co.uk/programmes/b006qykl" metadata["official_site"], "http://www.bbc.co.uk/programmes/b006qykl"
) )
self.assertEqual(metadata["genre"], ["History"]) self.assertEqual(metadata["genre"], ["History"])
self.assertEqual(metadata["hosts"], ["BBC Radio 4"]) self.assertEqual(metadata["host"], ["BBC Radio 4"])
self.assertIsNotNone(site.get_item().recent_episodes[0].title) self.assertIsNotNone(site.get_item().recent_episodes[0].title)
self.assertIsNotNone(site.get_item().recent_episodes[0].link) self.assertIsNotNone(site.get_item().recent_episodes[0].link)
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url) self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
@ -108,7 +108,7 @@ class PodcastRSSFeedTestCase(TestCase):
metadata["official_site"], "https://www.ximalaya.com/qita/51101122/" metadata["official_site"], "https://www.ximalaya.com/qita/51101122/"
) )
self.assertEqual(metadata["genre"], ["人文国学"]) self.assertEqual(metadata["genre"], ["人文国学"])
self.assertEqual(metadata["hosts"], ["看理想vistopia"]) self.assertEqual(metadata["host"], ["看理想vistopia"])
self.assertIsNotNone(site.get_item().recent_episodes[0].title) self.assertIsNotNone(site.get_item().recent_episodes[0].title)
self.assertIsNotNone(site.get_item().recent_episodes[0].link) self.assertIsNotNone(site.get_item().recent_episodes[0].link)
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url) self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
@ -123,7 +123,7 @@ class PodcastRSSFeedTestCase(TestCase):
self.assertEqual(metadata["title"], "跳岛FM") self.assertEqual(metadata["title"], "跳岛FM")
self.assertEqual(metadata["official_site"], "https://tiaodao.typlog.io/") self.assertEqual(metadata["official_site"], "https://tiaodao.typlog.io/")
self.assertEqual(metadata["genre"], ["Arts", "Books"]) self.assertEqual(metadata["genre"], ["Arts", "Books"])
self.assertEqual(metadata["hosts"], ["中信出版·大方"]) self.assertEqual(metadata["host"], ["中信出版·大方"])
self.assertIsNotNone(site.get_item().recent_episodes[0].title) self.assertIsNotNone(site.get_item().recent_episodes[0].title)
self.assertIsNotNone(site.get_item().recent_episodes[0].link) self.assertIsNotNone(site.get_item().recent_episodes[0].link)
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url) self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)

View file

@ -11,7 +11,7 @@ from lxml import html
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from catalog.sites.spotify import get_spotify_token from catalog.sites.spotify import get_spotify_token
from catalog.sites.tmdb import get_language_code from catalog.sites.tmdb import TMDB_DEFAULT_LANG
SEARCH_PAGE_SIZE = 5 # not all apis support page size SEARCH_PAGE_SIZE = 5 # not all apis support page size
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -173,7 +173,7 @@ class TheMovieDatabase:
@classmethod @classmethod
def search(cls, q, page=1): def search(cls, q, page=1):
results = [] results = []
api_url = f"https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&include_adult=true" api_url = f"https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&include_adult=true"
try: try:
j = requests.get(api_url, timeout=2).json() j = requests.get(api_url, timeout=2).json()
if j.get("results"): if j.get("results"):

View file

@ -10,11 +10,18 @@ Scraping the website directly.
import json import json
import logging import logging
from threading import local
import dateparser import dateparser
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import (
DEFAULT_CATALOG_LANGUAGE,
PREFERRED_LANGUAGES,
detect_language,
)
from common.models.misc import uniq
from .douban import * from .douban import *
@ -47,27 +54,58 @@ class AppleMusic(AbstractSite):
def id_to_url(cls, id_value): def id_to_url(cls, id_value):
return f"https://music.apple.com/album/{id_value}" return f"https://music.apple.com/album/{id_value}"
def get_localized_urls(self): def get_locales(self):
return [ locales = {}
f"https://music.apple.com/{locale}/album/{self.id_value}" for l in PREFERRED_LANGUAGES:
for locale in ["hk", "tw", "us", "sg", "jp", "cn", "gb", "ca", "fr"] match l:
] case "zh":
locales.update({"zh": ["cn", "tw", "hk", "sg"]})
case "en":
locales.update({"en": ["us", "gb", "ca"]})
case "ja":
locales.update({"ja": ["jp"]})
case "ko":
locales.update({"ko": ["kr"]})
case "fr":
locales.update({"fr": ["fr", "ca"]})
if not locales:
locales = {"en": ["us"]}
return locales
def scrape(self): def scrape(self):
content = None matched_content = None
# it's less than ideal to waterfall thru locales, a better solution localized_title = []
# would be change ExternalResource to store preferred locale, localized_desc = []
# or to find an AppleMusic API to get available locales for an album for lang, locales in self.get_locales().items():
for url in self.get_localized_urls(): for loc in locales: # waterfall thru all locales
url = f"https://music.apple.com/{loc}/album/{self.id_value}"
try: try:
content = BasicDownloader(url, headers=self.headers).download().html() content = (
BasicDownloader(url, headers=self.headers).download().html()
)
_logger.info(f"got localized content from {url}") _logger.info(f"got localized content from {url}")
elem = content.xpath(
"//script[@id='serialized-server-data']/text()"
)
txt: str = elem[0] # type:ignore
page_data = json.loads(txt)[0]
album_data = page_data["data"]["sections"][0]["items"][0]
title = album_data["title"]
brief = album_data.get("modalPresentationDescriptor", {}).get(
"paragraphText", ""
)
l = detect_language(title + " " + brief)
localized_title.append({"lang": l, "text": title})
if brief:
localized_desc.append({"lang": l, "text": brief})
if lang == DEFAULT_CATALOG_LANGUAGE or not matched_content:
matched_content = content
break break
except Exception: except Exception:
pass pass
if content is None: if matched_content is None:
raise ParseError(self, f"localized content for {self.url}") raise ParseError(self, f"localized content for {self.url}")
elem = content.xpath("//script[@id='serialized-server-data']/text()") elem = matched_content.xpath("//script[@id='serialized-server-data']/text()")
txt: str = elem[0] # type:ignore txt: str = elem[0] # type:ignore
page_data = json.loads(txt)[0] page_data = json.loads(txt)[0]
album_data = page_data["data"]["sections"][0]["items"][0] album_data = page_data["data"]["sections"][0]["items"][0]
@ -99,12 +137,14 @@ class AppleMusic(AbstractSite):
genre[0] genre[0]
] # apple treat "Music" as a genre. Thus, only the first genre is obtained. ] # apple treat "Music" as a genre. Thus, only the first genre is obtained.
images = content.xpath("//source[@type='image/jpeg']/@srcset") images = matched_content.xpath("//source[@type='image/jpeg']/@srcset")
image_elem: str = images[0] if images else "" # type:ignore image_elem: str = images[0] if images else "" # type:ignore
image_url = image_elem.split(" ")[0] if image_elem else None image_url = image_elem.split(" ")[0] if image_elem else None
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": uniq(localized_title),
"localized_description": uniq(localized_desc),
"title": title, "title": title,
"brief": brief, "brief": brief,
"artist": artist, "artist": artist,

View file

@ -26,11 +26,12 @@ class ApplePodcast(AbstractSite):
resp = dl.download() resp = dl.download()
r = resp.json()["results"][0] r = resp.json()["results"][0]
feed_url = r["feedUrl"] feed_url = r["feedUrl"]
title = r["trackName"]
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"title": r["trackName"], "title": title,
"feed_url": feed_url, "feed_url": feed_url,
"hosts": [r["artistName"]], "host": [r["artistName"]],
"genres": r["genres"], "genres": r["genres"],
"cover_image_url": r["artworkUrl600"], "cover_image_url": r["artworkUrl600"],
} }

View file

@ -8,6 +8,7 @@ import dns.resolver
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import detect_language
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -76,14 +77,19 @@ class Bandcamp(AbstractSite):
duration = None duration = None
company = None company = None
brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()") brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()")
brief = "".join(brief_nodes) if brief_nodes else None # type:ignore brief = "".join(brief_nodes) if brief_nodes else "" # type:ignore
cover_url = self.query_str(content, "//div[@id='tralbumArt']/a/@href") cover_url = self.query_str(content, "//div[@id='tralbumArt']/a/@href")
bandcamp_page_data = json.loads( bandcamp_page_data = json.loads(
self.query_str(content, "//meta[@name='bc-page-properties']/@content") self.query_str(content, "//meta[@name='bc-page-properties']/@content")
) )
bandcamp_album_id = bandcamp_page_data["item_id"] bandcamp_album_id = bandcamp_page_data["item_id"]
localized_title = [{"lang": detect_language(title), "text": title}]
localized_desc = (
[{"lang": detect_language(brief), "text": brief}] if brief else []
)
data = { data = {
"localized_title": localized_title,
"localized_description": localized_desc,
"title": title, "title": title,
"artist": artist, "artist": artist,
"genre": genre, "genre": genre,

View file

@ -3,6 +3,7 @@ import logging
from catalog.book.utils import detect_isbn_asin from catalog.book.utils import detect_isbn_asin
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import detect_language
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -101,7 +102,14 @@ class Bangumi(AbstractSite):
raw_img, ext = BasicImageDownloader.download_image( raw_img, ext = BasicImageDownloader.download_image(
img_url, None, headers={} img_url, None, headers={}
) )
titles = set(
[title] + (other_title or []) + ([orig_title] if orig_title else [])
)
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
localized_desc = [{"lang": detect_language(brief), "text": brief}]
data = { data = {
"localized_title": localized_title,
"localized_description": localized_desc,
"preferred_model": model, "preferred_model": model,
"title": title, "title": title,
"orig_title": orig_title, "orig_title": orig_title,

View file

@ -11,13 +11,7 @@ from loguru import logger
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import detect_language
def _lang(s: str) -> str:
try:
return detect(s)
except Exception:
return "en"
@SiteManager.register @SiteManager.register
@ -43,14 +37,13 @@ class BoardGameGeek(AbstractSite):
item = items[0] item = items[0]
title = self.query_str(item, "name[@type='primary']/@value") title = self.query_str(item, "name[@type='primary']/@value")
other_title = self.query_list(item, "name[@type='alternate']/@value") other_title = self.query_list(item, "name[@type='alternate']/@value")
zh_title = [ localized_title = [
t for t in other_title if _lang(t) in ["zh", "jp", "ko", "zh-cn", "zh-tw"] {"lang": detect_language(t), "text": t} for t in [title] + other_title
] ]
if zh_title: zh_title = [
for z in zh_title: t["text"] for t in localized_title if t["lang"] in ["zh", "zh-cn", "zh-tw"]
other_title.remove(z) ]
other_title = zh_title + other_title title = zh_title[0] if zh_title else other_title[0]
cover_image_url = self.query_str(item, "image/text()") cover_image_url = self.query_str(item, "image/text()")
brief = html.unescape(self.query_str(item, "description/text()")) brief = html.unescape(self.query_str(item, "description/text()"))
year = self.query_str(item, "yearpublished/@value") year = self.query_str(item, "yearpublished/@value")
@ -62,6 +55,8 @@ class BoardGameGeek(AbstractSite):
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": localized_title,
"localized_description": [{"lang": "en", "text": brief}],
"title": title, "title": title,
"other_title": other_title, "other_title": other_title,
"genre": category, "genre": category,

View file

@ -3,6 +3,7 @@ import logging
from catalog.book.models import * from catalog.book.models import *
from catalog.book.utils import * from catalog.book.utils import *
from catalog.common import * from catalog.common import *
from common.models.lang import detect_language
from .douban import * from .douban import *
@ -40,7 +41,7 @@ class BooksTW(AbstractSite):
if not title: if not title:
raise ParseError(self, "title") raise ParseError(self, "title")
subtitle = None subtitle = None
orig_title = content.xpath("string(//h1/following-sibling::h2)") orig_title = str(content.xpath("string(//h1/following-sibling::h2)"))
authors = content.xpath("string(//div/ul/li[contains(text(),'作者:')])") authors = content.xpath("string(//div/ul/li[contains(text(),'作者:')])")
authors = authors.strip().split("", 1)[1].split(",") if authors else [] # type: ignore authors = authors.strip().split("", 1)[1].split(",") if authors else [] # type: ignore
@ -116,9 +117,14 @@ class BooksTW(AbstractSite):
"string(//div[contains(@class,'cover_img')]//img[contains(@class,'cover')]/@src)" "string(//div[contains(@class,'cover_img')]//img[contains(@class,'cover')]/@src)"
) )
img_url = re.sub(r"&[wh]=\d+", "", img_url) if img_url else None # type: ignore img_url = re.sub(r"&[wh]=\d+", "", img_url) if img_url else None # type: ignore
localized_title = [{"lang": "zh-tw", "text": title}]
if orig_title:
localized_title.append(
{"lang": detect_language(orig_title), "text": orig_title}
)
data = { data = {
"title": title, "title": title,
"localized_title": localized_title,
"subtitle": subtitle, "subtitle": subtitle,
"orig_title": orig_title, "orig_title": orig_title,
"author": authors, "author": authors,

View file

@ -63,10 +63,11 @@ class DiscogsRelease(AbstractSite):
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"title": title, "title": title,
"localized_title": [{"lang": "en", "text": title}],
"artist": artist, "artist": artist,
"genre": genre, "genre": genre,
"track_list": "\n".join(track_list), "track_list": "\n".join(track_list),
"release_date": None, # only year provided by API # "release_date": None, # only year provided by API
"company": company, "company": company,
"media": media, "media": media,
"disc_count": disc_count, "disc_count": disc_count,

View file

@ -6,6 +6,7 @@ from lxml import html
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import detect_language
from .douban import DoubanDownloader from .douban import DoubanDownloader
@ -64,6 +65,7 @@ class DoubanDramaVersion(AbstractSite):
raise ParseError(self, "title") raise ParseError(self, "title")
data = { data = {
"title": title, "title": title,
"localized_title": [{"lang": "zh-cn", "text": title}],
"director": [x.strip() for x in h.xpath(q.format("导演"))], "director": [x.strip() for x in h.xpath(q.format("导演"))],
"playwright": [x.strip() for x in h.xpath(q.format("编剧"))], "playwright": [x.strip() for x in h.xpath(q.format("编剧"))],
# "actor": [x.strip() for x in h.xpath(q.format("主演"))], # "actor": [x.strip() for x in h.xpath(q.format("主演"))],
@ -238,6 +240,21 @@ class DoubanDrama(AbstractSite):
) )
img_url_elem = h.xpath("//img[@itemprop='image']/@src") img_url_elem = h.xpath("//img[@itemprop='image']/@src")
data["cover_image_url"] = img_url_elem[0].strip() if img_url_elem else None data["cover_image_url"] = img_url_elem[0].strip() if img_url_elem else None
data["localized_title"] = (
[{"lang": "zh-cn", "text": data["title"]}]
+ (
[
{
"lang": detect_language(data["orig_title"]),
"text": data["orig_title"],
}
]
if data["orig_title"]
else []
)
+ [{"lang": detect_language(t), "text": t} for t in data["other_title"]]
)
data["localized_description"] = [{"lang": "zh-cn", "text": data["brief"]}]
pd = ResourceContent(metadata=data) pd = ResourceContent(metadata=data)
if pd.metadata["cover_image_url"]: if pd.metadata["cover_image_url"]:

View file

@ -4,6 +4,7 @@ import dateparser
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import detect_language
from .douban import DoubanDownloader from .douban import DoubanDownloader
@ -34,51 +35,68 @@ class DoubanGame(AbstractSite):
if not title: if not title:
raise ParseError(self, "title") raise ParseError(self, "title")
elem = content.xpath("//div[@id='comments']//h2/text()")
title2 = elem[0].strip() if len(elem) else ""
if title2:
sp = title2.strip().rsplit("的短评", 1)
title2 = sp[0] if len(sp) > 1 else ""
if title2 and title.startswith(title2):
orig_title = title[len(title2) :].strip()
title = title2
else:
orig_title = ""
other_title_elem = content.xpath( other_title_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()" "//dl[@class='thing-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()"
) )
other_title = ( other_title = (
other_title_elem[0].strip().split(" / ") if other_title_elem else None other_title_elem[0].strip().split(" / ") if other_title_elem else []
) )
developer_elem = content.xpath( developer_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()" "//dl[@class='thing-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()"
) )
developer = developer_elem[0].strip().split(" / ") if developer_elem else None developer = developer_elem[0].strip().split(" / ") if developer_elem else None
publisher_elem = content.xpath( publisher_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()" "//dl[@class='thing-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()"
) )
publisher = publisher_elem[0].strip().split(" / ") if publisher_elem else None publisher = publisher_elem[0].strip().split(" / ") if publisher_elem else None
platform_elem = content.xpath( platform_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()" "//dl[@class='thing-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()"
) )
platform = platform_elem if platform_elem else None platform = platform_elem if platform_elem else None
genre_elem = content.xpath( genre_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()" "//dl[@class='thing-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()"
) )
genre = None genre = None
if genre_elem: if genre_elem:
genre = [g for g in genre_elem if g != "游戏"] genre = [g for g in genre_elem if g != "游戏"]
date_elem = content.xpath( date_elem = content.xpath(
"//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()" "//dl[@class='thing-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()"
) )
release_date = dateparser.parse(date_elem[0].strip()) if date_elem else None release_date = dateparser.parse(date_elem[0].strip()) if date_elem else None
release_date = release_date.strftime("%Y-%m-%d") if release_date else None release_date = release_date.strftime("%Y-%m-%d") if release_date else None
brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()") brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()")
brief = "\n".join(brief_elem) if brief_elem else None brief = "\n".join(brief_elem) if brief_elem else ""
img_url_elem = content.xpath( img_url_elem = content.xpath(
"//div[@class='item-subject-info']/div[@class='pic']//img/@src" "//div[@class='item-subject-info']/div[@class='pic']//img/@src"
) )
img_url = img_url_elem[0].strip() if img_url_elem else None img_url = img_url_elem[0].strip() if img_url_elem else None
titles = set([title] + other_title + ([orig_title] if orig_title else []))
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
localized_desc = [{"lang": detect_language(brief), "text": brief}]
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": localized_title,
"localized_description": localized_desc,
"title": title, "title": title,
"other_title": other_title, "other_title": other_title,
"developer": developer, "developer": developer,

View file

@ -4,6 +4,7 @@ import logging
from catalog.common import * from catalog.common import *
from catalog.movie.models import * from catalog.movie.models import *
from catalog.tv.models import * from catalog.tv.models import *
from common.models.lang import detect_language
from .douban import * from .douban import *
from .tmdb import TMDB_TV, TMDB_TVSeason, query_tmdb_tv_episode, search_tmdb_by_imdb_id from .tmdb import TMDB_TV, TMDB_TVSeason, query_tmdb_tv_episode, search_tmdb_by_imdb_id
@ -205,9 +206,18 @@ class DoubanMovie(AbstractSite):
img_url_elem = content.xpath("//img[@rel='v:image']/@src") img_url_elem = content.xpath("//img[@rel='v:image']/@src")
img_url = img_url_elem[0].strip() if img_url_elem else None img_url = img_url_elem[0].strip() if img_url_elem else None
titles = set(
[title]
+ ([orig_title] if orig_title else [])
+ (other_title if other_title else [])
)
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
localized_desc = [{"lang": detect_language(brief), "text": brief}]
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"title": title, "title": title,
"localized_title": localized_title,
"localized_description": localized_desc,
"orig_title": orig_title, "orig_title": orig_title,
"other_title": other_title, "other_title": other_title,
"imdb_code": imdb_code, "imdb_code": imdb_code,

View file

@ -5,6 +5,7 @@ import dateparser
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from catalog.music.utils import upc_to_gtin_13 from catalog.music.utils import upc_to_gtin_13
from common.models.lang import detect_language
from .douban import DoubanDownloader from .douban import DoubanDownloader
@ -77,9 +78,19 @@ class DoubanMusic(AbstractSite):
img_url_elem = content.xpath("//div[@id='mainpic']//img/@src") img_url_elem = content.xpath("//div[@id='mainpic']//img/@src")
img_url = img_url_elem[0].strip() if img_url_elem else None img_url = img_url_elem[0].strip() if img_url_elem else None
other_elem = content.xpath(
"//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]"
)
other_title = other_elem[0].strip().split(" / ") if other_elem else []
lang = detect_language(f"{title} {brief}")
localized_title = [{"lang": lang, "text": title}]
localized_title += [
{"lang": detect_language(t), "text": t} for t in other_title
]
data = { data = {
"title": title, "title": title,
"localized_title": localized_title,
"localized_description": [{"lang": lang, "text": brief}],
"artist": artist, "artist": artist,
"genre": genre, "genre": genre,
"release_date": release_date, "release_date": release_date,
@ -91,11 +102,6 @@ class DoubanMusic(AbstractSite):
} }
gtin = None gtin = None
isrc = None isrc = None
other_elem = content.xpath(
"//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]"
)
if other_elem:
data["other_title"] = other_elem[0].strip().split(" / ")
other_elem = content.xpath( other_elem = content.xpath(
"//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]" "//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]"
) )

View file

@ -129,6 +129,8 @@ class IGDB(AbstractSite):
steam_url = website["url"] steam_url = website["url"]
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": [{"lang": "en", "text": r["name"]}],
"localized_description": [{"lang": "en", "text": brief}],
"title": r["name"], "title": r["name"],
"other_title": [], "other_title": [],
"developer": [developer] if developer else [], "developer": [developer] if developer else [],

View file

@ -91,6 +91,8 @@ class IMDB(AbstractSite):
d["primaryImage"].get("url") if d.get("primaryImage") else None d["primaryImage"].get("url") if d.get("primaryImage") else None
), ),
} }
data["localized_title"] = [{"lang": "en", "text": data["title"]}]
data["localized_description"] = [{"lang": "en", "text": data["brief"]}]
if d.get("series"): if d.get("series"):
episode_info = d["series"].get("episodeNumber") episode_info = d["series"].get("episodeNumber")
if episode_info: if episode_info:
@ -133,14 +135,11 @@ class IMDB(AbstractSite):
url = f"https://m.imdb.com{show_url}episodes/?season={season_id}" url = f"https://m.imdb.com{show_url}episodes/?season={season_id}"
h = BasicDownloader(url).download().html() h = BasicDownloader(url).download().html()
episodes = [] episodes = []
for e in h.xpath('//div[@id="eplist"]/div/a'): # type: ignore for e in h.xpath('//article//a[@class="ipc-title-link-wrapper"]'): # type: ignore
episode_number = e.xpath( title = e.xpath('div[@class="ipc-title__text"]/text()')[0].split("", 1)
'./span[contains(@class,"episode-list__title")]/text()' episode_id = title[0].strip()
)[0].strip() episode_number = int(episode_id.split(".")[1][1:])
episode_number = int(episode_number.split(".")[0]) episode_title = title[1].strip()
episode_title = " ".join(
e.xpath('.//strong[@class="episode-list__title-text"]/text()')
).strip()
episode_url = e.xpath("./@href")[0] episode_url = e.xpath("./@href")[0]
episode_url = "https://www.imdb.com" + episode_url episode_url = "https://www.imdb.com" + episode_url
episodes.append( episodes.append(

View file

@ -19,6 +19,8 @@ from catalog.common.downloaders import (
) )
from catalog.models import * from catalog.models import *
from catalog.podcast.models import PodcastEpisode from catalog.podcast.models import PodcastEpisode
from common.models.lang import detect_language
from journal.models.renderers import html_to_text
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -86,11 +88,16 @@ class RSS(AbstractSite):
feed = self.parse_feed_from_url(self.url) feed = self.parse_feed_from_url(self.url)
if not feed: if not feed:
raise ValueError(f"no feed avaialble in {self.url}") raise ValueError(f"no feed avaialble in {self.url}")
title = feed["title"]
desc = html_to_text(feed["description"])
lang = detect_language(title + " " + desc)
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"title": feed["title"], "title": title,
"brief": bleach.clean(feed["description"], strip=True), "brief": desc,
"hosts": ( "localized_title": [{"lang": lang, "text": title}],
"localized_description": [{"lang": lang, "text": desc}],
"host": (
[feed.get("itunes_author")] if feed.get("itunes_author") else [] [feed.get("itunes_author")] if feed.get("itunes_author") else []
), ),
"official_site": feed.get("link"), "official_site": feed.get("link"),

View file

@ -13,6 +13,7 @@ from django.conf import settings
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from catalog.music.utils import upc_to_gtin_13 from catalog.music.utils import upc_to_gtin_13
from common.models.lang import detect_language
from .douban import * from .douban import *
@ -83,10 +84,11 @@ class Spotify(AbstractSite):
isrc = None isrc = None
if res_data["external_ids"].get("isrc"): if res_data["external_ids"].get("isrc"):
isrc = res_data["external_ids"].get("isrc") isrc = res_data["external_ids"].get("isrc")
lang = detect_language(title)
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"title": title, "title": title,
"localized_title": [{"lang": lang, "text": title}],
"artist": artist, "artist": artist,
"genre": genre, "genre": genre,
"track_list": track_list, "track_list": track_list,

View file

@ -1,15 +1,33 @@
import logging import logging
import re
import dateparser import dateparser
from django.conf import settings
from catalog.common import * from catalog.common import *
from catalog.models import * from catalog.models import *
from common.models.lang import PREFERRED_LANGUAGES
from journal.models.renderers import html_to_text
from .igdb import search_igdb_by_3p_url from .igdb import search_igdb_by_3p_url
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
def _get_preferred_languages():
langs = {}
for l in PREFERRED_LANGUAGES:
if l == "zh":
langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW"})
# zh-HK data is not good
else:
langs[l] = l
return langs
STEAM_PREFERRED_LANGS = _get_preferred_languages()
@SiteManager.register @SiteManager.register
class Steam(AbstractSite): class Steam(AbstractSite):
SITE_NAME = SiteName.Steam SITE_NAME = SiteName.Steam
@ -22,69 +40,72 @@ class Steam(AbstractSite):
def id_to_url(cls, id_value): def id_to_url(cls, id_value):
return "https://store.steampowered.com/app/" + str(id_value) return "https://store.steampowered.com/app/" + str(id_value)
def download(self, lang):
api_url = (
f"https://store.steampowered.com/api/appdetails?appids={self.id_value}"
)
headers = {
"User-Agent": settings.NEODB_USER_AGENT,
"Accept": "application/json",
"Accept-Language": STEAM_PREFERRED_LANGS[lang],
}
return BasicDownloader(api_url, headers=headers).download().json()
def scrape(self): def scrape(self):
i = search_igdb_by_3p_url(self.url) i = search_igdb_by_3p_url(self.url)
pd = i.scrape() if i else ResourceContent() pd = i.scrape() if i else ResourceContent()
headers = BasicDownloader.headers.copy() en_data = {}
headers["Host"] = "store.steampowered.com" localized_title = []
headers["Cookie"] = "wants_mature_content=1; birthtime=754700401;" localized_desc = []
content = BasicDownloader(self.url, headers=headers).download().html() for lang in STEAM_PREFERRED_LANGS.keys():
data = self.download(lang).get(self.id_value, {}).get("data", {})
title = self.query_str(content, "//div[@class='apphub_AppName']/text()") if lang == "en":
developer = content.xpath("//div[@id='developers_list']/a/text()") en_data = data
publisher = content.xpath( localized_title.append({"lang": lang, "text": data["name"]})
"//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()" desc = html_to_text(data["detailed_description"])
) localized_desc.append({"lang": lang, "text": desc})
dts = self.query_str( if not en_data:
content, "//div[@class='release_date']/div[@class='date']/text()" en_data = self.download("en")
) if not en_data:
dt = dateparser.parse(dts.replace(" ", "")) if dts else None raise ParseError(self, "id")
release_date = dt.strftime("%Y-%m-%d") if dt else None
genre = content.xpath(
"//div[@class='details_block']/b[2]/following-sibling::a/text()"
)
platform = ["PC"]
try:
brief = self.query_str(
content, "//div[@class='game_description_snippet']/text()"
)
except Exception:
brief = ""
# try Steam images if no image from IGDB
if pd.cover_image is None:
pd.metadata["cover_image_url"] = self.query_str(
content, "//img[@class='game_header_image_full']/@src"
).replace("header.jpg", "library_600x900.jpg")
(
pd.cover_image,
pd.cover_image_extention,
) = BasicImageDownloader.download_image(
pd.metadata["cover_image_url"], self.url
)
if pd.cover_image is None:
pd.metadata["cover_image_url"] = self.query_str(
content, "//img[@class='game_header_image_full']/@src"
)
(
pd.cover_image,
pd.cover_image_extention,
) = BasicImageDownloader.download_image(
pd.metadata["cover_image_url"], self.url
)
# merge data from IGDB, use localized Steam data if available # merge data from IGDB, use localized Steam data if available
d = { d = {
"developer": developer, "developer": en_data["developers"],
"publisher": publisher, "publisher": en_data["publishers"],
"release_date": release_date, "release_date": en_data["release_date"].get("date"),
"genre": genre, "genre": [g["description"] for g in en_data["genres"]],
"platform": platform, "platform": ["PC"],
} }
if en_data["release_date"].get("date"):
d["release_date"] = en_data["release_date"].get("date")
d.update(pd.metadata) d.update(pd.metadata)
d.update(
{
"localized_title": localized_title,
"localized_description": localized_desc,
}
)
pd.metadata = d pd.metadata = d
if title:
pd.metadata["title"] = title # try Steam images if no image from IGDB
if brief: header = en_data.get("header_image")
pd.metadata["brief"] = brief if header:
if pd.cover_image is None:
cover = header.replace("header.jpg", "library_600x900_2x.jpg")
pd.metadata["cover_image_url"] = cover
(
pd.cover_image,
pd.cover_image_extention,
) = BasicImageDownloader.download_image(
pd.metadata["cover_image_url"], self.url
)
if pd.cover_image is None:
pd.metadata["cover_image_url"] = header
(
pd.cover_image,
pd.cover_image_extention,
) = BasicImageDownloader.download_image(
pd.metadata["cover_image_url"], self.url
)
return pd return pd

View file

@ -1,5 +1,13 @@
""" """
The Movie Database The Movie Database
these language code from TMDB are not in currently iso-639-1
{'iso_639_1': 'xx', 'english_name': 'No Language', 'name': 'No Language'}
{'iso_639_1': 'sh', 'english_name': 'Serbo-Croatian', 'name': ''} - deprecated for several
{'iso_639_1': 'mo', 'english_name': 'Moldavian', 'name': ''} - deprecated for ro-MD
{'iso_639_1': 'cn', 'english_name': 'Cantonese', 'name': '粤语'} - faked for yue
""" """
import logging import logging
@ -10,13 +18,14 @@ from django.conf import settings
from catalog.common import * from catalog.common import *
from catalog.movie.models import * from catalog.movie.models import *
from catalog.tv.models import * from catalog.tv.models import *
from common.models.lang import PREFERRED_LANGUAGES
from .douban import * from .douban import *
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
def get_language_code(): def _get_language_code():
match settings.LANGUAGE_CODE: match settings.LANGUAGE_CODE:
case "zh-hans": case "zh-hans":
return "zh-CN" return "zh-CN"
@ -26,14 +35,28 @@ def get_language_code():
return "en-US" return "en-US"
def _get_preferred_languages():
langs = {}
for l in PREFERRED_LANGUAGES:
if l == "zh":
langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW", "zh-hk": "zh-HK"})
else:
langs[l] = l
return langs
TMDB_DEFAULT_LANG = _get_language_code()
TMDB_PREFERRED_LANGS = _get_preferred_languages()
def search_tmdb_by_imdb_id(imdb_id): def search_tmdb_by_imdb_id(imdb_id):
tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&external_source=imdb_id" tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&external_source=imdb_id"
res_data = BasicDownloader(tmdb_api_url).download().json() res_data = BasicDownloader(tmdb_api_url).download().json()
return res_data return res_data
def query_tmdb_tv_episode(tv, season, episode): def query_tmdb_tv_episode(tv, season, episode):
tmdb_api_url = f"https://api.themoviedb.org/3/tv/{tv}/season/{season}/episode/{episode}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids" tmdb_api_url = f"https://api.themoviedb.org/3/tv/{tv}/season/{season}/episode/{episode}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids"
res_data = BasicDownloader(tmdb_api_url).download().json() res_data = BasicDownloader(tmdb_api_url).download().json()
return res_data return res_data
@ -58,30 +81,16 @@ class TMDB_Movie(AbstractSite):
return f"https://www.themoviedb.org/movie/{id_value}" return f"https://www.themoviedb.org/movie/{id_value}"
def scrape(self): def scrape(self):
is_series = False res_data = {}
if is_series: localized_title = []
api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" localized_desc = []
else: # GET api urls in all locales
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" # btw it seems no way to tell if TMDB does not have a certain translation
for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
res_data = BasicDownloader(api_url).download().json() res_data = BasicDownloader(api_url).download().json()
localized_title.append({"lang": lang, "text": res_data["title"]})
if is_series: localized_desc.append({"lang": lang, "text": res_data["overview"]})
title = res_data["name"]
orig_title = res_data["original_name"]
year = (
int(res_data["first_air_date"].split("-")[0])
if res_data["first_air_date"]
else None
)
imdb_code = res_data["external_ids"]["imdb_id"]
showtime = (
[{"time": res_data["first_air_date"], "region": "首播日期"}]
if res_data["first_air_date"]
else None
)
duration = None
else:
title = res_data["title"] title = res_data["title"]
orig_title = res_data["original_title"] orig_title = res_data["original_title"]
year = ( year = (
@ -102,15 +111,10 @@ class TMDB_Movie(AbstractSite):
language = list(map(lambda x: x["name"], res_data["spoken_languages"])) language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
brief = res_data["overview"] brief = res_data["overview"]
if is_series:
director = list(map(lambda x: x["name"], res_data["created_by"]))
else:
director = list( director = list(
map( map(
lambda x: x["name"], lambda x: x["name"],
filter( filter(lambda c: c["job"] == "Director", res_data["credits"]["crew"]),
lambda c: c["job"] == "Director", res_data["credits"]["crew"]
),
) )
) )
playwright = list( playwright = list(
@ -128,19 +132,21 @@ class TMDB_Movie(AbstractSite):
# other_info['Metacritic评分'] = res_data['metacriticRating'] # other_info['Metacritic评分'] = res_data['metacriticRating']
# other_info['奖项'] = res_data['awards'] # other_info['奖项'] = res_data['awards']
# other_info['TMDB_ID'] = id # other_info['TMDB_ID'] = id
if is_series: # if is_series:
other_info["Seasons"] = res_data["number_of_seasons"] # other_info["Seasons"] = res_data["number_of_seasons"]
other_info["Episodes"] = res_data["number_of_episodes"] # other_info["Episodes"] = res_data["number_of_episodes"]
# TODO: use GET /configuration to get base url # TODO: use GET /configuration to get base url
img_url = ( img_url = (
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"]) ("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
if res_data["poster_path"] is not None if res_data.get("poster_path") is not None
else None else None
) )
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": localized_title,
"localized_description": localized_desc,
"title": title, "title": title,
"orig_title": orig_title, "orig_title": orig_title,
"other_title": [], "other_title": [],
@ -192,15 +198,15 @@ class TMDB_TV(AbstractSite):
return f"https://www.themoviedb.org/tv/{id_value}" return f"https://www.themoviedb.org/tv/{id_value}"
def scrape(self): def scrape(self):
is_series = True res_data = {}
if is_series: localized_title = []
api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" localized_desc = []
else: for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
res_data = BasicDownloader(api_url).download().json() res_data = BasicDownloader(api_url).download().json()
localized_title.append({"lang": lang, "text": res_data["name"]})
localized_desc.append({"lang": lang, "text": res_data["overview"]})
if is_series:
title = res_data["name"] title = res_data["name"]
orig_title = res_data["original_name"] orig_title = res_data["original_name"]
year = ( year = (
@ -215,39 +221,10 @@ class TMDB_TV(AbstractSite):
else None else None
) )
duration = None duration = None
else:
title = res_data["title"]
orig_title = res_data["original_title"]
year = (
int(res_data["release_date"].split("-")[0])
if res_data["release_date"]
else None
)
showtime = (
[{"time": res_data["release_date"], "region": "发布日期"}]
if res_data["release_date"]
else None
)
imdb_code = res_data["imdb_id"]
# in minutes
duration = res_data["runtime"] if res_data["runtime"] else None
genre = [x["name"] for x in res_data["genres"]] genre = [x["name"] for x in res_data["genres"]]
language = list(map(lambda x: x["name"], res_data["spoken_languages"])) language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
brief = res_data["overview"] brief = res_data["overview"]
if is_series:
director = list(map(lambda x: x["name"], res_data["created_by"])) director = list(map(lambda x: x["name"], res_data["created_by"]))
else:
director = list(
map(
lambda x: x["name"],
filter(
lambda c: c["job"] == "Director", res_data["credits"]["crew"]
),
)
)
playwright = list( playwright = list(
map( map(
lambda x: x["name"], lambda x: x["name"],
@ -256,24 +233,15 @@ class TMDB_TV(AbstractSite):
) )
actor = list(map(lambda x: x["name"], res_data["credits"]["cast"])) actor = list(map(lambda x: x["name"], res_data["credits"]["cast"]))
area = [] area = []
other_info = {} other_info = {}
# other_info['TMDB评分'] = res_data['vote_average']
# other_info['分级'] = res_data['contentRating']
# other_info['Metacritic评分'] = res_data['metacriticRating']
# other_info['奖项'] = res_data['awards']
# other_info['TMDB_ID'] = id
if is_series:
other_info["Seasons"] = res_data["number_of_seasons"] other_info["Seasons"] = res_data["number_of_seasons"]
other_info["Episodes"] = res_data["number_of_episodes"] other_info["Episodes"] = res_data["number_of_episodes"]
# TODO: use GET /configuration to get base url # TODO: use GET /configuration to get base url
img_url = ( img_url = (
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"]) ("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
if res_data["poster_path"] is not None if res_data.get("poster_path") is not None
else None else None
) )
season_links = list( season_links = list(
map( map(
lambda s: { lambda s: {
@ -288,6 +256,8 @@ class TMDB_TV(AbstractSite):
) )
pd = ResourceContent( pd = ResourceContent(
metadata={ metadata={
"localized_title": localized_title,
"localized_description": localized_desc,
"title": title, "title": title,
"orig_title": orig_title, "orig_title": orig_title,
"other_title": [], "other_title": [],
@ -313,7 +283,6 @@ class TMDB_TV(AbstractSite):
) )
if imdb_code: if imdb_code:
pd.lookup_ids[IdType.IMDB] = imdb_code pd.lookup_ids[IdType.IMDB] = imdb_code
if pd.metadata["cover_image_url"]: if pd.metadata["cover_image_url"]:
imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url)
try: try:
@ -357,13 +326,20 @@ class TMDB_TVSeason(AbstractSite):
show_resource = site.get_resource_ready(auto_create=False, auto_link=False) show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
if not show_resource: if not show_resource:
raise ValueError(f"TMDB: failed to get show for season {self.url}") raise ValueError(f"TMDB: failed to get show for season {self.url}")
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
d = BasicDownloader(api_url).download().json() res_data = {}
if not d.get("id"): localized_title = []
localized_desc = []
for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
res_data = BasicDownloader(api_url).download().json()
localized_title.append({"lang": lang, "text": res_data["name"]})
localized_desc.append({"lang": lang, "text": res_data["overview"]})
if not res_data.get("id"):
raise ParseError(self, "id") raise ParseError(self, "id")
pd = ResourceContent( d = res_data
metadata=_copy_dict( r = _copy_dict(
d, res_data,
{ {
"name": "title", "name": "title",
"overview": "brief", "overview": "brief",
@ -372,7 +348,9 @@ class TMDB_TVSeason(AbstractSite):
"external_ids": [], "external_ids": [],
}, },
) )
) r["localized_title"] = localized_title
r["localized_description"] = localized_desc
pd = ResourceContent(metadata=r)
pd.metadata["title"] = ( pd.metadata["title"] = (
show_resource.metadata["title"] + " " + pd.metadata["title"] show_resource.metadata["title"] + " " + pd.metadata["title"]
) )
@ -388,12 +366,12 @@ class TMDB_TVSeason(AbstractSite):
pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id") pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id")
pd.metadata["cover_image_url"] = ( pd.metadata["cover_image_url"] = (
("https://image.tmdb.org/t/p/original/" + d["poster_path"]) ("https://image.tmdb.org/t/p/original/" + d["poster_path"])
if d["poster_path"] if d.get("poster_path")
else None else None
) )
pd.metadata["title"] = ( pd.metadata["title"] = (
pd.metadata["title"] pd.metadata["title"]
if pd.metadata["title"] if pd.metadata.get("title")
else f'Season {d["season_number"]}' else f'Season {d["season_number"]}'
) )
pd.metadata["episode_number_list"] = list( pd.metadata["episode_number_list"] = list(
@ -429,7 +407,7 @@ class TMDB_TVSeason(AbstractSite):
) )
else: else:
ep = pd.metadata["episode_number_list"][0] ep = pd.metadata["episode_number_list"][0]
api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
d2 = BasicDownloader(api_url2).download().json() d2 = BasicDownloader(api_url2).download().json()
if not d2.get("id"): if not d2.get("id"):
raise ParseError(self, "first episode id for season") raise ParseError(self, "first episode id for season")
@ -469,7 +447,7 @@ class TMDB_TVEpisode(AbstractSite):
episode_id = v[2] episode_id = v[2]
site = TMDB_TV(TMDB_TV.id_to_url(show_id)) site = TMDB_TV(TMDB_TV.id_to_url(show_id))
show_resource = site.get_resource_ready(auto_create=False, auto_link=False) show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits" api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
d = BasicDownloader(api_url).download().json() d = BasicDownloader(api_url).download().json()
if not d.get("id"): if not d.get("id"):
raise ParseError(self, "id") raise ParseError(self, "id")

View file

@ -44,7 +44,7 @@
</div> </div>
<div> <div>
{% if item.language %} {% if item.language %}
{% trans 'language' %}: {{ item.language }} {% trans 'language' %}: {{ item.get_language_display }}
{% endif %} {% endif %}
</div> </div>
<div> <div>

View file

@ -30,7 +30,7 @@
<div id="item-title" class="middle"> <div id="item-title" class="middle">
{% if item.is_deleted %}[DELETED]{% endif %} {% if item.is_deleted %}[DELETED]{% endif %}
{% if item.merged_to_item %} {% if item.merged_to_item %}
[MERGED TO <a href="{{ item.merged_to_item.url }}">{{ item.merged_to_item.title }}</a>] [MERGED TO <a href="{{ item.merged_to_item.url }}">{{ item.merged_to_item.display_title }}</a>]
{% endif %} {% endif %}
<h1> <h1>
{{ item.display_title }} {{ item.display_title }}

View file

@ -47,6 +47,7 @@ from catalog.common import (
PrimaryLookupIdDescriptor, PrimaryLookupIdDescriptor,
jsondata, jsondata,
) )
from catalog.common.models import LANGUAGE_CHOICES_JSONFORM, LanguageListField
class TVShowInSchema(ItemInSchema): class TVShowInSchema(ItemInSchema):
@ -112,14 +113,16 @@ class TVShow(Item):
) )
METADATA_COPY_LIST = [ METADATA_COPY_LIST = [
"title", # "title",
"localized_title",
"season_count", "season_count",
"orig_title", "orig_title",
"other_title", # "other_title",
"director", "director",
"playwright", "playwright",
"actor", "actor",
"brief", # "brief",
"localized_description",
"genre", "genre",
"showtime", "showtime",
"site", "site",
@ -210,17 +213,8 @@ class TVShow(Item):
blank=True, blank=True,
default=list, default=list,
) )
language = jsondata.ArrayField( language = LanguageListField()
verbose_name=_("language"),
base_field=models.CharField(
blank=True,
default="",
max_length=100,
),
null=True,
blank=True,
default=list,
)
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True) year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
single_episode_length = jsondata.IntegerField( single_episode_length = jsondata.IntegerField(
verbose_name=_("episode length"), null=True, blank=True verbose_name=_("episode length"), null=True, blank=True
@ -374,16 +368,16 @@ class TVSeason(Item):
blank=True, blank=True,
default=list, default=list,
) )
language = jsondata.ArrayField( language = jsondata.JSONField(
verbose_name=_("language"), verbose_name=_("language"),
base_field=models.CharField( # base_field=models.CharField(blank=True, default="", max_length=100, choices=LANGUAGE_CHOICES ),
blank=True,
default="",
max_length=100,
),
null=True, null=True,
blank=True, blank=True,
default=list, default=list,
schema={
"type": "list",
"items": {"type": "string", "choices": LANGUAGE_CHOICES_JSONFORM},
},
) )
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True) year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
single_episode_length = jsondata.IntegerField( single_episode_length = jsondata.IntegerField(

View file

@ -6,6 +6,8 @@ from catalog.tv.models import *
class JSONFieldTestCase(TestCase): class JSONFieldTestCase(TestCase):
databases = "__all__"
def test_legacy_data(self): def test_legacy_data(self):
o = TVShow() o = TVShow()
self.assertEqual(o.other_title, []) self.assertEqual(o.other_title, [])
@ -18,6 +20,8 @@ class JSONFieldTestCase(TestCase):
class TMDBTVTestCase(TestCase): class TMDBTVTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id = "57243" t_id = "57243"
t_url = "https://www.themoviedb.org/tv/57243-doctor-who" t_url = "https://www.themoviedb.org/tv/57243-doctor-who"
@ -43,13 +47,15 @@ class TMDBTVTestCase(TestCase):
self.assertEqual(site.id_value, "57243") self.assertEqual(site.id_value, "57243")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "神秘博士") self.assertEqual(site.resource.metadata["title"], "Doctor Who")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.__class__.__name__, "TVShow") self.assertEqual(site.resource.item.__class__.__name__, "TVShow")
self.assertEqual(site.resource.item.imdb, "tt0436992") self.assertEqual(site.resource.item.imdb, "tt0436992")
class TMDBTVSeasonTestCase(TestCase): class TMDBTVSeasonTestCase(TestCase):
databases = "__all__"
def test_parse(self): def test_parse(self):
t_id = "57243-11" t_id = "57243-11"
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/11" t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/11"
@ -70,7 +76,7 @@ class TMDBTVSeasonTestCase(TestCase):
self.assertEqual(site.id_value, "57243-4") self.assertEqual(site.id_value, "57243-4")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "神秘博士 第 4 季") self.assertEqual(site.resource.metadata["title"], "Doctor Who Series 4")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.__class__.__name__, "TVSeason") self.assertEqual(site.resource.item.__class__.__name__, "TVSeason")
self.assertEqual(site.resource.item.imdb, "tt1159991") self.assertEqual(site.resource.item.imdb, "tt1159991")
@ -79,6 +85,8 @@ class TMDBTVSeasonTestCase(TestCase):
class TMDBEpisodeTestCase(TestCase): class TMDBEpisodeTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_scrape_tmdb(self): def test_scrape_tmdb(self):
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/4/episode/1" t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/4/episode/1"
@ -87,7 +95,7 @@ class TMDBEpisodeTestCase(TestCase):
self.assertEqual(site.id_value, "57243-4-1") self.assertEqual(site.id_value, "57243-4-1")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "活宝搭档") self.assertEqual(site.resource.metadata["title"], "Partners in Crime")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode") self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
self.assertEqual(site.resource.item.imdb, "tt1159991") self.assertEqual(site.resource.item.imdb, "tt1159991")
@ -98,6 +106,8 @@ class TMDBEpisodeTestCase(TestCase):
class DoubanMovieTVTestCase(TestCase): class DoubanMovieTVTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_scrape(self): def test_scrape(self):
url3 = "https://movie.douban.com/subject/3627919/" url3 = "https://movie.douban.com/subject/3627919/"
@ -122,6 +132,8 @@ class DoubanMovieTVTestCase(TestCase):
class MultiTVSitesTestCase(TestCase): class MultiTVSitesTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_tvshows(self): def test_tvshows(self):
url1 = "https://www.themoviedb.org/tv/57243-doctor-who" url1 = "https://www.themoviedb.org/tv/57243-doctor-who"
@ -170,6 +182,8 @@ class MultiTVSitesTestCase(TestCase):
class MovieTVModelRecastTestCase(TestCase): class MovieTVModelRecastTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_recast(self): def test_recast(self):
from catalog.models import Movie, TVShow from catalog.models import Movie, TVShow
@ -178,13 +192,15 @@ class MovieTVModelRecastTestCase(TestCase):
p2 = SiteManager.get_site_by_url(url2).get_resource_ready() p2 = SiteManager.get_site_by_url(url2).get_resource_ready()
tv = p2.item tv = p2.item
self.assertEqual(tv.class_name, "tvshow") self.assertEqual(tv.class_name, "tvshow")
self.assertEqual(tv.title, "神秘博士") self.assertEqual(tv.display_title, "Doctor Who")
movie = tv.recast_to(Movie) movie = tv.recast_to(Movie)
self.assertEqual(movie.class_name, "movie") self.assertEqual(movie.class_name, "movie")
self.assertEqual(movie.title, "神秘博士") self.assertEqual(movie.display_title, "Doctor Who")
class IMDBTestCase(TestCase): class IMDBTestCase(TestCase):
databases = "__all__"
@use_local_response @use_local_response
def test_fetch_episodes(self): def test_fetch_episodes(self):
t_url = "https://movie.douban.com/subject/1920763/" t_url = "https://movie.douban.com/subject/1920763/"
@ -243,7 +259,7 @@ class IMDBTestCase(TestCase):
self.assertEqual(site.id_value, "tt1159991") self.assertEqual(site.id_value, "tt1159991")
site.get_resource_ready() site.get_resource_ready()
self.assertEqual(site.ready, True) self.assertEqual(site.ready, True)
self.assertEqual(site.resource.metadata["title"], "活宝搭档") self.assertEqual(site.resource.metadata["title"], "Partners in Crime")
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode") self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
self.assertEqual(site.resource.item.imdb, "tt1159991") self.assertEqual(site.resource.item.imdb, "tt1159991")

View file

@ -0,0 +1,9 @@
from .cron import BaseJob, JobManager
from .lang import (
DEFAULT_CATALOG_LANGUAGE,
LANGUAGE_CHOICES,
LOCALE_CHOICES,
SCRIPT_CHOICES,
detect_language,
)
from .misc import uniq

319
common/models/lang.py Normal file
View file

@ -0,0 +1,319 @@
"""
language support utilities
https://en.wikipedia.org/wiki/IETF_language_tag
"""
import re
from typing import Any
from django.conf import settings
from django.utils.translation import get_language
from django.utils.translation import gettext_lazy as _
from langdetect import detect
from loguru import logger
PREFERRED_LANGUAGES: list[str] = settings.PREFERRED_LANGUAGES
DEFAULT_CATALOG_LANGUAGE = PREFERRED_LANGUAGES[0] if PREFERRED_LANGUAGES else "en"
ISO_639_1 = {
"aa": _("Afar"),
"af": _("Afrikaans"),
"ak": _("Akan"),
"an": _("Aragonese"),
"as": _("Assamese"),
"av": _("Avaric"),
"ae": _("Avestan"),
"ay": _("Aymara"),
"az": _("Azerbaijani"),
"ba": _("Bashkir"),
"bm": _("Bambara"),
"bi": _("Bislama"),
"bo": _("Tibetan"),
"br": _("Breton"),
"ca": _("Catalan"),
"cs": _("Czech"),
"ce": _("Chechen"),
"cu": _("Slavic"),
"cv": _("Chuvash"),
"kw": _("Cornish"),
"co": _("Corsican"),
"cr": _("Cree"),
"cy": _("Welsh"),
"da": _("Danish"),
"de": _("German"),
"dv": _("Divehi"),
"dz": _("Dzongkha"),
"eo": _("Esperanto"),
"et": _("Estonian"),
"eu": _("Basque"),
"fo": _("Faroese"),
"fj": _("Fijian"),
"fi": _("Finnish"),
"fr": _("French"),
"fy": _("Frisian"),
"ff": _("Fulah"),
"gd": _("Gaelic"),
"ga": _("Irish"),
"gl": _("Galician"),
"gv": _("Manx"),
"gn": _("Guarani"),
"gu": _("Gujarati"),
"ht": _("Haitian; Haitian Creole"),
"ha": _("Hausa"),
"sh": _("Serbo-Croatian"),
"hz": _("Herero"),
"ho": _("Hiri Motu"),
"hr": _("Croatian"),
"hu": _("Hungarian"),
"ig": _("Igbo"),
"io": _("Ido"),
"ii": _("Yi"),
"iu": _("Inuktitut"),
"ie": _("Interlingue"),
"ia": _("Interlingua"),
"id": _("Indonesian"),
"ik": _("Inupiaq"),
"is": _("Icelandic"),
"it": _("Italian"),
"jv": _("Javanese"),
"ja": _("Japanese"),
"kl": _("Kalaallisut"),
"kn": _("Kannada"),
"ks": _("Kashmiri"),
"kr": _("Kanuri"),
"kk": _("Kazakh"),
"km": _("Khmer"),
"ki": _("Kikuyu"),
"rw": _("Kinyarwanda"),
"ky": _("Kirghiz"),
"kv": _("Komi"),
"kg": _("Kongo"),
"ko": _("Korean"),
"kj": _("Kuanyama"),
"ku": _("Kurdish"),
"lo": _("Lao"),
"la": _("Latin"),
"lv": _("Latvian"),
"li": _("Limburgish"),
"ln": _("Lingala"),
"lt": _("Lithuanian"),
"lb": _("Letzeburgesch"),
"lu": _("Luba-Katanga"),
"lg": _("Ganda"),
"mh": _("Marshall"),
"ml": _("Malayalam"),
"mr": _("Marathi"),
"mg": _("Malagasy"),
"mt": _("Maltese"),
"mo": _("Moldavian"),
"mn": _("Mongolian"),
"mi": _("Maori"),
"ms": _("Malay"),
"my": _("Burmese"),
"na": _("Nauru"),
"nv": _("Navajo"),
"nr": _("Ndebele"),
"nd": _("Ndebele"),
"ng": _("Ndonga"),
"ne": _("Nepali"),
"nl": _("Dutch"),
"nn": _("Norwegian Nynorsk"),
"nb": _("Norwegian Bokmål"),
"no": _("Norwegian"),
"ny": _("Chichewa; Nyanja"),
"oc": _("Occitan"),
"oj": _("Ojibwa"),
"or": _("Oriya"),
"om": _("Oromo"),
"os": _("Ossetian; Ossetic"),
"pi": _("Pali"),
"pl": _("Polish"),
"pt": _("Portuguese"),
"qu": _("Quechua"),
"rm": _("Raeto-Romance"),
"ro": _("Romanian"),
"rn": _("Rundi"),
"ru": _("Russian"),
"sg": _("Sango"),
"sa": _("Sanskrit"),
"si": _("Sinhalese"),
"sk": _("Slovak"),
"sl": _("Slovenian"),
"se": _("Northern Sami"),
"sm": _("Samoan"),
"sn": _("Shona"),
"sd": _("Sindhi"),
"so": _("Somali"),
"st": _("Sotho"),
"es": _("Spanish"),
"sq": _("Albanian"),
"sc": _("Sardinian"),
"sr": _("Serbian"),
"ss": _("Swati"),
"su": _("Sundanese"),
"sw": _("Swahili"),
"sv": _("Swedish"),
"ty": _("Tahitian"),
"ta": _("Tamil"),
"tt": _("Tatar"),
"te": _("Telugu"),
"tg": _("Tajik"),
"tl": _("Tagalog"),
"th": _("Thai"),
"ti": _("Tigrinya"),
"to": _("Tonga"),
"tn": _("Tswana"),
"ts": _("Tsonga"),
"tk": _("Turkmen"),
"tr": _("Turkish"),
"tw": _("Twi"),
"ug": _("Uighur"),
"uk": _("Ukrainian"),
"ur": _("Urdu"),
"uz": _("Uzbek"),
"ve": _("Venda"),
"vi": _("Vietnamese"),
"vo": _("Volapük"),
"wa": _("Walloon"),
"wo": _("Wolof"),
"xh": _("Xhosa"),
"yi": _("Yiddish"),
"za": _("Zhuang"),
"zu": _("Zulu"),
"ab": _("Abkhazian"),
"zh": _("Chinese"),
"ps": _("Pushto"),
"am": _("Amharic"),
"ar": _("Arabic"),
"bg": _("Bulgarian"),
"mk": _("Macedonian"),
"el": _("Greek"),
"fa": _("Persian"),
"he": _("Hebrew"),
"hi": _("Hindi"),
"hy": _("Armenian"),
"en": _("English"),
"ee": _("Ewe"),
"ka": _("Georgian"),
"pa": _("Punjabi"),
"bn": _("Bengali"),
"bs": _("Bosnian"),
"ch": _("Chamorro"),
"be": _("Belarusian"),
"yo": _("Yoruba"),
"x": _("Unknown or Other"),
}
TOP_USED_LANG = [
"en",
"de",
"es",
"zh",
"fr",
"ja",
"it",
"ru",
"pt",
"nl",
"kr",
"hi",
"ar",
"bn",
]
ZH_LOCALE_SUBTAGS_PRIO = {
"zh-cn": _("Simplified Chinese (Mainland)"),
"zh-tw": _("Traditional Chinese (Taiwan)"),
"zh-hk": _("Traditional Chinese (Hongkong)"),
}
ZH_LOCALE_SUBTAGS = {
"zh-sg": _("Simplified Chinese (Singapore)"),
"zh-my": _("Simplified Chinese (Malaysia)"),
"zh-mo": _("Traditional Chinese (Taiwan)"),
}
ZH_LANGUAGE_SUBTAGS_PRIO = {
"cmn": _("Mandarin Chinese"),
"yue": _("Yue Chinese"),
}
ZH_LANGUAGE_SUBTAGS = {
"nan": _("Min Nan Chinese"),
"wuu": _("Wu Chinese"),
"hak": _("Hakka Chinese"),
}
ZH_LOCALE_SUBTAGS_PRIO.keys()
def get_base_lang_list():
langs = {}
for k in PREFERRED_LANGUAGES + TOP_USED_LANG:
if k not in langs:
if k in ISO_639_1:
langs[k] = ISO_639_1[k]
else:
logger.error(f"{k} is not a supported ISO-639-1 language tag")
for k, v in ISO_639_1.items():
if k not in langs:
langs[k] = v
return langs
BASE_LANG_LIST: dict[str, Any] = get_base_lang_list()
def get_locale_choices():
choices = []
for k, v in BASE_LANG_LIST.items():
if k == "zh":
choices += ZH_LOCALE_SUBTAGS_PRIO.items()
else:
choices.append((k, v))
choices += ZH_LOCALE_SUBTAGS.items()
return choices
def get_script_choices():
return list(BASE_LANG_LIST.items())
def get_language_choices():
choices = []
for k, v in BASE_LANG_LIST.items():
if k == "zh":
choices += ZH_LANGUAGE_SUBTAGS_PRIO.items()
else:
choices.append((k, v))
choices += ZH_LANGUAGE_SUBTAGS.items()
return choices
LOCALE_CHOICES: list[tuple[str, Any]] = get_locale_choices()
SCRIPT_CHOICES: list[tuple[str, Any]] = get_script_choices()
LANGUAGE_CHOICES: list[tuple[str, Any]] = get_language_choices()
def get_current_locales() -> list[str]:
lang = get_language().lower()
if lang == "zh-hans":
return ["zh-cn", "zh-sg", "zh-my", "zh-hk", "zh-tw", "zh-mo", "en"]
elif lang == "zh-hant":
return ["zh-tw", "zh-hk", "zh-mo", "zh-cn", "zh-sg", "zh-my", "en"]
else:
lng = lang.split("-")
return ["en"] if lng[0] == "en" else [lng[0], "en"]
_eng = re.compile(r"^[A-Za-z0-9\s]{1,13}$")
def detect_language(s: str) -> str:
try:
if _eng.match(s):
return "en"
return detect(s).lower()
except Exception:
return "x"
def migrate_languages(languages: list[str]) -> list[str]:
return []

6
common/models/misc.py Normal file
View file

@ -0,0 +1,6 @@
def uniq(ls: list) -> list:
r = []
for i in ls:
if i not in r:
r.append(i)
return r

View file

@ -8,7 +8,11 @@
} }
} }
// override django_jsonform/react-json-form styles // override django_jsonform/react-json-form styles
.rjf-form-group-wrapper{
max-width: unset !important;
}
.rjf-form-wrapper { .rjf-form-wrapper {
max-width: unset !important;
input[type="text"] { input[type="text"] {
max-width: unset !important; max-width: unset !important;
margin-top: 0 !important; margin-top: 0 !important;
@ -33,15 +37,15 @@
} }
.rjf-form-row-inner>div { .rjf-form-row-inner>div {
display: grid !important; // display: grid !important;
grid-template-columns: repeat(auto-fit, minmax(0%, 1fr)); // grid-template-columns: repeat(auto-fit, minmax(0%, 1fr));
>label { >label {
margin-top: var(--pico-form-element-spacing-vertical); margin-top: var(--pico-form-element-spacing-vertical);
} }
>* { >* {
width: max-content !important; // width: max-content !important;
button { button {

View file

@ -21,7 +21,7 @@
<script src="{{ cdn_url }}/npm/hyperscript.org@0.9.12"></script> <script src="{{ cdn_url }}/npm/hyperscript.org@0.9.12"></script>
<link rel="stylesheet" <link rel="stylesheet"
href="{{ cdn_url }}/npm/@picocss/pico@2/css/pico.min.css" /> href="{{ cdn_url }}/npm/@picocss/pico@2/css/pico.min.css" />
<link href="{% sass_src 'scss/neodb.scss' %}" <link href="{% sass_src 'scss/neodb.scss' %}?xxddddddddxdd"
rel="stylesheet" rel="stylesheet"
type="text/css" /> type="text/css" />
<link href="{{ cdn_url }}/npm/@fortawesome/fontawesome-free@6.5.2/css/all.min.css" <link href="{{ cdn_url }}/npm/@fortawesome/fontawesome-free@6.5.2/css/all.min.css"

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
from discord import SyncWebhook from discord import SyncWebhook
from django.conf import settings from django.conf import settings
from django.conf.locale import LANG_INFO
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.core.signing import b62_decode, b62_encode from django.core.signing import b62_decode, b62_encode
from django.http import Http404, HttpRequest, HttpResponseRedirect, QueryDict from django.http import Http404, HttpRequest, HttpResponseRedirect, QueryDict

View file

@ -509,7 +509,7 @@ class Content(Piece):
raise NotImplementedError("subclass should override this") raise NotImplementedError("subclass should override this")
@property @property
def display_description(self) -> str: def brief_description(self) -> str:
raise NotImplementedError("subclass should override this") raise NotImplementedError("subclass should override this")
class Meta: class Meta:

View file

@ -1,4 +1,5 @@
import re import re
from html import unescape
from typing import cast from typing import cast
import mistune import mistune
@ -38,6 +39,13 @@ def render_md(s: str) -> str:
return cast(str, _markdown(s)) return cast(str, _markdown(s))
_RE_HTML_TAG = re.compile(r"<[^>]*>")
def html_to_text(h: str) -> str:
return unescape(_RE_HTML_TAG.sub(" ", h.replace("\r", "")))
def _spolier(s: str) -> str: def _spolier(s: str) -> str:
sl = s.split(">!", 1) sl = s.split(">!", 1)
if len(sl) == 1: if len(sl) == 1:

View file

@ -32,7 +32,7 @@ class Review(Content):
return self.title return self.title
@property @property
def display_description(self): def brief_description(self):
return self.plain_content[:155] return self.plain_content[:155]
@property @property

View file

@ -146,7 +146,7 @@ class TagTest(TestCase):
def test_cleanup(self): def test_cleanup(self):
self.assertEqual(Tag.cleanup_title("# "), "_") self.assertEqual(Tag.cleanup_title("# "), "_")
self.assertEqual(Tag.deep_cleanup_title("# C "), "c") self.assertEqual(Tag.deep_cleanup_title("# C "), "text")
def test_user_tag(self): def test_user_tag(self):
t1 = "tag 1" t1 = "tag 1"
@ -183,7 +183,7 @@ class MarkTest(TestCase):
mark = Mark(self.user1.identity, self.book1) mark = Mark(self.user1.identity, self.book1)
self.assertEqual(mark.shelf_type, ShelfType.WISHLIST) self.assertEqual(mark.shelf_type, ShelfType.WISHLIST)
self.assertEqual(mark.shelf_label, "想读的书") self.assertEqual(mark.shelf_label, "books to read")
self.assertEqual(mark.comment_text, "a gentle comment") self.assertEqual(mark.comment_text, "a gentle comment")
self.assertEqual(mark.rating_grade, 9) self.assertEqual(mark.rating_grade, 9)
self.assertEqual(mark.visibility, 1) self.assertEqual(mark.visibility, 1)

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -27,10 +27,11 @@ class MastodonSiteCheck(BaseJob):
try: try:
api_domain = site.api_domain or site.domain_name api_domain = site.api_domain or site.domain_name
domain, api_domain, v = detect_server_info(api_domain) domain, api_domain, v = detect_server_info(api_domain)
site.server_version = v
site.last_reachable_date = timezone.now() site.last_reachable_date = timezone.now()
site.detect_configurations() site.detect_configurations()
except Exception as e: except Exception as e:
logger.error( logger.warning(
f"Failed to detect server info for {site.domain_name}/{site.api_domain}", f"Failed to detect server info for {site.domain_name}/{site.api_domain}",
extra={"exception": e}, extra={"exception": e},
) )
@ -40,12 +41,16 @@ class MastodonSiteCheck(BaseJob):
if timezone.now() > site.last_reachable_date + timedelta( if timezone.now() > site.last_reachable_date + timedelta(
days=self.max_unreachable_days days=self.max_unreachable_days
): ):
logger.error(
f"Failed to detect server info for {site.domain_name}/{site.api_domain} disabling it."
)
site.disabled = True site.disabled = True
count_disabled += 1 count_disabled += 1
finally: finally:
site.save( site.save(
update_fields=[ update_fields=[
"star_mode", "star_mode",
"server_version",
"max_status_len", "max_status_len",
"last_reachable_date", "last_reachable_date",
"disabled", "disabled",

View file

@ -257,7 +257,7 @@ class BlueskyAccount(SocialAccount):
embed = models.AppBskyEmbedExternal.Main( embed = models.AppBskyEmbedExternal.Main(
external=models.AppBskyEmbedExternal.External( external=models.AppBskyEmbedExternal.External(
title=obj.display_title, title=obj.display_title,
description=obj.display_description, description=obj.brief_description,
uri=obj.absolute_url, uri=obj.absolute_url,
) )
) )

View file

@ -57,7 +57,6 @@ dependencies = [
"deepmerge>=1.1.1", "deepmerge>=1.1.1",
"django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git", "django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git",
"atproto>=0.0.49", "atproto>=0.0.49",
"pyright>=1.1.370",
] ]
[tool.rye] [tool.rye]
@ -70,7 +69,7 @@ dev-dependencies = [
"djlint~=1.34.1", "djlint~=1.34.1",
"isort~=5.13.2", "isort~=5.13.2",
"lxml-stubs", "lxml-stubs",
"pyright>=1.1.369", "pyright>=1.1.371",
"ruff", "ruff",
"mkdocs-material>=9.5.25", "mkdocs-material>=9.5.25",
] ]

View file

@ -229,7 +229,7 @@ pygments==2.18.0
# via mkdocs-material # via mkdocs-material
pymdown-extensions==10.8.1 pymdown-extensions==10.8.1
# via mkdocs-material # via mkdocs-material
pyright==1.1.370 pyright==1.1.371
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
# via dateparser # via dateparser
# via django-auditlog # via django-auditlog

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/zJkLznQNzeLfKuD238Czo6jk65X.jpg","id":71365,"name":"Battlestar Galactica","original_name":"Battlestar Galactica","overview":"A re-imagining of the original series in which a \"rag-tag fugitive fleet\" of the last remnants of mankind flees pursuing robots while simultaneously searching for their true home, Earth.","poster_path":"/imTQ4nBdA68TVpLaWhhQJnb7NQh.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[10759,18,10765],"popularity":51.919,"first_air_date":"2003-12-08","vote_average":8.184,"vote_count":801,"origin_country":["CA"]}],"tv_episode_results":[],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/nfH8SZJVOxcBlFaqqtoqS5hHizG.jpg","id":57243,"name":"Doctor Who","original_name":"Doctor Who","overview":"The Doctor is a Time Lord: a 900 year old alien with 2 hearts, part of a gifted civilization who mastered time travel. The Doctor saves planets for a living—more of a hobby actually, and the Doctor's very, very good at it.","poster_path":"/4edFyasCrkH4MKs6H4mHqlrxA6b.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[10759,18,10765],"popularity":1090.391,"first_air_date":"2005-03-26","vote_average":7.519,"vote_count":2930,"origin_country":["GB"]}],"tv_episode_results":[],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[{"backdrop_path":"/eCTWG4HzsOQomghw8sRd1qpeOlA.jpg","id":282758,"title":"Doctor Who: The Runaway Bride","original_title":"Doctor Who: The Runaway Bride","overview":"A young bride in the midst of her wedding finds herself mysteriously transported to the TARDIS. The Doctor must discover what her connection is with the Empress of the Racnoss's plan to destroy the world.","poster_path":"/dy7JzhXnDFhQsHRiPXxpu62j3yQ.jpg","media_type":"movie","adult":false,"original_language":"en","genre_ids":[878],"popularity":17.25,"release_date":"2006-12-25","video":false,"vote_average":7.728,"vote_count":224}],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1008547,"name":"The Runaway Bride","overview":"Bride-to-be Donna vanishes as she walks down the aisle to marry boyfriend Lance. To her complete astonishment - and the Doctor's - she reappears in the Tardis. As the Time Lord, still reeling from Rose's departure, investigates how Donna came to be there, the duo uncover a terrifying enemy. How far will the Doctor go to save Earth from the latest alien threat?","media_type":"tv_episode","vote_average":6.925,"vote_count":20,"air_date":"2006-12-25","episode_number":4,"episode_type":"standard","production_code":"NCFT094N","runtime":64,"season_number":0,"show_id":57243,"still_path":"/pncNamTuydXWinybPuMTsBUVjSD.jpg"}],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":941505,"name":"Partners in Crime","overview":"During an alien emergency in London, a woman called Donna Noble must search for an old friend who can save the day - a man named the Doctor. But can even the Doctor halt the plans of the mysterious Miss Foster?","media_type":"tv_episode","vote_average":7.26,"vote_count":52,"air_date":"2008-04-05","episode_number":1,"episode_type":"standard","production_code":"","runtime":51,"season_number":4,"show_id":57243,"still_path":"/vg5oP1tOzivl4EV7iHiEaKwiZkK.jpg"}],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[{"backdrop_path":"/8ZTVqvKDQ8emSGUEMjsS4yHAwrp.jpg","id":27205,"title":"Inception","original_title":"Inception","overview":"Cobb, a skilled thief who commits corporate espionage by infiltrating the subconscious of his targets is offered a chance to regain his old life as payment for a task considered to be impossible: \"inception\", the implantation of another person's idea into a target's subconscious.","poster_path":"/oYuLEt3zVCKq57qu2F8dT7NIa6f.jpg","media_type":"movie","adult":false,"original_language":"en","genre_ids":[28,878,12],"popularity":92.871,"release_date":"2010-07-15","video":false,"vote_average":8.369,"vote_count":35987}],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1305550,"name":"Part 1","overview":"In a distant part of the galaxy lie The Twelve Colonies of Man, a civilization that has been at peace for some forty years with an empire of machines, the Cylons, who were created generations before as worker drones for mankind, but became independent, rose in rebellion, and launched war on their masters. Now, the Cylons have evolved into more human form, into machine-created biological beings, who seek to exterminate true biological humans. To this end they use a human scientist, Gaius, to help one of their infiltrators, known as #6, penetrate the Colonies' master ...","media_type":"tv_episode","vote_average":8.1,"vote_count":20,"air_date":"2003-12-08","episode_number":1,"episode_type":"standard","production_code":"","runtime":95,"season_number":1,"show_id":71365,"still_path":"/mBkKJW9ppIEjkD4CXGaAntekQNm.jpg"}],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1305551,"name":"Part 2","overview":"After forty years of armistice, the Cylons attacks the Twelve Colonies of Kobol. Their strategy: a virus implanted into the mankind defense system. The former Battlestar Galactica, which is being adapted into a museum, is not connected with the defense system and becomes the only warship capable of fighting against the Cylons in the hopes of leading the survivors to planet 'Earth'.","media_type":"tv_episode","vote_average":8.118,"vote_count":17,"air_date":"2003-12-09","episode_number":2,"episode_type":"finale","production_code":"","runtime":90,"season_number":1,"show_id":71365,"still_path":"/77kEx9Zw6yI69oCrffQc1hIGOjC.jpg"}],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":3891529,"name":"Solaricks","overview":"The Smiths deal with last season's fallout, and Rick and Morty are stranded in space.","media_type":"tv_episode","vote_average":8.091,"vote_count":55,"air_date":"2022-09-04","episode_number":1,"episode_type":"standard","production_code":"","runtime":23,"season_number":6,"show_id":60625,"still_path":"/5tiOEjp03nvaGiKT73knretU8e8.jpg"}],"tv_season_results":[]}

View file

@ -0,0 +1 @@
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/8IC1q0lHFwi5m8VtChLzIfmpaZH.jpg","id":86941,"name":"The North Water","original_name":"The North Water","overview":"Henry Drax is a harpooner and brutish killer whose amorality has been shaped to fit the harshness of his world, who will set sail on a whaling expedition to the Arctic with Patrick Sumner, a disgraced ex-army surgeon who signs up as the ships doctor. Hoping to escape the horrors of his past, Sumner finds himself on an ill-fated journey with a murderous psychopath. In search of redemption, his story becomes a harsh struggle for survival in the Arctic wasteland.","poster_path":"/9CM0ca8pX1os3SJ24hsIc0nN8ph.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[18,9648],"popularity":40.783,"first_air_date":"2021-07-14","vote_average":7.392,"vote_count":120,"origin_country":["US"]}],"tv_episode_results":[],"tv_season_results":[]}

Some files were not shown because too many files have changed in this diff Show more