supports localized title
This commit is contained in:
parent
9b13e8028e
commit
65098a137d
151 changed files with 16260 additions and 12811 deletions
1
.github/workflows/tests.yml
vendored
1
.github/workflows/tests.yml
vendored
|
@ -52,7 +52,6 @@ jobs:
|
||||||
NEODB_SITE_NAME: test
|
NEODB_SITE_NAME: test
|
||||||
NEODB_SITE_DOMAIN: test.domain
|
NEODB_SITE_DOMAIN: test.domain
|
||||||
NEODB_SECRET_KEY: test
|
NEODB_SECRET_KEY: test
|
||||||
NEODB_LANGUAGE: zh-hans
|
|
||||||
run: |
|
run: |
|
||||||
python manage.py compilemessages -i .venv -l zh_Hans
|
python manage.py compilemessages -i .venv -l zh_Hans
|
||||||
python manage.py test
|
python manage.py test
|
||||||
|
|
|
@ -46,8 +46,8 @@ env = environ.FileAwareEnv(
|
||||||
NEODB_SITE_LINKS=(dict, {}),
|
NEODB_SITE_LINKS=(dict, {}),
|
||||||
# Alternative domains
|
# Alternative domains
|
||||||
NEODB_ALTERNATIVE_DOMAINS=(list, []),
|
NEODB_ALTERNATIVE_DOMAINS=(list, []),
|
||||||
# Default language
|
# Preferred languages in catalog
|
||||||
NEODB_LANGUAGE=(str, "zh-hans"),
|
NEODB_PREFERRED_LANGUAGES=(list, ["en", "zh"]), # , "ja", "ko", "de", "fr", "es"
|
||||||
# Invite only mode
|
# Invite only mode
|
||||||
# when True: user will not be able to register unless with invite token
|
# when True: user will not be able to register unless with invite token
|
||||||
# (generated by `neodb-manage invite --create`)
|
# (generated by `neodb-manage invite --create`)
|
||||||
|
@ -408,14 +408,38 @@ if SLACK_TOKEN:
|
||||||
|
|
||||||
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md"
|
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md"
|
||||||
|
|
||||||
LANGUAGE_CODE = env("NEODB_LANGUAGE", default="zh-hans") # type: ignore
|
SUPPORTED_UI_LANGUAGES = {
|
||||||
LOCALE_PATHS = [os.path.join(BASE_DIR, "locale")]
|
"en": _("English"),
|
||||||
LANGUAGES = (
|
"zh-hans": _("Simplified Chinese"),
|
||||||
("en", _("English")),
|
"zh-hant": _("Traditional Chinese"),
|
||||||
("zh-hans", _("Simplified Chinese")),
|
}
|
||||||
("zh-hant", _("Traditional Chinese")),
|
|
||||||
|
LANGUAGES = SUPPORTED_UI_LANGUAGES.items()
|
||||||
|
|
||||||
|
|
||||||
|
def _init_language_settings(preferred_lanugages_env):
|
||||||
|
default_language = None
|
||||||
|
preferred_lanugages = []
|
||||||
|
for pl in preferred_lanugages_env:
|
||||||
|
lang = pl.strip().lower()
|
||||||
|
if not default_language:
|
||||||
|
if lang in SUPPORTED_UI_LANGUAGES:
|
||||||
|
default_language = lang
|
||||||
|
elif lang == "zh":
|
||||||
|
default_language = "zh-hans"
|
||||||
|
if lang.startswith("zh-"):
|
||||||
|
lang = "zh"
|
||||||
|
if lang not in preferred_lanugages:
|
||||||
|
preferred_lanugages.append(lang)
|
||||||
|
return default_language or "en", preferred_lanugages or ["en"]
|
||||||
|
|
||||||
|
|
||||||
|
LANGUAGE_CODE, PREFERRED_LANGUAGES = _init_language_settings(
|
||||||
|
env("NEODB_PREFERRED_LANGUAGES")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
LOCALE_PATHS = [os.path.join(BASE_DIR, "locale")]
|
||||||
|
|
||||||
TIME_ZONE = env("NEODB_TIMEZONE", default="Asia/Shanghai") # type: ignore
|
TIME_ZONE = env("NEODB_TIMEZONE", default="Asia/Shanghai") # type: ignore
|
||||||
|
|
||||||
USE_I18N = True
|
USE_I18N = True
|
||||||
|
|
|
@ -17,7 +17,6 @@ work data seems asymmetric (a book links to a work, but may not listed in that w
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from os.path import exists
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from django.core.validators import MaxValueValidator, MinValueValidator
|
from django.core.validators import MaxValueValidator, MinValueValidator
|
||||||
|
@ -37,6 +36,7 @@ from catalog.common import (
|
||||||
PrimaryLookupIdDescriptor,
|
PrimaryLookupIdDescriptor,
|
||||||
jsondata,
|
jsondata,
|
||||||
)
|
)
|
||||||
|
from catalog.common.models import SCRIPT_CHOICES
|
||||||
|
|
||||||
from .utils import *
|
from .utils import *
|
||||||
|
|
||||||
|
@ -90,6 +90,7 @@ class Edition(Item):
|
||||||
"pages",
|
"pages",
|
||||||
"price",
|
"price",
|
||||||
"brief",
|
"brief",
|
||||||
|
"localized_description",
|
||||||
"contents",
|
"contents",
|
||||||
]
|
]
|
||||||
subtitle = jsondata.CharField(
|
subtitle = jsondata.CharField(
|
||||||
|
@ -113,7 +114,12 @@ class Edition(Item):
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.CharField(
|
language = jsondata.CharField(
|
||||||
_("language"), null=True, blank=True, default=None, max_length=500
|
_("language"),
|
||||||
|
null=False,
|
||||||
|
blank=True,
|
||||||
|
default=None,
|
||||||
|
max_length=500,
|
||||||
|
choices=SCRIPT_CHOICES,
|
||||||
)
|
)
|
||||||
pub_house = jsondata.CharField(
|
pub_house = jsondata.CharField(
|
||||||
_("publishing house"), null=True, blank=False, default=None, max_length=500
|
_("publishing house"), null=True, blank=False, default=None, max_length=500
|
||||||
|
|
|
@ -13,12 +13,16 @@ from django.db import connection, models
|
||||||
from django.db.models import QuerySet, Value
|
from django.db.models import QuerySet, Value
|
||||||
from django.template.defaultfilters import default
|
from django.template.defaultfilters import default
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
from django.utils.translation import get_language
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from ninja import Field, Schema
|
from ninja import Field, Schema
|
||||||
from polymorphic.models import PolymorphicModel
|
from polymorphic.models import PolymorphicModel
|
||||||
|
|
||||||
from catalog.common import jsondata
|
from catalog.common import jsondata
|
||||||
|
from common.models import LANGUAGE_CHOICES, LOCALE_CHOICES, SCRIPT_CHOICES
|
||||||
|
from common.models.lang import get_current_locales
|
||||||
|
from common.models.misc import uniq
|
||||||
|
|
||||||
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
|
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
|
||||||
|
|
||||||
|
@ -258,9 +262,16 @@ class BaseSchema(Schema):
|
||||||
external_resources: list[ExternalResourceSchema] | None
|
external_resources: list[ExternalResourceSchema] | None
|
||||||
|
|
||||||
|
|
||||||
|
class LocalizedTitleSchema(Schema):
|
||||||
|
lang: str
|
||||||
|
text: str
|
||||||
|
|
||||||
|
|
||||||
class ItemInSchema(Schema):
|
class ItemInSchema(Schema):
|
||||||
title: str
|
title: str
|
||||||
brief: str
|
brief: str
|
||||||
|
localized_title: list[LocalizedTitleSchema] = []
|
||||||
|
localized_description: list[LocalizedTitleSchema] = []
|
||||||
cover_image_url: str | None
|
cover_image_url: str | None
|
||||||
rating: float | None
|
rating: float | None
|
||||||
rating_count: int | None
|
rating_count: int | None
|
||||||
|
@ -270,6 +281,63 @@ class ItemSchema(BaseSchema, ItemInSchema):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_locale_choices_for_jsonform(choices):
|
||||||
|
"""return list for jsonform schema"""
|
||||||
|
return [{"title": v, "value": k} for k, v in choices]
|
||||||
|
|
||||||
|
|
||||||
|
LOCALE_CHOICES_JSONFORM = get_locale_choices_for_jsonform(LOCALE_CHOICES)
|
||||||
|
LANGUAGE_CHOICES_JSONFORM = get_locale_choices_for_jsonform(LANGUAGE_CHOICES)
|
||||||
|
|
||||||
|
LOCALIZED_TITLE_SCHEMA = {
|
||||||
|
"type": "list",
|
||||||
|
"items": {
|
||||||
|
"type": "dict",
|
||||||
|
"keys": {
|
||||||
|
"lang": {
|
||||||
|
"type": "string",
|
||||||
|
"title": _("language"),
|
||||||
|
"choices": LOCALE_CHOICES_JSONFORM,
|
||||||
|
},
|
||||||
|
"text": {"type": "string", "title": _("content")},
|
||||||
|
},
|
||||||
|
"required": ["lang", "s"],
|
||||||
|
},
|
||||||
|
"uniqueItems": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
LOCALIZED_DESCRIPTION_SCHEMA = {
|
||||||
|
"type": "list",
|
||||||
|
"items": {
|
||||||
|
"type": "dict",
|
||||||
|
"keys": {
|
||||||
|
"lang": {
|
||||||
|
"type": "string",
|
||||||
|
"title": _("language"),
|
||||||
|
"choices": LOCALE_CHOICES_JSONFORM,
|
||||||
|
},
|
||||||
|
"text": {"type": "string", "title": _("content"), "widget": "textarea"},
|
||||||
|
},
|
||||||
|
"required": ["lang", "s"],
|
||||||
|
},
|
||||||
|
"uniqueItems": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def LanguageListField():
|
||||||
|
return jsondata.ArrayField(
|
||||||
|
verbose_name=_("language"),
|
||||||
|
base_field=models.CharField(blank=True, default="", max_length=100),
|
||||||
|
null=True,
|
||||||
|
blank=True,
|
||||||
|
default=list,
|
||||||
|
# schema={
|
||||||
|
# "type": "list",
|
||||||
|
# "items": {"type": "string", "choices": LANGUAGE_CHOICES_JSONFORM},
|
||||||
|
# },
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Item(PolymorphicModel):
|
class Item(PolymorphicModel):
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
external_resources: QuerySet["ExternalResource"]
|
external_resources: QuerySet["ExternalResource"]
|
||||||
|
@ -308,6 +376,22 @@ class Item(PolymorphicModel):
|
||||||
related_name="merged_from_items",
|
related_name="merged_from_items",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
localized_title = jsondata.JSONField(
|
||||||
|
verbose_name=_("title"),
|
||||||
|
null=False,
|
||||||
|
blank=True,
|
||||||
|
default=list,
|
||||||
|
schema=LOCALIZED_TITLE_SCHEMA,
|
||||||
|
)
|
||||||
|
|
||||||
|
localized_description = jsondata.JSONField(
|
||||||
|
verbose_name=_("description"),
|
||||||
|
null=False,
|
||||||
|
blank=True,
|
||||||
|
default=list,
|
||||||
|
schema=LOCALIZED_DESCRIPTION_SCHEMA,
|
||||||
|
)
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
index_together = [
|
index_together = [
|
||||||
[
|
[
|
||||||
|
@ -494,12 +578,52 @@ class Item(PolymorphicModel):
|
||||||
def class_name(self) -> str:
|
def class_name(self) -> str:
|
||||||
return self.__class__.__name__.lower()
|
return self.__class__.__name__.lower()
|
||||||
|
|
||||||
@property
|
def get_localized_title(self) -> str | None:
|
||||||
def display_title(self) -> str:
|
if self.localized_title:
|
||||||
return self.title
|
locales = get_current_locales()
|
||||||
|
for loc in locales:
|
||||||
|
v = next(
|
||||||
|
filter(lambda t: t["lang"] == loc, self.localized_title), {}
|
||||||
|
).get("text")
|
||||||
|
if v:
|
||||||
|
return v
|
||||||
|
|
||||||
|
def get_localized_description(self) -> str | None:
|
||||||
|
if self.localized_description:
|
||||||
|
locales = get_current_locales()
|
||||||
|
for loc in locales:
|
||||||
|
v = next(
|
||||||
|
filter(lambda t: t["lang"] == loc, self.localized_description), {}
|
||||||
|
).get("text")
|
||||||
|
if v:
|
||||||
|
return v
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def display_description(self):
|
def display_title(self) -> str:
|
||||||
|
return (
|
||||||
|
self.get_localized_title()
|
||||||
|
or self.title
|
||||||
|
or (
|
||||||
|
self.orig_title # type:ignore
|
||||||
|
if hasattr(self, "orig_title")
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
) or (self.localized_title[0]["text"] if self.localized_title else "")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def display_description(self) -> str:
|
||||||
|
return (
|
||||||
|
self.get_localized_description()
|
||||||
|
or self.brief
|
||||||
|
or (
|
||||||
|
self.localized_description[0]["text"]
|
||||||
|
if self.localized_description
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def brief_description(self):
|
||||||
return self.brief[:155]
|
return self.brief[:155]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
@ -547,7 +671,13 @@ class Item(PolymorphicModel):
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
"title",
|
||||||
"brief",
|
"brief",
|
||||||
|
"localized_title",
|
||||||
|
"localized_description",
|
||||||
] # list of metadata keys to copy from resource to item
|
] # list of metadata keys to copy from resource to item
|
||||||
|
METADATA_MERGE_LIST = [
|
||||||
|
"localized_title",
|
||||||
|
"localized_description",
|
||||||
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def copy_metadata(cls, metadata: dict[str, Any]) -> dict[str, Any]:
|
def copy_metadata(cls, metadata: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
@ -568,19 +698,26 @@ class Item(PolymorphicModel):
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def merge_data_from_external_resource(
|
||||||
|
self, p: "ExternalResource", ignore_existing_content: bool = False
|
||||||
|
):
|
||||||
|
for k in self.METADATA_COPY_LIST:
|
||||||
|
v = p.metadata.get(k)
|
||||||
|
if v:
|
||||||
|
if not getattr(self, k) or ignore_existing_content:
|
||||||
|
setattr(self, k, v)
|
||||||
|
elif k in self.METADATA_MERGE_LIST:
|
||||||
|
setattr(self, k, uniq((v or []) + getattr(self, k, [])))
|
||||||
|
if p.cover and (not self.has_cover() or ignore_existing_content):
|
||||||
|
self.cover = p.cover
|
||||||
|
|
||||||
def merge_data_from_external_resources(self, ignore_existing_content: bool = False):
|
def merge_data_from_external_resources(self, ignore_existing_content: bool = False):
|
||||||
"""Subclass may override this"""
|
"""Subclass may override this"""
|
||||||
lookup_ids = []
|
lookup_ids = []
|
||||||
for p in self.external_resources.all():
|
for p in self.external_resources.all():
|
||||||
lookup_ids.append((p.id_type, p.id_value))
|
lookup_ids.append((p.id_type, p.id_value))
|
||||||
lookup_ids += p.other_lookup_ids.items()
|
lookup_ids += p.other_lookup_ids.items()
|
||||||
for k in self.METADATA_COPY_LIST:
|
self.merge_data_from_external_resource(p, ignore_existing_content)
|
||||||
if p.metadata.get(k) and (
|
|
||||||
not getattr(self, k) or ignore_existing_content
|
|
||||||
):
|
|
||||||
setattr(self, k, p.metadata.get(k))
|
|
||||||
if p.cover and (not self.has_cover() or ignore_existing_content):
|
|
||||||
self.cover = p.cover
|
|
||||||
self.update_lookup_ids(list(set(lookup_ids)))
|
self.update_lookup_ids(list(set(lookup_ids)))
|
||||||
|
|
||||||
def update_linked_items_from_external_resource(self, resource: "ExternalResource"):
|
def update_linked_items_from_external_resource(self, resource: "ExternalResource"):
|
||||||
|
|
|
@ -349,8 +349,12 @@ def crawl_related_resources_task(resource_pk):
|
||||||
if not site and w.get("url"):
|
if not site and w.get("url"):
|
||||||
site = SiteManager.get_site_by_url(w["url"])
|
site = SiteManager.get_site_by_url(w["url"])
|
||||||
if site:
|
if site:
|
||||||
site.get_resource_ready(ignore_existing_content=False, auto_link=True)
|
res = site.get_resource_ready(
|
||||||
|
ignore_existing_content=False, auto_link=True
|
||||||
|
)
|
||||||
item = site.get_item()
|
item = site.get_item()
|
||||||
|
if item and res and w in resource.prematched_resources:
|
||||||
|
item.merge_data_from_external_resource(res)
|
||||||
if item:
|
if item:
|
||||||
logger.info(f"crawled {w} {item}")
|
logger.info(f"crawled {w} {item}")
|
||||||
else:
|
else:
|
||||||
|
|
5
catalog/common/tests.py
Normal file
5
catalog/common/tests.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
from django.test import TestCase as DjangoTestCase
|
||||||
|
|
||||||
|
|
||||||
|
class TestCase(DjangoTestCase):
|
||||||
|
databases = "__all__"
|
|
@ -3,6 +3,7 @@ from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
from common.forms import PreviewImageInput
|
from common.forms import PreviewImageInput
|
||||||
|
from common.models import DEFAULT_CATALOG_LANGUAGE, detect_language, uniq
|
||||||
|
|
||||||
CatalogForms = {}
|
CatalogForms = {}
|
||||||
|
|
||||||
|
@ -39,6 +40,73 @@ def _EditForm(item_model):
|
||||||
"cover": PreviewImageInput(),
|
"cover": PreviewImageInput(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.migrate_initial()
|
||||||
|
|
||||||
|
# {'id': 547, 'primary_lookup_id_type': 'imdb', 'primary_lookup_id_value': 'tt0056923', 'cover': <ImageFieldFile: item/tmdb_movie/2024/01/12/10973d2b-1d20-4e37-8c3c-ecc89e671a80.jpg>, 'orig_title': 'Charade', 'other_title': [], 'director': ['Stanley Donen'], 'playwright': ['Peter Stone'], 'actor': ['Cary Grant', 'Audrey Hepburn', 'Walter Matthau', 'James Coburn', 'George Kennedy', 'Dominique Minot', 'Ned Glass', 'Jacques Marin', 'Paul Bonifas', 'Thomas Chelimsky', 'Marc Arian', 'Claudine Berg', 'Marcel Bernier', 'Albert Daumergue', 'Raoul Delfosse', 'Stanley Donen', 'Jean Gold', 'Chantal Goya', 'Clément Harari', 'Monte Landis', 'Bernard Musson', 'Antonio Passalia', 'Jacques Préboist', 'Peter Stone', 'Michel Thomass', 'Roger Trapp', 'Louis Viret'], 'genre': ['喜剧', '悬疑', '爱情'], 'showtime': [{'time': '1963-12-05', 'region': ''}], 'site': '', 'area': [], 'language': ['English', 'Français', 'Deutsch', 'Italiano'], 'year': 1963, 'duration': '', 'localized_title': [], 'localized_description': []}
|
||||||
|
|
||||||
|
def migrate_initial(self):
|
||||||
|
if self.initial and self.instance:
|
||||||
|
if (
|
||||||
|
"localized_title" in self.Meta.fields
|
||||||
|
and not self.initial["localized_title"]
|
||||||
|
):
|
||||||
|
titles = []
|
||||||
|
if self.instance.title:
|
||||||
|
titles.append(
|
||||||
|
{
|
||||||
|
"lang": detect_language(self.instance.title),
|
||||||
|
"text": self.instance.title,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
hasattr(self.instance, "orig_title")
|
||||||
|
and self.instance.orig_title
|
||||||
|
):
|
||||||
|
titles.append(
|
||||||
|
{
|
||||||
|
"lang": detect_language(self.instance.orig_title),
|
||||||
|
"text": self.instance.orig_title,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
hasattr(self.instance, "other_title")
|
||||||
|
and self.instance.other_title
|
||||||
|
):
|
||||||
|
for t in self.instance.other_title:
|
||||||
|
titles.append(
|
||||||
|
{
|
||||||
|
"lang": detect_language(t),
|
||||||
|
"text": self.instance.orig_title,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not titles:
|
||||||
|
titles = [
|
||||||
|
{"lang": DEFAULT_CATALOG_LANGUAGE, "text": "<no title>"}
|
||||||
|
]
|
||||||
|
self.initial["localized_title"] = uniq(titles) # type:ignore
|
||||||
|
if (
|
||||||
|
"localized_description" in self.Meta.fields
|
||||||
|
and not self.initial["localized_description"]
|
||||||
|
):
|
||||||
|
if self.instance.brief:
|
||||||
|
d = {
|
||||||
|
"lang": detect_language(self.instance.brief),
|
||||||
|
"text": self.instance.brief,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
d = {
|
||||||
|
"lang": self.initial["localized_title"][0]["lang"],
|
||||||
|
"text": "",
|
||||||
|
}
|
||||||
|
self.initial["localized_description"] = [d] # type:ignore
|
||||||
|
# if (
|
||||||
|
# "language" in self.Meta.fields
|
||||||
|
# and self.initial["language"]
|
||||||
|
# ):
|
||||||
|
# if isinstance(self.initial["language"], str):
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
data = super().clean() or {}
|
data = super().clean() or {}
|
||||||
t, v = self.Meta.model.lookup_id_cleanup(
|
t, v = self.Meta.model.lookup_id_cleanup(
|
||||||
|
|
|
@ -39,9 +39,11 @@ class Game(Item):
|
||||||
douban_game = PrimaryLookupIdDescriptor(IdType.DoubanGame)
|
douban_game = PrimaryLookupIdDescriptor(IdType.DoubanGame)
|
||||||
|
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
"brief",
|
# "brief",
|
||||||
"other_title",
|
# "other_title",
|
||||||
|
"localized_title",
|
||||||
|
"localized_description",
|
||||||
"designer",
|
"designer",
|
||||||
"artist",
|
"artist",
|
||||||
"developer",
|
"developer",
|
||||||
|
|
|
@ -5,6 +5,8 @@ from catalog.models import *
|
||||||
|
|
||||||
|
|
||||||
class IGDBTestCase(TestCase):
|
class IGDBTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id_type = IdType.IGDB
|
t_id_type = IdType.IGDB
|
||||||
t_id_value = "portal-2"
|
t_id_value = "portal-2"
|
||||||
|
@ -42,7 +44,9 @@ class IGDBTestCase(TestCase):
|
||||||
)
|
)
|
||||||
self.assertIsInstance(site.resource.item, Game)
|
self.assertIsInstance(site.resource.item, Game)
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IGDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IGDB)
|
||||||
self.assertEqual(site.resource.item.genre, ["Role-playing (RPG)", "Adventure"])
|
self.assertEqual(
|
||||||
|
site.resource.item.genre, ["Puzzle", "Role-playing (RPG)", "Adventure"]
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
site.resource.item.primary_lookup_id_value,
|
site.resource.item.primary_lookup_id_value,
|
||||||
"the-legend-of-zelda-breath-of-the-wild",
|
"the-legend-of-zelda-breath-of-the-wild",
|
||||||
|
@ -50,6 +54,8 @@ class IGDBTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class SteamTestCase(TestCase):
|
class SteamTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id_type = IdType.Steam
|
t_id_type = IdType.Steam
|
||||||
t_id_value = "620"
|
t_id_value = "620"
|
||||||
|
@ -70,10 +76,7 @@ class SteamTestCase(TestCase):
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "Portal 2")
|
self.assertEqual(site.resource.metadata["title"], "Portal 2")
|
||||||
self.assertEqual(
|
self.assertEqual(site.resource.metadata["brief"][:6], "Sequel")
|
||||||
site.resource.metadata["brief"],
|
|
||||||
"“终身测试计划”现已升级,您可以为您自己或您的好友设计合作谜题!",
|
|
||||||
)
|
|
||||||
self.assertIsInstance(site.resource.item, Game)
|
self.assertIsInstance(site.resource.item, Game)
|
||||||
self.assertEqual(site.resource.item.steam, "620")
|
self.assertEqual(site.resource.item.steam, "620")
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
|
@ -82,6 +85,8 @@ class SteamTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class DoubanGameTestCase(TestCase):
|
class DoubanGameTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id_type = IdType.DoubanGame
|
t_id_type = IdType.DoubanGame
|
||||||
t_id_value = "10734307"
|
t_id_value = "10734307"
|
||||||
|
@ -100,16 +105,16 @@ class DoubanGameTestCase(TestCase):
|
||||||
self.assertEqual(site.ready, False)
|
self.assertEqual(site.ready, False)
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "传送门2 Portal 2")
|
|
||||||
self.assertIsInstance(site.resource.item, Game)
|
self.assertIsInstance(site.resource.item, Game)
|
||||||
|
self.assertEqual(site.resource.item.display_title, "Portal 2")
|
||||||
self.assertEqual(site.resource.item.douban_game, "10734307")
|
self.assertEqual(site.resource.item.douban_game, "10734307")
|
||||||
self.assertEqual(
|
self.assertEqual(site.resource.item.genre, ["第一人称射击", "益智"])
|
||||||
site.resource.item.genre, ["第一人称射击", "益智", "射击", "动作"]
|
|
||||||
)
|
|
||||||
self.assertEqual(site.resource.item.other_title, [])
|
self.assertEqual(site.resource.item.other_title, [])
|
||||||
|
|
||||||
|
|
||||||
class BangumiGameTestCase(TestCase):
|
class BangumiGameTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id_type = IdType.Bangumi
|
t_id_type = IdType.Bangumi
|
||||||
|
@ -124,6 +129,8 @@ class BangumiGameTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class BoardGameGeekTestCase(TestCase):
|
class BoardGameGeekTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_scrape(self):
|
def test_scrape(self):
|
||||||
t_url = "https://boardgamegeek.com/boardgame/167791"
|
t_url = "https://boardgamegeek.com/boardgame/167791"
|
||||||
|
@ -134,15 +141,17 @@ class BoardGameGeekTestCase(TestCase):
|
||||||
self.assertEqual(site.ready, False)
|
self.assertEqual(site.ready, False)
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "Terraforming Mars")
|
|
||||||
self.assertIsInstance(site.resource.item, Game)
|
self.assertIsInstance(site.resource.item, Game)
|
||||||
|
self.assertEqual(site.resource.item.display_title, "Terraforming Mars")
|
||||||
self.assertEqual(site.resource.item.platform, ["Boardgame"])
|
self.assertEqual(site.resource.item.platform, ["Boardgame"])
|
||||||
self.assertEqual(site.resource.item.genre[0], "Economic")
|
self.assertEqual(site.resource.item.genre[0], "Economic")
|
||||||
self.assertEqual(site.resource.item.other_title[0], "殖民火星")
|
# self.assertEqual(site.resource.item.other_title[0], "殖民火星")
|
||||||
self.assertEqual(site.resource.item.designer, ["Jacob Fryxelius"])
|
self.assertEqual(site.resource.item.designer, ["Jacob Fryxelius"])
|
||||||
|
|
||||||
|
|
||||||
class MultiGameSitesTestCase(TestCase):
|
class MultiGameSitesTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_games(self):
|
def test_games(self):
|
||||||
url1 = "https://www.igdb.com/games/portal-2"
|
url1 = "https://www.igdb.com/games/portal-2"
|
||||||
|
|
|
@ -13,6 +13,7 @@ from catalog.common import (
|
||||||
PrimaryLookupIdDescriptor,
|
PrimaryLookupIdDescriptor,
|
||||||
jsondata,
|
jsondata,
|
||||||
)
|
)
|
||||||
|
from catalog.common.models import LanguageListField
|
||||||
|
|
||||||
|
|
||||||
class MovieInSchema(ItemInSchema):
|
class MovieInSchema(ItemInSchema):
|
||||||
|
@ -43,9 +44,10 @@ class Movie(Item):
|
||||||
douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie)
|
douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie)
|
||||||
|
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
|
"localized_title",
|
||||||
"orig_title",
|
"orig_title",
|
||||||
"other_title",
|
# "other_title",
|
||||||
"director",
|
"director",
|
||||||
"playwright",
|
"playwright",
|
||||||
"actor",
|
"actor",
|
||||||
|
@ -59,7 +61,8 @@ class Movie(Item):
|
||||||
# "season_number",
|
# "season_number",
|
||||||
# "episodes",
|
# "episodes",
|
||||||
# "single_episode_length",
|
# "single_episode_length",
|
||||||
"brief",
|
"localized_description",
|
||||||
|
# "brief",
|
||||||
]
|
]
|
||||||
orig_title = jsondata.CharField(
|
orig_title = jsondata.CharField(
|
||||||
verbose_name=_("original title"), blank=True, default="", max_length=500
|
verbose_name=_("original title"), blank=True, default="", max_length=500
|
||||||
|
@ -141,17 +144,7 @@ class Movie(Item):
|
||||||
blank=True,
|
blank=True,
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.ArrayField(
|
language = LanguageListField()
|
||||||
verbose_name=_("language"),
|
|
||||||
base_field=models.CharField(
|
|
||||||
blank=True,
|
|
||||||
default="",
|
|
||||||
max_length=100,
|
|
||||||
),
|
|
||||||
null=True,
|
|
||||||
blank=True,
|
|
||||||
default=list,
|
|
||||||
)
|
|
||||||
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
||||||
duration = jsondata.CharField(
|
duration = jsondata.CharField(
|
||||||
verbose_name=_("length"), blank=True, default="", max_length=200
|
verbose_name=_("length"), blank=True, default="", max_length=200
|
||||||
|
|
|
@ -4,6 +4,8 @@ from catalog.common import *
|
||||||
|
|
||||||
|
|
||||||
class DoubanMovieTestCase(TestCase):
|
class DoubanMovieTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id = "3541415"
|
t_id = "3541415"
|
||||||
t_url = "https://movie.douban.com/subject/3541415/"
|
t_url = "https://movie.douban.com/subject/3541415/"
|
||||||
|
@ -28,6 +30,8 @@ class DoubanMovieTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class TMDBMovieTestCase(TestCase):
|
class TMDBMovieTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id = "293767"
|
t_id = "293767"
|
||||||
t_url = (
|
t_url = (
|
||||||
|
@ -49,13 +53,17 @@ class TMDBMovieTestCase(TestCase):
|
||||||
self.assertEqual(site.ready, False)
|
self.assertEqual(site.ready, False)
|
||||||
self.assertEqual(site.id_value, "293767")
|
self.assertEqual(site.id_value, "293767")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.resource.metadata["title"], "比利·林恩的中场战事")
|
self.assertEqual(
|
||||||
|
site.resource.metadata["title"], "Billy Lynn's Long Halftime Walk"
|
||||||
|
)
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.__class__.__name__, "Movie")
|
self.assertEqual(site.resource.item.__class__.__name__, "Movie")
|
||||||
self.assertEqual(site.resource.item.imdb, "tt2513074")
|
self.assertEqual(site.resource.item.imdb, "tt2513074")
|
||||||
|
|
||||||
|
|
||||||
class IMDBMovieTestCase(TestCase):
|
class IMDBMovieTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id = "tt1375666"
|
t_id = "tt1375666"
|
||||||
t_url = "https://www.imdb.com/title/tt1375666/"
|
t_url = "https://www.imdb.com/title/tt1375666/"
|
||||||
|
@ -75,24 +83,28 @@ class IMDBMovieTestCase(TestCase):
|
||||||
self.assertEqual(site.ready, False)
|
self.assertEqual(site.ready, False)
|
||||||
self.assertEqual(site.id_value, "tt1375666")
|
self.assertEqual(site.id_value, "tt1375666")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.resource.metadata["title"], "盗梦空间")
|
self.assertEqual(site.resource.metadata["title"], "Inception")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.imdb, "tt1375666")
|
self.assertEqual(site.resource.item.imdb, "tt1375666")
|
||||||
|
|
||||||
|
|
||||||
class BangumiMovieTestCase(TestCase):
|
class BangumiMovieTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_scrape(self):
|
def test_scrape(self):
|
||||||
url = "https://bgm.tv/subject/237"
|
url = "https://bgm.tv/subject/237"
|
||||||
site = SiteManager.get_site_by_url(url)
|
site = SiteManager.get_site_by_url(url)
|
||||||
self.assertEqual(site.id_value, "237")
|
self.assertEqual(site.id_value, "237")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.resource.metadata["title"], "攻壳机动队")
|
self.assertEqual(site.resource.item.display_title, "GHOST IN THE SHELL")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.imdb, "tt0113568")
|
self.assertEqual(site.resource.item.imdb, "tt0113568")
|
||||||
|
|
||||||
|
|
||||||
class MultiMovieSitesTestCase(TestCase):
|
class MultiMovieSitesTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_movies(self):
|
def test_movies(self):
|
||||||
url1 = "https://www.themoviedb.org/movie/27205-inception"
|
url1 = "https://www.themoviedb.org/movie/27205-inception"
|
||||||
|
|
|
@ -41,12 +41,14 @@ class Album(Item):
|
||||||
douban_music = PrimaryLookupIdDescriptor(IdType.DoubanMusic)
|
douban_music = PrimaryLookupIdDescriptor(IdType.DoubanMusic)
|
||||||
spotify_album = PrimaryLookupIdDescriptor(IdType.Spotify_Album)
|
spotify_album = PrimaryLookupIdDescriptor(IdType.Spotify_Album)
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
"other_title",
|
# "other_title",
|
||||||
|
"localized_title",
|
||||||
"artist",
|
"artist",
|
||||||
"company",
|
"company",
|
||||||
"track_list",
|
"track_list",
|
||||||
"brief",
|
# "brief",
|
||||||
|
"localized_description",
|
||||||
"album_type",
|
"album_type",
|
||||||
"media",
|
"media",
|
||||||
"disc_count",
|
"disc_count",
|
||||||
|
|
|
@ -69,7 +69,7 @@ class DoubanMusicTestCase(TestCase):
|
||||||
self.assertIsInstance(site.resource.item, Album)
|
self.assertIsInstance(site.resource.item, Album)
|
||||||
self.assertEqual(site.resource.item.barcode, "0077774644020")
|
self.assertEqual(site.resource.item.barcode, "0077774644020")
|
||||||
self.assertEqual(site.resource.item.genre, ["摇滚"])
|
self.assertEqual(site.resource.item.genre, ["摇滚"])
|
||||||
self.assertEqual(site.resource.item.other_title, ["橡胶灵魂"])
|
self.assertEqual(len(site.resource.item.localized_title), 2)
|
||||||
|
|
||||||
|
|
||||||
class MultiMusicSitesTestCase(TestCase):
|
class MultiMusicSitesTestCase(TestCase):
|
||||||
|
|
|
@ -14,6 +14,7 @@ from catalog.common import (
|
||||||
ItemType,
|
ItemType,
|
||||||
jsondata,
|
jsondata,
|
||||||
)
|
)
|
||||||
|
from catalog.common.models import LanguageListField
|
||||||
from catalog.common.utils import DEFAULT_ITEM_COVER
|
from catalog.common.utils import DEFAULT_ITEM_COVER
|
||||||
|
|
||||||
|
|
||||||
|
@ -124,13 +125,7 @@ class Performance(Item):
|
||||||
blank=False,
|
blank=False,
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.ArrayField(
|
language = LanguageListField()
|
||||||
verbose_name=_("language"),
|
|
||||||
base_field=models.CharField(blank=False, default="", max_length=200),
|
|
||||||
null=False,
|
|
||||||
blank=True,
|
|
||||||
default=list,
|
|
||||||
)
|
|
||||||
director = jsondata.ArrayField(
|
director = jsondata.ArrayField(
|
||||||
verbose_name=_("director"),
|
verbose_name=_("director"),
|
||||||
base_field=models.CharField(blank=False, default="", max_length=500),
|
base_field=models.CharField(blank=False, default="", max_length=500),
|
||||||
|
@ -211,10 +206,12 @@ class Performance(Item):
|
||||||
verbose_name=_("website"), max_length=1000, null=True, blank=True
|
verbose_name=_("website"), max_length=1000, null=True, blank=True
|
||||||
)
|
)
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
"brief",
|
# "brief",
|
||||||
|
"localized_title",
|
||||||
|
"localized_description",
|
||||||
"orig_title",
|
"orig_title",
|
||||||
"other_title",
|
# "other_title",
|
||||||
"genre",
|
"genre",
|
||||||
"language",
|
"language",
|
||||||
"opening_date",
|
"opening_date",
|
||||||
|
@ -266,13 +263,7 @@ class PerformanceProduction(Item):
|
||||||
blank=True,
|
blank=True,
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.ArrayField(
|
language = LanguageListField()
|
||||||
verbose_name=_("language"),
|
|
||||||
base_field=models.CharField(blank=False, default="", max_length=200),
|
|
||||||
null=False,
|
|
||||||
blank=True,
|
|
||||||
default=list,
|
|
||||||
)
|
|
||||||
director = jsondata.ArrayField(
|
director = jsondata.ArrayField(
|
||||||
verbose_name=_("director"),
|
verbose_name=_("director"),
|
||||||
base_field=models.CharField(blank=False, default="", max_length=500),
|
base_field=models.CharField(blank=False, default="", max_length=500),
|
||||||
|
@ -353,10 +344,12 @@ class PerformanceProduction(Item):
|
||||||
verbose_name=_("website"), max_length=1000, null=True, blank=True
|
verbose_name=_("website"), max_length=1000, null=True, blank=True
|
||||||
)
|
)
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
"localized_title",
|
||||||
"brief",
|
"localized_description",
|
||||||
|
# "title",
|
||||||
|
# "brief",
|
||||||
"orig_title",
|
"orig_title",
|
||||||
"other_title",
|
# "other_title",
|
||||||
"language",
|
"language",
|
||||||
"opening_date",
|
"opening_date",
|
||||||
"closing_date",
|
"closing_date",
|
||||||
|
@ -389,7 +382,9 @@ class PerformanceProduction(Item):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def display_title(self):
|
def display_title(self):
|
||||||
return f"{self.show.title if self.show else '♢'} {self.title}"
|
return (
|
||||||
|
f"{self.show.display_title if self.show else '♢'} {super().display_title}"
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def cover_image_url(self) -> str | None:
|
def cover_image_url(self) -> str | None:
|
||||||
|
|
|
@ -5,6 +5,8 @@ from catalog.common.sites import crawl_related_resources_task
|
||||||
|
|
||||||
|
|
||||||
class DoubanDramaTestCase(TestCase):
|
class DoubanDramaTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -29,10 +31,10 @@ class DoubanDramaTestCase(TestCase):
|
||||||
site = SiteManager.get_site_by_url(t_url)
|
site = SiteManager.get_site_by_url(t_url)
|
||||||
resource = site.get_resource_ready()
|
resource = site.get_resource_ready()
|
||||||
item = site.get_item()
|
item = site.get_item()
|
||||||
self.assertEqual(item.title, "不眠之人·拿破仑")
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
item.other_title, ["眠らない男・ナポレオン ―愛と栄光の涯(はて)に―"]
|
item.display_title, "眠らない男・ナポレオン ―愛と栄光の涯(はて)に―"
|
||||||
)
|
)
|
||||||
|
self.assertEqual(len(item.localized_title), 2)
|
||||||
self.assertEqual(item.genre, ["音乐剧"])
|
self.assertEqual(item.genre, ["音乐剧"])
|
||||||
self.assertEqual(item.troupe, ["宝塚歌剧团"])
|
self.assertEqual(item.troupe, ["宝塚歌剧团"])
|
||||||
self.assertEqual(item.composer, ["ジェラール・プレスギュルヴィック"])
|
self.assertEqual(item.composer, ["ジェラール・プレスギュルヴィック"])
|
||||||
|
@ -41,7 +43,7 @@ class DoubanDramaTestCase(TestCase):
|
||||||
site = SiteManager.get_site_by_url(t_url)
|
site = SiteManager.get_site_by_url(t_url)
|
||||||
resource = site.get_resource_ready()
|
resource = site.get_resource_ready()
|
||||||
item = site.get_item()
|
item = site.get_item()
|
||||||
self.assertEqual(item.title, "相声说垮鬼子们")
|
self.assertEqual(item.display_title, "相聲說垮鬼子們")
|
||||||
self.assertEqual(item.opening_date, "1997-05")
|
self.assertEqual(item.opening_date, "1997-05")
|
||||||
self.assertEqual(item.location, ["臺北新舞臺"])
|
self.assertEqual(item.location, ["臺北新舞臺"])
|
||||||
|
|
||||||
|
@ -54,7 +56,8 @@ class DoubanDramaTestCase(TestCase):
|
||||||
if item is None:
|
if item is None:
|
||||||
raise ValueError()
|
raise ValueError()
|
||||||
self.assertEqual(item.orig_title, "Iphigenie auf Tauris")
|
self.assertEqual(item.orig_title, "Iphigenie auf Tauris")
|
||||||
self.assertEqual(sorted(item.other_title), ["死而复生的伊菲格尼"])
|
print(item.localized_title)
|
||||||
|
self.assertEqual(len(item.localized_title), 3)
|
||||||
self.assertEqual(item.opening_date, "1974-04-21")
|
self.assertEqual(item.opening_date, "1974-04-21")
|
||||||
self.assertEqual(item.choreographer, ["Pina Bausch"])
|
self.assertEqual(item.choreographer, ["Pina Bausch"])
|
||||||
|
|
||||||
|
@ -68,9 +71,9 @@ class DoubanDramaTestCase(TestCase):
|
||||||
item = site.get_item()
|
item = site.get_item()
|
||||||
if item is None:
|
if item is None:
|
||||||
raise ValueError()
|
raise ValueError()
|
||||||
self.assertEqual(item.title, "红花侠")
|
self.assertEqual(item.display_title, "THE SCARLET PIMPERNEL")
|
||||||
self.assertEqual(sorted(item.other_title), ["THE SCARLET PIMPERNEL"])
|
self.assertEqual(len(item.localized_title), 3)
|
||||||
self.assertEqual(len(item.brief), 545)
|
self.assertEqual(len(item.display_description), 545)
|
||||||
self.assertEqual(item.genre, ["音乐剧"])
|
self.assertEqual(item.genre, ["音乐剧"])
|
||||||
# self.assertEqual(
|
# self.assertEqual(
|
||||||
# item.version, ["08星组公演版", "10年月組公演版", "17年星組公演版", "ュージカル(2017年)版"]
|
# item.version, ["08星组公演版", "10年月組公演版", "17年星組公演版", "ュージカル(2017年)版"]
|
||||||
|
@ -80,7 +83,7 @@ class DoubanDramaTestCase(TestCase):
|
||||||
item.playwright, ["小池修一郎", "Baroness Orczy(原作)", "小池 修一郎"]
|
item.playwright, ["小池修一郎", "Baroness Orczy(原作)", "小池 修一郎"]
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
item.actor,
|
sorted(item.actor, key=lambda a: a["name"]),
|
||||||
[
|
[
|
||||||
{"name": "安蘭けい", "role": ""},
|
{"name": "安蘭けい", "role": ""},
|
||||||
{"name": "柚希礼音", "role": ""},
|
{"name": "柚希礼音", "role": ""},
|
||||||
|
@ -110,7 +113,10 @@ class DoubanDramaTestCase(TestCase):
|
||||||
self.assertEqual(productions[2].closing_date, "2017-03-17")
|
self.assertEqual(productions[2].closing_date, "2017-03-17")
|
||||||
self.assertEqual(productions[3].opening_date, "2017-11-13")
|
self.assertEqual(productions[3].opening_date, "2017-11-13")
|
||||||
self.assertEqual(productions[3].closing_date, None)
|
self.assertEqual(productions[3].closing_date, None)
|
||||||
self.assertEqual(productions[3].title, "ミュージカル(2017年)版")
|
self.assertEqual(
|
||||||
|
productions[3].display_title,
|
||||||
|
"THE SCARLET PIMPERNEL ミュージカル(2017年)版",
|
||||||
|
)
|
||||||
self.assertEqual(len(productions[3].actor), 6)
|
self.assertEqual(len(productions[3].actor), 6)
|
||||||
self.assertEqual(productions[3].language, ["日语"])
|
self.assertEqual(productions[3].language, ["日语"])
|
||||||
self.assertEqual(productions[3].opening_date, "2017-11-13")
|
self.assertEqual(productions[3].opening_date, "2017-11-13")
|
||||||
|
|
|
@ -15,6 +15,7 @@ from catalog.common import (
|
||||||
PrimaryLookupIdDescriptor,
|
PrimaryLookupIdDescriptor,
|
||||||
jsondata,
|
jsondata,
|
||||||
)
|
)
|
||||||
|
from catalog.common.models import LanguageListField
|
||||||
|
|
||||||
|
|
||||||
class PodcastInSchema(ItemInSchema):
|
class PodcastInSchema(ItemInSchema):
|
||||||
|
@ -44,6 +45,8 @@ class Podcast(Item):
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
language = LanguageListField()
|
||||||
|
|
||||||
hosts = jsondata.ArrayField(
|
hosts = jsondata.ArrayField(
|
||||||
verbose_name=_("host"),
|
verbose_name=_("host"),
|
||||||
base_field=models.CharField(blank=True, default="", max_length=200),
|
base_field=models.CharField(blank=True, default="", max_length=200),
|
||||||
|
@ -55,8 +58,11 @@ class Podcast(Item):
|
||||||
)
|
)
|
||||||
|
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
"brief",
|
# "brief",
|
||||||
|
"localized_title",
|
||||||
|
"localized_description",
|
||||||
|
"language",
|
||||||
"hosts",
|
"hosts",
|
||||||
"genre",
|
"genre",
|
||||||
"official_site",
|
"official_site",
|
||||||
|
|
|
@ -91,7 +91,7 @@ class PodcastRSSFeedTestCase(TestCase):
|
||||||
metadata["official_site"], "http://www.bbc.co.uk/programmes/b006qykl"
|
metadata["official_site"], "http://www.bbc.co.uk/programmes/b006qykl"
|
||||||
)
|
)
|
||||||
self.assertEqual(metadata["genre"], ["History"])
|
self.assertEqual(metadata["genre"], ["History"])
|
||||||
self.assertEqual(metadata["hosts"], ["BBC Radio 4"])
|
self.assertEqual(metadata["host"], ["BBC Radio 4"])
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
||||||
|
@ -108,7 +108,7 @@ class PodcastRSSFeedTestCase(TestCase):
|
||||||
metadata["official_site"], "https://www.ximalaya.com/qita/51101122/"
|
metadata["official_site"], "https://www.ximalaya.com/qita/51101122/"
|
||||||
)
|
)
|
||||||
self.assertEqual(metadata["genre"], ["人文国学"])
|
self.assertEqual(metadata["genre"], ["人文国学"])
|
||||||
self.assertEqual(metadata["hosts"], ["看理想vistopia"])
|
self.assertEqual(metadata["host"], ["看理想vistopia"])
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
||||||
|
@ -123,7 +123,7 @@ class PodcastRSSFeedTestCase(TestCase):
|
||||||
self.assertEqual(metadata["title"], "跳岛FM")
|
self.assertEqual(metadata["title"], "跳岛FM")
|
||||||
self.assertEqual(metadata["official_site"], "https://tiaodao.typlog.io/")
|
self.assertEqual(metadata["official_site"], "https://tiaodao.typlog.io/")
|
||||||
self.assertEqual(metadata["genre"], ["Arts", "Books"])
|
self.assertEqual(metadata["genre"], ["Arts", "Books"])
|
||||||
self.assertEqual(metadata["hosts"], ["中信出版·大方"])
|
self.assertEqual(metadata["host"], ["中信出版·大方"])
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].title)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].link)
|
||||||
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
self.assertIsNotNone(site.get_item().recent_episodes[0].media_url)
|
||||||
|
|
|
@ -11,7 +11,7 @@ from lxml import html
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
from catalog.sites.spotify import get_spotify_token
|
from catalog.sites.spotify import get_spotify_token
|
||||||
from catalog.sites.tmdb import get_language_code
|
from catalog.sites.tmdb import TMDB_DEFAULT_LANG
|
||||||
|
|
||||||
SEARCH_PAGE_SIZE = 5 # not all apis support page size
|
SEARCH_PAGE_SIZE = 5 # not all apis support page size
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -173,7 +173,7 @@ class TheMovieDatabase:
|
||||||
@classmethod
|
@classmethod
|
||||||
def search(cls, q, page=1):
|
def search(cls, q, page=1):
|
||||||
results = []
|
results = []
|
||||||
api_url = f"https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&include_adult=true"
|
api_url = f"https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&include_adult=true"
|
||||||
try:
|
try:
|
||||||
j = requests.get(api_url, timeout=2).json()
|
j = requests.get(api_url, timeout=2).json()
|
||||||
if j.get("results"):
|
if j.get("results"):
|
||||||
|
|
|
@ -10,11 +10,18 @@ Scraping the website directly.
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
from threading import local
|
||||||
|
|
||||||
import dateparser
|
import dateparser
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import (
|
||||||
|
DEFAULT_CATALOG_LANGUAGE,
|
||||||
|
PREFERRED_LANGUAGES,
|
||||||
|
detect_language,
|
||||||
|
)
|
||||||
|
from common.models.misc import uniq
|
||||||
|
|
||||||
from .douban import *
|
from .douban import *
|
||||||
|
|
||||||
|
@ -47,27 +54,58 @@ class AppleMusic(AbstractSite):
|
||||||
def id_to_url(cls, id_value):
|
def id_to_url(cls, id_value):
|
||||||
return f"https://music.apple.com/album/{id_value}"
|
return f"https://music.apple.com/album/{id_value}"
|
||||||
|
|
||||||
def get_localized_urls(self):
|
def get_locales(self):
|
||||||
return [
|
locales = {}
|
||||||
f"https://music.apple.com/{locale}/album/{self.id_value}"
|
for l in PREFERRED_LANGUAGES:
|
||||||
for locale in ["hk", "tw", "us", "sg", "jp", "cn", "gb", "ca", "fr"]
|
match l:
|
||||||
]
|
case "zh":
|
||||||
|
locales.update({"zh": ["cn", "tw", "hk", "sg"]})
|
||||||
|
case "en":
|
||||||
|
locales.update({"en": ["us", "gb", "ca"]})
|
||||||
|
case "ja":
|
||||||
|
locales.update({"ja": ["jp"]})
|
||||||
|
case "ko":
|
||||||
|
locales.update({"ko": ["kr"]})
|
||||||
|
case "fr":
|
||||||
|
locales.update({"fr": ["fr", "ca"]})
|
||||||
|
if not locales:
|
||||||
|
locales = {"en": ["us"]}
|
||||||
|
return locales
|
||||||
|
|
||||||
def scrape(self):
|
def scrape(self):
|
||||||
content = None
|
matched_content = None
|
||||||
# it's less than ideal to waterfall thru locales, a better solution
|
localized_title = []
|
||||||
# would be change ExternalResource to store preferred locale,
|
localized_desc = []
|
||||||
# or to find an AppleMusic API to get available locales for an album
|
for lang, locales in self.get_locales().items():
|
||||||
for url in self.get_localized_urls():
|
for loc in locales: # waterfall thru all locales
|
||||||
|
url = f"https://music.apple.com/{loc}/album/{self.id_value}"
|
||||||
try:
|
try:
|
||||||
content = BasicDownloader(url, headers=self.headers).download().html()
|
content = (
|
||||||
|
BasicDownloader(url, headers=self.headers).download().html()
|
||||||
|
)
|
||||||
_logger.info(f"got localized content from {url}")
|
_logger.info(f"got localized content from {url}")
|
||||||
|
elem = content.xpath(
|
||||||
|
"//script[@id='serialized-server-data']/text()"
|
||||||
|
)
|
||||||
|
txt: str = elem[0] # type:ignore
|
||||||
|
page_data = json.loads(txt)[0]
|
||||||
|
album_data = page_data["data"]["sections"][0]["items"][0]
|
||||||
|
title = album_data["title"]
|
||||||
|
brief = album_data.get("modalPresentationDescriptor", {}).get(
|
||||||
|
"paragraphText", ""
|
||||||
|
)
|
||||||
|
l = detect_language(title + " " + brief)
|
||||||
|
localized_title.append({"lang": l, "text": title})
|
||||||
|
if brief:
|
||||||
|
localized_desc.append({"lang": l, "text": brief})
|
||||||
|
if lang == DEFAULT_CATALOG_LANGUAGE or not matched_content:
|
||||||
|
matched_content = content
|
||||||
break
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
if content is None:
|
if matched_content is None:
|
||||||
raise ParseError(self, f"localized content for {self.url}")
|
raise ParseError(self, f"localized content for {self.url}")
|
||||||
elem = content.xpath("//script[@id='serialized-server-data']/text()")
|
elem = matched_content.xpath("//script[@id='serialized-server-data']/text()")
|
||||||
txt: str = elem[0] # type:ignore
|
txt: str = elem[0] # type:ignore
|
||||||
page_data = json.loads(txt)[0]
|
page_data = json.loads(txt)[0]
|
||||||
album_data = page_data["data"]["sections"][0]["items"][0]
|
album_data = page_data["data"]["sections"][0]["items"][0]
|
||||||
|
@ -99,12 +137,14 @@ class AppleMusic(AbstractSite):
|
||||||
genre[0]
|
genre[0]
|
||||||
] # apple treat "Music" as a genre. Thus, only the first genre is obtained.
|
] # apple treat "Music" as a genre. Thus, only the first genre is obtained.
|
||||||
|
|
||||||
images = content.xpath("//source[@type='image/jpeg']/@srcset")
|
images = matched_content.xpath("//source[@type='image/jpeg']/@srcset")
|
||||||
image_elem: str = images[0] if images else "" # type:ignore
|
image_elem: str = images[0] if images else "" # type:ignore
|
||||||
image_url = image_elem.split(" ")[0] if image_elem else None
|
image_url = image_elem.split(" ")[0] if image_elem else None
|
||||||
|
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": uniq(localized_title),
|
||||||
|
"localized_description": uniq(localized_desc),
|
||||||
"title": title,
|
"title": title,
|
||||||
"brief": brief,
|
"brief": brief,
|
||||||
"artist": artist,
|
"artist": artist,
|
||||||
|
|
|
@ -26,11 +26,12 @@ class ApplePodcast(AbstractSite):
|
||||||
resp = dl.download()
|
resp = dl.download()
|
||||||
r = resp.json()["results"][0]
|
r = resp.json()["results"][0]
|
||||||
feed_url = r["feedUrl"]
|
feed_url = r["feedUrl"]
|
||||||
|
title = r["trackName"]
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
"title": r["trackName"],
|
"title": title,
|
||||||
"feed_url": feed_url,
|
"feed_url": feed_url,
|
||||||
"hosts": [r["artistName"]],
|
"host": [r["artistName"]],
|
||||||
"genres": r["genres"],
|
"genres": r["genres"],
|
||||||
"cover_image_url": r["artworkUrl600"],
|
"cover_image_url": r["artworkUrl600"],
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import dns.resolver
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -76,14 +77,19 @@ class Bandcamp(AbstractSite):
|
||||||
duration = None
|
duration = None
|
||||||
company = None
|
company = None
|
||||||
brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()")
|
brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()")
|
||||||
brief = "".join(brief_nodes) if brief_nodes else None # type:ignore
|
brief = "".join(brief_nodes) if brief_nodes else "" # type:ignore
|
||||||
cover_url = self.query_str(content, "//div[@id='tralbumArt']/a/@href")
|
cover_url = self.query_str(content, "//div[@id='tralbumArt']/a/@href")
|
||||||
bandcamp_page_data = json.loads(
|
bandcamp_page_data = json.loads(
|
||||||
self.query_str(content, "//meta[@name='bc-page-properties']/@content")
|
self.query_str(content, "//meta[@name='bc-page-properties']/@content")
|
||||||
)
|
)
|
||||||
bandcamp_album_id = bandcamp_page_data["item_id"]
|
bandcamp_album_id = bandcamp_page_data["item_id"]
|
||||||
|
localized_title = [{"lang": detect_language(title), "text": title}]
|
||||||
|
localized_desc = (
|
||||||
|
[{"lang": detect_language(brief), "text": brief}] if brief else []
|
||||||
|
)
|
||||||
data = {
|
data = {
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"title": title,
|
"title": title,
|
||||||
"artist": artist,
|
"artist": artist,
|
||||||
"genre": genre,
|
"genre": genre,
|
||||||
|
|
|
@ -3,6 +3,7 @@ import logging
|
||||||
from catalog.book.utils import detect_isbn_asin
|
from catalog.book.utils import detect_isbn_asin
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -101,7 +102,14 @@ class Bangumi(AbstractSite):
|
||||||
raw_img, ext = BasicImageDownloader.download_image(
|
raw_img, ext = BasicImageDownloader.download_image(
|
||||||
img_url, None, headers={}
|
img_url, None, headers={}
|
||||||
)
|
)
|
||||||
|
titles = set(
|
||||||
|
[title] + (other_title or []) + ([orig_title] if orig_title else [])
|
||||||
|
)
|
||||||
|
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
|
||||||
|
localized_desc = [{"lang": detect_language(brief), "text": brief}]
|
||||||
data = {
|
data = {
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"preferred_model": model,
|
"preferred_model": model,
|
||||||
"title": title,
|
"title": title,
|
||||||
"orig_title": orig_title,
|
"orig_title": orig_title,
|
||||||
|
|
|
@ -11,13 +11,7 @@ from loguru import logger
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
def _lang(s: str) -> str:
|
|
||||||
try:
|
|
||||||
return detect(s)
|
|
||||||
except Exception:
|
|
||||||
return "en"
|
|
||||||
|
|
||||||
|
|
||||||
@SiteManager.register
|
@SiteManager.register
|
||||||
|
@ -43,14 +37,13 @@ class BoardGameGeek(AbstractSite):
|
||||||
item = items[0]
|
item = items[0]
|
||||||
title = self.query_str(item, "name[@type='primary']/@value")
|
title = self.query_str(item, "name[@type='primary']/@value")
|
||||||
other_title = self.query_list(item, "name[@type='alternate']/@value")
|
other_title = self.query_list(item, "name[@type='alternate']/@value")
|
||||||
zh_title = [
|
localized_title = [
|
||||||
t for t in other_title if _lang(t) in ["zh", "jp", "ko", "zh-cn", "zh-tw"]
|
{"lang": detect_language(t), "text": t} for t in [title] + other_title
|
||||||
]
|
]
|
||||||
if zh_title:
|
zh_title = [
|
||||||
for z in zh_title:
|
t["text"] for t in localized_title if t["lang"] in ["zh", "zh-cn", "zh-tw"]
|
||||||
other_title.remove(z)
|
]
|
||||||
other_title = zh_title + other_title
|
title = zh_title[0] if zh_title else other_title[0]
|
||||||
|
|
||||||
cover_image_url = self.query_str(item, "image/text()")
|
cover_image_url = self.query_str(item, "image/text()")
|
||||||
brief = html.unescape(self.query_str(item, "description/text()"))
|
brief = html.unescape(self.query_str(item, "description/text()"))
|
||||||
year = self.query_str(item, "yearpublished/@value")
|
year = self.query_str(item, "yearpublished/@value")
|
||||||
|
@ -62,6 +55,8 @@ class BoardGameGeek(AbstractSite):
|
||||||
|
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": [{"lang": "en", "text": brief}],
|
||||||
"title": title,
|
"title": title,
|
||||||
"other_title": other_title,
|
"other_title": other_title,
|
||||||
"genre": category,
|
"genre": category,
|
||||||
|
|
|
@ -3,6 +3,7 @@ import logging
|
||||||
from catalog.book.models import *
|
from catalog.book.models import *
|
||||||
from catalog.book.utils import *
|
from catalog.book.utils import *
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import *
|
from .douban import *
|
||||||
|
|
||||||
|
@ -40,7 +41,7 @@ class BooksTW(AbstractSite):
|
||||||
if not title:
|
if not title:
|
||||||
raise ParseError(self, "title")
|
raise ParseError(self, "title")
|
||||||
subtitle = None
|
subtitle = None
|
||||||
orig_title = content.xpath("string(//h1/following-sibling::h2)")
|
orig_title = str(content.xpath("string(//h1/following-sibling::h2)"))
|
||||||
|
|
||||||
authors = content.xpath("string(//div/ul/li[contains(text(),'作者:')])")
|
authors = content.xpath("string(//div/ul/li[contains(text(),'作者:')])")
|
||||||
authors = authors.strip().split(":", 1)[1].split(",") if authors else [] # type: ignore
|
authors = authors.strip().split(":", 1)[1].split(",") if authors else [] # type: ignore
|
||||||
|
@ -116,9 +117,14 @@ class BooksTW(AbstractSite):
|
||||||
"string(//div[contains(@class,'cover_img')]//img[contains(@class,'cover')]/@src)"
|
"string(//div[contains(@class,'cover_img')]//img[contains(@class,'cover')]/@src)"
|
||||||
)
|
)
|
||||||
img_url = re.sub(r"&[wh]=\d+", "", img_url) if img_url else None # type: ignore
|
img_url = re.sub(r"&[wh]=\d+", "", img_url) if img_url else None # type: ignore
|
||||||
|
localized_title = [{"lang": "zh-tw", "text": title}]
|
||||||
|
if orig_title:
|
||||||
|
localized_title.append(
|
||||||
|
{"lang": detect_language(orig_title), "text": orig_title}
|
||||||
|
)
|
||||||
data = {
|
data = {
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": localized_title,
|
||||||
"subtitle": subtitle,
|
"subtitle": subtitle,
|
||||||
"orig_title": orig_title,
|
"orig_title": orig_title,
|
||||||
"author": authors,
|
"author": authors,
|
||||||
|
|
|
@ -63,10 +63,11 @@ class DiscogsRelease(AbstractSite):
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": [{"lang": "en", "text": title}],
|
||||||
"artist": artist,
|
"artist": artist,
|
||||||
"genre": genre,
|
"genre": genre,
|
||||||
"track_list": "\n".join(track_list),
|
"track_list": "\n".join(track_list),
|
||||||
"release_date": None, # only year provided by API
|
# "release_date": None, # only year provided by API
|
||||||
"company": company,
|
"company": company,
|
||||||
"media": media,
|
"media": media,
|
||||||
"disc_count": disc_count,
|
"disc_count": disc_count,
|
||||||
|
|
|
@ -6,6 +6,7 @@ from lxml import html
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import DoubanDownloader
|
from .douban import DoubanDownloader
|
||||||
|
|
||||||
|
@ -64,6 +65,7 @@ class DoubanDramaVersion(AbstractSite):
|
||||||
raise ParseError(self, "title")
|
raise ParseError(self, "title")
|
||||||
data = {
|
data = {
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": [{"lang": "zh-cn", "text": title}],
|
||||||
"director": [x.strip() for x in h.xpath(q.format("导演"))],
|
"director": [x.strip() for x in h.xpath(q.format("导演"))],
|
||||||
"playwright": [x.strip() for x in h.xpath(q.format("编剧"))],
|
"playwright": [x.strip() for x in h.xpath(q.format("编剧"))],
|
||||||
# "actor": [x.strip() for x in h.xpath(q.format("主演"))],
|
# "actor": [x.strip() for x in h.xpath(q.format("主演"))],
|
||||||
|
@ -238,6 +240,21 @@ class DoubanDrama(AbstractSite):
|
||||||
)
|
)
|
||||||
img_url_elem = h.xpath("//img[@itemprop='image']/@src")
|
img_url_elem = h.xpath("//img[@itemprop='image']/@src")
|
||||||
data["cover_image_url"] = img_url_elem[0].strip() if img_url_elem else None
|
data["cover_image_url"] = img_url_elem[0].strip() if img_url_elem else None
|
||||||
|
data["localized_title"] = (
|
||||||
|
[{"lang": "zh-cn", "text": data["title"]}]
|
||||||
|
+ (
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"lang": detect_language(data["orig_title"]),
|
||||||
|
"text": data["orig_title"],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
if data["orig_title"]
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
+ [{"lang": detect_language(t), "text": t} for t in data["other_title"]]
|
||||||
|
)
|
||||||
|
data["localized_description"] = [{"lang": "zh-cn", "text": data["brief"]}]
|
||||||
|
|
||||||
pd = ResourceContent(metadata=data)
|
pd = ResourceContent(metadata=data)
|
||||||
if pd.metadata["cover_image_url"]:
|
if pd.metadata["cover_image_url"]:
|
||||||
|
|
|
@ -4,6 +4,7 @@ import dateparser
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import DoubanDownloader
|
from .douban import DoubanDownloader
|
||||||
|
|
||||||
|
@ -34,51 +35,68 @@ class DoubanGame(AbstractSite):
|
||||||
if not title:
|
if not title:
|
||||||
raise ParseError(self, "title")
|
raise ParseError(self, "title")
|
||||||
|
|
||||||
|
elem = content.xpath("//div[@id='comments']//h2/text()")
|
||||||
|
title2 = elem[0].strip() if len(elem) else ""
|
||||||
|
if title2:
|
||||||
|
sp = title2.strip().rsplit("的短评", 1)
|
||||||
|
title2 = sp[0] if len(sp) > 1 else ""
|
||||||
|
if title2 and title.startswith(title2):
|
||||||
|
orig_title = title[len(title2) :].strip()
|
||||||
|
title = title2
|
||||||
|
else:
|
||||||
|
orig_title = ""
|
||||||
|
|
||||||
other_title_elem = content.xpath(
|
other_title_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()"
|
"//dl[@class='thing-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()"
|
||||||
)
|
)
|
||||||
other_title = (
|
other_title = (
|
||||||
other_title_elem[0].strip().split(" / ") if other_title_elem else None
|
other_title_elem[0].strip().split(" / ") if other_title_elem else []
|
||||||
)
|
)
|
||||||
|
|
||||||
developer_elem = content.xpath(
|
developer_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()"
|
"//dl[@class='thing-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()"
|
||||||
)
|
)
|
||||||
developer = developer_elem[0].strip().split(" / ") if developer_elem else None
|
developer = developer_elem[0].strip().split(" / ") if developer_elem else None
|
||||||
|
|
||||||
publisher_elem = content.xpath(
|
publisher_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()"
|
"//dl[@class='thing-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()"
|
||||||
)
|
)
|
||||||
publisher = publisher_elem[0].strip().split(" / ") if publisher_elem else None
|
publisher = publisher_elem[0].strip().split(" / ") if publisher_elem else None
|
||||||
|
|
||||||
platform_elem = content.xpath(
|
platform_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()"
|
"//dl[@class='thing-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()"
|
||||||
)
|
)
|
||||||
platform = platform_elem if platform_elem else None
|
platform = platform_elem if platform_elem else None
|
||||||
|
|
||||||
genre_elem = content.xpath(
|
genre_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()"
|
"//dl[@class='thing-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()"
|
||||||
)
|
)
|
||||||
genre = None
|
genre = None
|
||||||
if genre_elem:
|
if genre_elem:
|
||||||
genre = [g for g in genre_elem if g != "游戏"]
|
genre = [g for g in genre_elem if g != "游戏"]
|
||||||
|
|
||||||
date_elem = content.xpath(
|
date_elem = content.xpath(
|
||||||
"//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()"
|
"//dl[@class='thing-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()"
|
||||||
)
|
)
|
||||||
release_date = dateparser.parse(date_elem[0].strip()) if date_elem else None
|
release_date = dateparser.parse(date_elem[0].strip()) if date_elem else None
|
||||||
release_date = release_date.strftime("%Y-%m-%d") if release_date else None
|
release_date = release_date.strftime("%Y-%m-%d") if release_date else None
|
||||||
|
|
||||||
brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()")
|
brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()")
|
||||||
brief = "\n".join(brief_elem) if brief_elem else None
|
brief = "\n".join(brief_elem) if brief_elem else ""
|
||||||
|
|
||||||
img_url_elem = content.xpath(
|
img_url_elem = content.xpath(
|
||||||
"//div[@class='item-subject-info']/div[@class='pic']//img/@src"
|
"//div[@class='item-subject-info']/div[@class='pic']//img/@src"
|
||||||
)
|
)
|
||||||
img_url = img_url_elem[0].strip() if img_url_elem else None
|
img_url = img_url_elem[0].strip() if img_url_elem else None
|
||||||
|
|
||||||
|
titles = set([title] + other_title + ([orig_title] if orig_title else []))
|
||||||
|
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
|
||||||
|
localized_desc = [{"lang": detect_language(brief), "text": brief}]
|
||||||
|
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"title": title,
|
"title": title,
|
||||||
"other_title": other_title,
|
"other_title": other_title,
|
||||||
"developer": developer,
|
"developer": developer,
|
||||||
|
|
|
@ -4,6 +4,7 @@ import logging
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.movie.models import *
|
from catalog.movie.models import *
|
||||||
from catalog.tv.models import *
|
from catalog.tv.models import *
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import *
|
from .douban import *
|
||||||
from .tmdb import TMDB_TV, TMDB_TVSeason, query_tmdb_tv_episode, search_tmdb_by_imdb_id
|
from .tmdb import TMDB_TV, TMDB_TVSeason, query_tmdb_tv_episode, search_tmdb_by_imdb_id
|
||||||
|
@ -205,9 +206,18 @@ class DoubanMovie(AbstractSite):
|
||||||
img_url_elem = content.xpath("//img[@rel='v:image']/@src")
|
img_url_elem = content.xpath("//img[@rel='v:image']/@src")
|
||||||
img_url = img_url_elem[0].strip() if img_url_elem else None
|
img_url = img_url_elem[0].strip() if img_url_elem else None
|
||||||
|
|
||||||
|
titles = set(
|
||||||
|
[title]
|
||||||
|
+ ([orig_title] if orig_title else [])
|
||||||
|
+ (other_title if other_title else [])
|
||||||
|
)
|
||||||
|
localized_title = [{"lang": detect_language(t), "text": t} for t in titles]
|
||||||
|
localized_desc = [{"lang": detect_language(brief), "text": brief}]
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"orig_title": orig_title,
|
"orig_title": orig_title,
|
||||||
"other_title": other_title,
|
"other_title": other_title,
|
||||||
"imdb_code": imdb_code,
|
"imdb_code": imdb_code,
|
||||||
|
|
|
@ -5,6 +5,7 @@ import dateparser
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
from catalog.music.utils import upc_to_gtin_13
|
from catalog.music.utils import upc_to_gtin_13
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import DoubanDownloader
|
from .douban import DoubanDownloader
|
||||||
|
|
||||||
|
@ -77,9 +78,19 @@ class DoubanMusic(AbstractSite):
|
||||||
|
|
||||||
img_url_elem = content.xpath("//div[@id='mainpic']//img/@src")
|
img_url_elem = content.xpath("//div[@id='mainpic']//img/@src")
|
||||||
img_url = img_url_elem[0].strip() if img_url_elem else None
|
img_url = img_url_elem[0].strip() if img_url_elem else None
|
||||||
|
other_elem = content.xpath(
|
||||||
|
"//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]"
|
||||||
|
)
|
||||||
|
other_title = other_elem[0].strip().split(" / ") if other_elem else []
|
||||||
|
lang = detect_language(f"{title} {brief}")
|
||||||
|
localized_title = [{"lang": lang, "text": title}]
|
||||||
|
localized_title += [
|
||||||
|
{"lang": detect_language(t), "text": t} for t in other_title
|
||||||
|
]
|
||||||
data = {
|
data = {
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": [{"lang": lang, "text": brief}],
|
||||||
"artist": artist,
|
"artist": artist,
|
||||||
"genre": genre,
|
"genre": genre,
|
||||||
"release_date": release_date,
|
"release_date": release_date,
|
||||||
|
@ -91,11 +102,6 @@ class DoubanMusic(AbstractSite):
|
||||||
}
|
}
|
||||||
gtin = None
|
gtin = None
|
||||||
isrc = None
|
isrc = None
|
||||||
other_elem = content.xpath(
|
|
||||||
"//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]"
|
|
||||||
)
|
|
||||||
if other_elem:
|
|
||||||
data["other_title"] = other_elem[0].strip().split(" / ")
|
|
||||||
other_elem = content.xpath(
|
other_elem = content.xpath(
|
||||||
"//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]"
|
"//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]"
|
||||||
)
|
)
|
||||||
|
|
|
@ -129,6 +129,8 @@ class IGDB(AbstractSite):
|
||||||
steam_url = website["url"]
|
steam_url = website["url"]
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": [{"lang": "en", "text": r["name"]}],
|
||||||
|
"localized_description": [{"lang": "en", "text": brief}],
|
||||||
"title": r["name"],
|
"title": r["name"],
|
||||||
"other_title": [],
|
"other_title": [],
|
||||||
"developer": [developer] if developer else [],
|
"developer": [developer] if developer else [],
|
||||||
|
|
|
@ -91,6 +91,8 @@ class IMDB(AbstractSite):
|
||||||
d["primaryImage"].get("url") if d.get("primaryImage") else None
|
d["primaryImage"].get("url") if d.get("primaryImage") else None
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
data["localized_title"] = [{"lang": "en", "text": data["title"]}]
|
||||||
|
data["localized_description"] = [{"lang": "en", "text": data["brief"]}]
|
||||||
if d.get("series"):
|
if d.get("series"):
|
||||||
episode_info = d["series"].get("episodeNumber")
|
episode_info = d["series"].get("episodeNumber")
|
||||||
if episode_info:
|
if episode_info:
|
||||||
|
@ -133,14 +135,11 @@ class IMDB(AbstractSite):
|
||||||
url = f"https://m.imdb.com{show_url}episodes/?season={season_id}"
|
url = f"https://m.imdb.com{show_url}episodes/?season={season_id}"
|
||||||
h = BasicDownloader(url).download().html()
|
h = BasicDownloader(url).download().html()
|
||||||
episodes = []
|
episodes = []
|
||||||
for e in h.xpath('//div[@id="eplist"]/div/a'): # type: ignore
|
for e in h.xpath('//article//a[@class="ipc-title-link-wrapper"]'): # type: ignore
|
||||||
episode_number = e.xpath(
|
title = e.xpath('div[@class="ipc-title__text"]/text()')[0].split("∙", 1)
|
||||||
'./span[contains(@class,"episode-list__title")]/text()'
|
episode_id = title[0].strip()
|
||||||
)[0].strip()
|
episode_number = int(episode_id.split(".")[1][1:])
|
||||||
episode_number = int(episode_number.split(".")[0])
|
episode_title = title[1].strip()
|
||||||
episode_title = " ".join(
|
|
||||||
e.xpath('.//strong[@class="episode-list__title-text"]/text()')
|
|
||||||
).strip()
|
|
||||||
episode_url = e.xpath("./@href")[0]
|
episode_url = e.xpath("./@href")[0]
|
||||||
episode_url = "https://www.imdb.com" + episode_url
|
episode_url = "https://www.imdb.com" + episode_url
|
||||||
episodes.append(
|
episodes.append(
|
||||||
|
|
|
@ -19,6 +19,8 @@ from catalog.common.downloaders import (
|
||||||
)
|
)
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
from catalog.podcast.models import PodcastEpisode
|
from catalog.podcast.models import PodcastEpisode
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
from journal.models.renderers import html_to_text
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -86,11 +88,16 @@ class RSS(AbstractSite):
|
||||||
feed = self.parse_feed_from_url(self.url)
|
feed = self.parse_feed_from_url(self.url)
|
||||||
if not feed:
|
if not feed:
|
||||||
raise ValueError(f"no feed avaialble in {self.url}")
|
raise ValueError(f"no feed avaialble in {self.url}")
|
||||||
|
title = feed["title"]
|
||||||
|
desc = html_to_text(feed["description"])
|
||||||
|
lang = detect_language(title + " " + desc)
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
"title": feed["title"],
|
"title": title,
|
||||||
"brief": bleach.clean(feed["description"], strip=True),
|
"brief": desc,
|
||||||
"hosts": (
|
"localized_title": [{"lang": lang, "text": title}],
|
||||||
|
"localized_description": [{"lang": lang, "text": desc}],
|
||||||
|
"host": (
|
||||||
[feed.get("itunes_author")] if feed.get("itunes_author") else []
|
[feed.get("itunes_author")] if feed.get("itunes_author") else []
|
||||||
),
|
),
|
||||||
"official_site": feed.get("link"),
|
"official_site": feed.get("link"),
|
||||||
|
|
|
@ -13,6 +13,7 @@ from django.conf import settings
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
from catalog.music.utils import upc_to_gtin_13
|
from catalog.music.utils import upc_to_gtin_13
|
||||||
|
from common.models.lang import detect_language
|
||||||
|
|
||||||
from .douban import *
|
from .douban import *
|
||||||
|
|
||||||
|
@ -83,10 +84,11 @@ class Spotify(AbstractSite):
|
||||||
isrc = None
|
isrc = None
|
||||||
if res_data["external_ids"].get("isrc"):
|
if res_data["external_ids"].get("isrc"):
|
||||||
isrc = res_data["external_ids"].get("isrc")
|
isrc = res_data["external_ids"].get("isrc")
|
||||||
|
lang = detect_language(title)
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
"title": title,
|
"title": title,
|
||||||
|
"localized_title": [{"lang": lang, "text": title}],
|
||||||
"artist": artist,
|
"artist": artist,
|
||||||
"genre": genre,
|
"genre": genre,
|
||||||
"track_list": track_list,
|
"track_list": track_list,
|
||||||
|
|
|
@ -1,15 +1,33 @@
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
import dateparser
|
import dateparser
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.models import *
|
from catalog.models import *
|
||||||
|
from common.models.lang import PREFERRED_LANGUAGES
|
||||||
|
from journal.models.renderers import html_to_text
|
||||||
|
|
||||||
from .igdb import search_igdb_by_3p_url
|
from .igdb import search_igdb_by_3p_url
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_preferred_languages():
|
||||||
|
langs = {}
|
||||||
|
for l in PREFERRED_LANGUAGES:
|
||||||
|
if l == "zh":
|
||||||
|
langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW"})
|
||||||
|
# zh-HK data is not good
|
||||||
|
else:
|
||||||
|
langs[l] = l
|
||||||
|
return langs
|
||||||
|
|
||||||
|
|
||||||
|
STEAM_PREFERRED_LANGS = _get_preferred_languages()
|
||||||
|
|
||||||
|
|
||||||
@SiteManager.register
|
@SiteManager.register
|
||||||
class Steam(AbstractSite):
|
class Steam(AbstractSite):
|
||||||
SITE_NAME = SiteName.Steam
|
SITE_NAME = SiteName.Steam
|
||||||
|
@ -22,69 +40,72 @@ class Steam(AbstractSite):
|
||||||
def id_to_url(cls, id_value):
|
def id_to_url(cls, id_value):
|
||||||
return "https://store.steampowered.com/app/" + str(id_value)
|
return "https://store.steampowered.com/app/" + str(id_value)
|
||||||
|
|
||||||
|
def download(self, lang):
|
||||||
|
api_url = (
|
||||||
|
f"https://store.steampowered.com/api/appdetails?appids={self.id_value}"
|
||||||
|
)
|
||||||
|
headers = {
|
||||||
|
"User-Agent": settings.NEODB_USER_AGENT,
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Accept-Language": STEAM_PREFERRED_LANGS[lang],
|
||||||
|
}
|
||||||
|
return BasicDownloader(api_url, headers=headers).download().json()
|
||||||
|
|
||||||
def scrape(self):
|
def scrape(self):
|
||||||
i = search_igdb_by_3p_url(self.url)
|
i = search_igdb_by_3p_url(self.url)
|
||||||
pd = i.scrape() if i else ResourceContent()
|
pd = i.scrape() if i else ResourceContent()
|
||||||
|
|
||||||
headers = BasicDownloader.headers.copy()
|
en_data = {}
|
||||||
headers["Host"] = "store.steampowered.com"
|
localized_title = []
|
||||||
headers["Cookie"] = "wants_mature_content=1; birthtime=754700401;"
|
localized_desc = []
|
||||||
content = BasicDownloader(self.url, headers=headers).download().html()
|
for lang in STEAM_PREFERRED_LANGS.keys():
|
||||||
|
data = self.download(lang).get(self.id_value, {}).get("data", {})
|
||||||
title = self.query_str(content, "//div[@class='apphub_AppName']/text()")
|
if lang == "en":
|
||||||
developer = content.xpath("//div[@id='developers_list']/a/text()")
|
en_data = data
|
||||||
publisher = content.xpath(
|
localized_title.append({"lang": lang, "text": data["name"]})
|
||||||
"//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()"
|
desc = html_to_text(data["detailed_description"])
|
||||||
)
|
localized_desc.append({"lang": lang, "text": desc})
|
||||||
dts = self.query_str(
|
if not en_data:
|
||||||
content, "//div[@class='release_date']/div[@class='date']/text()"
|
en_data = self.download("en")
|
||||||
)
|
if not en_data:
|
||||||
dt = dateparser.parse(dts.replace(" ", "")) if dts else None
|
raise ParseError(self, "id")
|
||||||
release_date = dt.strftime("%Y-%m-%d") if dt else None
|
|
||||||
|
|
||||||
genre = content.xpath(
|
|
||||||
"//div[@class='details_block']/b[2]/following-sibling::a/text()"
|
|
||||||
)
|
|
||||||
platform = ["PC"]
|
|
||||||
try:
|
|
||||||
brief = self.query_str(
|
|
||||||
content, "//div[@class='game_description_snippet']/text()"
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
brief = ""
|
|
||||||
# try Steam images if no image from IGDB
|
|
||||||
if pd.cover_image is None:
|
|
||||||
pd.metadata["cover_image_url"] = self.query_str(
|
|
||||||
content, "//img[@class='game_header_image_full']/@src"
|
|
||||||
).replace("header.jpg", "library_600x900.jpg")
|
|
||||||
(
|
|
||||||
pd.cover_image,
|
|
||||||
pd.cover_image_extention,
|
|
||||||
) = BasicImageDownloader.download_image(
|
|
||||||
pd.metadata["cover_image_url"], self.url
|
|
||||||
)
|
|
||||||
if pd.cover_image is None:
|
|
||||||
pd.metadata["cover_image_url"] = self.query_str(
|
|
||||||
content, "//img[@class='game_header_image_full']/@src"
|
|
||||||
)
|
|
||||||
(
|
|
||||||
pd.cover_image,
|
|
||||||
pd.cover_image_extention,
|
|
||||||
) = BasicImageDownloader.download_image(
|
|
||||||
pd.metadata["cover_image_url"], self.url
|
|
||||||
)
|
|
||||||
# merge data from IGDB, use localized Steam data if available
|
# merge data from IGDB, use localized Steam data if available
|
||||||
d = {
|
d = {
|
||||||
"developer": developer,
|
"developer": en_data["developers"],
|
||||||
"publisher": publisher,
|
"publisher": en_data["publishers"],
|
||||||
"release_date": release_date,
|
"release_date": en_data["release_date"].get("date"),
|
||||||
"genre": genre,
|
"genre": [g["description"] for g in en_data["genres"]],
|
||||||
"platform": platform,
|
"platform": ["PC"],
|
||||||
}
|
}
|
||||||
|
if en_data["release_date"].get("date"):
|
||||||
|
d["release_date"] = en_data["release_date"].get("date")
|
||||||
d.update(pd.metadata)
|
d.update(pd.metadata)
|
||||||
|
d.update(
|
||||||
|
{
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
|
}
|
||||||
|
)
|
||||||
pd.metadata = d
|
pd.metadata = d
|
||||||
if title:
|
|
||||||
pd.metadata["title"] = title
|
# try Steam images if no image from IGDB
|
||||||
if brief:
|
header = en_data.get("header_image")
|
||||||
pd.metadata["brief"] = brief
|
if header:
|
||||||
|
if pd.cover_image is None:
|
||||||
|
cover = header.replace("header.jpg", "library_600x900_2x.jpg")
|
||||||
|
pd.metadata["cover_image_url"] = cover
|
||||||
|
(
|
||||||
|
pd.cover_image,
|
||||||
|
pd.cover_image_extention,
|
||||||
|
) = BasicImageDownloader.download_image(
|
||||||
|
pd.metadata["cover_image_url"], self.url
|
||||||
|
)
|
||||||
|
if pd.cover_image is None:
|
||||||
|
pd.metadata["cover_image_url"] = header
|
||||||
|
(
|
||||||
|
pd.cover_image,
|
||||||
|
pd.cover_image_extention,
|
||||||
|
) = BasicImageDownloader.download_image(
|
||||||
|
pd.metadata["cover_image_url"], self.url
|
||||||
|
)
|
||||||
return pd
|
return pd
|
||||||
|
|
|
@ -1,5 +1,13 @@
|
||||||
"""
|
"""
|
||||||
The Movie Database
|
The Movie Database
|
||||||
|
|
||||||
|
|
||||||
|
these language code from TMDB are not in currently iso-639-1
|
||||||
|
{'iso_639_1': 'xx', 'english_name': 'No Language', 'name': 'No Language'}
|
||||||
|
{'iso_639_1': 'sh', 'english_name': 'Serbo-Croatian', 'name': ''} - deprecated for several
|
||||||
|
{'iso_639_1': 'mo', 'english_name': 'Moldavian', 'name': ''} - deprecated for ro-MD
|
||||||
|
{'iso_639_1': 'cn', 'english_name': 'Cantonese', 'name': '粤语'} - faked for yue
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
@ -10,13 +18,14 @@ from django.conf import settings
|
||||||
from catalog.common import *
|
from catalog.common import *
|
||||||
from catalog.movie.models import *
|
from catalog.movie.models import *
|
||||||
from catalog.tv.models import *
|
from catalog.tv.models import *
|
||||||
|
from common.models.lang import PREFERRED_LANGUAGES
|
||||||
|
|
||||||
from .douban import *
|
from .douban import *
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def get_language_code():
|
def _get_language_code():
|
||||||
match settings.LANGUAGE_CODE:
|
match settings.LANGUAGE_CODE:
|
||||||
case "zh-hans":
|
case "zh-hans":
|
||||||
return "zh-CN"
|
return "zh-CN"
|
||||||
|
@ -26,14 +35,28 @@ def get_language_code():
|
||||||
return "en-US"
|
return "en-US"
|
||||||
|
|
||||||
|
|
||||||
|
def _get_preferred_languages():
|
||||||
|
langs = {}
|
||||||
|
for l in PREFERRED_LANGUAGES:
|
||||||
|
if l == "zh":
|
||||||
|
langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW", "zh-hk": "zh-HK"})
|
||||||
|
else:
|
||||||
|
langs[l] = l
|
||||||
|
return langs
|
||||||
|
|
||||||
|
|
||||||
|
TMDB_DEFAULT_LANG = _get_language_code()
|
||||||
|
TMDB_PREFERRED_LANGS = _get_preferred_languages()
|
||||||
|
|
||||||
|
|
||||||
def search_tmdb_by_imdb_id(imdb_id):
|
def search_tmdb_by_imdb_id(imdb_id):
|
||||||
tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&external_source=imdb_id"
|
tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&external_source=imdb_id"
|
||||||
res_data = BasicDownloader(tmdb_api_url).download().json()
|
res_data = BasicDownloader(tmdb_api_url).download().json()
|
||||||
return res_data
|
return res_data
|
||||||
|
|
||||||
|
|
||||||
def query_tmdb_tv_episode(tv, season, episode):
|
def query_tmdb_tv_episode(tv, season, episode):
|
||||||
tmdb_api_url = f"https://api.themoviedb.org/3/tv/{tv}/season/{season}/episode/{episode}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids"
|
tmdb_api_url = f"https://api.themoviedb.org/3/tv/{tv}/season/{season}/episode/{episode}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids"
|
||||||
res_data = BasicDownloader(tmdb_api_url).download().json()
|
res_data = BasicDownloader(tmdb_api_url).download().json()
|
||||||
return res_data
|
return res_data
|
||||||
|
|
||||||
|
@ -58,30 +81,16 @@ class TMDB_Movie(AbstractSite):
|
||||||
return f"https://www.themoviedb.org/movie/{id_value}"
|
return f"https://www.themoviedb.org/movie/{id_value}"
|
||||||
|
|
||||||
def scrape(self):
|
def scrape(self):
|
||||||
is_series = False
|
res_data = {}
|
||||||
if is_series:
|
localized_title = []
|
||||||
api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
localized_desc = []
|
||||||
else:
|
# GET api urls in all locales
|
||||||
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
# btw it seems no way to tell if TMDB does not have a certain translation
|
||||||
|
for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
|
||||||
|
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
|
||||||
res_data = BasicDownloader(api_url).download().json()
|
res_data = BasicDownloader(api_url).download().json()
|
||||||
|
localized_title.append({"lang": lang, "text": res_data["title"]})
|
||||||
if is_series:
|
localized_desc.append({"lang": lang, "text": res_data["overview"]})
|
||||||
title = res_data["name"]
|
|
||||||
orig_title = res_data["original_name"]
|
|
||||||
year = (
|
|
||||||
int(res_data["first_air_date"].split("-")[0])
|
|
||||||
if res_data["first_air_date"]
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
imdb_code = res_data["external_ids"]["imdb_id"]
|
|
||||||
showtime = (
|
|
||||||
[{"time": res_data["first_air_date"], "region": "首播日期"}]
|
|
||||||
if res_data["first_air_date"]
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
duration = None
|
|
||||||
else:
|
|
||||||
title = res_data["title"]
|
title = res_data["title"]
|
||||||
orig_title = res_data["original_title"]
|
orig_title = res_data["original_title"]
|
||||||
year = (
|
year = (
|
||||||
|
@ -102,15 +111,10 @@ class TMDB_Movie(AbstractSite):
|
||||||
language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
|
language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
|
||||||
brief = res_data["overview"]
|
brief = res_data["overview"]
|
||||||
|
|
||||||
if is_series:
|
|
||||||
director = list(map(lambda x: x["name"], res_data["created_by"]))
|
|
||||||
else:
|
|
||||||
director = list(
|
director = list(
|
||||||
map(
|
map(
|
||||||
lambda x: x["name"],
|
lambda x: x["name"],
|
||||||
filter(
|
filter(lambda c: c["job"] == "Director", res_data["credits"]["crew"]),
|
||||||
lambda c: c["job"] == "Director", res_data["credits"]["crew"]
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
playwright = list(
|
playwright = list(
|
||||||
|
@ -128,19 +132,21 @@ class TMDB_Movie(AbstractSite):
|
||||||
# other_info['Metacritic评分'] = res_data['metacriticRating']
|
# other_info['Metacritic评分'] = res_data['metacriticRating']
|
||||||
# other_info['奖项'] = res_data['awards']
|
# other_info['奖项'] = res_data['awards']
|
||||||
# other_info['TMDB_ID'] = id
|
# other_info['TMDB_ID'] = id
|
||||||
if is_series:
|
# if is_series:
|
||||||
other_info["Seasons"] = res_data["number_of_seasons"]
|
# other_info["Seasons"] = res_data["number_of_seasons"]
|
||||||
other_info["Episodes"] = res_data["number_of_episodes"]
|
# other_info["Episodes"] = res_data["number_of_episodes"]
|
||||||
|
|
||||||
# TODO: use GET /configuration to get base url
|
# TODO: use GET /configuration to get base url
|
||||||
img_url = (
|
img_url = (
|
||||||
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
|
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
|
||||||
if res_data["poster_path"] is not None
|
if res_data.get("poster_path") is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"title": title,
|
"title": title,
|
||||||
"orig_title": orig_title,
|
"orig_title": orig_title,
|
||||||
"other_title": [],
|
"other_title": [],
|
||||||
|
@ -192,15 +198,15 @@ class TMDB_TV(AbstractSite):
|
||||||
return f"https://www.themoviedb.org/tv/{id_value}"
|
return f"https://www.themoviedb.org/tv/{id_value}"
|
||||||
|
|
||||||
def scrape(self):
|
def scrape(self):
|
||||||
is_series = True
|
res_data = {}
|
||||||
if is_series:
|
localized_title = []
|
||||||
api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
localized_desc = []
|
||||||
else:
|
for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
|
||||||
api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
|
||||||
|
|
||||||
res_data = BasicDownloader(api_url).download().json()
|
res_data = BasicDownloader(api_url).download().json()
|
||||||
|
localized_title.append({"lang": lang, "text": res_data["name"]})
|
||||||
|
localized_desc.append({"lang": lang, "text": res_data["overview"]})
|
||||||
|
|
||||||
if is_series:
|
|
||||||
title = res_data["name"]
|
title = res_data["name"]
|
||||||
orig_title = res_data["original_name"]
|
orig_title = res_data["original_name"]
|
||||||
year = (
|
year = (
|
||||||
|
@ -215,39 +221,10 @@ class TMDB_TV(AbstractSite):
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
duration = None
|
duration = None
|
||||||
else:
|
|
||||||
title = res_data["title"]
|
|
||||||
orig_title = res_data["original_title"]
|
|
||||||
year = (
|
|
||||||
int(res_data["release_date"].split("-")[0])
|
|
||||||
if res_data["release_date"]
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
showtime = (
|
|
||||||
[{"time": res_data["release_date"], "region": "发布日期"}]
|
|
||||||
if res_data["release_date"]
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
imdb_code = res_data["imdb_id"]
|
|
||||||
# in minutes
|
|
||||||
duration = res_data["runtime"] if res_data["runtime"] else None
|
|
||||||
|
|
||||||
genre = [x["name"] for x in res_data["genres"]]
|
genre = [x["name"] for x in res_data["genres"]]
|
||||||
|
|
||||||
language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
|
language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
|
||||||
brief = res_data["overview"]
|
brief = res_data["overview"]
|
||||||
|
|
||||||
if is_series:
|
|
||||||
director = list(map(lambda x: x["name"], res_data["created_by"]))
|
director = list(map(lambda x: x["name"], res_data["created_by"]))
|
||||||
else:
|
|
||||||
director = list(
|
|
||||||
map(
|
|
||||||
lambda x: x["name"],
|
|
||||||
filter(
|
|
||||||
lambda c: c["job"] == "Director", res_data["credits"]["crew"]
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
playwright = list(
|
playwright = list(
|
||||||
map(
|
map(
|
||||||
lambda x: x["name"],
|
lambda x: x["name"],
|
||||||
|
@ -256,24 +233,15 @@ class TMDB_TV(AbstractSite):
|
||||||
)
|
)
|
||||||
actor = list(map(lambda x: x["name"], res_data["credits"]["cast"]))
|
actor = list(map(lambda x: x["name"], res_data["credits"]["cast"]))
|
||||||
area = []
|
area = []
|
||||||
|
|
||||||
other_info = {}
|
other_info = {}
|
||||||
# other_info['TMDB评分'] = res_data['vote_average']
|
|
||||||
# other_info['分级'] = res_data['contentRating']
|
|
||||||
# other_info['Metacritic评分'] = res_data['metacriticRating']
|
|
||||||
# other_info['奖项'] = res_data['awards']
|
|
||||||
# other_info['TMDB_ID'] = id
|
|
||||||
if is_series:
|
|
||||||
other_info["Seasons"] = res_data["number_of_seasons"]
|
other_info["Seasons"] = res_data["number_of_seasons"]
|
||||||
other_info["Episodes"] = res_data["number_of_episodes"]
|
other_info["Episodes"] = res_data["number_of_episodes"]
|
||||||
|
|
||||||
# TODO: use GET /configuration to get base url
|
# TODO: use GET /configuration to get base url
|
||||||
img_url = (
|
img_url = (
|
||||||
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
|
("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
|
||||||
if res_data["poster_path"] is not None
|
if res_data.get("poster_path") is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
season_links = list(
|
season_links = list(
|
||||||
map(
|
map(
|
||||||
lambda s: {
|
lambda s: {
|
||||||
|
@ -288,6 +256,8 @@ class TMDB_TV(AbstractSite):
|
||||||
)
|
)
|
||||||
pd = ResourceContent(
|
pd = ResourceContent(
|
||||||
metadata={
|
metadata={
|
||||||
|
"localized_title": localized_title,
|
||||||
|
"localized_description": localized_desc,
|
||||||
"title": title,
|
"title": title,
|
||||||
"orig_title": orig_title,
|
"orig_title": orig_title,
|
||||||
"other_title": [],
|
"other_title": [],
|
||||||
|
@ -313,7 +283,6 @@ class TMDB_TV(AbstractSite):
|
||||||
)
|
)
|
||||||
if imdb_code:
|
if imdb_code:
|
||||||
pd.lookup_ids[IdType.IMDB] = imdb_code
|
pd.lookup_ids[IdType.IMDB] = imdb_code
|
||||||
|
|
||||||
if pd.metadata["cover_image_url"]:
|
if pd.metadata["cover_image_url"]:
|
||||||
imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url)
|
imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url)
|
||||||
try:
|
try:
|
||||||
|
@ -357,13 +326,20 @@ class TMDB_TVSeason(AbstractSite):
|
||||||
show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
|
show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
|
||||||
if not show_resource:
|
if not show_resource:
|
||||||
raise ValueError(f"TMDB: failed to get show for season {self.url}")
|
raise ValueError(f"TMDB: failed to get show for season {self.url}")
|
||||||
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
|
||||||
d = BasicDownloader(api_url).download().json()
|
res_data = {}
|
||||||
if not d.get("id"):
|
localized_title = []
|
||||||
|
localized_desc = []
|
||||||
|
for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
|
||||||
|
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
|
||||||
|
res_data = BasicDownloader(api_url).download().json()
|
||||||
|
localized_title.append({"lang": lang, "text": res_data["name"]})
|
||||||
|
localized_desc.append({"lang": lang, "text": res_data["overview"]})
|
||||||
|
if not res_data.get("id"):
|
||||||
raise ParseError(self, "id")
|
raise ParseError(self, "id")
|
||||||
pd = ResourceContent(
|
d = res_data
|
||||||
metadata=_copy_dict(
|
r = _copy_dict(
|
||||||
d,
|
res_data,
|
||||||
{
|
{
|
||||||
"name": "title",
|
"name": "title",
|
||||||
"overview": "brief",
|
"overview": "brief",
|
||||||
|
@ -372,7 +348,9 @@ class TMDB_TVSeason(AbstractSite):
|
||||||
"external_ids": [],
|
"external_ids": [],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
)
|
r["localized_title"] = localized_title
|
||||||
|
r["localized_description"] = localized_desc
|
||||||
|
pd = ResourceContent(metadata=r)
|
||||||
pd.metadata["title"] = (
|
pd.metadata["title"] = (
|
||||||
show_resource.metadata["title"] + " " + pd.metadata["title"]
|
show_resource.metadata["title"] + " " + pd.metadata["title"]
|
||||||
)
|
)
|
||||||
|
@ -388,12 +366,12 @@ class TMDB_TVSeason(AbstractSite):
|
||||||
pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id")
|
pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id")
|
||||||
pd.metadata["cover_image_url"] = (
|
pd.metadata["cover_image_url"] = (
|
||||||
("https://image.tmdb.org/t/p/original/" + d["poster_path"])
|
("https://image.tmdb.org/t/p/original/" + d["poster_path"])
|
||||||
if d["poster_path"]
|
if d.get("poster_path")
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
pd.metadata["title"] = (
|
pd.metadata["title"] = (
|
||||||
pd.metadata["title"]
|
pd.metadata["title"]
|
||||||
if pd.metadata["title"]
|
if pd.metadata.get("title")
|
||||||
else f'Season {d["season_number"]}'
|
else f'Season {d["season_number"]}'
|
||||||
)
|
)
|
||||||
pd.metadata["episode_number_list"] = list(
|
pd.metadata["episode_number_list"] = list(
|
||||||
|
@ -429,7 +407,7 @@ class TMDB_TVSeason(AbstractSite):
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
ep = pd.metadata["episode_number_list"][0]
|
ep = pd.metadata["episode_number_list"][0]
|
||||||
api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
|
||||||
d2 = BasicDownloader(api_url2).download().json()
|
d2 = BasicDownloader(api_url2).download().json()
|
||||||
if not d2.get("id"):
|
if not d2.get("id"):
|
||||||
raise ParseError(self, "first episode id for season")
|
raise ParseError(self, "first episode id for season")
|
||||||
|
@ -469,7 +447,7 @@ class TMDB_TVEpisode(AbstractSite):
|
||||||
episode_id = v[2]
|
episode_id = v[2]
|
||||||
site = TMDB_TV(TMDB_TV.id_to_url(show_id))
|
site = TMDB_TV(TMDB_TV.id_to_url(show_id))
|
||||||
show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
|
show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
|
||||||
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={get_language_code()}&append_to_response=external_ids,credits"
|
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
|
||||||
d = BasicDownloader(api_url).download().json()
|
d = BasicDownloader(api_url).download().json()
|
||||||
if not d.get("id"):
|
if not d.get("id"):
|
||||||
raise ParseError(self, "id")
|
raise ParseError(self, "id")
|
||||||
|
|
|
@ -44,7 +44,7 @@
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
{% if item.language %}
|
{% if item.language %}
|
||||||
{% trans 'language' %}: {{ item.language }}
|
{% trans 'language' %}: {{ item.get_language_display }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
|
|
|
@ -30,7 +30,7 @@
|
||||||
<div id="item-title" class="middle">
|
<div id="item-title" class="middle">
|
||||||
{% if item.is_deleted %}[DELETED]{% endif %}
|
{% if item.is_deleted %}[DELETED]{% endif %}
|
||||||
{% if item.merged_to_item %}
|
{% if item.merged_to_item %}
|
||||||
[MERGED TO <a href="{{ item.merged_to_item.url }}">{{ item.merged_to_item.title }}</a>]
|
[MERGED TO <a href="{{ item.merged_to_item.url }}">{{ item.merged_to_item.display_title }}</a>]
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<h1>
|
<h1>
|
||||||
{{ item.display_title }}
|
{{ item.display_title }}
|
||||||
|
|
|
@ -47,6 +47,7 @@ from catalog.common import (
|
||||||
PrimaryLookupIdDescriptor,
|
PrimaryLookupIdDescriptor,
|
||||||
jsondata,
|
jsondata,
|
||||||
)
|
)
|
||||||
|
from catalog.common.models import LANGUAGE_CHOICES_JSONFORM, LanguageListField
|
||||||
|
|
||||||
|
|
||||||
class TVShowInSchema(ItemInSchema):
|
class TVShowInSchema(ItemInSchema):
|
||||||
|
@ -112,14 +113,16 @@ class TVShow(Item):
|
||||||
)
|
)
|
||||||
|
|
||||||
METADATA_COPY_LIST = [
|
METADATA_COPY_LIST = [
|
||||||
"title",
|
# "title",
|
||||||
|
"localized_title",
|
||||||
"season_count",
|
"season_count",
|
||||||
"orig_title",
|
"orig_title",
|
||||||
"other_title",
|
# "other_title",
|
||||||
"director",
|
"director",
|
||||||
"playwright",
|
"playwright",
|
||||||
"actor",
|
"actor",
|
||||||
"brief",
|
# "brief",
|
||||||
|
"localized_description",
|
||||||
"genre",
|
"genre",
|
||||||
"showtime",
|
"showtime",
|
||||||
"site",
|
"site",
|
||||||
|
@ -210,17 +213,8 @@ class TVShow(Item):
|
||||||
blank=True,
|
blank=True,
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.ArrayField(
|
language = LanguageListField()
|
||||||
verbose_name=_("language"),
|
|
||||||
base_field=models.CharField(
|
|
||||||
blank=True,
|
|
||||||
default="",
|
|
||||||
max_length=100,
|
|
||||||
),
|
|
||||||
null=True,
|
|
||||||
blank=True,
|
|
||||||
default=list,
|
|
||||||
)
|
|
||||||
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
||||||
single_episode_length = jsondata.IntegerField(
|
single_episode_length = jsondata.IntegerField(
|
||||||
verbose_name=_("episode length"), null=True, blank=True
|
verbose_name=_("episode length"), null=True, blank=True
|
||||||
|
@ -374,16 +368,16 @@ class TVSeason(Item):
|
||||||
blank=True,
|
blank=True,
|
||||||
default=list,
|
default=list,
|
||||||
)
|
)
|
||||||
language = jsondata.ArrayField(
|
language = jsondata.JSONField(
|
||||||
verbose_name=_("language"),
|
verbose_name=_("language"),
|
||||||
base_field=models.CharField(
|
# base_field=models.CharField(blank=True, default="", max_length=100, choices=LANGUAGE_CHOICES ),
|
||||||
blank=True,
|
|
||||||
default="",
|
|
||||||
max_length=100,
|
|
||||||
),
|
|
||||||
null=True,
|
null=True,
|
||||||
blank=True,
|
blank=True,
|
||||||
default=list,
|
default=list,
|
||||||
|
schema={
|
||||||
|
"type": "list",
|
||||||
|
"items": {"type": "string", "choices": LANGUAGE_CHOICES_JSONFORM},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
year = jsondata.IntegerField(verbose_name=_("year"), null=True, blank=True)
|
||||||
single_episode_length = jsondata.IntegerField(
|
single_episode_length = jsondata.IntegerField(
|
||||||
|
|
|
@ -6,6 +6,8 @@ from catalog.tv.models import *
|
||||||
|
|
||||||
|
|
||||||
class JSONFieldTestCase(TestCase):
|
class JSONFieldTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_legacy_data(self):
|
def test_legacy_data(self):
|
||||||
o = TVShow()
|
o = TVShow()
|
||||||
self.assertEqual(o.other_title, [])
|
self.assertEqual(o.other_title, [])
|
||||||
|
@ -18,6 +20,8 @@ class JSONFieldTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class TMDBTVTestCase(TestCase):
|
class TMDBTVTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id = "57243"
|
t_id = "57243"
|
||||||
t_url = "https://www.themoviedb.org/tv/57243-doctor-who"
|
t_url = "https://www.themoviedb.org/tv/57243-doctor-who"
|
||||||
|
@ -43,13 +47,15 @@ class TMDBTVTestCase(TestCase):
|
||||||
self.assertEqual(site.id_value, "57243")
|
self.assertEqual(site.id_value, "57243")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "神秘博士")
|
self.assertEqual(site.resource.metadata["title"], "Doctor Who")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.__class__.__name__, "TVShow")
|
self.assertEqual(site.resource.item.__class__.__name__, "TVShow")
|
||||||
self.assertEqual(site.resource.item.imdb, "tt0436992")
|
self.assertEqual(site.resource.item.imdb, "tt0436992")
|
||||||
|
|
||||||
|
|
||||||
class TMDBTVSeasonTestCase(TestCase):
|
class TMDBTVSeasonTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
def test_parse(self):
|
def test_parse(self):
|
||||||
t_id = "57243-11"
|
t_id = "57243-11"
|
||||||
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/11"
|
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/11"
|
||||||
|
@ -70,7 +76,7 @@ class TMDBTVSeasonTestCase(TestCase):
|
||||||
self.assertEqual(site.id_value, "57243-4")
|
self.assertEqual(site.id_value, "57243-4")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "神秘博士 第 4 季")
|
self.assertEqual(site.resource.metadata["title"], "Doctor Who Series 4")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.__class__.__name__, "TVSeason")
|
self.assertEqual(site.resource.item.__class__.__name__, "TVSeason")
|
||||||
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
||||||
|
@ -79,6 +85,8 @@ class TMDBTVSeasonTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class TMDBEpisodeTestCase(TestCase):
|
class TMDBEpisodeTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_scrape_tmdb(self):
|
def test_scrape_tmdb(self):
|
||||||
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/4/episode/1"
|
t_url = "https://www.themoviedb.org/tv/57243-doctor-who/season/4/episode/1"
|
||||||
|
@ -87,7 +95,7 @@ class TMDBEpisodeTestCase(TestCase):
|
||||||
self.assertEqual(site.id_value, "57243-4-1")
|
self.assertEqual(site.id_value, "57243-4-1")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "活宝搭档")
|
self.assertEqual(site.resource.metadata["title"], "Partners in Crime")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
|
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
|
||||||
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
||||||
|
@ -98,6 +106,8 @@ class TMDBEpisodeTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class DoubanMovieTVTestCase(TestCase):
|
class DoubanMovieTVTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_scrape(self):
|
def test_scrape(self):
|
||||||
url3 = "https://movie.douban.com/subject/3627919/"
|
url3 = "https://movie.douban.com/subject/3627919/"
|
||||||
|
@ -122,6 +132,8 @@ class DoubanMovieTVTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class MultiTVSitesTestCase(TestCase):
|
class MultiTVSitesTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_tvshows(self):
|
def test_tvshows(self):
|
||||||
url1 = "https://www.themoviedb.org/tv/57243-doctor-who"
|
url1 = "https://www.themoviedb.org/tv/57243-doctor-who"
|
||||||
|
@ -170,6 +182,8 @@ class MultiTVSitesTestCase(TestCase):
|
||||||
|
|
||||||
|
|
||||||
class MovieTVModelRecastTestCase(TestCase):
|
class MovieTVModelRecastTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_recast(self):
|
def test_recast(self):
|
||||||
from catalog.models import Movie, TVShow
|
from catalog.models import Movie, TVShow
|
||||||
|
@ -178,13 +192,15 @@ class MovieTVModelRecastTestCase(TestCase):
|
||||||
p2 = SiteManager.get_site_by_url(url2).get_resource_ready()
|
p2 = SiteManager.get_site_by_url(url2).get_resource_ready()
|
||||||
tv = p2.item
|
tv = p2.item
|
||||||
self.assertEqual(tv.class_name, "tvshow")
|
self.assertEqual(tv.class_name, "tvshow")
|
||||||
self.assertEqual(tv.title, "神秘博士")
|
self.assertEqual(tv.display_title, "Doctor Who")
|
||||||
movie = tv.recast_to(Movie)
|
movie = tv.recast_to(Movie)
|
||||||
self.assertEqual(movie.class_name, "movie")
|
self.assertEqual(movie.class_name, "movie")
|
||||||
self.assertEqual(movie.title, "神秘博士")
|
self.assertEqual(movie.display_title, "Doctor Who")
|
||||||
|
|
||||||
|
|
||||||
class IMDBTestCase(TestCase):
|
class IMDBTestCase(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
@use_local_response
|
@use_local_response
|
||||||
def test_fetch_episodes(self):
|
def test_fetch_episodes(self):
|
||||||
t_url = "https://movie.douban.com/subject/1920763/"
|
t_url = "https://movie.douban.com/subject/1920763/"
|
||||||
|
@ -243,7 +259,7 @@ class IMDBTestCase(TestCase):
|
||||||
self.assertEqual(site.id_value, "tt1159991")
|
self.assertEqual(site.id_value, "tt1159991")
|
||||||
site.get_resource_ready()
|
site.get_resource_ready()
|
||||||
self.assertEqual(site.ready, True)
|
self.assertEqual(site.ready, True)
|
||||||
self.assertEqual(site.resource.metadata["title"], "活宝搭档")
|
self.assertEqual(site.resource.metadata["title"], "Partners in Crime")
|
||||||
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB)
|
||||||
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
|
self.assertEqual(site.resource.item.__class__.__name__, "TVEpisode")
|
||||||
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
self.assertEqual(site.resource.item.imdb, "tt1159991")
|
||||||
|
|
9
common/models/__init__.py
Normal file
9
common/models/__init__.py
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
from .cron import BaseJob, JobManager
|
||||||
|
from .lang import (
|
||||||
|
DEFAULT_CATALOG_LANGUAGE,
|
||||||
|
LANGUAGE_CHOICES,
|
||||||
|
LOCALE_CHOICES,
|
||||||
|
SCRIPT_CHOICES,
|
||||||
|
detect_language,
|
||||||
|
)
|
||||||
|
from .misc import uniq
|
319
common/models/lang.py
Normal file
319
common/models/lang.py
Normal file
|
@ -0,0 +1,319 @@
|
||||||
|
"""
|
||||||
|
language support utilities
|
||||||
|
|
||||||
|
https://en.wikipedia.org/wiki/IETF_language_tag
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.utils.translation import get_language
|
||||||
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
from langdetect import detect
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
PREFERRED_LANGUAGES: list[str] = settings.PREFERRED_LANGUAGES
|
||||||
|
|
||||||
|
DEFAULT_CATALOG_LANGUAGE = PREFERRED_LANGUAGES[0] if PREFERRED_LANGUAGES else "en"
|
||||||
|
|
||||||
|
ISO_639_1 = {
|
||||||
|
"aa": _("Afar"),
|
||||||
|
"af": _("Afrikaans"),
|
||||||
|
"ak": _("Akan"),
|
||||||
|
"an": _("Aragonese"),
|
||||||
|
"as": _("Assamese"),
|
||||||
|
"av": _("Avaric"),
|
||||||
|
"ae": _("Avestan"),
|
||||||
|
"ay": _("Aymara"),
|
||||||
|
"az": _("Azerbaijani"),
|
||||||
|
"ba": _("Bashkir"),
|
||||||
|
"bm": _("Bambara"),
|
||||||
|
"bi": _("Bislama"),
|
||||||
|
"bo": _("Tibetan"),
|
||||||
|
"br": _("Breton"),
|
||||||
|
"ca": _("Catalan"),
|
||||||
|
"cs": _("Czech"),
|
||||||
|
"ce": _("Chechen"),
|
||||||
|
"cu": _("Slavic"),
|
||||||
|
"cv": _("Chuvash"),
|
||||||
|
"kw": _("Cornish"),
|
||||||
|
"co": _("Corsican"),
|
||||||
|
"cr": _("Cree"),
|
||||||
|
"cy": _("Welsh"),
|
||||||
|
"da": _("Danish"),
|
||||||
|
"de": _("German"),
|
||||||
|
"dv": _("Divehi"),
|
||||||
|
"dz": _("Dzongkha"),
|
||||||
|
"eo": _("Esperanto"),
|
||||||
|
"et": _("Estonian"),
|
||||||
|
"eu": _("Basque"),
|
||||||
|
"fo": _("Faroese"),
|
||||||
|
"fj": _("Fijian"),
|
||||||
|
"fi": _("Finnish"),
|
||||||
|
"fr": _("French"),
|
||||||
|
"fy": _("Frisian"),
|
||||||
|
"ff": _("Fulah"),
|
||||||
|
"gd": _("Gaelic"),
|
||||||
|
"ga": _("Irish"),
|
||||||
|
"gl": _("Galician"),
|
||||||
|
"gv": _("Manx"),
|
||||||
|
"gn": _("Guarani"),
|
||||||
|
"gu": _("Gujarati"),
|
||||||
|
"ht": _("Haitian; Haitian Creole"),
|
||||||
|
"ha": _("Hausa"),
|
||||||
|
"sh": _("Serbo-Croatian"),
|
||||||
|
"hz": _("Herero"),
|
||||||
|
"ho": _("Hiri Motu"),
|
||||||
|
"hr": _("Croatian"),
|
||||||
|
"hu": _("Hungarian"),
|
||||||
|
"ig": _("Igbo"),
|
||||||
|
"io": _("Ido"),
|
||||||
|
"ii": _("Yi"),
|
||||||
|
"iu": _("Inuktitut"),
|
||||||
|
"ie": _("Interlingue"),
|
||||||
|
"ia": _("Interlingua"),
|
||||||
|
"id": _("Indonesian"),
|
||||||
|
"ik": _("Inupiaq"),
|
||||||
|
"is": _("Icelandic"),
|
||||||
|
"it": _("Italian"),
|
||||||
|
"jv": _("Javanese"),
|
||||||
|
"ja": _("Japanese"),
|
||||||
|
"kl": _("Kalaallisut"),
|
||||||
|
"kn": _("Kannada"),
|
||||||
|
"ks": _("Kashmiri"),
|
||||||
|
"kr": _("Kanuri"),
|
||||||
|
"kk": _("Kazakh"),
|
||||||
|
"km": _("Khmer"),
|
||||||
|
"ki": _("Kikuyu"),
|
||||||
|
"rw": _("Kinyarwanda"),
|
||||||
|
"ky": _("Kirghiz"),
|
||||||
|
"kv": _("Komi"),
|
||||||
|
"kg": _("Kongo"),
|
||||||
|
"ko": _("Korean"),
|
||||||
|
"kj": _("Kuanyama"),
|
||||||
|
"ku": _("Kurdish"),
|
||||||
|
"lo": _("Lao"),
|
||||||
|
"la": _("Latin"),
|
||||||
|
"lv": _("Latvian"),
|
||||||
|
"li": _("Limburgish"),
|
||||||
|
"ln": _("Lingala"),
|
||||||
|
"lt": _("Lithuanian"),
|
||||||
|
"lb": _("Letzeburgesch"),
|
||||||
|
"lu": _("Luba-Katanga"),
|
||||||
|
"lg": _("Ganda"),
|
||||||
|
"mh": _("Marshall"),
|
||||||
|
"ml": _("Malayalam"),
|
||||||
|
"mr": _("Marathi"),
|
||||||
|
"mg": _("Malagasy"),
|
||||||
|
"mt": _("Maltese"),
|
||||||
|
"mo": _("Moldavian"),
|
||||||
|
"mn": _("Mongolian"),
|
||||||
|
"mi": _("Maori"),
|
||||||
|
"ms": _("Malay"),
|
||||||
|
"my": _("Burmese"),
|
||||||
|
"na": _("Nauru"),
|
||||||
|
"nv": _("Navajo"),
|
||||||
|
"nr": _("Ndebele"),
|
||||||
|
"nd": _("Ndebele"),
|
||||||
|
"ng": _("Ndonga"),
|
||||||
|
"ne": _("Nepali"),
|
||||||
|
"nl": _("Dutch"),
|
||||||
|
"nn": _("Norwegian Nynorsk"),
|
||||||
|
"nb": _("Norwegian Bokmål"),
|
||||||
|
"no": _("Norwegian"),
|
||||||
|
"ny": _("Chichewa; Nyanja"),
|
||||||
|
"oc": _("Occitan"),
|
||||||
|
"oj": _("Ojibwa"),
|
||||||
|
"or": _("Oriya"),
|
||||||
|
"om": _("Oromo"),
|
||||||
|
"os": _("Ossetian; Ossetic"),
|
||||||
|
"pi": _("Pali"),
|
||||||
|
"pl": _("Polish"),
|
||||||
|
"pt": _("Portuguese"),
|
||||||
|
"qu": _("Quechua"),
|
||||||
|
"rm": _("Raeto-Romance"),
|
||||||
|
"ro": _("Romanian"),
|
||||||
|
"rn": _("Rundi"),
|
||||||
|
"ru": _("Russian"),
|
||||||
|
"sg": _("Sango"),
|
||||||
|
"sa": _("Sanskrit"),
|
||||||
|
"si": _("Sinhalese"),
|
||||||
|
"sk": _("Slovak"),
|
||||||
|
"sl": _("Slovenian"),
|
||||||
|
"se": _("Northern Sami"),
|
||||||
|
"sm": _("Samoan"),
|
||||||
|
"sn": _("Shona"),
|
||||||
|
"sd": _("Sindhi"),
|
||||||
|
"so": _("Somali"),
|
||||||
|
"st": _("Sotho"),
|
||||||
|
"es": _("Spanish"),
|
||||||
|
"sq": _("Albanian"),
|
||||||
|
"sc": _("Sardinian"),
|
||||||
|
"sr": _("Serbian"),
|
||||||
|
"ss": _("Swati"),
|
||||||
|
"su": _("Sundanese"),
|
||||||
|
"sw": _("Swahili"),
|
||||||
|
"sv": _("Swedish"),
|
||||||
|
"ty": _("Tahitian"),
|
||||||
|
"ta": _("Tamil"),
|
||||||
|
"tt": _("Tatar"),
|
||||||
|
"te": _("Telugu"),
|
||||||
|
"tg": _("Tajik"),
|
||||||
|
"tl": _("Tagalog"),
|
||||||
|
"th": _("Thai"),
|
||||||
|
"ti": _("Tigrinya"),
|
||||||
|
"to": _("Tonga"),
|
||||||
|
"tn": _("Tswana"),
|
||||||
|
"ts": _("Tsonga"),
|
||||||
|
"tk": _("Turkmen"),
|
||||||
|
"tr": _("Turkish"),
|
||||||
|
"tw": _("Twi"),
|
||||||
|
"ug": _("Uighur"),
|
||||||
|
"uk": _("Ukrainian"),
|
||||||
|
"ur": _("Urdu"),
|
||||||
|
"uz": _("Uzbek"),
|
||||||
|
"ve": _("Venda"),
|
||||||
|
"vi": _("Vietnamese"),
|
||||||
|
"vo": _("Volapük"),
|
||||||
|
"wa": _("Walloon"),
|
||||||
|
"wo": _("Wolof"),
|
||||||
|
"xh": _("Xhosa"),
|
||||||
|
"yi": _("Yiddish"),
|
||||||
|
"za": _("Zhuang"),
|
||||||
|
"zu": _("Zulu"),
|
||||||
|
"ab": _("Abkhazian"),
|
||||||
|
"zh": _("Chinese"),
|
||||||
|
"ps": _("Pushto"),
|
||||||
|
"am": _("Amharic"),
|
||||||
|
"ar": _("Arabic"),
|
||||||
|
"bg": _("Bulgarian"),
|
||||||
|
"mk": _("Macedonian"),
|
||||||
|
"el": _("Greek"),
|
||||||
|
"fa": _("Persian"),
|
||||||
|
"he": _("Hebrew"),
|
||||||
|
"hi": _("Hindi"),
|
||||||
|
"hy": _("Armenian"),
|
||||||
|
"en": _("English"),
|
||||||
|
"ee": _("Ewe"),
|
||||||
|
"ka": _("Georgian"),
|
||||||
|
"pa": _("Punjabi"),
|
||||||
|
"bn": _("Bengali"),
|
||||||
|
"bs": _("Bosnian"),
|
||||||
|
"ch": _("Chamorro"),
|
||||||
|
"be": _("Belarusian"),
|
||||||
|
"yo": _("Yoruba"),
|
||||||
|
"x": _("Unknown or Other"),
|
||||||
|
}
|
||||||
|
TOP_USED_LANG = [
|
||||||
|
"en",
|
||||||
|
"de",
|
||||||
|
"es",
|
||||||
|
"zh",
|
||||||
|
"fr",
|
||||||
|
"ja",
|
||||||
|
"it",
|
||||||
|
"ru",
|
||||||
|
"pt",
|
||||||
|
"nl",
|
||||||
|
"kr",
|
||||||
|
"hi",
|
||||||
|
"ar",
|
||||||
|
"bn",
|
||||||
|
]
|
||||||
|
ZH_LOCALE_SUBTAGS_PRIO = {
|
||||||
|
"zh-cn": _("Simplified Chinese (Mainland)"),
|
||||||
|
"zh-tw": _("Traditional Chinese (Taiwan)"),
|
||||||
|
"zh-hk": _("Traditional Chinese (Hongkong)"),
|
||||||
|
}
|
||||||
|
ZH_LOCALE_SUBTAGS = {
|
||||||
|
"zh-sg": _("Simplified Chinese (Singapore)"),
|
||||||
|
"zh-my": _("Simplified Chinese (Malaysia)"),
|
||||||
|
"zh-mo": _("Traditional Chinese (Taiwan)"),
|
||||||
|
}
|
||||||
|
ZH_LANGUAGE_SUBTAGS_PRIO = {
|
||||||
|
"cmn": _("Mandarin Chinese"),
|
||||||
|
"yue": _("Yue Chinese"),
|
||||||
|
}
|
||||||
|
ZH_LANGUAGE_SUBTAGS = {
|
||||||
|
"nan": _("Min Nan Chinese"),
|
||||||
|
"wuu": _("Wu Chinese"),
|
||||||
|
"hak": _("Hakka Chinese"),
|
||||||
|
}
|
||||||
|
|
||||||
|
ZH_LOCALE_SUBTAGS_PRIO.keys()
|
||||||
|
|
||||||
|
|
||||||
|
def get_base_lang_list():
|
||||||
|
langs = {}
|
||||||
|
for k in PREFERRED_LANGUAGES + TOP_USED_LANG:
|
||||||
|
if k not in langs:
|
||||||
|
if k in ISO_639_1:
|
||||||
|
langs[k] = ISO_639_1[k]
|
||||||
|
else:
|
||||||
|
logger.error(f"{k} is not a supported ISO-639-1 language tag")
|
||||||
|
for k, v in ISO_639_1.items():
|
||||||
|
if k not in langs:
|
||||||
|
langs[k] = v
|
||||||
|
return langs
|
||||||
|
|
||||||
|
|
||||||
|
BASE_LANG_LIST: dict[str, Any] = get_base_lang_list()
|
||||||
|
|
||||||
|
|
||||||
|
def get_locale_choices():
|
||||||
|
choices = []
|
||||||
|
for k, v in BASE_LANG_LIST.items():
|
||||||
|
if k == "zh":
|
||||||
|
choices += ZH_LOCALE_SUBTAGS_PRIO.items()
|
||||||
|
else:
|
||||||
|
choices.append((k, v))
|
||||||
|
choices += ZH_LOCALE_SUBTAGS.items()
|
||||||
|
return choices
|
||||||
|
|
||||||
|
|
||||||
|
def get_script_choices():
|
||||||
|
return list(BASE_LANG_LIST.items())
|
||||||
|
|
||||||
|
|
||||||
|
def get_language_choices():
|
||||||
|
choices = []
|
||||||
|
for k, v in BASE_LANG_LIST.items():
|
||||||
|
if k == "zh":
|
||||||
|
choices += ZH_LANGUAGE_SUBTAGS_PRIO.items()
|
||||||
|
else:
|
||||||
|
choices.append((k, v))
|
||||||
|
choices += ZH_LANGUAGE_SUBTAGS.items()
|
||||||
|
return choices
|
||||||
|
|
||||||
|
|
||||||
|
LOCALE_CHOICES: list[tuple[str, Any]] = get_locale_choices()
|
||||||
|
SCRIPT_CHOICES: list[tuple[str, Any]] = get_script_choices()
|
||||||
|
LANGUAGE_CHOICES: list[tuple[str, Any]] = get_language_choices()
|
||||||
|
|
||||||
|
|
||||||
|
def get_current_locales() -> list[str]:
|
||||||
|
lang = get_language().lower()
|
||||||
|
if lang == "zh-hans":
|
||||||
|
return ["zh-cn", "zh-sg", "zh-my", "zh-hk", "zh-tw", "zh-mo", "en"]
|
||||||
|
elif lang == "zh-hant":
|
||||||
|
return ["zh-tw", "zh-hk", "zh-mo", "zh-cn", "zh-sg", "zh-my", "en"]
|
||||||
|
else:
|
||||||
|
lng = lang.split("-")
|
||||||
|
return ["en"] if lng[0] == "en" else [lng[0], "en"]
|
||||||
|
|
||||||
|
|
||||||
|
_eng = re.compile(r"^[A-Za-z0-9\s]{1,13}$")
|
||||||
|
|
||||||
|
|
||||||
|
def detect_language(s: str) -> str:
|
||||||
|
try:
|
||||||
|
if _eng.match(s):
|
||||||
|
return "en"
|
||||||
|
return detect(s).lower()
|
||||||
|
except Exception:
|
||||||
|
return "x"
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_languages(languages: list[str]) -> list[str]:
|
||||||
|
return []
|
6
common/models/misc.py
Normal file
6
common/models/misc.py
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
def uniq(ls: list) -> list:
|
||||||
|
r = []
|
||||||
|
for i in ls:
|
||||||
|
if i not in r:
|
||||||
|
r.append(i)
|
||||||
|
return r
|
|
@ -8,7 +8,11 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// override django_jsonform/react-json-form styles
|
// override django_jsonform/react-json-form styles
|
||||||
|
.rjf-form-group-wrapper{
|
||||||
|
max-width: unset !important;
|
||||||
|
}
|
||||||
.rjf-form-wrapper {
|
.rjf-form-wrapper {
|
||||||
|
max-width: unset !important;
|
||||||
input[type="text"] {
|
input[type="text"] {
|
||||||
max-width: unset !important;
|
max-width: unset !important;
|
||||||
margin-top: 0 !important;
|
margin-top: 0 !important;
|
||||||
|
@ -33,15 +37,15 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
.rjf-form-row-inner>div {
|
.rjf-form-row-inner>div {
|
||||||
display: grid !important;
|
// display: grid !important;
|
||||||
grid-template-columns: repeat(auto-fit, minmax(0%, 1fr));
|
// grid-template-columns: repeat(auto-fit, minmax(0%, 1fr));
|
||||||
|
|
||||||
>label {
|
>label {
|
||||||
margin-top: var(--pico-form-element-spacing-vertical);
|
margin-top: var(--pico-form-element-spacing-vertical);
|
||||||
}
|
}
|
||||||
|
|
||||||
>* {
|
>* {
|
||||||
width: max-content !important;
|
// width: max-content !important;
|
||||||
|
|
||||||
|
|
||||||
button {
|
button {
|
||||||
|
|
|
@ -21,7 +21,7 @@
|
||||||
<script src="{{ cdn_url }}/npm/hyperscript.org@0.9.12"></script>
|
<script src="{{ cdn_url }}/npm/hyperscript.org@0.9.12"></script>
|
||||||
<link rel="stylesheet"
|
<link rel="stylesheet"
|
||||||
href="{{ cdn_url }}/npm/@picocss/pico@2/css/pico.min.css" />
|
href="{{ cdn_url }}/npm/@picocss/pico@2/css/pico.min.css" />
|
||||||
<link href="{% sass_src 'scss/neodb.scss' %}"
|
<link href="{% sass_src 'scss/neodb.scss' %}?xxddddddddxdd"
|
||||||
rel="stylesheet"
|
rel="stylesheet"
|
||||||
type="text/css" />
|
type="text/css" />
|
||||||
<link href="{{ cdn_url }}/npm/@fortawesome/fontawesome-free@6.5.2/css/all.min.css"
|
<link href="{{ cdn_url }}/npm/@fortawesome/fontawesome-free@6.5.2/css/all.min.css"
|
||||||
|
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from discord import SyncWebhook
|
from discord import SyncWebhook
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from django.conf.locale import LANG_INFO
|
||||||
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
|
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
|
||||||
from django.core.signing import b62_decode, b62_encode
|
from django.core.signing import b62_decode, b62_encode
|
||||||
from django.http import Http404, HttpRequest, HttpResponseRedirect, QueryDict
|
from django.http import Http404, HttpRequest, HttpResponseRedirect, QueryDict
|
||||||
|
|
|
@ -509,7 +509,7 @@ class Content(Piece):
|
||||||
raise NotImplementedError("subclass should override this")
|
raise NotImplementedError("subclass should override this")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def display_description(self) -> str:
|
def brief_description(self) -> str:
|
||||||
raise NotImplementedError("subclass should override this")
|
raise NotImplementedError("subclass should override this")
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import re
|
import re
|
||||||
|
from html import unescape
|
||||||
from typing import cast
|
from typing import cast
|
||||||
|
|
||||||
import mistune
|
import mistune
|
||||||
|
@ -38,6 +39,13 @@ def render_md(s: str) -> str:
|
||||||
return cast(str, _markdown(s))
|
return cast(str, _markdown(s))
|
||||||
|
|
||||||
|
|
||||||
|
_RE_HTML_TAG = re.compile(r"<[^>]*>")
|
||||||
|
|
||||||
|
|
||||||
|
def html_to_text(h: str) -> str:
|
||||||
|
return unescape(_RE_HTML_TAG.sub(" ", h.replace("\r", "")))
|
||||||
|
|
||||||
|
|
||||||
def _spolier(s: str) -> str:
|
def _spolier(s: str) -> str:
|
||||||
sl = s.split(">!", 1)
|
sl = s.split(">!", 1)
|
||||||
if len(sl) == 1:
|
if len(sl) == 1:
|
||||||
|
|
|
@ -32,7 +32,7 @@ class Review(Content):
|
||||||
return self.title
|
return self.title
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def display_description(self):
|
def brief_description(self):
|
||||||
return self.plain_content[:155]
|
return self.plain_content[:155]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
|
@ -146,7 +146,7 @@ class TagTest(TestCase):
|
||||||
|
|
||||||
def test_cleanup(self):
|
def test_cleanup(self):
|
||||||
self.assertEqual(Tag.cleanup_title("# "), "_")
|
self.assertEqual(Tag.cleanup_title("# "), "_")
|
||||||
self.assertEqual(Tag.deep_cleanup_title("# C "), "c")
|
self.assertEqual(Tag.deep_cleanup_title("# C "), "text")
|
||||||
|
|
||||||
def test_user_tag(self):
|
def test_user_tag(self):
|
||||||
t1 = "tag 1"
|
t1 = "tag 1"
|
||||||
|
@ -183,7 +183,7 @@ class MarkTest(TestCase):
|
||||||
|
|
||||||
mark = Mark(self.user1.identity, self.book1)
|
mark = Mark(self.user1.identity, self.book1)
|
||||||
self.assertEqual(mark.shelf_type, ShelfType.WISHLIST)
|
self.assertEqual(mark.shelf_type, ShelfType.WISHLIST)
|
||||||
self.assertEqual(mark.shelf_label, "想读的书")
|
self.assertEqual(mark.shelf_label, "books to read")
|
||||||
self.assertEqual(mark.comment_text, "a gentle comment")
|
self.assertEqual(mark.comment_text, "a gentle comment")
|
||||||
self.assertEqual(mark.rating_grade, 9)
|
self.assertEqual(mark.rating_grade, 9)
|
||||||
self.assertEqual(mark.visibility, 1)
|
self.assertEqual(mark.visibility, 1)
|
||||||
|
|
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -27,10 +27,11 @@ class MastodonSiteCheck(BaseJob):
|
||||||
try:
|
try:
|
||||||
api_domain = site.api_domain or site.domain_name
|
api_domain = site.api_domain or site.domain_name
|
||||||
domain, api_domain, v = detect_server_info(api_domain)
|
domain, api_domain, v = detect_server_info(api_domain)
|
||||||
|
site.server_version = v
|
||||||
site.last_reachable_date = timezone.now()
|
site.last_reachable_date = timezone.now()
|
||||||
site.detect_configurations()
|
site.detect_configurations()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.warning(
|
||||||
f"Failed to detect server info for {site.domain_name}/{site.api_domain}",
|
f"Failed to detect server info for {site.domain_name}/{site.api_domain}",
|
||||||
extra={"exception": e},
|
extra={"exception": e},
|
||||||
)
|
)
|
||||||
|
@ -40,12 +41,16 @@ class MastodonSiteCheck(BaseJob):
|
||||||
if timezone.now() > site.last_reachable_date + timedelta(
|
if timezone.now() > site.last_reachable_date + timedelta(
|
||||||
days=self.max_unreachable_days
|
days=self.max_unreachable_days
|
||||||
):
|
):
|
||||||
|
logger.error(
|
||||||
|
f"Failed to detect server info for {site.domain_name}/{site.api_domain} disabling it."
|
||||||
|
)
|
||||||
site.disabled = True
|
site.disabled = True
|
||||||
count_disabled += 1
|
count_disabled += 1
|
||||||
finally:
|
finally:
|
||||||
site.save(
|
site.save(
|
||||||
update_fields=[
|
update_fields=[
|
||||||
"star_mode",
|
"star_mode",
|
||||||
|
"server_version",
|
||||||
"max_status_len",
|
"max_status_len",
|
||||||
"last_reachable_date",
|
"last_reachable_date",
|
||||||
"disabled",
|
"disabled",
|
||||||
|
|
|
@ -257,7 +257,7 @@ class BlueskyAccount(SocialAccount):
|
||||||
embed = models.AppBskyEmbedExternal.Main(
|
embed = models.AppBskyEmbedExternal.Main(
|
||||||
external=models.AppBskyEmbedExternal.External(
|
external=models.AppBskyEmbedExternal.External(
|
||||||
title=obj.display_title,
|
title=obj.display_title,
|
||||||
description=obj.display_description,
|
description=obj.brief_description,
|
||||||
uri=obj.absolute_url,
|
uri=obj.absolute_url,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
@ -57,7 +57,6 @@ dependencies = [
|
||||||
"deepmerge>=1.1.1",
|
"deepmerge>=1.1.1",
|
||||||
"django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git",
|
"django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git",
|
||||||
"atproto>=0.0.49",
|
"atproto>=0.0.49",
|
||||||
"pyright>=1.1.370",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.rye]
|
[tool.rye]
|
||||||
|
@ -70,7 +69,7 @@ dev-dependencies = [
|
||||||
"djlint~=1.34.1",
|
"djlint~=1.34.1",
|
||||||
"isort~=5.13.2",
|
"isort~=5.13.2",
|
||||||
"lxml-stubs",
|
"lxml-stubs",
|
||||||
"pyright>=1.1.369",
|
"pyright>=1.1.371",
|
||||||
"ruff",
|
"ruff",
|
||||||
"mkdocs-material>=9.5.25",
|
"mkdocs-material>=9.5.25",
|
||||||
]
|
]
|
||||||
|
|
|
@ -229,7 +229,7 @@ pygments==2.18.0
|
||||||
# via mkdocs-material
|
# via mkdocs-material
|
||||||
pymdown-extensions==10.8.1
|
pymdown-extensions==10.8.1
|
||||||
# via mkdocs-material
|
# via mkdocs-material
|
||||||
pyright==1.1.370
|
pyright==1.1.371
|
||||||
python-dateutil==2.9.0.post0
|
python-dateutil==2.9.0.post0
|
||||||
# via dateparser
|
# via dateparser
|
||||||
# via django-auditlog
|
# via django-auditlog
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/zJkLznQNzeLfKuD238Czo6jk65X.jpg","id":71365,"name":"Battlestar Galactica","original_name":"Battlestar Galactica","overview":"A re-imagining of the original series in which a \"rag-tag fugitive fleet\" of the last remnants of mankind flees pursuing robots while simultaneously searching for their true home, Earth.","poster_path":"/imTQ4nBdA68TVpLaWhhQJnb7NQh.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[10759,18,10765],"popularity":51.919,"first_air_date":"2003-12-08","vote_average":8.184,"vote_count":801,"origin_country":["CA"]}],"tv_episode_results":[],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/nfH8SZJVOxcBlFaqqtoqS5hHizG.jpg","id":57243,"name":"Doctor Who","original_name":"Doctor Who","overview":"The Doctor is a Time Lord: a 900 year old alien with 2 hearts, part of a gifted civilization who mastered time travel. The Doctor saves planets for a living—more of a hobby actually, and the Doctor's very, very good at it.","poster_path":"/4edFyasCrkH4MKs6H4mHqlrxA6b.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[10759,18,10765],"popularity":1090.391,"first_air_date":"2005-03-26","vote_average":7.519,"vote_count":2930,"origin_country":["GB"]}],"tv_episode_results":[],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[{"backdrop_path":"/eCTWG4HzsOQomghw8sRd1qpeOlA.jpg","id":282758,"title":"Doctor Who: The Runaway Bride","original_title":"Doctor Who: The Runaway Bride","overview":"A young bride in the midst of her wedding finds herself mysteriously transported to the TARDIS. The Doctor must discover what her connection is with the Empress of the Racnoss's plan to destroy the world.","poster_path":"/dy7JzhXnDFhQsHRiPXxpu62j3yQ.jpg","media_type":"movie","adult":false,"original_language":"en","genre_ids":[878],"popularity":17.25,"release_date":"2006-12-25","video":false,"vote_average":7.728,"vote_count":224}],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1008547,"name":"The Runaway Bride","overview":"Bride-to-be Donna vanishes as she walks down the aisle to marry boyfriend Lance. To her complete astonishment - and the Doctor's - she reappears in the Tardis. As the Time Lord, still reeling from Rose's departure, investigates how Donna came to be there, the duo uncover a terrifying enemy. How far will the Doctor go to save Earth from the latest alien threat?","media_type":"tv_episode","vote_average":6.925,"vote_count":20,"air_date":"2006-12-25","episode_number":4,"episode_type":"standard","production_code":"NCFT094N","runtime":64,"season_number":0,"show_id":57243,"still_path":"/pncNamTuydXWinybPuMTsBUVjSD.jpg"}],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":941505,"name":"Partners in Crime","overview":"During an alien emergency in London, a woman called Donna Noble must search for an old friend who can save the day - a man named the Doctor. But can even the Doctor halt the plans of the mysterious Miss Foster?","media_type":"tv_episode","vote_average":7.26,"vote_count":52,"air_date":"2008-04-05","episode_number":1,"episode_type":"standard","production_code":"","runtime":51,"season_number":4,"show_id":57243,"still_path":"/vg5oP1tOzivl4EV7iHiEaKwiZkK.jpg"}],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[{"backdrop_path":"/8ZTVqvKDQ8emSGUEMjsS4yHAwrp.jpg","id":27205,"title":"Inception","original_title":"Inception","overview":"Cobb, a skilled thief who commits corporate espionage by infiltrating the subconscious of his targets is offered a chance to regain his old life as payment for a task considered to be impossible: \"inception\", the implantation of another person's idea into a target's subconscious.","poster_path":"/oYuLEt3zVCKq57qu2F8dT7NIa6f.jpg","media_type":"movie","adult":false,"original_language":"en","genre_ids":[28,878,12],"popularity":92.871,"release_date":"2010-07-15","video":false,"vote_average":8.369,"vote_count":35987}],"person_results":[],"tv_results":[],"tv_episode_results":[],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1305550,"name":"Part 1","overview":"In a distant part of the galaxy lie The Twelve Colonies of Man, a civilization that has been at peace for some forty years with an empire of machines, the Cylons, who were created generations before as worker drones for mankind, but became independent, rose in rebellion, and launched war on their masters. Now, the Cylons have evolved into more human form, into machine-created biological beings, who seek to exterminate true biological humans. To this end they use a human scientist, Gaius, to help one of their infiltrators, known as #6, penetrate the Colonies' master ...","media_type":"tv_episode","vote_average":8.1,"vote_count":20,"air_date":"2003-12-08","episode_number":1,"episode_type":"standard","production_code":"","runtime":95,"season_number":1,"show_id":71365,"still_path":"/mBkKJW9ppIEjkD4CXGaAntekQNm.jpg"}],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":1305551,"name":"Part 2","overview":"After forty years of armistice, the Cylons attacks the Twelve Colonies of Kobol. Their strategy: a virus implanted into the mankind defense system. The former Battlestar Galactica, which is being adapted into a museum, is not connected with the defense system and becomes the only warship capable of fighting against the Cylons in the hopes of leading the survivors to planet 'Earth'.","media_type":"tv_episode","vote_average":8.118,"vote_count":17,"air_date":"2003-12-09","episode_number":2,"episode_type":"finale","production_code":"","runtime":90,"season_number":1,"show_id":71365,"still_path":"/77kEx9Zw6yI69oCrffQc1hIGOjC.jpg"}],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[],"tv_episode_results":[{"id":3891529,"name":"Solaricks","overview":"The Smiths deal with last season's fallout, and Rick and Morty are stranded in space.","media_type":"tv_episode","vote_average":8.091,"vote_count":55,"air_date":"2022-09-04","episode_number":1,"episode_type":"standard","production_code":"","runtime":23,"season_number":6,"show_id":60625,"still_path":"/5tiOEjp03nvaGiKT73knretU8e8.jpg"}],"tv_season_results":[]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"movie_results":[],"person_results":[],"tv_results":[{"backdrop_path":"/8IC1q0lHFwi5m8VtChLzIfmpaZH.jpg","id":86941,"name":"The North Water","original_name":"The North Water","overview":"Henry Drax is a harpooner and brutish killer whose amorality has been shaped to fit the harshness of his world, who will set sail on a whaling expedition to the Arctic with Patrick Sumner, a disgraced ex-army surgeon who signs up as the ship’s doctor. Hoping to escape the horrors of his past, Sumner finds himself on an ill-fated journey with a murderous psychopath. In search of redemption, his story becomes a harsh struggle for survival in the Arctic wasteland.","poster_path":"/9CM0ca8pX1os3SJ24hsIc0nN8ph.jpg","media_type":"tv","adult":false,"original_language":"en","genre_ids":[18,9648],"popularity":40.783,"first_air_date":"2021-07-14","vote_average":7.392,"vote_count":120,"origin_country":["US"]}],"tv_episode_results":[],"tv_season_results":[]}
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue