lib.itmens/catalog/sites/tmdb.py

"""
The Movie Database


these language code from TMDB are not in currently iso-639-1
{'iso_639_1': 'xx', 'english_name': 'No Language', 'name': 'No Language'}
{'iso_639_1': 'sh', 'english_name': 'Serbo-Croatian', 'name': ''} - deprecated for several
{'iso_639_1': 'mo', 'english_name': 'Moldavian', 'name': ''} - deprecated for ro-MD
{'iso_639_1': 'cn', 'english_name': 'Cantonese', 'name': '粤语'} - faked for yue

"""

import logging
import re

from django.conf import settings

from catalog.common import *
from catalog.movie.models import *
from catalog.tv.models import *
from common.models.lang import SITE_PREFERRED_LANGUAGES

from .douban import *

_logger = logging.getLogger(__name__)


def _get_language_code():
    match settings.LANGUAGE_CODE:
        case "zh-hans":
            return "zh-CN"
        case "zh-hant":
            return "zh-TW"
        case _:
            return "en-US"


def _get_preferred_languages():
    langs = {}
    for lang in SITE_PREFERRED_LANGUAGES:
        if lang == "zh":
            langs.update({"zh-cn": "zh-CN", "zh-tw": "zh-TW", "zh-hk": "zh-HK"})
        else:
            langs[lang] = lang
    return langs


TMDB_DEFAULT_LANG = _get_language_code()
TMDB_PREFERRED_LANGS = _get_preferred_languages()


def search_tmdb_by_imdb_id(imdb_id):
    tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&external_source=imdb_id"
    try:
        return BasicDownloader(tmdb_api_url).download().json()
    except Exception:
        return {}


def query_tmdb_tv_episode(tv, season, episode):
    tmdb_api_url = f"https://api.themoviedb.org/3/tv/{tv}/season/{season}/episode/{episode}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids"
    res_data = BasicDownloader(tmdb_api_url).download().json()
    return res_data


def _copy_dict(s, key_map):
    d = {}
    for src, dst in key_map.items():
        d[dst if dst else src] = s.get(src)
    return d


@SiteManager.register
class TMDB_Movie(AbstractSite):
    SITE_NAME = SiteName.TMDB
    ID_TYPE = IdType.TMDB_Movie
    URL_PATTERNS = [r"^\w+://www.themoviedb.org/movie/(\d+)"]
    WIKI_PROPERTY_ID = "?"
    DEFAULT_MODEL = Movie

    @classmethod
    def id_to_url(cls, id_value):
        return f"https://www.themoviedb.org/movie/{id_value}"

    def scrape(self):
        res_data = {}
        localized_title = []
        localized_desc = []
        # GET api urls in all locales
        # btw it seems no way to tell if TMDB does not have a certain translation
        for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
            api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
            res_data = BasicDownloader(api_url).download().json()
            localized_title.append({"lang": lang, "text": res_data["title"]})
            if res_data.get("overview", "").strip():
                localized_desc.append({"lang": lang, "text": res_data["overview"]})
        title = res_data["title"]
        orig_title = res_data["original_title"]
        year = (
            int(res_data["release_date"].split("-")[0])
            if res_data["release_date"]
            else None
        )
        showtime = (
            [{"time": res_data["release_date"], "region": "发布日期"}]
            if res_data["release_date"]
            else None
        )
        imdb_code = res_data["imdb_id"]
        # in minutes
        duration = res_data["runtime"] if res_data["runtime"] else None

        genre = [x["name"] for x in res_data["genres"]]
        language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
        brief = res_data["overview"]

        director = list(
            map(
                lambda x: x["name"],
                filter(lambda c: c["job"] == "Director", res_data["credits"]["crew"]),
            )
        )
        playwright = list(
            map(
                lambda x: x["name"],
                filter(lambda c: c["job"] == "Screenplay", res_data["credits"]["crew"]),
            )
        )
        actor = list(map(lambda x: x["name"], res_data["credits"]["cast"]))
        area = []

        # other_info = {}
        # other_info['TMDB评分'] = res_data['vote_average']
        # other_info['分级'] = res_data['contentRating']
        # other_info['Metacritic评分'] = res_data['metacriticRating']
        # other_info['奖项'] = res_data['awards']
        # other_info['TMDB_ID'] = id
        # if is_series:
        #     other_info["Seasons"] = res_data["number_of_seasons"]
        #     other_info["Episodes"] = res_data["number_of_episodes"]

        # TODO: use GET /configuration to get base url
        img_url = (
            ("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
            if res_data.get("poster_path") is not None
            else None
        )

        pd = ResourceContent(
            metadata={
                "localized_title": localized_title,
                "localized_description": localized_desc,
                "title": title,
                "orig_title": orig_title,
                "other_title": [],
                "imdb_code": imdb_code,
                "director": director,
                "playwright": playwright,
                "actor": actor,
                "genre": genre,
                "showtime": showtime,
                "site": None,
                "area": area,
                "language": language,
                "year": year,
                "duration": duration,
                "season": None,
                "episodes": None,
                "single_episode_length": None,
                "brief": brief,
                "cover_image_url": img_url,
            }
        )
        if imdb_code:
            pd.lookup_ids[IdType.IMDB] = imdb_code
        return pd


@SiteManager.register
class TMDB_TV(AbstractSite):
    SITE_NAME = SiteName.TMDB
    ID_TYPE = IdType.TMDB_TV
    URL_PATTERNS = [
        r"^\w+://www.themoviedb.org/tv/(\d+)[^/]*/?$",
        r"^\w+://www.themoviedb.org/tv/(\d+)[^/]*/seasons$",
    ]
    WIKI_PROPERTY_ID = "?"
    DEFAULT_MODEL = TVShow

    @classmethod
    def id_to_url(cls, id_value):
        return f"https://www.themoviedb.org/tv/{id_value}"

    def scrape(self):
        res_data = {}
        localized_title = []
        localized_desc = []
        for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
            api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
            res_data = BasicDownloader(api_url).download().json()
            localized_title.append({"lang": lang, "text": res_data["name"]})
            if res_data.get("overview", "").strip():
                localized_desc.append({"lang": lang, "text": res_data["overview"]})

        title = res_data["name"]
        orig_title = res_data["original_name"]
        year = (
            int(res_data["first_air_date"].split("-")[0])
            if res_data["first_air_date"]
            else None
        )
        imdb_code = res_data["external_ids"]["imdb_id"]
        showtime = (
            [{"time": res_data["first_air_date"], "region": "首播日期"}]
            if res_data["first_air_date"]
            else None
        )
        duration = None
        genre = [x["name"] for x in res_data["genres"]]
        language = list(map(lambda x: x["name"], res_data["spoken_languages"]))
        brief = res_data["overview"]
        director = list(map(lambda x: x["name"], res_data["created_by"]))
        playwright = list(
            map(
                lambda x: x["name"],
                filter(lambda c: c["job"] == "Screenplay", res_data["credits"]["crew"]),
            )
        )
        actor = list(map(lambda x: x["name"], res_data["credits"]["cast"]))
        area = []
        # other_info = {}
        # other_info["Seasons"] = res_data["number_of_seasons"]
        # other_info["Episodes"] = res_data["number_of_episodes"]
        # TODO: use GET /configuration to get base url
        img_url = (
            ("https://image.tmdb.org/t/p/original/" + res_data["poster_path"])
            if res_data.get("poster_path") is not None
            else None
        )
        season_links = list(
            map(
                lambda s: {
                    "model": "TVSeason",
                    "id_type": IdType.TMDB_TVSeason,
                    "id_value": f'{self.id_value}-{s["season_number"]}',
                    "title": s["name"],
                    "url": f'{self.url}/season/{s["season_number"]}',
                },
                res_data["seasons"],
            )
        )
        pd = ResourceContent(
            metadata={
                "localized_title": localized_title,
                "localized_description": localized_desc,
                "title": title,
                "orig_title": orig_title,
                "other_title": [],
                "imdb_code": imdb_code,
                "director": director,
                "playwright": playwright,
                "actor": actor,
                "genre": genre,
                "showtime": showtime,
                "site": None,
                "area": area,
                "language": language,
                "year": year,
                "duration": duration,
                "season_count": res_data["number_of_seasons"],
                "season": None,
                "episodes": None,
                "single_episode_length": None,
                "brief": brief,
                "cover_image_url": img_url,
                "related_resources": season_links,  # FIXME crawling them for now, but many douban tv season data may have wrong imdb links
            }
        )
        if imdb_code:
            pd.lookup_ids[IdType.IMDB] = imdb_code
        return pd


@SiteManager.register
class TMDB_TVSeason(AbstractSite):
    SITE_NAME = SiteName.TMDB
    ID_TYPE = IdType.TMDB_TVSeason
    URL_PATTERNS = [r"^\w+://www.themoviedb.org/tv/(\d+)[^/]*/season/(\d+)[^/]*/?$"]
    WIKI_PROPERTY_ID = "?"
    DEFAULT_MODEL = TVSeason
    ID_PATTERN = r"^(\d+)-(\d+)$"

    @classmethod
    def url_to_id(cls, url: str):
        u = next(
            iter([re.match(p, url) for p in cls.URL_PATTERNS if re.match(p, url)]), None
        )
        return u[1] + "-" + u[2] if u else None

    @classmethod
    def id_to_url(cls, id_value):
        v = id_value.split("-")
        return f"https://www.themoviedb.org/tv/{v[0]}/season/{v[1]}"

    def scrape(self):
        if not self.id_value:
            raise ParseError(self, "id_value")
        v = self.id_value.split("-")
        show_id = v[0]
        season_id = v[1]
        site = TMDB_TV(TMDB_TV.id_to_url(show_id))
        show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
        if not show_resource:
            raise ValueError(f"TMDB: failed to get show for season {self.url}")

        res_data = {}
        localized_title = []
        localized_desc = []
        for lang, lang_param in reversed(TMDB_PREFERRED_LANGS.items()):
            api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language={lang_param}&append_to_response=external_ids,credits"
            res_data = BasicDownloader(api_url).download().json()
            localized_title.append({"lang": lang, "text": res_data["name"]})
            if res_data.get("overview", "").strip():
                localized_desc.append({"lang": lang, "text": res_data["overview"]})
        if not res_data.get("id"):
            raise ParseError(self, "id")
        d = res_data
        r = _copy_dict(
            res_data,
            {
                "name": "title",
                "overview": "brief",
                "air_date": "air_date",
                "season_number": 0,
                "external_ids": [],
            },
        )
        r["localized_title"] = localized_title
        r["localized_description"] = localized_desc
        pd = ResourceContent(metadata=r)
        pd.metadata["title"] = (
            show_resource.metadata["title"] + " " + pd.metadata["title"]
        )
        pd.metadata["required_resources"] = [
            {
                "model": "TVShow",
                "id_type": IdType.TMDB_TV,
                "id_value": show_id,
                "title": f"TMDB TV Show {show_id}",
                "url": f"https://www.themoviedb.org/tv/{show_id}",
            }
        ]
        pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id")
        pd.metadata["cover_image_url"] = (
            ("https://image.tmdb.org/t/p/original/" + d["poster_path"])
            if d.get("poster_path")
            else None
        )
        pd.metadata["title"] = (
            pd.metadata["title"]
            if pd.metadata.get("title")
            else f'Season {d["season_number"]}'
        )
        pd.metadata["episode_number_list"] = list(
            map(lambda ep: ep["episode_number"], d["episodes"])
        )
        pd.metadata["episode_count"] = len(pd.metadata["episode_number_list"])

        # use show's IMDB (for Season 1) or 1st episode's IMDB (if not Season 1) as this season's IMDB so that it can be compatible with TVSeason data from Douban
        if pd.lookup_ids.get(IdType.IMDB):
            # this should not happen
            _logger.warning("Unexpected IMDB id for TMDB tv season")
        elif pd.metadata.get("season_number") == 1:
            site = SiteManager.get_site_by_url(
                f"https://www.themoviedb.org/tv/{show_id}"
            )
            if site:
                res = site.get_resource_ready()
                pd.lookup_ids[IdType.IMDB] = (
                    res.other_lookup_ids.get(IdType.IMDB) if res else None
                )
        elif len(pd.metadata["episode_number_list"]) == 0:
            _logger.warning(
                "Unable to lookup IMDB id for TMDB tv season with zero episodes"
            )
        else:
            ep = pd.metadata["episode_number_list"][0]
            api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
            d2 = BasicDownloader(api_url2).download().json()
            if not d2.get("id"):
                raise ParseError(self, "first episode id for season")
            pd.lookup_ids[IdType.IMDB] = d2["external_ids"].get("imdb_id")
        return pd


@SiteManager.register
class TMDB_TVEpisode(AbstractSite):
    SITE_NAME = SiteName.TMDB
    ID_TYPE = IdType.TMDB_TVEpisode
    URL_PATTERNS = [
        r"\w+://www.themoviedb.org/tv/(\d+)[^/]*/season/(\d+)/episode/(\d+)[^/]*$"
    ]
    WIKI_PROPERTY_ID = "?"
    DEFAULT_MODEL = TVEpisode
    ID_PATTERN = r"^(\d+)-(\d+)-(\d+)$"

    @classmethod
    def url_to_id(cls, url: str):
        u = next(
            iter([re.match(p, url) for p in cls.URL_PATTERNS if re.match(p, url)]), None
        )
        return u[1] + "-" + u[2] + "-" + u[3] if u else None

    @classmethod
    def id_to_url(cls, id_value):
        v = id_value.split("-")
        return f"https://www.themoviedb.org/tv/{v[0]}/season/{v[1]}/episode/{v[2]}"

    def scrape(self):
        if not self.id_value:
            raise ParseError(self, "id_value")
        v = self.id_value.split("-")
        show_id = v[0]
        season_id = v[1]
        episode_id = v[2]
        site = TMDB_TV(TMDB_TV.id_to_url(show_id))
        site.get_resource_ready(auto_create=False, auto_link=False)
        api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}/episode/{episode_id}?api_key={settings.TMDB_API3_KEY}&language={TMDB_DEFAULT_LANG}&append_to_response=external_ids,credits"
        d = BasicDownloader(api_url).download().json()
        if not d.get("id"):
            raise ParseError(self, "id")
        pd = ResourceContent(
            metadata=_copy_dict(
                d,
                {
                    "name": "title",
                    "overview": "brief",
                    # "air_date": "air_date",
                    "season_number": 0,
                    "episode_number": 0,
                    "external_ids": [],
                },
            )
        )
        pd.metadata["required_resources"] = [
            {
                "model": "TVSeason",
                "id_type": IdType.TMDB_TVSeason,
                "id_value": f"{show_id}-{season_id}",
                "title": f"TMDB TV Season {show_id}-{season_id}",
                "url": f"https://www.themoviedb.org/tv/{show_id}/season/{season_id}",
            }
        ]
        pd.lookup_ids[IdType.IMDB] = d["external_ids"].get("imdb_id")
        pd.metadata["cover_image_url"] = (
            ("https://image.tmdb.org/t/p/original/" + d["poster_path"])
            if d.get("poster_path")
            else None
        )
        pd.metadata["title"] = (
            pd.metadata["title"]
            if pd.metadata["title"]
            else f'S{d["season_number"]} E{d["episode_number"]}'
        )

        if pd.lookup_ids.get(IdType.IMDB):
            pd.lookup_ids[IdType.IMDB] = pd.lookup_ids[IdType.IMDB]
        return pd