diff --git a/.gitignore b/.gitignore index bbbed5da..e8fbbaa8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/catalog/__init__.py b/catalog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalog/admin.py b/catalog/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/catalog/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/catalog/api.py b/catalog/api.py new file mode 100644 index 00000000..467b0811 --- /dev/null +++ b/catalog/api.py @@ -0,0 +1,11 @@ +from ninja import NinjaAPI +from .models import Podcast +from django.conf import settings + + +api = NinjaAPI(title=settings.SITE_INFO['site_name'], version="1.0.0", description=settings.SITE_INFO['site_name']) + + +@api.get("/podcasts/{item_id}") +def get_item(request, item_id: int): + return Podcast.objects.filter(pk=item_id).first() diff --git a/catalog/apps.py b/catalog/apps.py new file mode 100644 index 00000000..a5993c68 --- /dev/null +++ b/catalog/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CatalogConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'catalog' diff --git a/catalog/book/models.py b/catalog/book/models.py new file mode 100644 index 00000000..131c89de --- /dev/null +++ b/catalog/book/models.py @@ -0,0 +1,77 @@ +""" +Models for Book + +Series -> Work -> Edition + +Series is not fully implemented at the moment + +Goodreads +Famous works have many editions + +Google Books: +only has Edition level ("volume") data + +Douban: +old editions has only CUBN(Chinese Unified Book Number) +work data seems asymmetric (a book links to a work, but may not listed in that work as one of its editions) + +""" + +from django.db import models +from django.utils.translation import gettext_lazy as _ +from catalog.common import * +from .utils import * + + +class Edition(Item): + isbn = PrimaryLookupIdDescriptor(IdType.ISBN) + asin = PrimaryLookupIdDescriptor(IdType.ASIN) + cubn = PrimaryLookupIdDescriptor(IdType.CUBN) + # douban_book = LookupIdDescriptor(IdType.DoubanBook) + # goodreads = LookupIdDescriptor(IdType.Goodreads) + languages = jsondata.ArrayField(_("语言"), null=True, blank=True, default=list) + publish_year = jsondata.IntegerField(_("发表年份"), null=True, blank=True) + publish_month = jsondata.IntegerField(_("发表月份"), null=True, blank=True) + pages = jsondata.IntegerField(blank=True, default=None) + authors = jsondata.ArrayField(_('作者'), null=False, blank=False, default=list) + translaters = jsondata.ArrayField(_('译者'), null=True, blank=True, default=list) + publishers = jsondata.ArrayField(_('出版方'), null=True, blank=True, default=list) + + @property + def isbn10(self): + return isbn_13_to_10(self.isbn) + + @isbn10.setter + def isbn10(self, value): + self.isbn = isbn_10_to_13(value) + + def update_linked_items_from_external_resource(self, resource): + """add Work from resource.metadata['work'] if not yet""" + links = resource.required_resources + resource.related_resources + for w in links: + if w['model'] == 'Work': + work = Work.objects.filter(primary_lookup_id_type=w['id_type'], primary_lookup_id_value=w['id_value']).first() + if work and work not in self.works.all(): + self.works.add(work) + # if not work: + # _logger.info(f'Unable to find link for {w["url"]}') + + +class Work(Item): + # douban_work = PrimaryLookupIdDescriptor(IdType.DoubanBook_Work) + # goodreads_work = PrimaryLookupIdDescriptor(IdType.Goodreads_Work) + editions = models.ManyToManyField(Edition, related_name='works') # , through='WorkEdition' + + # def __str__(self): + # return self.title + + # class Meta: + # proxy = True + + +class Series(Item): + # douban_serie = LookupIdDescriptor(IdType.DoubanBook_Serie) + # goodreads_serie = LookupIdDescriptor(IdType.Goodreads_Serie) + + class Meta: + proxy = True diff --git a/catalog/book/tests.py b/catalog/book/tests.py new file mode 100644 index 00000000..51e55353 --- /dev/null +++ b/catalog/book/tests.py @@ -0,0 +1,237 @@ +from django.test import TestCase +from catalog.book.models import * +from catalog.common import * + + +class BookTestCase(TestCase): + def setUp(self): + hyperion = Edition.objects.create(title="Hyperion") + hyperion.pages = 500 + hyperion.isbn = '9780553283686' + hyperion.save() + # hyperion.isbn10 = '0553283685' + + def test_properties(self): + hyperion = Edition.objects.get(title="Hyperion") + self.assertEqual(hyperion.title, "Hyperion") + self.assertEqual(hyperion.pages, 500) + self.assertEqual(hyperion.primary_lookup_id_type, IdType.ISBN) + self.assertEqual(hyperion.primary_lookup_id_value, '9780553283686') + andymion = Edition(title="Andymion", pages=42) + self.assertEqual(andymion.pages, 42) + + def test_lookupids(self): + hyperion = Edition.objects.get(title="Hyperion") + hyperion.asin = 'B004G60EHS' + self.assertEqual(hyperion.primary_lookup_id_type, IdType.ASIN) + self.assertEqual(hyperion.primary_lookup_id_value, 'B004G60EHS') + self.assertEqual(hyperion.isbn, None) + self.assertEqual(hyperion.isbn10, None) + + def test_isbn(self): + hyperion = Edition.objects.get(title="Hyperion") + self.assertEqual(hyperion.isbn, '9780553283686') + self.assertEqual(hyperion.isbn10, '0553283685') + hyperion.isbn10 = '0575099437' + self.assertEqual(hyperion.isbn, '9780575099432') + self.assertEqual(hyperion.isbn10, '0575099437') + + def test_work(self): + hyperion_print = Edition.objects.get(title="Hyperion") + hyperion_ebook = Edition(title="Hyperion") + hyperion_ebook.save() + hyperion_ebook.asin = 'B0043M6780' + hyperion = Work(title="Hyperion") + hyperion.save() + hyperion.editions.add(hyperion_print) + hyperion.editions.add(hyperion_ebook) + # andymion = Edition(title="Andymion", pages=42) + # serie = Serie(title="Hyperion Cantos") + + +class GoodreadsTestCase(TestCase): + def setUp(self): + pass + + def test_parse(self): + t_type = IdType.Goodreads + t_id = '77566' + t_url = 'https://www.goodreads.com/zh/book/show/77566.Hyperion' + t_url2 = 'https://www.goodreads.com/book/show/77566' + p1 = SiteList.get_site_by_id_type(t_type) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url2) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.goodreads.com/book/show/77566.Hyperion' + t_url2 = 'https://www.goodreads.com/book/show/77566' + isbn = '9780553283686' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.url, t_url2) + site.get_resource() + self.assertEqual(site.ready, False) + self.assertIsNotNone(site.resource) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata.get('title'), 'Hyperion') + self.assertEqual(site.resource.metadata.get('isbn'), isbn) + self.assertEqual(site.resource.required_resources[0]['id_value'], '1383900') + edition = Edition.objects.get(primary_lookup_id_type=IdType.ISBN, primary_lookup_id_value=isbn) + resource = edition.external_resources.all().first() + self.assertEqual(resource.id_type, IdType.Goodreads) + self.assertEqual(resource.id_value, '77566') + self.assertNotEqual(resource.cover, '/media/item/default.svg') + self.assertEqual(edition.isbn, '9780553283686') + self.assertEqual(edition.title, 'Hyperion') + + edition.delete() + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.url, t_url2) + site.get_resource() + self.assertEqual(site.ready, True, 'previous resource should still exist with data') + + @use_local_response + def test_asin(self): + t_url = 'https://www.goodreads.com/book/show/45064996-hyperion' + site = SiteList.get_site_by_url(t_url) + site.get_resource_ready() + self.assertEqual(site.resource.item.title, 'Hyperion') + self.assertEqual(site.resource.item.asin, 'B004G60EHS') + + @use_local_response + def test_work(self): + url = 'https://www.goodreads.com/work/editions/153313' + p = SiteList.get_site_by_url(url).get_resource_ready() + self.assertEqual(p.item.title, '1984') + url1 = 'https://www.goodreads.com/book/show/3597767-rok-1984' + url2 = 'https://www.goodreads.com/book/show/40961427-1984' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + w1 = p1.item.works.all().first() + w2 = p2.item.works.all().first() + self.assertEqual(w1, w2) + + +class GoogleBooksTestCase(TestCase): + def test_parse(self): + t_type = IdType.GoogleBooks + t_id = 'hV--zQEACAAJ' + t_url = 'https://books.google.com.bn/books?id=hV--zQEACAAJ&hl=ms' + t_url2 = 'https://books.google.com/books?id=hV--zQEACAAJ' + p1 = SiteList.get_site_by_url(t_url) + p2 = SiteList.get_site_by_url(t_url2) + self.assertIsNotNone(p1) + self.assertEqual(p1.url, t_url2) + self.assertEqual(p1.ID_TYPE, t_type) + self.assertEqual(p1.id_value, t_id) + self.assertEqual(p2.url, t_url2) + + @use_local_response + def test_scrape(self): + t_url = 'https://books.google.com.bn/books?id=hV--zQEACAAJ' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata.get('title'), '1984 Nineteen Eighty-Four') + self.assertEqual(site.resource.metadata.get('isbn'), '9781847498571') + self.assertEqual(site.resource.id_type, IdType.GoogleBooks) + self.assertEqual(site.resource.id_value, 'hV--zQEACAAJ') + self.assertEqual(site.resource.item.isbn, '9781847498571') + self.assertEqual(site.resource.item.title, '1984 Nineteen Eighty-Four') + + +class DoubanBookTestCase(TestCase): + def setUp(self): + pass + + def test_parse(self): + t_type = IdType.DoubanBook + t_id = '35902899' + t_url = 'https://m.douban.com/book/subject/35902899/' + t_url2 = 'https://book.douban.com/subject/35902899/' + p1 = SiteList.get_site_by_url(t_url) + p2 = SiteList.get_site_by_url(t_url2) + self.assertEqual(p1.url, t_url2) + self.assertEqual(p1.ID_TYPE, t_type) + self.assertEqual(p1.id_value, t_id) + self.assertEqual(p2.url, t_url2) + + @use_local_response + def test_scrape(self): + t_url = 'https://book.douban.com/subject/35902899/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata.get('title'), '1984 Nineteen Eighty-Four') + self.assertEqual(site.resource.metadata.get('isbn'), '9781847498571') + self.assertEqual(site.resource.id_type, IdType.DoubanBook) + self.assertEqual(site.resource.id_value, '35902899') + self.assertEqual(site.resource.item.isbn, '9781847498571') + self.assertEqual(site.resource.item.title, '1984 Nineteen Eighty-Four') + + @use_local_response + def test_work(self): + # url = 'https://www.goodreads.com/work/editions/153313' + url1 = 'https://book.douban.com/subject/1089243/' + url2 = 'https://book.douban.com/subject/2037260/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + w1 = p1.item.works.all().first() + w2 = p2.item.works.all().first() + self.assertEqual(w1.title, '黄金时代') + self.assertEqual(w2.title, '黄金时代') + self.assertEqual(w1, w2) + editions = w1.editions.all().order_by('title') + self.assertEqual(editions.count(), 2) + self.assertEqual(editions[0].title, 'Wang in Love and Bondage') + self.assertEqual(editions[1].title, '黄金时代') + + +class MultiBookSitesTestCase(TestCase): + @use_local_response + def test_editions(self): + # isbn = '9781847498571' + url1 = 'https://www.goodreads.com/book/show/56821625-1984' + url2 = 'https://book.douban.com/subject/35902899/' + url3 = 'https://books.google.com/books?id=hV--zQEACAAJ' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p1.item.id, p2.item.id) + self.assertEqual(p2.item.id, p3.item.id) + + @use_local_response + def test_works(self): + # url1 and url4 has same ISBN, hence they share same Edition instance, which belongs to 2 Work instances + url1 = 'https://book.douban.com/subject/1089243/' + url2 = 'https://book.douban.com/subject/2037260/' + url3 = 'https://www.goodreads.com/book/show/59952545-golden-age' + url4 = 'https://www.goodreads.com/book/show/11798823' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() # lxml bug may break this + w1 = p1.item.works.all().first() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + w2 = p2.item.works.all().first() + self.assertEqual(w1, w2) + self.assertEqual(p1.item.works.all().count(), 1) + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + w3 = p3.item.works.all().first() + self.assertNotEqual(w3, w2) + p4 = SiteList.get_site_by_url(url4).get_resource_ready() + self.assertEqual(p4.item.works.all().count(), 2) + self.assertEqual(p1.item.works.all().count(), 2) + w2e = w2.editions.all().order_by('title') + self.assertEqual(w2e.count(), 2) + self.assertEqual(w2e[0].title, 'Wang in Love and Bondage') + self.assertEqual(w2e[1].title, '黄金时代') + w3e = w3.editions.all().order_by('title') + self.assertEqual(w3e.count(), 2) + self.assertEqual(w3e[0].title, 'Golden Age: A Novel') + self.assertEqual(w3e[1].title, '黄金时代') + e = Edition.objects.get(primary_lookup_id_value=9781662601217) + self.assertEqual(e.title, 'Golden Age: A Novel') diff --git a/catalog/book/utils.py b/catalog/book/utils.py new file mode 100644 index 00000000..fe3e50fc --- /dev/null +++ b/catalog/book/utils.py @@ -0,0 +1,45 @@ +def check_digit_10(isbn): + assert len(isbn) == 9 + sum = 0 + for i in range(len(isbn)): + c = int(isbn[i]) + w = i + 1 + sum += w * c + r = sum % 11 + return 'X' if r == 10 else str(r) + + +def check_digit_13(isbn): + assert len(isbn) == 12 + sum = 0 + for i in range(len(isbn)): + c = int(isbn[i]) + w = 3 if i % 2 else 1 + sum += w * c + r = 10 - (sum % 10) + return '0' if r == 10 else str(r) + + +def isbn_10_to_13(isbn): + if not isbn or len(isbn) != 10: + return None + return '978' + isbn[:-1] + check_digit_13('978' + isbn[:-1]) + + +def isbn_13_to_10(isbn): + if not isbn or len(isbn) != 13 or isbn[:3] != '978': + return None + else: + return isbn[3:12] + check_digit_10(isbn[3:12]) + + +def is_isbn_13(isbn): + return len(isbn) == 13 + + +def is_isbn_10(isbn): + return len(isbn) == 10 and isbn[0] >= '0' and isbn[0] <= '9' + + +def is_asin(asin): + return len(asin) == 10 and asin[0].lower == 'b' diff --git a/catalog/common/__init__.py b/catalog/common/__init__.py new file mode 100644 index 00000000..9a0a165b --- /dev/null +++ b/catalog/common/__init__.py @@ -0,0 +1,8 @@ +from .models import * +from .sites import * +from .downloaders import * +from .scrapers import * +from . import jsondata + + +__all__ = ('IdType', 'Item', 'ExternalResource', 'ResourceContent', 'ParseError', 'AbstractSite', 'SiteList', 'jsondata', 'PrimaryLookupIdDescriptor', 'LookupIdDescriptor', 'get_mock_mode', 'get_mock_file', 'use_local_response', 'RetryDownloader', 'BasicDownloader', 'ProxiedDownloader', 'BasicImageDownloader', 'RESPONSE_OK', 'RESPONSE_NETWORK_ERROR', 'RESPONSE_INVALID_CONTENT', 'RESPONSE_CENSORSHIP') diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py new file mode 100644 index 00000000..f7d1ed83 --- /dev/null +++ b/catalog/common/downloaders.py @@ -0,0 +1,245 @@ +import requests +import filetype +from PIL import Image +from io import BytesIO +from requests.exceptions import RequestException +from django.conf import settings +from pathlib import Path +import json +from io import StringIO +import re +import time +import logging +from lxml import html + + +_logger = logging.getLogger(__name__) + + +RESPONSE_OK = 0 # response is ready for pasring +RESPONSE_INVALID_CONTENT = -1 # content not valid but no need to retry +RESPONSE_NETWORK_ERROR = -2 # network error, retry next proxied url +RESPONSE_CENSORSHIP = -3 # censored, try sth special if possible + +_mock_mode = False + + +def use_local_response(func): + def _func(args): + set_mock_mode(True) + func(args) + set_mock_mode(False) + return _func + + +def set_mock_mode(enabled): + global _mock_mode + _mock_mode = enabled + + +def get_mock_mode(): + global _mock_mode + return _mock_mode + + +def get_mock_file(url): + fn = re.sub(r'[^\w]', '_', url) + return re.sub(r'_key_[A-Za-z0-9]+', '_key_19890604', fn) + + +class DownloadError(Exception): + def __init__(self, downloader, msg=None): + self.url = downloader.url + self.logs = downloader.logs + if downloader.response_type == RESPONSE_INVALID_CONTENT: + error = "Invalid Response" + elif downloader.response_type == RESPONSE_NETWORK_ERROR: + error = "Network Error" + elif downloader.response_type == RESPONSE_NETWORK_ERROR: + error = "Censored Content" + else: + error = "Unknown Error" + self.message = f"Download Failed: {error}{', ' + msg if msg else ''}, url: {self.url}" + super().__init__(self.message) + + +class BasicDownloader: + headers = { + # 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0', + 'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', + 'Accept-Encoding': 'gzip, deflate', + 'Connection': 'keep-alive', + 'DNT': '1', + 'Upgrade-Insecure-Requests': '1', + 'Cache-Control': 'no-cache', + } + + def __init__(self, url, headers=None): + self.url = url + self.response_type = RESPONSE_OK + self.logs = [] + if headers: + self.headers = headers + + def get_timeout(self): + return settings.SCRAPING_TIMEOUT + + def validate_response(self, response): + if response is None: + return RESPONSE_NETWORK_ERROR + elif response.status_code == 200: + return RESPONSE_OK + else: + return RESPONSE_INVALID_CONTENT + + def _download(self, url): + try: + if not _mock_mode: + # TODO cache = get/set from redis + resp = requests.get(url, headers=self.headers, timeout=self.get_timeout()) + if settings.DOWNLOADER_SAVEDIR: + with open(settings.DOWNLOADER_SAVEDIR + '/' + get_mock_file(url), 'w', encoding='utf-8') as fp: + fp.write(resp.text) + else: + resp = MockResponse(self.url) + response_type = self.validate_response(resp) + self.logs.append({'response_type': response_type, 'url': url, 'exception': None}) + + return resp, response_type + except RequestException as e: + self.logs.append({'response_type': RESPONSE_NETWORK_ERROR, 'url': url, 'exception': e}) + return None, RESPONSE_NETWORK_ERROR + + def download(self): + resp, self.response_type = self._download(self.url) + if self.response_type == RESPONSE_OK: + return resp + else: + raise DownloadError(self) + + +class ProxiedDownloader(BasicDownloader): + def get_proxied_urls(self): + urls = [] + if settings.PROXYCRAWL_KEY is not None: + urls.append(f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={self.url}') + if settings.SCRAPESTACK_KEY is not None: + # urls.append(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={self.url}') + urls.append(f'http://api.scrapestack.com/scrape?keep_headers=1&access_key={settings.SCRAPESTACK_KEY}&url={self.url}') + if settings.SCRAPERAPI_KEY is not None: + urls.append(f'http://api.scraperapi.com/?api_key={settings.SCRAPERAPI_KEY}&url={self.url}') + return urls + + def get_special_proxied_url(self): + return f'{settings.LOCAL_PROXY}?url={self.url}' if settings.LOCAL_PROXY is not None else None + + def download(self): + urls = self.get_proxied_urls() + last_try = False + url = urls.pop(0) if len(urls) else None + resp = None + while url: + resp, resp_type = self._download(url) + if resp_type == RESPONSE_OK or resp_type == RESPONSE_INVALID_CONTENT or last_try: + url = None + elif resp_type == RESPONSE_CENSORSHIP: + url = self.get_special_proxied_url() + last_try = True + else: # resp_type == RESPONSE_NETWORK_ERROR: + url = urls.pop(0) if len(urls) else None + self.response_type = resp_type + if self.response_type == RESPONSE_OK: + return resp + else: + raise DownloadError(self) + + +class RetryDownloader(BasicDownloader): + def download(self): + retries = settings.DOWNLOADER_RETRIES + while retries: + retries -= 1 + resp, self.response_type = self._download(self.url) + if self.response_type == RESPONSE_OK: + return resp + elif self.response_type != RESPONSE_NETWORK_ERROR and retries == 0: + raise DownloadError(self) + elif retries > 0: + _logger.debug('Retry ' + self.url) + time.sleep((settings.DOWNLOADER_RETRIES - retries) * 0.5) + raise DownloadError(self, 'max out of retries') + + +class ImageDownloaderMixin: + def __init__(self, url, referer=None): + if referer is not None: + self.headers['Referer'] = referer + super().__init__(url) + + def validate_response(self, response): + if response and response.status_code == 200: + try: + raw_img = response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = response.headers.get('Content-Type') + self.extention = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + return RESPONSE_OK + except Exception: + return RESPONSE_NETWORK_ERROR + if response and response.status_code >= 400 and response.status_code < 500: + return RESPONSE_INVALID_CONTENT + else: + return RESPONSE_NETWORK_ERROR + + +class BasicImageDownloader(ImageDownloaderMixin, BasicDownloader): + @classmethod + def download_image(cls, image_url, page_url): + imgdl = cls(image_url, page_url) + try: + image = imgdl.download().content + image_extention = imgdl.extention + return image, image_extention + except Exception: + return None, None + + +class ProxiedImageDownloader(ImageDownloaderMixin, ProxiedDownloader): + pass + + +_local_response_path = str(Path(__file__).parent.parent.parent.absolute()) + '/test_data/' + + +class MockResponse: + def __init__(self, url): + self.url = url + fn = _local_response_path + get_mock_file(url) + try: + self.content = Path(fn).read_bytes() + self.status_code = 200 + _logger.debug(f"use local response for {url} from {fn}") + except Exception: + self.content = b'Error: response file not found' + self.status_code = 404 + _logger.debug(f"local response not found for {url} at {fn}") + + @property + def text(self): + return self.content.decode('utf-8') + + def json(self): + return json.load(StringIO(self.text)) + + def html(self): + return html.fromstring(self.text) # may throw exception unexpectedly due to OS bug + + @property + def headers(self): + return {'Content-Type': 'image/jpeg' if self.url.endswith('jpg') else 'text/html'} + + +requests.Response.html = MockResponse.html diff --git a/catalog/common/jsondata.py b/catalog/common/jsondata.py new file mode 100644 index 00000000..ade72570 --- /dev/null +++ b/catalog/common/jsondata.py @@ -0,0 +1,201 @@ +import copy +from datetime import date, datetime +from importlib import import_module + +import django +from django.conf import settings +from django.core.exceptions import FieldError +from django.db.models import fields +from django.utils import dateparse, timezone + +from functools import partialmethod +from django.db.models import JSONField + + +__all__ = ('BooleanField', 'CharField', 'DateField', 'DateTimeField', 'DecimalField', 'EmailField', 'FloatField', 'IntegerField', 'IPAddressField', 'GenericIPAddressField', 'NullBooleanField', 'TextField', 'TimeField', 'URLField', 'ArrayField') + + +class JSONFieldDescriptor(object): + def __init__(self, field): + self.field = field + + def __get__(self, instance, cls=None): + if instance is None: + return self + json_value = getattr(instance, self.field.json_field_name) + if isinstance(json_value, dict): + if self.field.attname in json_value or not self.field.has_default(): + value = json_value.get(self.field.attname, None) + if hasattr(self.field, 'from_json'): + value = self.field.from_json(value) + return value + else: + default = self.field.get_default() + if hasattr(self.field, 'to_json'): + json_value[self.field.attname] = self.field.to_json(default) + else: + json_value[self.field.attname] = default + return default + return None + + def __set__(self, instance, value): + json_value = getattr(instance, self.field.json_field_name) + if json_value: + assert isinstance(json_value, dict) + else: + json_value = {} + + if hasattr(self.field, 'to_json'): + value = self.field.to_json(value) + + if not value and self.field.blank and not self.field.null: + try: + del json_value[self.field.attname] + except KeyError: + pass + else: + json_value[self.field.attname] = value + + setattr(instance, self.field.json_field_name, json_value) + + +class JSONFieldMixin(object): + """ + Override django.db.model.fields.Field.contribute_to_class + to make a field always private, and register custom access descriptor + """ + + def __init__(self, *args, **kwargs): + self.json_field_name = kwargs.pop('json_field_name', 'metadata') + super(JSONFieldMixin, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name, private_only=False): + self.set_attributes_from_name(name) + self.model = cls + self.concrete = False + self.column = self.json_field_name + cls._meta.add_field(self, private=True) + + if not getattr(cls, self.attname, None): + descriptor = JSONFieldDescriptor(self) + setattr(cls, self.attname, descriptor) + + if self.choices is not None: + setattr(cls, 'get_%s_display' % self.name, + partialmethod(cls._get_FIELD_display, field=self)) + + def get_lookup(self, lookup_name): + # Always return None, to make get_transform been called + return None + + def get_transform(self, name): + class TransformFactoryWrapper: + def __init__(self, json_field, transform, original_lookup): + self.json_field = json_field + self.transform = transform + self.original_lookup = original_lookup + + def __call__(self, lhs, **kwargs): + lhs = copy.copy(lhs) + lhs.target = self.json_field + lhs.output_field = self.json_field + transform = self.transform(lhs, **kwargs) + transform._original_get_lookup = transform.get_lookup + transform.get_lookup = lambda name: transform._original_get_lookup(self.original_lookup) + return transform + + json_field = self.model._meta.get_field(self.json_field_name) + transform = json_field.get_transform(self.name) + if transform is None: + raise FieldError( + "JSONField '%s' has no support for key '%s' %s lookup" % + (self.json_field_name, self.name, name) + ) + + return TransformFactoryWrapper(json_field, transform, name) + + +class BooleanField(JSONFieldMixin, fields.BooleanField): + def __init__(self, *args, **kwargs): + super(BooleanField, self).__init__(*args, **kwargs) + if django.VERSION < (2, ): + self.blank = False + + +class CharField(JSONFieldMixin, fields.CharField): + pass + + +class DateField(JSONFieldMixin, fields.DateField): + def to_json(self, value): + if value: + assert isinstance(value, (datetime, date)) + return value.strftime('%Y-%m-%d') + + def from_json(self, value): + if value is not None: + return dateparse.parse_date(value) + + +class DateTimeField(JSONFieldMixin, fields.DateTimeField): + def to_json(self, value): + if value: + if not timezone.is_aware(value): + value = timezone.make_aware(value) + return value.isoformat() + + def from_json(self, value): + if value: + return dateparse.parse_datetime(value) + + +class DecimalField(JSONFieldMixin, fields.DecimalField): + pass + + +class EmailField(JSONFieldMixin, fields.EmailField): + pass + + +class FloatField(JSONFieldMixin, fields.FloatField): + pass + + +class IntegerField(JSONFieldMixin, fields.IntegerField): + pass + + +class IPAddressField(JSONFieldMixin, fields.IPAddressField): + pass + + +class GenericIPAddressField(JSONFieldMixin, fields.GenericIPAddressField): + pass + + +class NullBooleanField(JSONFieldMixin, fields.NullBooleanField): + pass + + +class TextField(JSONFieldMixin, fields.TextField): + pass + + +class TimeField(JSONFieldMixin, fields.TimeField): + def to_json(self, value): + if value: + if not timezone.is_aware(value): + value = timezone.make_aware(value) + return value.isoformat() + + def from_json(self, value): + if value: + return dateparse.parse_time(value) + + +class URLField(JSONFieldMixin, fields.URLField): + pass + + +class ArrayField(JSONFieldMixin, JSONField): + pass diff --git a/catalog/common/models.py b/catalog/common/models.py new file mode 100644 index 00000000..586e7d95 --- /dev/null +++ b/catalog/common/models.py @@ -0,0 +1,268 @@ +from polymorphic.models import PolymorphicModel +from django.db import models +from catalog.common import jsondata +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from django.core.files.uploadedfile import SimpleUploadedFile +from django.contrib.contenttypes.models import ContentType +import uuid +from .utils import DEFAULT_ITEM_COVER, item_cover_path +# from django.conf import settings + + +class IdType(models.TextChoices): + WikiData = 'wikidata', _('维基数据') + ISBN10 = 'isbn10', _('ISBN10') + ISBN = 'isbn', _('ISBN') # ISBN 13 + ASIN = 'asin', _('ASIN') + ISSN = 'issn', _('ISSN') + CUBN = 'cubn', _('统一书号') + ISRC = 'isrc', _('ISRC') # only for songs + GTIN = 'gtin', _('GTIN UPC EAN码') # ISBN is separate + Feed = 'feed', _('Feed URL') + IMDB = 'imdb', _('IMDb') + TMDB_TV = 'tmdb_tv', _('TMDB剧集') + TMDB_TVSeason = 'tmdb_tvseason', _('TMDB剧集') + TMDB_TVEpisode = 'tmdb_tvepisode', _('TMDB剧集') + TMDB_Movie = 'tmdb_movie', _('TMDB电影') + Goodreads = 'goodreads', _('Goodreads') + Goodreads_Work = 'goodreads_work', _('Goodreads著作') + GoogleBooks = 'googlebooks', _('谷歌图书') + DoubanBook = 'doubanbook', _('豆瓣读书') + DoubanBook_Work = 'doubanbook_work', _('豆瓣读书著作') + DoubanMovie = 'doubanmovie', _('豆瓣电影') + DoubanMusic = 'doubanmusic', _('豆瓣音乐') + DoubanGame = 'doubangame', _('豆瓣游戏') + DoubanDrama = 'doubandrama', _('豆瓣舞台剧') + Bandcamp = 'bandcamp', _('Bandcamp') + Spotify_Album = 'spotify_album', _('Spotify专辑') + Spotify_Show = 'spotify_show', _('Spotify播客') + Discogs_Release = 'discogs_release', ('Discogs Release') + Discogs_Master = 'discogs_master', ('Discogs Master') + MusicBrainz = 'musicbrainz', ('MusicBrainz ID') + DoubanBook_Author = 'doubanbook_author', _('豆瓣读书作者') + DoubanCelebrity = 'doubanmovie_celebrity', _('豆瓣电影影人') + Goodreads_Author = 'goodreads_author', _('Goodreads作者') + Spotify_Artist = 'spotify_artist', _('Spotify艺术家') + TMDB_Person = 'tmdb_person', _('TMDB影人') + IGDB = 'igdb', _('IGDB游戏') + Steam = 'steam', _('Steam游戏') + Bangumi = 'bangumi', _('Bangumi') + ApplePodcast = 'apple_podcast', _('苹果播客') + + +class ItemType(models.TextChoices): + Book = 'book', _('书') + TV = 'tv', _('剧集') + TVSeason = 'tvseason', _('剧集分季') + TVEpisode = 'tvepisode', _('剧集分集') + Movie = 'movie', _('电影') + Music = 'music', _('音乐') + Game = 'game', _('游戏') + Boardgame = 'boardgame', _('桌游') + Podcast = 'podcast', _('播客') + FanFic = 'fanfic', _('网文') + Performance = 'performance', _('演出') + Exhibition = 'exhibition', _('展览') + + +class SubItemType(models.TextChoices): + Season = 'season', _('剧集分季') + Episode = 'episode', _('剧集分集') + Version = 'version', _('版本') + +# class CreditType(models.TextChoices): +# Author = 'author', _('作者') +# Translater = 'translater', _('译者') +# Producer = 'producer', _('出品人') +# Director = 'director', _('电影') +# Actor = 'actor', _('演员') +# Playwright = 'playwright', _('播客') +# VoiceActor = 'voiceactor', _('配音') +# Host = 'host', _('主持人') +# Developer = 'developer', _('开发者') +# Publisher = 'publisher', _('出版方') + + +class PrimaryLookupIdDescriptor(object): # TODO make it mixin of Field + def __init__(self, id_type): + self.id_type = id_type + + def __get__(self, instance, cls=None): + if instance is None: + return self + if self.id_type != instance.primary_lookup_id_type: + return None + return instance.primary_lookup_id_value + + def __set__(self, instance, id_value): + if id_value: + instance.primary_lookup_id_type = self.id_type + instance.primary_lookup_id_value = id_value + else: + instance.primary_lookup_id_type = None + instance.primary_lookup_id_value = None + + +class LookupIdDescriptor(object): # TODO make it mixin of Field + def __init__(self, id_type): + self.id_type = id_type + + def __get__(self, instance, cls=None): + if instance is None: + return self + return instance.get_lookup_id(self.id_type) + + def __set__(self, instance, value): + instance.set_lookup_id(self.id_type, value) + + +# class ItemId(models.Model): +# item = models.ForeignKey('Item', models.CASCADE) +# id_type = models.CharField(_("源网站"), blank=False, choices=IdType.choices, max_length=50) +# id_value = models.CharField(_("源网站ID"), blank=False, max_length=1000) + + +# class ItemCredit(models.Model): +# item = models.ForeignKey('Item', models.CASCADE) +# credit_type = models.CharField(_("类型"), choices=CreditType.choices, blank=False, max_length=50) +# name = models.CharField(_("名字"), blank=False, max_length=1000) + + +# def check_source_id(sid): +# if not sid: +# return True +# s = sid.split(':') +# if len(s) < 2: +# return False +# return sid[0] in IdType.values() + + +class Item(PolymorphicModel): + uid = models.UUIDField(default=uuid.uuid4, editable=False) + # item_type = models.CharField(_("类型"), choices=ItemType.choices, blank=False, max_length=50) + title = models.CharField(_("title in primary language"), max_length=1000, default="") + # title_ml = models.JSONField(_("title in different languages {['lang':'zh-cn', 'text':'', primary:True], ...}"), null=True, blank=True, default=list) + brief = models.TextField(_("简介"), blank=True, default="") + # brief_ml = models.JSONField(_("brief in different languages {['lang':'zh-cn', 'text':'', primary:True], ...}"), null=True, blank=True, default=list) + genres = models.JSONField(_("分类"), null=True, blank=True, default=list) + primary_lookup_id_type = models.CharField(_("isbn/cubn/imdb"), blank=False, null=True, max_length=50) + primary_lookup_id_value = models.CharField(_("1234/tt789"), blank=False, null=True, max_length=1000) + metadata = models.JSONField(_("其他信息"), blank=True, null=True, default=dict) + cover = models.ImageField(upload_to=item_cover_path, default=DEFAULT_ITEM_COVER, blank=True) + created_time = models.DateTimeField(auto_now_add=True) + edited_time = models.DateTimeField(auto_now=True) + # parent_item = models.ForeignKey('Item', null=True, on_delete=models.SET_NULL, related_name='child_items') + # identical_item = models.ForeignKey('Item', null=True, on_delete=models.SET_NULL, related_name='identical_items') + # def get_lookup_id(self, id_type: str) -> str: + # prefix = id_type.strip().lower() + ':' + # return next((x[len(prefix):] for x in self.lookup_ids if x.startswith(prefix)), None) + + class Meta: + unique_together = [['polymorphic_ctype_id', 'primary_lookup_id_type', 'primary_lookup_id_value']] + + def __str__(self): + return f"{self.id}{' ' + self.primary_lookup_id_type + ':' + self.primary_lookup_id_value if self.primary_lookup_id_value else ''} ({self.title})" + + @classmethod + def get_best_lookup_id(cls, lookup_ids): + """ get best available lookup id, ideally commonly used """ + best_id_types = [ + IdType.ISBN, IdType.CUBN, IdType.ASIN, + IdType.GTIN, IdType.ISRC, IdType.MusicBrainz, + IdType.Feed, + IdType.IMDB, IdType.TMDB_TVSeason + ] + for t in best_id_types: + if lookup_ids.get(t): + return t, lookup_ids[t] + return list(lookup_ids.items())[0] + + def update_lookup_ids(self, lookup_ids): + # TODO + # ll = set(lookup_ids) + # ll = list(filter(lambda a, b: b, ll)) + # print(ll) + pass + + METADATA_COPY_LIST = ['title', 'brief'] # list of metadata keys to copy from resource to item + + @classmethod + def copy_metadata(cls, metadata): + return dict((k, v) for k, v in metadata.items() if k in cls.METADATA_COPY_LIST and v is not None) + + def merge_data_from_external_resources(self): + """Subclass may override this""" + lookup_ids = [] + for p in self.external_resources.all(): + lookup_ids.append((p.id_type, p.id_value)) + lookup_ids += p.other_lookup_ids.items() + for k in self.METADATA_COPY_LIST: + if not getattr(self, k) and p.metadata.get(k): + setattr(self, k, p.metadata.get(k)) + if not self.cover and p.cover: + self.cover = p.cover + self.update_lookup_ids(lookup_ids) + + def update_linked_items_from_external_resource(self, resource): + """Subclass should override this""" + pass + + +class ItemLookupId(models.Model): + item = models.ForeignKey(Item, null=True, on_delete=models.SET_NULL, related_name='lookup_ids') + id_type = models.CharField(_("源网站"), blank=True, choices=IdType.choices, max_length=50) + id_value = models.CharField(_("源网站ID"), blank=True, max_length=1000) + raw_url = models.CharField(_("源网站ID"), blank=True, max_length=1000, unique=True) + + class Meta: + unique_together = [['id_type', 'id_value']] + + +class ExternalResource(models.Model): + item = models.ForeignKey(Item, null=True, on_delete=models.SET_NULL, related_name='external_resources') + id_type = models.CharField(_("IdType of the source site"), blank=False, choices=IdType.choices, max_length=50) + id_value = models.CharField(_("Primary Id on the source site"), blank=False, max_length=1000) + url = models.CharField(_("url to the resource"), blank=False, max_length=1000, unique=True) + cover = models.ImageField(upload_to=item_cover_path, default=DEFAULT_ITEM_COVER, blank=True) + other_lookup_ids = models.JSONField(default=dict) + metadata = models.JSONField(default=dict) + scraped_time = models.DateTimeField(null=True) + created_time = models.DateTimeField(auto_now_add=True) + edited_time = models.DateTimeField(auto_now=True) + required_resources = jsondata.ArrayField(null=False, blank=False, default=list) + related_resources = jsondata.ArrayField(null=False, blank=False, default=list) + + class Meta: + unique_together = [['id_type', 'id_value']] + + def __str__(self): + return f"{self.id}{':' + self.id_type + ':' + self.id_value if self.id_value else ''} ({self.url})" + + def update_content(self, resource_content): + self.other_lookup_ids = resource_content.lookup_ids + self.metadata = resource_content.metadata + if resource_content.cover_image and resource_content.cover_image_extention: + self.cover = SimpleUploadedFile('temp.' + resource_content.cover_image_extention, resource_content.cover_image) + self.scraped_time = timezone.now() + self.save() + + @property + def ready(self): + return bool(self.metadata and self.scraped_time) + + def get_all_lookup_ids(self): + d = self.other_lookup_ids.copy() + d[self.id_type] = self.id_value + d = {k: v for k, v in d.items() if bool(v)} + return d + + def get_preferred_model(self): + model = self.metadata.get('preferred_model') + if model: + m = ContentType.objects.filter(app_label='catalog', model=model.lower()).first() + if m: + return m.model_class() + else: + raise ValueError(f'preferred model {model} does not exist') + return None diff --git a/catalog/common/scrapers.py b/catalog/common/scrapers.py new file mode 100644 index 00000000..b97f6d0d --- /dev/null +++ b/catalog/common/scrapers.py @@ -0,0 +1,4 @@ +class ParseError(Exception): + def __init__(self, scraper, field): + msg = f'{type(scraper).__name__}: Error parsing field "{field}" for url {scraper.url}' + super().__init__(msg) diff --git a/catalog/common/sites.py b/catalog/common/sites.py new file mode 100644 index 00000000..d23db01e --- /dev/null +++ b/catalog/common/sites.py @@ -0,0 +1,155 @@ +""" +Site and SiteList + +Site should inherite from AbstractSite +a Site should map to a unique set of url patterns. +a Site may scrape a url and store result in ResourceContent +ResourceContent persists as an ExternalResource which may link to an Item +""" +from typing import * +import re +from .models import ExternalResource +from dataclasses import dataclass, field +import logging + + +_logger = logging.getLogger(__name__) + + +@dataclass +class ResourceContent: + lookup_ids: dict = field(default_factory=dict) + metadata: dict = field(default_factory=dict) + cover_image: bytes = None + cover_image_extention: str = None + + +class AbstractSite: + """ + Abstract class to represent a site + """ + ID_TYPE = None + WIKI_PROPERTY_ID = 'P0undefined0' + DEFAULT_MODEL = None + URL_PATTERNS = [r"\w+://undefined/(\d+)"] + + @classmethod + def validate_url(self, url: str): + u = next(iter([re.match(p, url) for p in self.URL_PATTERNS if re.match(p, url)]), None) + return u is not None + + @classmethod + def id_to_url(self, id_value): + return 'https://undefined/' + id_value + + @classmethod + def url_to_id(self, url: str): + u = next(iter([re.match(p, url) for p in self.URL_PATTERNS if re.match(p, url)]), None) + return u[1] if u else None + + def __str__(self): + return f'<{self.__class__.__name__}: {self.url}>' + + def __init__(self, url=None): + self.id_value = self.url_to_id(url) if url else None + self.url = self.id_to_url(self.id_value) if url else None + self.resource = None + + def get_resource(self): + if not self.resource: + self.resource = ExternalResource.objects.filter(url=self.url).first() + if self.resource is None: + self.resource = ExternalResource(id_type=self.ID_TYPE, id_value=self.id_value, url=self.url) + return self.resource + + def bypass_scrape(self, data_from_link) -> ResourceContent | None: + """subclass may implement this to use data from linked resource and bypass actual scrape""" + return None + + def scrape(self) -> ResourceContent: + """subclass should implement this, return ResourceContent object""" + data = ResourceContent() + return data + + def get_item(self): + p = self.get_resource() + if not p: + raise ValueError(f'resource not available for {self.url}') + model = p.get_preferred_model() + if not model: + model = self.DEFAULT_MODEL + t, v = model.get_best_lookup_id(p.get_all_lookup_ids()) + if t is not None: + p.item = model.objects.filter(primary_lookup_id_type=t, primary_lookup_id_value=v).first() + if p.item is None: + obj = model.copy_metadata(p.metadata) + obj['primary_lookup_id_type'] = t + obj['primary_lookup_id_value'] = v + p.item = model.objects.create(**obj) + return p.item + + @property + def ready(self): + return bool(self.resource and self.resource.ready) + + def get_resource_ready(self, auto_save=True, auto_create=True, auto_link=True, data_from_link=None): + """return a resource scraped, or scrape if not yet""" + if auto_link: + auto_create = True + if auto_create: + auto_save = True + p = self.get_resource() + resource_content = {} + if not self.resource: + return None + if not p.ready: + resource_content = self.bypass_scrape(data_from_link) + if not resource_content: + resource_content = self.scrape() + p.update_content(resource_content) + if not p.ready: + _logger.error(f'unable to get resource {self.url} ready') + return None + if auto_create and p.item is None: + self.get_item() + if auto_save: + p.save() + if p.item: + p.item.merge_data_from_external_resources() + p.item.save() + if auto_link: + for linked_resources in p.required_resources: + linked_site = SiteList.get_site_by_url(linked_resources['url']) + if linked_site: + linked_site.get_resource_ready(auto_link=False) + else: + _logger.error(f'unable to get site for {linked_resources["url"]}') + p.item.update_linked_items_from_external_resource(p) + p.item.save() + return p + + +class SiteList: + registry = {} + + @classmethod + def register(cls, target) -> Callable: + id_type = target.ID_TYPE + if id_type in cls.registry: + raise ValueError(f'Site for {id_type} already exists') + cls.registry[id_type] = target + return target + + @classmethod + def get_site_by_id_type(cls, typ: str): + return cls.registry[typ]() if typ in cls.registry else None + + @classmethod + def get_site_by_url(cls, url: str): + cls = next(filter(lambda p: p.validate_url(url), cls.registry.values()), None) + return cls(url) if cls else None + + @classmethod + def get_id_by_url(cls, url: str): + site = cls.get_site_by_url(url) + return site.url_to_id(url) if site else None diff --git a/catalog/common/utils.py b/catalog/common/utils.py new file mode 100644 index 00000000..39b115a9 --- /dev/null +++ b/catalog/common/utils.py @@ -0,0 +1,14 @@ +import logging +from django.utils import timezone +import uuid + + +_logger = logging.getLogger(__name__) + + +DEFAULT_ITEM_COVER = 'item/default.svg' + + +def item_cover_path(resource, filename): + fn = timezone.now().strftime('%Y/%m/%d/') + str(uuid.uuid4()) + '.' + filename.split('.')[-1] + return 'items/' + resource.id_type + '/' + fn diff --git a/catalog/game/models.py b/catalog/game/models.py new file mode 100644 index 00000000..08c07dc3 --- /dev/null +++ b/catalog/game/models.py @@ -0,0 +1,8 @@ +from catalog.common import * + + +class Game(Item): + igdb = PrimaryLookupIdDescriptor(IdType.IGDB) + steam = PrimaryLookupIdDescriptor(IdType.Steam) + douban_game = PrimaryLookupIdDescriptor(IdType.DoubanGame) + platforms = jsondata.ArrayField(default=list) diff --git a/catalog/game/tests.py b/catalog/game/tests.py new file mode 100644 index 00000000..cf7437d6 --- /dev/null +++ b/catalog/game/tests.py @@ -0,0 +1,117 @@ +from django.test import TestCase +from catalog.common import * +from catalog.models import * + + +class IGDBTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.IGDB + t_id_value = 'portal-2' + t_url = 'https://www.igdb.com/games/portal-2' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.igdb.com/games/portal-2' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], 'Portal 2') + self.assertIsInstance(site.resource.item, Game) + self.assertEqual(site.resource.item.steam, '620') + + @use_local_response + def test_scrape_non_steam(self): + t_url = 'https://www.igdb.com/games/the-legend-of-zelda-breath-of-the-wild' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], 'The Legend of Zelda: Breath of the Wild') + self.assertIsInstance(site.resource.item, Game) + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IGDB) + self.assertEqual(site.resource.item.primary_lookup_id_value, 'the-legend-of-zelda-breath-of-the-wild') + + +class SteamTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.Steam + t_id_value = '620' + t_url = 'https://store.steampowered.com/app/620/Portal_2/' + t_url2 = 'https://store.steampowered.com/app/620' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url2) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + t_url = 'https://store.steampowered.com/app/620/Portal_2/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], 'Portal 2') + self.assertEqual(site.resource.metadata['brief'], '“终身测试计划”现已升级，您可以为您自己或您的好友设计合作谜题！') + self.assertIsInstance(site.resource.item, Game) + self.assertEqual(site.resource.item.steam, '620') + + +class DoubanGameTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.DoubanGame + t_id_value = '10734307' + t_url = 'https://www.douban.com/game/10734307/' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.douban.com/game/10734307/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], '传送门2 Portal 2') + self.assertIsInstance(site.resource.item, Game) + self.assertEqual(site.resource.item.douban_game, '10734307') + + +class BangumiGameTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.Bangumi + t_id_value = '15912' + t_url = 'https://bgm.tv/subject/15912' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + # TODO + pass + + +class MultiGameSitesTestCase(TestCase): + @use_local_response + def test_games(self): + url1 = 'https://www.igdb.com/games/portal-2' + url2 = 'https://store.steampowered.com/app/620/Portal_2/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + self.assertEqual(p1.item.id, p2.item.id) diff --git a/catalog/management/commands/cat.py b/catalog/management/commands/cat.py new file mode 100644 index 00000000..1f3ed236 --- /dev/null +++ b/catalog/management/commands/cat.py @@ -0,0 +1,22 @@ +from django.core.management.base import BaseCommand +import pprint +from catalog.common import SiteList +from catalog.sites import * + + +class Command(BaseCommand): + help = 'Scrape a catalog item from external resource (but not save it)' + + def add_arguments(self, parser): + parser.add_argument('url', type=str, help='URL to scrape') + + def handle(self, *args, **options): + url = str(options['url']) + site = SiteList.get_site_by_url(url) + if site is None: + self.stdout.write(self.style.ERROR(f'Unknown site for {url}')) + return + self.stdout.write(f'Fetching from {site}') + resource = site.get_resource_ready(auto_link=False, auto_save=False) + self.stdout.write(self.style.SUCCESS(f'Done.')) + pprint.pp(resource.metadata) diff --git a/catalog/models.py b/catalog/models.py new file mode 100644 index 00000000..68af2b67 --- /dev/null +++ b/catalog/models.py @@ -0,0 +1,25 @@ +from .book.models import Edition, Work, Series +from .movie.models import Movie +from .tv.models import TVShow, TVSeason, TVEpisode +from .music.models import Album +from .game.models import Game +from .podcast.models import Podcast +from .performance.models import Performance + + +# class Exhibition(Item): + +# class Meta: +# proxy = True + + +# class Fanfic(Item): + +# class Meta: +# proxy = True + + +# class Boardgame(Item): + +# class Meta: +# proxy = True diff --git a/catalog/movie/models.py b/catalog/movie/models.py new file mode 100644 index 00000000..9679a18c --- /dev/null +++ b/catalog/movie/models.py @@ -0,0 +1,8 @@ +from catalog.common import * + + +class Movie(Item): + imdb = PrimaryLookupIdDescriptor(IdType.IMDB) + tmdb_movie = PrimaryLookupIdDescriptor(IdType.TMDB_Movie) + douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie) + duration = jsondata.IntegerField(blank=True, default=None) diff --git a/catalog/movie/tests.py b/catalog/movie/tests.py new file mode 100644 index 00000000..b3deacce --- /dev/null +++ b/catalog/movie/tests.py @@ -0,0 +1,90 @@ +from django.test import TestCase +from catalog.common import * + + +class DoubanMovieTestCase(TestCase): + def test_parse(self): + t_id = '3541415' + t_url = 'https://movie.douban.com/subject/3541415/' + p1 = SiteList.get_site_by_id_type(IdType.DoubanMovie) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://movie.douban.com/subject/3541415/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, '3541415') + site.get_resource_ready() + self.assertEqual(site.resource.metadata['title'], '盗梦空间') + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) + self.assertEqual(site.resource.item.__class__.__name__, 'Movie') + self.assertEqual(site.resource.item.imdb, 'tt1375666') + + +class TMDBMovieTestCase(TestCase): + def test_parse(self): + t_id = '293767' + t_url = 'https://www.themoviedb.org/movie/293767-billy-lynn-s-long-halftime-walk' + t_url2 = 'https://www.themoviedb.org/movie/293767' + p1 = SiteList.get_site_by_id_type(IdType.TMDB_Movie) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + self.assertEqual(p1.validate_url(t_url2), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url2) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.themoviedb.org/movie/293767' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, '293767') + site.get_resource_ready() + self.assertEqual(site.resource.metadata['title'], '比利·林恩的中场战事') + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) + self.assertEqual(site.resource.item.__class__.__name__, 'Movie') + self.assertEqual(site.resource.item.imdb, 'tt2513074') + + +class IMDBMovieTestCase(TestCase): + def test_parse(self): + t_id = 'tt1375666' + t_url = 'https://www.imdb.com/title/tt1375666/' + t_url2 = 'https://www.imdb.com/title/tt1375666/' + p1 = SiteList.get_site_by_id_type(IdType.IMDB) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + self.assertEqual(p1.validate_url(t_url2), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url2) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.imdb.com/title/tt1375666/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, 'tt1375666') + site.get_resource_ready() + self.assertEqual(site.resource.metadata['title'], '盗梦空间') + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) + self.assertEqual(site.resource.item.imdb, 'tt1375666') + + +class MultiMovieSitesTestCase(TestCase): + @use_local_response + def test_movies(self): + url1 = 'https://www.themoviedb.org/movie/27205-inception' + url2 = 'https://movie.douban.com/subject/3541415/' + url3 = 'https://www.imdb.com/title/tt1375666/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p1.item.id, p2.item.id) + self.assertEqual(p2.item.id, p3.item.id) diff --git a/catalog/music/models.py b/catalog/music/models.py new file mode 100644 index 00000000..11d008ba --- /dev/null +++ b/catalog/music/models.py @@ -0,0 +1,10 @@ +from catalog.common import * + + +class Album(Item): + barcode = PrimaryLookupIdDescriptor(IdType.GTIN) + douban_music = PrimaryLookupIdDescriptor(IdType.DoubanMusic) + spotify_album = PrimaryLookupIdDescriptor(IdType.Spotify_Album) + + class Meta: + proxy = True diff --git a/catalog/music/tests.py b/catalog/music/tests.py new file mode 100644 index 00000000..d035382d --- /dev/null +++ b/catalog/music/tests.py @@ -0,0 +1,61 @@ +from django.test import TestCase +from catalog.common import * +from catalog.models import * + + +class SpotifyTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.Spotify_Album + t_id_value = '65KwtzkJXw7oT819NFWmEP' + t_url = 'https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + t_url = 'https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], 'The Race For Space') + self.assertIsInstance(site.resource.item, Album) + self.assertEqual(site.resource.item.barcode, '3610159662676') + + +class DoubanMusicTestCase(TestCase): + def test_parse(self): + t_id_type = IdType.DoubanMusic + t_id_value = '33551231' + t_url = 'https://music.douban.com/subject/33551231/' + site = SiteList.get_site_by_id_type(t_id_type) + self.assertIsNotNone(site) + self.assertEqual(site.validate_url(t_url), True) + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.url, t_url) + self.assertEqual(site.id_value, t_id_value) + + @use_local_response + def test_scrape(self): + t_url = 'https://music.douban.com/subject/33551231/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], 'The Race For Space') + self.assertIsInstance(site.resource.item, Album) + self.assertEqual(site.resource.item.barcode, '3610159662676') + + +class MultiMusicSitesTestCase(TestCase): + @use_local_response + def test_albums(self): + url1 = 'https://music.douban.com/subject/33551231/' + url2 = 'https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + self.assertEqual(p1.item.id, p2.item.id) diff --git a/catalog/performance/models.py b/catalog/performance/models.py new file mode 100644 index 00000000..68760eb6 --- /dev/null +++ b/catalog/performance/models.py @@ -0,0 +1,13 @@ +from catalog.common import * +from django.utils.translation import gettext_lazy as _ + + +class Performance(Item): + douban_drama = LookupIdDescriptor(IdType.DoubanDrama) + versions = jsondata.ArrayField(_('版本'), null=False, blank=False, default=list) + directors = jsondata.ArrayField(_('导演'), null=False, blank=False, default=list) + playwrights = jsondata.ArrayField(_('编剧'), null=False, blank=False, default=list) + actors = jsondata.ArrayField(_('主演'), null=False, blank=False, default=list) + + class Meta: + proxy = True diff --git a/catalog/performance/tests.py b/catalog/performance/tests.py new file mode 100644 index 00000000..9d3302ea --- /dev/null +++ b/catalog/performance/tests.py @@ -0,0 +1,37 @@ +from django.test import TestCase +from catalog.common import * + + +class DoubanDramaTestCase(TestCase): + def setUp(self): + pass + + def test_parse(self): + t_id = '24849279' + t_url = 'https://www.douban.com/location/drama/24849279/' + p1 = SiteList.get_site_by_id_type(IdType.DoubanDrama) + self.assertIsNotNone(p1) + p1 = SiteList.get_site_by_url(t_url) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + self.assertEqual(p1.id_to_url(t_id), t_url) + self.assertEqual(p1.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.douban.com/location/drama/24849279/' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + resource = site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(resource.metadata['title'], '红花侠') + item = site.get_item() + self.assertEqual(item.title, '红花侠') + + # self.assertEqual(i.other_titles, ['スカーレットピンパーネル', 'THE SCARLET PIMPERNEL']) + # self.assertEqual(len(i.brief), 545) + # self.assertEqual(i.genres, ['音乐剧']) + # self.assertEqual(i.versions, ['08星组公演版', '10年月組公演版', '17年星組公演版', 'ュージカル（2017年）版']) + # self.assertEqual(i.directors, ['小池修一郎', '小池 修一郎', '石丸さち子']) + # self.assertEqual(i.playwrights, ['小池修一郎', 'Baroness Orczy（原作）', '小池 修一郎']) + # self.assertEqual(i.actors, ['安蘭けい', '柚希礼音', '遠野あすか', '霧矢大夢', '龍真咲']) diff --git a/catalog/podcast/models.py b/catalog/podcast/models.py new file mode 100644 index 00000000..1df67b49 --- /dev/null +++ b/catalog/podcast/models.py @@ -0,0 +1,13 @@ +from catalog.common import * + + +class Podcast(Item): + feed_url = PrimaryLookupIdDescriptor(IdType.Feed) + apple_podcast = PrimaryLookupIdDescriptor(IdType.ApplePodcast) + # ximalaya = LookupIdDescriptor(IdType.Ximalaya) + # xiaoyuzhou = LookupIdDescriptor(IdType.Xiaoyuzhou) + hosts = jsondata.ArrayField(default=list) + + +# class PodcastEpisode(Item): +# pass diff --git a/catalog/podcast/tests.py b/catalog/podcast/tests.py new file mode 100644 index 00000000..0e70b3e1 --- /dev/null +++ b/catalog/podcast/tests.py @@ -0,0 +1,30 @@ +from django.test import TestCase +from catalog.podcast.models import * +from catalog.common import * + + +class ApplePodcastTestCase(TestCase): + def setUp(self): + pass + + def test_parse(self): + t_id = '657765158' + t_url = 'https://podcasts.apple.com/us/podcast/%E5%A4%A7%E5%86%85%E5%AF%86%E8%B0%88/id657765158' + t_url2 = 'https://podcasts.apple.com/us/podcast/id657765158' + p1 = SiteList.get_site_by_id_type(IdType.ApplePodcast) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url2) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://podcasts.apple.com/gb/podcast/the-new-yorker-radio-hour/id1050430296' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, '1050430296') + site.get_resource_ready() + self.assertEqual(site.resource.metadata['title'], 'The New Yorker Radio Hour') + # self.assertEqual(site.resource.metadata['feed_url'], 'http://feeds.wnyc.org/newyorkerradiohour') + self.assertEqual(site.resource.metadata['feed_url'], 'http://feeds.feedburner.com/newyorkerradiohour') diff --git a/catalog/sites/__init__.py b/catalog/sites/__init__.py new file mode 100644 index 00000000..5f103943 --- /dev/null +++ b/catalog/sites/__init__.py @@ -0,0 +1,15 @@ +from ..common.sites import SiteList +from .apple_podcast import ApplePodcast +from .douban_book import DoubanBook +from .douban_movie import DoubanMovie +from .douban_music import DoubanMusic +from .douban_game import DoubanGame +from .douban_drama import DoubanDrama +from .goodreads import Goodreads +from .google_books import GoogleBooks +from .tmdb import TMDB_Movie +from .imdb import IMDB +from .spotify import Spotify +from .igdb import IGDB +from .steam import Steam +from .bangumi import Bangumi diff --git a/catalog/sites/apple_podcast.py b/catalog/sites/apple_podcast.py new file mode 100644 index 00000000..ae2b3b54 --- /dev/null +++ b/catalog/sites/apple_podcast.py @@ -0,0 +1,40 @@ +from catalog.common import * +from catalog.models import * +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class ApplePodcast(AbstractSite): + ID_TYPE = IdType.ApplePodcast + URL_PATTERNS = [r"https://[^.]+.apple.com/\w+/podcast/*[^/?]*/id(\d+)"] + WIKI_PROPERTY_ID = 'P5842' + DEFAULT_MODEL = Podcast + + @classmethod + def id_to_url(self, id_value): + return "https://podcasts.apple.com/us/podcast/id" + id_value + + def scrape(self): + api_url = f'https://itunes.apple.com/lookup?id={self.id_value}' + dl = BasicDownloader(api_url) + resp = dl.download() + r = resp.json()['results'][0] + pd = ResourceContent(metadata={ + 'title': r['trackName'], + 'feed_url': r['feedUrl'], + 'hosts': [r['artistName']], + 'genres': r['genres'], + 'cover_image_url': r['artworkUrl600'], + }) + pd.lookup_ids[IdType.Feed] = pd.metadata.get('feed_url') + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd diff --git a/catalog/sites/bangumi.py b/catalog/sites/bangumi.py new file mode 100644 index 00000000..21875536 --- /dev/null +++ b/catalog/sites/bangumi.py @@ -0,0 +1,24 @@ +from catalog.common import * +from catalog.models import * +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class Bangumi(AbstractSite): + ID_TYPE = IdType.Bangumi + URL_PATTERNS = [ + r"https://bgm\.tv/subject/(\d+)", + ] + WIKI_PROPERTY_ID = '' + DEFAULT_MODEL = None + + @classmethod + def id_to_url(self, id_value): + return f"https://bgm.tv/subject/{id_value}" + + def scrape(self): + # TODO rewrite with bangumi api https://bangumi.github.io/api/ + pass diff --git a/catalog/sites/douban.py b/catalog/sites/douban.py new file mode 100644 index 00000000..b26d42fc --- /dev/null +++ b/catalog/sites/douban.py @@ -0,0 +1,28 @@ +import re +from catalog.common import * + + +RE_NUMBERS = re.compile(r"\d+\d*") +RE_WHITESPACES = re.compile(r"\s+") + + +class DoubanDownloader(ProxiedDownloader): + def validate_response(self, response): + if response is None: + return RESPONSE_NETWORK_ERROR + elif response.status_code == 204: + return RESPONSE_CENSORSHIP + elif response.status_code == 200: + content = response.content.decode('utf-8') + if content.find('关于豆瓣') == -1: + # if content.find('你的 IP 发出') == -1: + # error = error + 'Content not authentic' # response is garbage + # else: + # error = error + 'IP banned' + return RESPONSE_NETWORK_ERROR + elif content.find('页面不存在') != -1 or content.find('呃... 你想访问的条目豆瓣不收录。') != -1: # re.search('不存在[^<]+', content, re.MULTILINE): + return RESPONSE_CENSORSHIP + else: + return RESPONSE_OK + else: + return RESPONSE_INVALID_CONTENT diff --git a/catalog/sites/douban_book.py b/catalog/sites/douban_book.py new file mode 100644 index 00000000..021857c5 --- /dev/null +++ b/catalog/sites/douban_book.py @@ -0,0 +1,180 @@ +from catalog.common import * +from .douban import * +from catalog.book.models import * +from catalog.book.utils import * +import logging + + +_logger = logging.getLogger(__name__) + + +class ScraperMixin: + def set_field(self, field, value=None): + self.data[field] = value + + def parse_str(self, query): + elem = self.html.xpath(query) + return elem[0].strip() if elem else None + + def parse_field(self, field, query, error_when_missing=False): + elem = self.html.xpath(query) + if elem: + self.data[field] = elem[0].strip() + elif error_when_missing: + raise ParseError(self, field) + else: + self.data[field] = None + return elem + + +@SiteList.register +class DoubanBook(AbstractSite, ScraperMixin): + ID_TYPE = IdType.DoubanBook + URL_PATTERNS = [r"\w+://book\.douban\.com/subject/(\d+)/{0,1}", r"\w+://m.douban.com/book/subject/(\d+)/{0,1}"] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = Edition + + @classmethod + def id_to_url(self, id_value): + return "https://book.douban.com/subject/" + id_value + "/" + + def scrape(self): + self.data = {} + self.html = DoubanDownloader(self.url).download().html() + self.parse_field('title', "/html/body//h1/span/text()") + self.parse_field('isbn', "//div[@id='info']//span[text()='ISBN:']/following::text()") + # TODO does douban store ASIN as ISBN, need more cleanup if so + if not self.data['title']: + if self.data['isbn']: + self.data['title'] = 'isbn: ' + isbn + else: + raise ParseError(self, 'title') + + self.parse_field('cover_image_url', "//*[@id='mainpic']/a/img/@src") + self.parse_field('brief', "//h2/span[text()='内容简介']/../following-sibling::div[1]//div[@class='intro'][not(ancestor::span[@class='short'])]/p/text()") + self.parse_field('series', "//div[@id='info']//span[text()='丛书:']/following-sibling::a[1]/text()") + self.parse_field('producer', "//div[@id='info']//span[text()='出品方:']/following-sibling::a[1]/text()") + self.parse_field('cubn', "//div[@id='info']//span[text()='统一书号:']/following::text()") + self.parse_field('subtitle', "//div[@id='info']//span[text()='副标题:']/following::text()") + self.parse_field('orig_title', "//div[@id='info']//span[text()='原作名:']/following::text()") + self.parse_field('language', "//div[@id='info']//span[text()='语言:']/following::text()") + self.parse_field('pub_house', "//div[@id='info']//span[text()='出版社:']/following::text()") + self.parse_field('pub_date', "//div[@id='info']//span[text()='出版年:']/following::text()") + year_month_day = RE_NUMBERS.findall(self.data['pub_date']) if self.data['pub_date'] else [] + if len(year_month_day) in (2, 3): + pub_year = int(year_month_day[0]) + pub_month = int(year_month_day[1]) + elif len(year_month_day) == 1: + pub_year = int(year_month_day[0]) + pub_month = None + else: + pub_year = None + pub_month = None + if pub_year and pub_month and pub_year < pub_month: + pub_year, pub_month = pub_month, pub_year + pub_year = None if pub_year is not None and pub_year not in range( + 0, 3000) else pub_year + pub_month = None if pub_month is not None and pub_month not in range( + 1, 12) else pub_month + + self.parse_field('binding', "//div[@id='info']//span[text()='装帧:']/following::text()") + self.parse_field('price', "//div[@id='info']//span[text()='定价:']/following::text()") + self.parse_field('pages', "//div[@id='info']//span[text()='页数:']/following::text()") + if self.data['pages'] is not None: + self.data['pages'] = int(RE_NUMBERS.findall(self.data['pages'])[0]) if RE_NUMBERS.findall(self.data['pages']) else None + if self.data['pages'] and (self.data['pages'] > 999999 or self.data['pages'] < 1): + self.data['pages'] = None + + contents = None + try: + contents_elem = self.html.xpath( + "//h2/span[text()='目录']/../following-sibling::div[1]")[0] + # if next the id of next sibling contains `dir`, that would be the full contents + if "dir" in contents_elem.getnext().xpath("@id")[0]: + contents_elem = contents_elem.getnext() + contents = '\n'.join(p.strip() for p in contents_elem.xpath("text()")[:-2]) if len(contents_elem) else None + else: + contents = '\n'.join(p.strip() for p in contents_elem.xpath("text()")) if len(contents_elem) else None + except Exception: + pass + self.data['contents'] = contents + + # there are two html formats for authors and translators + authors_elem = self.html.xpath("""//div[@id='info']//span[text()='作者:']/following-sibling::br[1]/ + preceding-sibling::a[preceding-sibling::span[text()='作者:']]/text()""") + if not authors_elem: + authors_elem = self.html.xpath( + """//div[@id='info']//span[text()=' 作者']/following-sibling::a/text()""") + if authors_elem: + authors = [] + for author in authors_elem: + authors.append(RE_WHITESPACES.sub(' ', author.strip())[:200]) + else: + authors = None + self.data['authors'] = authors + + translators_elem = self.html.xpath("""//div[@id='info']//span[text()='译者:']/following-sibling::br[1]/ + preceding-sibling::a[preceding-sibling::span[text()='译者:']]/text()""") + if not translators_elem: + translators_elem = self.html.xpath( + """//div[@id='info']//span[text()=' 译者']/following-sibling::a/text()""") + if translators_elem: + translators = [] + for translator in translators_elem: + translators.append(RE_WHITESPACES.sub(' ', translator.strip())) + else: + translators = None + self.data['translators'] = translators + + work_link = self.parse_str('//h2/span[text()="这本书的其他版本"]/following-sibling::span[@class="pl"]/a/@href') + if work_link: + r = re.match(r'\w+://book.douban.com/works/(\d+)', work_link) + self.data['required_resources'] = [{ + 'model': 'Work', + 'id_type': IdType.DoubanBook_Work, + 'id_value': r[1] if r else None, + 'title': self.data['title'], + 'url': work_link, + }] + pd = ResourceContent(metadata=self.data) + pd.lookup_ids[IdType.ISBN] = self.data.get('isbn') + pd.lookup_ids[IdType.CUBN] = self.data.get('cubn') + if self.data["cover_image_url"]: + imgdl = BasicImageDownloader(self.data["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {self.data["cover_image_url"]}') + return pd + + +@SiteList.register +class DoubanBook_Work(AbstractSite): + ID_TYPE = IdType.DoubanBook_Work + URL_PATTERNS = [r"\w+://book\.douban\.com/works/(\d+)"] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = Work + + @classmethod + def id_to_url(self, id_value): + return "https://book.douban.com/works/" + id_value + "/" + + def bypass_scrape(self, data_from_link): + if not data_from_link: + return None + pd = ResourceContent(metadata={ + 'title': data_from_link['title'], + }) + return pd + + def scrape(self): + content = DoubanDownloader(self.url).download().html() + title_elem = content.xpath("//h1/text()") + title = title_elem[0].split('全部版本(')[0].strip() if title_elem else None + if not title: + raise ParseError(self, 'title') + pd = ResourceContent(metadata={ + 'title': title, + }) + return pd diff --git a/catalog/sites/douban_drama.py b/catalog/sites/douban_drama.py new file mode 100644 index 00000000..4a0c27b7 --- /dev/null +++ b/catalog/sites/douban_drama.py @@ -0,0 +1,58 @@ +from catalog.common import * +from catalog.models import * +from .douban import DoubanDownloader +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class DoubanDrama(AbstractSite): + ID_TYPE = IdType.DoubanDrama + URL_PATTERNS = [r"\w+://www.douban.com/location/drama/(\d+)/"] + WIKI_PROPERTY_ID = 'P6443' + DEFAULT_MODEL = Performance + + @classmethod + def id_to_url(self, id_value): + return "https://www.douban.com/location/drama/" + id_value + "/" + + def scrape(self): + h = DoubanDownloader(self.url).download().html() + data = {} + + title_elem = h.xpath("/html/body//h1/span/text()") + if title_elem: + data["title"] = title_elem[0].strip() + else: + raise ParseError(self, "title") + + data['other_titles'] = [s.strip() for s in title_elem[1:]] + other_title_elem = h.xpath("//dl//dt[text()='又名：']/following::dd[@itemprop='name']/text()") + if len(other_title_elem) > 0: + data['other_titles'].append(other_title_elem[0].strip()) + + plot_elem = h.xpath("//div[@id='link-report']/text()") + if len(plot_elem) == 0: + plot_elem = h.xpath("//div[@class='abstract']/text()") + data['brief'] = '\n'.join(plot_elem) if len(plot_elem) > 0 else '' + + data['genres'] = [s.strip() for s in h.xpath("//dl//dt[text()='类型：']/following-sibling::dd[@itemprop='genre']/text()")] + data['versions'] = [s.strip() for s in h.xpath("//dl//dt[text()='版本：']/following-sibling::dd[@class='titles']/a//text()")] + data['directors'] = [s.strip() for s in h.xpath("//div[@class='meta']/dl//dt[text()='导演：']/following-sibling::dd/a[@itemprop='director']//text()")] + data['playwrights'] = [s.strip() for s in h.xpath("//div[@class='meta']/dl//dt[text()='编剧：']/following-sibling::dd/a[@itemprop='author']//text()")] + data['actors'] = [s.strip() for s in h.xpath("//div[@class='meta']/dl//dt[text()='主演：']/following-sibling::dd/a[@itemprop='actor']//text()")] + + img_url_elem = h.xpath("//img[@itemprop='image']/@src") + data['cover_image_url'] = img_url_elem[0].strip() if img_url_elem else None + + pd = ResourceContent(metadata=data) + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd diff --git a/catalog/sites/douban_game.py b/catalog/sites/douban_game.py new file mode 100644 index 00000000..4c50a42e --- /dev/null +++ b/catalog/sites/douban_game.py @@ -0,0 +1,76 @@ +from catalog.common import * +from catalog.models import * +from .douban import DoubanDownloader +import dateparser +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class DoubanGame(AbstractSite): + ID_TYPE = IdType.DoubanGame + URL_PATTERNS = [r"\w+://www\.douban\.com/game/(\d+)/{0,1}", r"\w+://m.douban.com/game/subject/(\d+)/{0,1}"] + WIKI_PROPERTY_ID = '' + DEFAULT_MODEL = Game + + @classmethod + def id_to_url(self, id_value): + return "https://www.douban.com/game/" + id_value + "/" + + def scrape(self): + content = DoubanDownloader(self.url).download().html() + + elem = content.xpath("//div[@id='content']/h1/text()") + title = elem[0].strip() if len(elem) else None + if not title: + raise ParseError(self, "title") + + other_title_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()") + other_title = other_title_elem[0].strip().split(' / ') if other_title_elem else None + + developer_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()") + developer = developer_elem[0].strip().split(' / ') if developer_elem else None + + publisher_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()") + publisher = publisher_elem[0].strip().split(' / ') if publisher_elem else None + + platform_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()") + platform = platform_elem if platform_elem else None + + genre_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()") + genre = None + if genre_elem: + genre = [g for g in genre_elem if g != '游戏'] + + date_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()") + release_date = dateparser.parse(date_elem[0].strip()).strftime('%Y-%m-%d') if date_elem else None + + brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()") + brief = '\n'.join(brief_elem) if brief_elem else None + + img_url_elem = content.xpath( + "//div[@class='item-subject-info']/div[@class='pic']//img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + + pd = ResourceContent(metadata={ + 'title': title, + 'other_title': other_title, + 'developer': developer, + 'publisher': publisher, + 'release_date': release_date, + 'genre': genre, + 'platform': platform, + 'brief': brief, + 'cover_image_url': img_url + }) + if pd.metadata["cover_image_url"]: + pd.cover_image, pd.cover_image_extention = BasicImageDownloader.download_image(pd.metadata['cover_image_url'], self.url) + return pd diff --git a/catalog/sites/douban_movie.py b/catalog/sites/douban_movie.py new file mode 100644 index 00000000..d2a971c5 --- /dev/null +++ b/catalog/sites/douban_movie.py @@ -0,0 +1,275 @@ +from catalog.common import * +from .douban import * +from catalog.movie.models import * +from catalog.tv.models import * +import logging +from django.db import models +from django.utils.translation import gettext_lazy as _ +from .tmdb import TMDB_TV, search_tmdb_by_imdb_id + + +_logger = logging.getLogger(__name__) + + +class MovieGenreEnum(models.TextChoices): + DRAMA = 'Drama', _('剧情') + KIDS = 'Kids', _('儿童') + COMEDY = 'Comedy', _('喜剧') + BIOGRAPHY = 'Biography', _('传记') + ACTION = 'Action', _('动作') + HISTORY = 'History', _('历史') + ROMANCE = 'Romance', _('爱情') + WAR = 'War', _('战争') + SCI_FI = 'Sci-Fi', _('科幻') + CRIME = 'Crime', _('犯罪') + ANIMATION = 'Animation', _('动画') + WESTERN = 'Western', _('西部') + MYSTERY = 'Mystery', _('悬疑') + FANTASY = 'Fantasy', _('奇幻') + THRILLER = 'Thriller', _('惊悚') + ADVENTURE = 'Adventure', _('冒险') + HORROR = 'Horror', _('恐怖') + DISASTER = 'Disaster', _('灾难') + DOCUMENTARY = 'Documentary', _('纪录片') + MARTIAL_ARTS = 'Martial-Arts', _('武侠') + SHORT = 'Short', _('短片') + ANCIENT_COSTUM = 'Ancient-Costum', _('古装') + EROTICA = 'Erotica', _('情色') + SPORT = 'Sport', _('运动') + GAY_LESBIAN = 'Gay/Lesbian', _('同性') + OPERA = 'Opera', _('戏曲') + MUSIC = 'Music', _('音乐') + FILM_NOIR = 'Film-Noir', _('黑色电影') + MUSICAL = 'Musical', _('歌舞') + REALITY_TV = 'Reality-TV', _('真人秀') + FAMILY = 'Family', _('家庭') + TALK_SHOW = 'Talk-Show', _('脱口秀') + NEWS = 'News', _('新闻') + SOAP = 'Soap', _('肥皂剧') + TV_MOVIE = 'TV Movie', _('电视电影') + THEATRE = 'Theatre', _('舞台艺术') + OTHER = 'Other', _('其他') + + +# MovieGenreTranslator = ChoicesDictGenerator(MovieGenreEnum) + + +@SiteList.register +class DoubanMovie(AbstractSite): + ID_TYPE = IdType.DoubanMovie + URL_PATTERNS = [r"\w+://movie\.douban\.com/subject/(\d+)/{0,1}", r"\w+://m.douban.com/movie/subject/(\d+)/{0,1}"] + WIKI_PROPERTY_ID = '?' + # no DEFAULT_MODEL as it may be either TV Season and Movie + + @classmethod + def id_to_url(self, id_value): + return "https://movie.douban.com/subject/" + id_value + "/" + + def scrape(self): + content = DoubanDownloader(self.url).download().html() + + try: + raw_title = content.xpath( + "//span[@property='v:itemreviewed']/text()")[0].strip() + except IndexError: + raise ParseError(self, 'title') + + orig_title = content.xpath( + "//img[@rel='v:image']/@alt")[0].strip() + title = raw_title.split(orig_title)[0].strip() + # if has no chinese title + if title == '': + title = orig_title + + if title == orig_title: + orig_title = None + + # there are two html formats for authors and translators + other_title_elem = content.xpath( + "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") + other_title = other_title_elem[0].strip().split( + ' / ') if other_title_elem else None + + imdb_elem = content.xpath( + "//div[@id='info']//span[text()='IMDb链接:']/following-sibling::a[1]/text()") + if not imdb_elem: + imdb_elem = content.xpath( + "//div[@id='info']//span[text()='IMDb:']/following-sibling::text()[1]") + imdb_code = imdb_elem[0].strip() if imdb_elem else None + + director_elem = content.xpath( + "//div[@id='info']//span[text()='导演']/following-sibling::span[1]/a/text()") + director = director_elem if director_elem else None + + playwright_elem = content.xpath( + "//div[@id='info']//span[text()='编剧']/following-sibling::span[1]/a/text()") + playwright = list(map(lambda a: a[:200], playwright_elem)) if playwright_elem else None + + actor_elem = content.xpath( + "//div[@id='info']//span[text()='主演']/following-sibling::span[1]/a/text()") + actor = list(map(lambda a: a[:200], actor_elem)) if actor_elem else None + + # construct genre translator + genre_translator = {} + attrs = [attr for attr in dir(MovieGenreEnum) if '__' not in attr] + for attr in attrs: + genre_translator[getattr(MovieGenreEnum, attr).label] = getattr( + MovieGenreEnum, attr).value + + genre_elem = content.xpath("//span[@property='v:genre']/text()") + if genre_elem: + genre = [] + for g in genre_elem: + g = g.split(' ')[0] + if g == '紀錄片': # likely some original data on douban was corrupted + g = '纪录片' + elif g == '鬼怪': + g = '惊悚' + if g in genre_translator: + genre.append(genre_translator[g]) + elif g in genre_translator.values(): + genre.append(g) + else: + _logger.error(f'unable to map genre {g}') + else: + genre = None + + showtime_elem = content.xpath( + "//span[@property='v:initialReleaseDate']/text()") + if showtime_elem: + showtime = [] + for st in showtime_elem: + parts = st.split('(') + if len(parts) == 1: + time = st.split('(')[0] + region = '' + else: + time = st.split('(')[0] + region = st.split('(')[1][0:-1] + showtime.append({time: region}) + else: + showtime = None + + site_elem = content.xpath( + "//div[@id='info']//span[text()='官方网站:']/following-sibling::a[1]/@href") + site = site_elem[0].strip()[:200] if site_elem else None + if site and not re.match(r'http.+', site): + site = None + + area_elem = content.xpath( + "//div[@id='info']//span[text()='制片国家/地区:']/following-sibling::text()[1]") + if area_elem: + area = [a.strip()[:100] for a in area_elem[0].split('/')] + else: + area = None + + language_elem = content.xpath( + "//div[@id='info']//span[text()='语言:']/following-sibling::text()[1]") + if language_elem: + language = [a.strip() for a in language_elem[0].split(' / ')] + else: + language = None + + year_elem = content.xpath("//span[@class='year']/text()") + year = int(re.search(r'\d+', year_elem[0])[0]) if year_elem and re.search(r'\d+', year_elem[0]) else None + + duration_elem = content.xpath("//span[@property='v:runtime']/text()") + other_duration_elem = content.xpath( + "//span[@property='v:runtime']/following-sibling::text()[1]") + if duration_elem: + duration = duration_elem[0].strip() + if other_duration_elem: + duration += other_duration_elem[0].rstrip() + duration = duration.split('/')[0].strip() + else: + duration = None + + season_elem = content.xpath( + "//*[@id='season']/option[@selected='selected']/text()") + if not season_elem: + season_elem = content.xpath( + "//div[@id='info']//span[text()='季数:']/following-sibling::text()[1]") + season = int(season_elem[0].strip()) if season_elem else None + else: + season = int(season_elem[0].strip()) + + episodes_elem = content.xpath( + "//div[@id='info']//span[text()='集数:']/following-sibling::text()[1]") + episodes = int(episodes_elem[0].strip()) if episodes_elem and episodes_elem[0].strip().isdigit() else None + + single_episode_length_elem = content.xpath( + "//div[@id='info']//span[text()='单集片长:']/following-sibling::text()[1]") + single_episode_length = single_episode_length_elem[0].strip( + )[:100] if single_episode_length_elem else None + + # if has field `episodes` not none then must be series + is_series = True if episodes else False + + brief_elem = content.xpath("//span[@class='all hidden']") + if not brief_elem: + brief_elem = content.xpath("//span[@property='v:summary']") + brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( + './text()')]) if brief_elem else None + + img_url_elem = content.xpath("//img[@rel='v:image']/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + + pd = ResourceContent(metadata={ + 'title': title, + 'orig_title': orig_title, + 'other_title': other_title, + 'imdb_code': imdb_code, + 'director': director, + 'playwright': playwright, + 'actor': actor, + 'genre': genre, + 'showtime': showtime, + 'site': site, + 'area': area, + 'language': language, + 'year': year, + 'duration': duration, + 'season_number': season, + 'episodes': episodes, + 'single_episode_length': single_episode_length, + 'brief': brief, + 'is_series': is_series, + 'cover_image_url': img_url, + }) + pd.metadata['preferred_model'] = ('TVSeason' if season else 'TVShow') if is_series else 'Movie' + + if imdb_code: + res_data = search_tmdb_by_imdb_id(imdb_code) + tmdb_show_id = None + if 'movie_results' in res_data and len(res_data['movie_results']) > 0: + pd.metadata['preferred_model'] = 'Movie' + elif 'tv_results' in res_data and len(res_data['tv_results']) > 0: + pd.metadata['preferred_model'] = 'TVShow' + elif 'tv_season_results' in res_data and len(res_data['tv_season_results']) > 0: + pd.metadata['preferred_model'] = 'TVSeason' + tmdb_show_id = res_data['tv_season_results'][0]['show_id'] + elif 'tv_episode_results' in res_data and len(res_data['tv_episode_results']) > 0: + pd.metadata['preferred_model'] = 'TVSeason' + tmdb_show_id = res_data['tv_episode_results'][0]['show_id'] + if res_data['tv_episode_results'][0]['episode_number'] != 1: + _logger.error(f'Douban Movie {self.url} mapping to unexpected imdb episode {imdb_code}') + # TODO correct the IMDB id + pd.lookup_ids[IdType.IMDB] = imdb_code + if tmdb_show_id: + pd.metadata['required_resources'] = [{ + 'model': 'TVShow', + 'id_type': IdType.TMDB_TV, + 'id_value': tmdb_show_id, + 'title': title, + 'url': TMDB_TV.id_to_url(tmdb_show_id), + }] + # TODO parse sister seasons + # pd.metadata['related_resources'] = [] + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd diff --git a/catalog/sites/douban_music.py b/catalog/sites/douban_music.py new file mode 100644 index 00000000..1aa157f2 --- /dev/null +++ b/catalog/sites/douban_music.py @@ -0,0 +1,115 @@ +from catalog.common import * +from catalog.models import * +from .douban import DoubanDownloader +import dateparser +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class DoubanMusic(AbstractSite): + ID_TYPE = IdType.DoubanMusic + URL_PATTERNS = [r"\w+://music\.douban\.com/subject/(\d+)/{0,1}", r"\w+://m.douban.com/music/subject/(\d+)/{0,1}"] + WIKI_PROPERTY_ID = '' + DEFAULT_MODEL = Album + + @classmethod + def id_to_url(self, id_value): + return "https://music.douban.com/subject/" + id_value + "/" + + def scrape(self): + content = DoubanDownloader(self.url).download().html() + + elem = content.xpath("//h1/span/text()") + title = elem[0].strip() if len(elem) else None + if not title: + raise ParseError(self, "title") + + artists_elem = content.xpath("//div[@id='info']/span/span[@class='pl']/a/text()") + artist = None if not artists_elem else list(map(lambda a: a[:200], artists_elem)) + + genre_elem = content.xpath( + "//div[@id='info']//span[text()='流派:']/following::text()[1]") + genre = genre_elem[0].strip() if genre_elem else None + + date_elem = content.xpath( + "//div[@id='info']//span[text()='发行时间:']/following::text()[1]") + release_date = dateparser.parse(date_elem[0].strip()).strftime('%Y-%m-%d') if date_elem else None + + company_elem = content.xpath( + "//div[@id='info']//span[text()='出版者:']/following::text()[1]") + company = company_elem[0].strip() if company_elem else None + + track_list_elem = content.xpath( + "//div[@class='track-list']/div[@class='indent']/div/text()" + ) + if track_list_elem: + track_list = '\n'.join([track.strip() for track in track_list_elem]) + else: + track_list = None + + brief_elem = content.xpath("//span[@class='all hidden']") + if not brief_elem: + brief_elem = content.xpath("//span[@property='v:summary']") + brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( + './text()')]) if brief_elem else None + + gtin = None + isrc = None + other_info = {} + other_elem = content.xpath( + "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") + if other_elem: + other_info['又名'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]") + if other_elem: + other_info['专辑类型'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='介质:']/following-sibling::text()[1]") + if other_elem: + other_info['介质'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='ISRC:']/following-sibling::text()[1]") + if other_elem: + other_info['ISRC'] = other_elem[0].strip() + isrc = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='条形码:']/following-sibling::text()[1]") + if other_elem: + other_info['条形码'] = other_elem[0].strip() + gtin = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='碟片数:']/following-sibling::text()[1]") + if other_elem: + other_info['碟片数'] = other_elem[0].strip() + + img_url_elem = content.xpath("//div[@id='mainpic']//img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + + pd = ResourceContent(metadata={ + 'title': title, + 'artist': artist, + 'genre': genre, + 'release_date': release_date, + 'duration': None, + 'company': company, + 'track_list': track_list, + 'brief': brief, + 'other_info': other_info, + 'cover_image_url': img_url + }) + if gtin: + pd.lookup_ids[IdType.GTIN] = gtin + if isrc: + pd.lookup_ids[IdType.ISRC] = isrc + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd diff --git a/catalog/sites/goodreads.py b/catalog/sites/goodreads.py new file mode 100644 index 00000000..be3d4c26 --- /dev/null +++ b/catalog/sites/goodreads.py @@ -0,0 +1,116 @@ +from catalog.book.models import Edition, Work +from catalog.common import * +from lxml import html +import json +import logging + + +_logger = logging.getLogger(__name__) + + +class GoodreadsDownloader(RetryDownloader): + def validate_response(self, response): + if response is None: + return RESPONSE_NETWORK_ERROR + elif response.status_code == 200: + if response.text.find('__NEXT_DATA__') != -1: + return RESPONSE_OK + else: + # Goodreads may return legacy version for a/b testing + # retry if so + return RESPONSE_NETWORK_ERROR + else: + return RESPONSE_INVALID_CONTENT + + +@SiteList.register +class Goodreads(AbstractSite): + ID_TYPE = IdType.Goodreads + WIKI_PROPERTY_ID = 'P2968' + DEFAULT_MODEL = Edition + URL_PATTERNS = [r".+goodreads.com/.*book/show/(\d+)", r".+goodreads.com/.*book/(\d+)"] + + @classmethod + def id_to_url(self, id_value): + return "https://www.goodreads.com/book/show/" + id_value + + def scrape(self, response=None): + data = {} + if response is not None: + h = html.fromstring(response.text.strip()) + else: + dl = GoodreadsDownloader(self.url) + h = dl.download().html() + # Next.JS version of GoodReads + # JSON.parse(document.getElementById('__NEXT_DATA__').innerHTML)['props']['pageProps']['apolloState'] + elem = h.xpath('//script[@id="__NEXT_DATA__"]/text()') + src = elem[0].strip() if elem else None + if not src: + raise ParseError(self, '__NEXT_DATA__ element') + d = json.loads(src)['props']['pageProps']['apolloState'] + o = {'Book': [], 'Work': [], 'Series': [], 'Contributor': []} + for v in d.values(): + t = v.get('__typename') + if t and t in o: + o[t].append(v) + b = next(filter(lambda x: x.get('title'), o['Book']), None) + if not b: + # Goodreads may return empty page template when internal service timeouts + raise ParseError(self, 'Book in __NEXT_DATA__ json') + data['title'] = b['title'] + data['brief'] = b['description'] + data['isbn'] = b['details'].get('isbn13') + asin = b['details'].get('asin') + if asin and asin != data['isbn']: + data['asin'] = asin + data['pages'] = b['details'].get('numPages') + data['cover_image_url'] = b['imageUrl'] + w = next(filter(lambda x: x.get('details'), o['Work']), None) + if w: + data['required_resources'] = [{ + 'model': 'Work', + 'id_type': IdType.Goodreads_Work, + 'id_value': str(w['legacyId']), + 'title': w['details']['originalTitle'], + 'url': w['editions']['webUrl'], + }] + pd = ResourceContent(metadata=data) + pd.lookup_ids[IdType.ISBN] = data.get('isbn') + pd.lookup_ids[IdType.ASIN] = data.get('asin') + if data["cover_image_url"]: + imgdl = BasicImageDownloader(data["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {data["cover_image_url"]}') + return pd + + +@SiteList.register +class Goodreads_Work(AbstractSite): + ID_TYPE = IdType.Goodreads_Work + WIKI_PROPERTY_ID = '' + DEFAULT_MODEL = Work + URL_PATTERNS = [r".+goodreads.com/work/editions/(\d+)"] + + @classmethod + def id_to_url(self, id_value): + return "https://www.goodreads.com/work/editions/" + id_value + + def scrape(self, response=None): + content = BasicDownloader(self.url).download().html() + title_elem = content.xpath("//h1/a/text()") + title = title_elem[0].strip() if title_elem else None + if not title: + raise ParseError(self, 'title') + author_elem = content.xpath("//h2/a/text()") + author = author_elem[0].strip() if author_elem else None + first_published_elem = content.xpath("//h2/span/text()") + first_published = first_published_elem[0].strip() if first_published_elem else None + pd = ResourceContent(metadata={ + 'title': title, + 'author': author, + 'first_published': first_published + }) + return pd diff --git a/catalog/sites/google_books.py b/catalog/sites/google_books.py new file mode 100644 index 00000000..0554d3d8 --- /dev/null +++ b/catalog/sites/google_books.py @@ -0,0 +1,79 @@ +from catalog.common import * +from catalog.models import * +import re +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class GoogleBooks(AbstractSite): + ID_TYPE = IdType.GoogleBooks + URL_PATTERNS = [ + r"https://books\.google\.co[^/]+/books\?id=([^&#]+)", + r"https://www\.google\.co[^/]+/books/edition/[^/]+/([^&#?]+)", + r"https://books\.google\.co[^/]+/books/about/[^?]+?id=([^&#?]+)", + ] + WIKI_PROPERTY_ID = '' + DEFAULT_MODEL = Edition + + @classmethod + def id_to_url(self, id_value): + return "https://books.google.com/books?id=" + id_value + + def scrape(self): + api_url = f'https://www.googleapis.com/books/v1/volumes/{self.id_value}' + b = BasicDownloader(api_url).download().json() + other = {} + title = b['volumeInfo']['title'] + subtitle = b['volumeInfo']['subtitle'] if 'subtitle' in b['volumeInfo'] else None + pub_year = None + pub_month = None + if 'publishedDate' in b['volumeInfo']: + pub_date = b['volumeInfo']['publishedDate'].split('-') + pub_year = pub_date[0] + pub_month = pub_date[1] if len(pub_date) > 1 else None + pub_house = b['volumeInfo']['publisher'] if 'publisher' in b['volumeInfo'] else None + language = b['volumeInfo']['language'] if 'language' in b['volumeInfo'] else None + pages = b['volumeInfo']['pageCount'] if 'pageCount' in b['volumeInfo'] else None + if 'mainCategory' in b['volumeInfo']: + other['分类'] = b['volumeInfo']['mainCategory'] + authors = b['volumeInfo']['authors'] if 'authors' in b['volumeInfo'] else None + if 'description' in b['volumeInfo']: + brief = b['volumeInfo']['description'] + elif 'textSnippet' in b['volumeInfo']: + brief = b["volumeInfo"]["textSnippet"]["searchInfo"] + else: + brief = '' + brief = re.sub(r'<.*?>', '', brief.replace(' 0: + url = f"https://www.themoviedb.org/movie/{res_data['movie_results'][0]['id']}" + elif 'tv_results' in res_data and len(res_data['tv_results']) > 0: + url = f"https://www.themoviedb.org/tv/{res_data['tv_results'][0]['id']}" + elif 'tv_season_results' in res_data and len(res_data['tv_season_results']) > 0: + # this should not happen given IMDB only has ids for either show or episode + tv_id = res_data['tv_season_results'][0]['show_id'] + season_number = res_data['tv_season_results'][0]['season_number'] + url = f"https://www.themoviedb.org/tv/{tv_id}/season/{season_number}/episode/{episode_number}" + elif 'tv_episode_results' in res_data and len(res_data['tv_episode_results']) > 0: + tv_id = res_data['tv_episode_results'][0]['show_id'] + season_number = res_data['tv_episode_results'][0]['season_number'] + episode_number = res_data['tv_episode_results'][0]['episode_number'] + if season_number == 0: + url = f"https://www.themoviedb.org/tv/{tv_id}/season/{season_number}/episode/{episode_number}" + elif episode_number == 1: + url = f"https://www.themoviedb.org/tv/{tv_id}/season/{season_number}" + else: + raise ParseError(self, "IMDB id matching TMDB but not first episode, this is not supported") + else: + raise ParseError(self, "IMDB id not found in TMDB") + tmdb = SiteList.get_site_by_url(url) + pd = tmdb.scrape() + pd.metadata['preferred_model'] = tmdb.DEFAULT_MODEL.__name__ + return pd diff --git a/catalog/sites/spotify.py b/catalog/sites/spotify.py new file mode 100644 index 00000000..c29c32dd --- /dev/null +++ b/catalog/sites/spotify.py @@ -0,0 +1,145 @@ +""" +Spotify +""" +from django.conf import settings +from catalog.common import * +from catalog.models import * +from .douban import * +import time +import datetime +import requests +import dateparser +import logging + + +_logger = logging.getLogger(__name__) + + +spotify_token = None +spotify_token_expire_time = time.time() + + +@SiteList.register +class Spotify(AbstractSite): + ID_TYPE = IdType.Spotify_Album + URL_PATTERNS = [r'\w+://open\.spotify\.com/album/([a-zA-Z0-9]+)'] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = Album + + @classmethod + def id_to_url(self, id_value): + return f"https://open.spotify.com/album/{id_value}" + + def scrape(self): + api_url = "https://api.spotify.com/v1/albums/" + self.id_value + headers = { + 'Authorization': f"Bearer {get_spotify_token()}" + } + res_data = BasicDownloader(api_url, headers=headers).download().json() + artist = [] + for artist_dict in res_data['artists']: + artist.append(artist_dict['name']) + + title = res_data['name'] + + genre = ', '.join(res_data['genres']) + + company = [] + for com in res_data['copyrights']: + company.append(com['text']) + + duration = 0 + track_list = [] + track_urls = [] + for track in res_data['tracks']['items']: + track_urls.append(track['external_urls']['spotify']) + duration += track['duration_ms'] + if res_data['tracks']['items'][-1]['disc_number'] > 1: + # more than one disc + track_list.append(str( + track['disc_number']) + '-' + str(track['track_number']) + '. ' + track['name']) + else: + track_list.append(str(track['track_number']) + '. ' + track['name']) + track_list = '\n'.join(track_list) + + release_date = dateparser.parse(res_data['release_date']).strftime('%Y-%m-%d') + + gtin = None + if res_data['external_ids'].get('upc'): + gtin = res_data['external_ids'].get('upc') + if res_data['external_ids'].get('ean'): + gtin = res_data['external_ids'].get('ean') + isrc = None + if res_data['external_ids'].get('isrc'): + isrc = res_data['external_ids'].get('isrc') + + pd = ResourceContent(metadata={ + 'title': title, + 'artist': artist, + 'genre': genre, + 'track_list': track_list, + 'release_date': release_date, + 'duration': duration, + 'company': company, + 'brief': None, + 'cover_image_url': res_data['images'][0]['url'] + }) + if gtin: + pd.lookup_ids[IdType.GTIN] = gtin + if isrc: + pd.lookup_ids[IdType.ISRC] = isrc + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd + + +def get_spotify_token(): + global spotify_token, spotify_token_expire_time + if get_mock_mode(): + return 'mocked' + if spotify_token is None or is_spotify_token_expired(): + invoke_spotify_token() + return spotify_token + + +def is_spotify_token_expired(): + global spotify_token_expire_time + return True if spotify_token_expire_time <= time.time() else False + + +def invoke_spotify_token(): + global spotify_token, spotify_token_expire_time + r = requests.post( + "https://accounts.spotify.com/api/token", + data={ + "grant_type": "client_credentials" + }, + headers={ + "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}" + } + ) + data = r.json() + if r.status_code == 401: + # token expired, try one more time + # this maybe caused by external operations, + # for example debugging using a http client + r = requests.post( + "https://accounts.spotify.com/api/token", + data={ + "grant_type": "client_credentials" + }, + headers={ + "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}" + } + ) + data = r.json() + elif r.status_code != 200: + raise Exception(f"Request to spotify API fails. Reason: {r.reason}") + # minus 2 for execution time error + spotify_token_expire_time = int(data['expires_in']) + time.time() - 2 + spotify_token = data['access_token'] diff --git a/catalog/sites/steam.py b/catalog/sites/steam.py new file mode 100644 index 00000000..c80bc769 --- /dev/null +++ b/catalog/sites/steam.py @@ -0,0 +1,64 @@ +from catalog.common import * +from catalog.models import * +from .igdb import search_igdb_by_3p_url +import dateparser +import logging + + +_logger = logging.getLogger(__name__) + + +@SiteList.register +class Steam(AbstractSite): + ID_TYPE = IdType.Steam + URL_PATTERNS = [r"\w+://store\.steampowered\.com/app/(\d+)"] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = Game + + @classmethod + def id_to_url(self, id_value): + return "https://store.steampowered.com/app/" + str(id_value) + + def scrape(self): + i = search_igdb_by_3p_url(self.url) + pd = i.scrape() if i else ResourceContent() + + headers = BasicDownloader.headers.copy() + headers['Host'] = 'store.steampowered.com' + headers['Cookie'] = "wants_mature_content=1; birthtime=754700401;" + content = BasicDownloader(self.url, headers=headers).download().html() + + title = content.xpath("//div[@class='apphub_AppName']/text()")[0] + developer = content.xpath("//div[@id='developers_list']/a/text()") + publisher = content.xpath("//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()") + release_date = dateparser.parse( + content.xpath( + "//div[@class='release_date']/div[@class='date']/text()")[0] + ).strftime('%Y-%m-%d') + genre = content.xpath( + "//div[@class='details_block']/b[2]/following-sibling::a/text()") + platform = ['PC'] + brief = content.xpath( + "//div[@class='game_description_snippet']/text()")[0].strip() + # try Steam images if no image from IGDB + if pd.cover_image is None: + pd.metadata['cover_image_url'] = content.xpath("//img[@class='game_header_image_full']/@src")[0].replace("header.jpg", "library_600x900.jpg") + pd.cover_image, pd.cover_image_extention = BasicImageDownloader.download_image(pd.metadata['cover_image_url'], self.url) + if pd.cover_image is None: + pd.metadata['cover_image_url'] = content.xpath("//img[@class='game_header_image_full']/@src")[0] + pd.cover_image, pd.cover_image_extention = BasicImageDownloader.download_image(pd.metadata['cover_image_url'], self.url) + # merge data from IGDB, use localized Steam data if available + d = { + 'developer': developer, + 'publisher': publisher, + 'release_date': release_date, + 'genre': genre, + 'platform': platform, + } + d.update(pd.metadata) + pd.metadata = d + if title: + pd.metadata['title'] = title + if brief: + pd.metadata['brief'] = brief + return pd diff --git a/catalog/sites/tmdb.py b/catalog/sites/tmdb.py new file mode 100644 index 00000000..f0c65c90 --- /dev/null +++ b/catalog/sites/tmdb.py @@ -0,0 +1,328 @@ +""" +The Movie Database +""" + +import re +from django.conf import settings +from catalog.common import * +from .douban import * +from catalog.movie.models import * +from catalog.tv.models import * +import logging + + +_logger = logging.getLogger(__name__) + + +def search_tmdb_by_imdb_id(imdb_id): + tmdb_api_url = f"https://api.themoviedb.org/3/find/{imdb_id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&external_source=imdb_id" + res_data = BasicDownloader(tmdb_api_url).download().json() + return res_data + + +def _copy_dict(s, key_map): + d = {} + for src, dst in key_map.items(): + d[dst if dst else src] = s.get(src) + return d + + +genre_map = { + 'Sci-Fi & Fantasy': 'Sci-Fi', + 'War & Politics': 'War', + '儿童': 'Kids', + '冒险': 'Adventure', + '剧情': 'Drama', + '动作': 'Action', + '动作冒险': 'Action', + '动画': 'Animation', + '历史': 'History', + '喜剧': 'Comedy', + '奇幻': 'Fantasy', + '家庭': 'Family', + '恐怖': 'Horror', + '悬疑': 'Mystery', + '惊悚': 'Thriller', + '战争': 'War', + '新闻': 'News', + '爱情': 'Romance', + '犯罪': 'Crime', + '电视电影': 'TV Movie', + '真人秀': 'Reality-TV', + '科幻': 'Sci-Fi', + '纪录': 'Documentary', + '肥皂剧': 'Soap', + '脱口秀': 'Talk-Show', + '西部': 'Western', + '音乐': 'Music', +} + + +@SiteList.register +class TMDB_Movie(AbstractSite): + ID_TYPE = IdType.TMDB_Movie + URL_PATTERNS = [r'\w+://www.themoviedb.org/movie/(\d+)'] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = Movie + + @classmethod + def id_to_url(self, id_value): + return f"https://www.themoviedb.org/movie/{id_value}" + + def scrape(self): + is_series = False + if is_series: + api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + else: + api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + + res_data = BasicDownloader(api_url).download().json() + + if is_series: + title = res_data['name'] + orig_title = res_data['original_name'] + year = int(res_data['first_air_date'].split( + '-')[0]) if res_data['first_air_date'] else None + imdb_code = res_data['external_ids']['imdb_id'] + showtime = [{res_data['first_air_date']: "首播日期"} + ] if res_data['first_air_date'] else None + duration = None + else: + title = res_data['title'] + orig_title = res_data['original_title'] + year = int(res_data['release_date'].split('-') + [0]) if res_data['release_date'] else None + showtime = [{res_data['release_date']: "发布日期"} + ] if res_data['release_date'] else None + imdb_code = res_data['imdb_id'] + # in minutes + duration = res_data['runtime'] if res_data['runtime'] else None + + genre = list(map(lambda x: genre_map[x['name']] if x['name'] + in genre_map else 'Other', res_data['genres'])) + language = list(map(lambda x: x['name'], res_data['spoken_languages'])) + brief = res_data['overview'] + + if is_series: + director = list(map(lambda x: x['name'], res_data['created_by'])) + else: + director = list(map(lambda x: x['name'], filter( + lambda c: c['job'] == 'Director', res_data['credits']['crew']))) + playwright = list(map(lambda x: x['name'], filter( + lambda c: c['job'] == 'Screenplay', res_data['credits']['crew']))) + actor = list(map(lambda x: x['name'], res_data['credits']['cast'])) + area = [] + + other_info = {} + # other_info['TMDB评分'] = res_data['vote_average'] + # other_info['分级'] = res_data['contentRating'] + # other_info['Metacritic评分'] = res_data['metacriticRating'] + # other_info['奖项'] = res_data['awards'] + # other_info['TMDB_ID'] = id + if is_series: + other_info['Seasons'] = res_data['number_of_seasons'] + other_info['Episodes'] = res_data['number_of_episodes'] + + # TODO: use GET /configuration to get base url + img_url = ('https://image.tmdb.org/t/p/original/' + res_data['poster_path']) if res_data['poster_path'] is not None else None + + pd = ResourceContent(metadata={ + 'title': title, + 'orig_title': orig_title, + 'other_title': None, + 'imdb_code': imdb_code, + 'director': director, + 'playwright': playwright, + 'actor': actor, + 'genre': genre, + 'showtime': showtime, + 'site': None, + 'area': area, + 'language': language, + 'year': year, + 'duration': duration, + 'season': None, + 'episodes': None, + 'single_episode_length': None, + 'brief': brief, + 'cover_image_url': img_url, + }) + if imdb_code: + pd.lookup_ids[IdType.IMDB] = imdb_code + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd + + +@SiteList.register +class TMDB_TV(AbstractSite): + ID_TYPE = IdType.TMDB_TV + URL_PATTERNS = [r'\w+://www.themoviedb.org/tv/(\d+)[^/]*$', r'\w+://www.themoviedb.org/tv/(\d+)[^/]*/seasons'] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = TVShow + + @classmethod + def id_to_url(self, id_value): + return f"https://www.themoviedb.org/tv/{id_value}" + + def scrape(self): + is_series = True + if is_series: + api_url = f"https://api.themoviedb.org/3/tv/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + else: + api_url = f"https://api.themoviedb.org/3/movie/{self.id_value}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + + res_data = BasicDownloader(api_url).download().json() + + if is_series: + title = res_data['name'] + orig_title = res_data['original_name'] + year = int(res_data['first_air_date'].split( + '-')[0]) if res_data['first_air_date'] else None + imdb_code = res_data['external_ids']['imdb_id'] + showtime = [{res_data['first_air_date']: "首播日期"} + ] if res_data['first_air_date'] else None + duration = None + else: + title = res_data['title'] + orig_title = res_data['original_title'] + year = int(res_data['release_date'].split('-') + [0]) if res_data['release_date'] else None + showtime = [{res_data['release_date']: "发布日期"} + ] if res_data['release_date'] else None + imdb_code = res_data['imdb_id'] + # in minutes + duration = res_data['runtime'] if res_data['runtime'] else None + + genre = list(map(lambda x: genre_map[x['name']] if x['name'] + in genre_map else 'Other', res_data['genres'])) + language = list(map(lambda x: x['name'], res_data['spoken_languages'])) + brief = res_data['overview'] + + if is_series: + director = list(map(lambda x: x['name'], res_data['created_by'])) + else: + director = list(map(lambda x: x['name'], filter( + lambda c: c['job'] == 'Director', res_data['credits']['crew']))) + playwright = list(map(lambda x: x['name'], filter( + lambda c: c['job'] == 'Screenplay', res_data['credits']['crew']))) + actor = list(map(lambda x: x['name'], res_data['credits']['cast'])) + area = [] + + other_info = {} + # other_info['TMDB评分'] = res_data['vote_average'] + # other_info['分级'] = res_data['contentRating'] + # other_info['Metacritic评分'] = res_data['metacriticRating'] + # other_info['奖项'] = res_data['awards'] + # other_info['TMDB_ID'] = id + if is_series: + other_info['Seasons'] = res_data['number_of_seasons'] + other_info['Episodes'] = res_data['number_of_episodes'] + + # TODO: use GET /configuration to get base url + img_url = ('https://image.tmdb.org/t/p/original/' + res_data['poster_path']) if res_data['poster_path'] is not None else None + + season_links = list(map(lambda s: { + 'model': 'TVSeason', + 'id_type': IdType.TMDB_TVSeason, + 'id_value': f'{self.id_value}-{s["season_number"]}', + 'title': s['name'], + 'url': f'{self.url}/season/{s["season_number"]}'}, res_data['seasons'])) + pd = ResourceContent(metadata={ + 'title': title, + 'orig_title': orig_title, + 'other_title': None, + 'imdb_code': imdb_code, + 'director': director, + 'playwright': playwright, + 'actor': actor, + 'genre': genre, + 'showtime': showtime, + 'site': None, + 'area': area, + 'language': language, + 'year': year, + 'duration': duration, + 'season': None, + 'episodes': None, + 'single_episode_length': None, + 'brief': brief, + 'cover_image_url': img_url, + 'related_resources': season_links, + }) + if imdb_code: + pd.lookup_ids[IdType.IMDB] = imdb_code + + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + return pd + + +@SiteList.register +class TMDB_TVSeason(AbstractSite): + ID_TYPE = IdType.TMDB_TVSeason + URL_PATTERNS = [r'\w+://www.themoviedb.org/tv/(\d+)[^/]*/season/(\d+)[^/]*$'] + WIKI_PROPERTY_ID = '?' + DEFAULT_MODEL = TVSeason + ID_PATTERN = r'^(\d+)-(\d+)$' + + @classmethod + def url_to_id(cls, url: str): + u = next(iter([re.match(p, url) for p in cls.URL_PATTERNS if re.match(p, url)]), None) + return u[1] + '-' + u[2] if u else None + + @classmethod + def id_to_url(cls, id_value): + v = id_value.split('-') + return f"https://www.themoviedb.org/tv/{v[0]}/season/{v[1]}" + + def scrape(self): + v = self.id_value.split('-') + api_url = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + d = BasicDownloader(api_url).download().json() + if not d.get('id'): + raise ParseError('id') + pd = ResourceContent(metadata=_copy_dict(d, {'name': 'title', 'overview': 'brief', 'air_date': 'air_date', 'season_number': 0, 'external_ids': 0})) + pd.metadata['required_resources'] = [{ + 'model': 'TVShow', + 'id_type': IdType.TMDB_TV, + 'id_value': v[0], + 'title': f'TMDB TV Show {v[0]}', + 'url': f"https://www.themoviedb.org/tv/{v[0]}", + }] + pd.lookup_ids[IdType.IMDB] = d['external_ids'].get('imdb_id') + pd.metadata['cover_image_url'] = ('https://image.tmdb.org/t/p/original/' + d['poster_path']) if d['poster_path'] else None + pd.metadata['title'] = pd.metadata['title'] if pd.metadata['title'] else f'Season {d["season_number"]}' + pd.metadata['episode_number_list'] = list(map(lambda ep: ep['episode_number'], d['episodes'])) + pd.metadata['episode_count'] = len(pd.metadata['episode_number_list']) + if pd.metadata["cover_image_url"]: + imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url) + try: + pd.cover_image = imgdl.download().content + pd.cover_image_extention = imgdl.extention + except Exception: + _logger.debug(f'failed to download cover for {self.url} from {pd.metadata["cover_image_url"]}') + + # get external id from 1st episode + if pd.lookup_ids[IdType.IMDB]: + _logger.warning("Unexpected IMDB id for TMDB tv season") + elif len(pd.metadata['episode_number_list']) == 0: + _logger.warning("Unable to lookup IMDB id for TMDB tv season with zero episodes") + else: + ep = pd.metadata['episode_number_list'][0] + api_url2 = f"https://api.themoviedb.org/3/tv/{v[0]}/season/{v[1]}/episode/{ep}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + d2 = BasicDownloader(api_url2).download().json() + if not d2.get('id'): + raise ParseError('episode id for season') + pd.lookup_ids[IdType.IMDB] = d2['external_ids'].get('imdb_id') + return pd diff --git a/catalog/tests.py b/catalog/tests.py new file mode 100644 index 00000000..952d0993 --- /dev/null +++ b/catalog/tests.py @@ -0,0 +1,10 @@ +from django.test import TestCase +from catalog.book.tests import * +from catalog.movie.tests import * +from catalog.tv.tests import * +from catalog.music.tests import * +from catalog.game.tests import * +from catalog.podcast.tests import * +from catalog.performance.tests import * + +# imported tests with same name might be ignored silently diff --git a/catalog/tv/models.py b/catalog/tv/models.py new file mode 100644 index 00000000..ea1044f1 --- /dev/null +++ b/catalog/tv/models.py @@ -0,0 +1,62 @@ +""" +Models for TV + +TVShow -> TVSeason -> TVEpisode + +TVEpisode is not fully implemented at the moment + +Three way linking between Douban / IMDB / TMDB are quite messy + +IMDB: +most widely used. +no ID for Season, only for Show and Episode + +TMDB: +most friendly API. +for some TV specials, both shown as an Episode of Season 0 and a Movie, with same IMDB id + +Douban: +most wanted by our users. +for single season show, IMDB id of the show id used +for multi season show, IMDB id for Ep 1 will be used to repensent that season +tv specials are are shown as movies + +For now, we follow Douban convention, but keep an eye on it in case it breaks its own rules... + +""" +from catalog.common import * +from django.db import models + + +class TVShow(Item): + imdb = PrimaryLookupIdDescriptor(IdType.IMDB) + tmdb_tv = PrimaryLookupIdDescriptor(IdType.TMDB_TV) + imdb = PrimaryLookupIdDescriptor(IdType.IMDB) + season_count = jsondata.IntegerField(blank=True, default=None) + + +class TVSeason(Item): + douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie) + imdb = PrimaryLookupIdDescriptor(IdType.IMDB) + tmdb_tvseason = PrimaryLookupIdDescriptor(IdType.TMDB_TVSeason) + show = models.ForeignKey(TVShow, null=True, on_delete=models.SET_NULL, related_name='seasons') + season_number = models.PositiveIntegerField() + episode_count = jsondata.IntegerField(blank=True, default=None) + METADATA_COPY_LIST = ['title', 'brief', 'season_number', 'episode_count'] + + def update_linked_items_from_external_resource(self, resource): + """add Work from resource.metadata['work'] if not yet""" + links = resource.required_resources + resource.related_resources + for w in links: + if w['model'] == 'TVShow': + p = ExternalResource.objects.filter(id_type=w['id_type'], id_value=w['id_value']).first() + if p and p.item and self.show != p.item: + self.show = p.item + + +class TVEpisode(Item): + show = models.ForeignKey(TVShow, null=True, on_delete=models.SET_NULL, related_name='episodes') + season = models.ForeignKey(TVSeason, null=True, on_delete=models.SET_NULL, related_name='episodes') + episode_number = models.PositiveIntegerField() + imdb = PrimaryLookupIdDescriptor(IdType.IMDB) + METADATA_COPY_LIST = ['title', 'brief', 'episode_number'] diff --git a/catalog/tv/tests.py b/catalog/tv/tests.py new file mode 100644 index 00000000..a25c45aa --- /dev/null +++ b/catalog/tv/tests.py @@ -0,0 +1,128 @@ +from django.test import TestCase +from catalog.common import * +from catalog.tv.models import * + + +class TMDBTVTestCase(TestCase): + def test_parse(self): + t_id = '57243' + t_url = 'https://www.themoviedb.org/tv/57243-doctor-who' + t_url1 = 'https://www.themoviedb.org/tv/57243-doctor-who/seasons' + t_url2 = 'https://www.themoviedb.org/tv/57243' + p1 = SiteList.get_site_by_id_type(IdType.TMDB_TV) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + self.assertEqual(p1.validate_url(t_url1), True) + self.assertEqual(p1.validate_url(t_url2), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url2) + self.assertEqual(p2.url_to_id(t_url), t_id) + wrong_url = 'https://www.themoviedb.org/tv/57243-doctor-who/season/13' + s1 = SiteList.get_site_by_url(wrong_url) + self.assertNotIsInstance(s1, TVShow) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.themoviedb.org/tv/57243-doctor-who' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, '57243') + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], '神秘博士') + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) + self.assertEqual(site.resource.item.__class__.__name__, 'TVShow') + self.assertEqual(site.resource.item.imdb, 'tt0436992') + + +class TMDBTVSeasonTestCase(TestCase): + def test_parse(self): + t_id = '57243-11' + t_url = 'https://www.themoviedb.org/tv/57243-doctor-who/season/11' + t_url_unique = 'https://www.themoviedb.org/tv/57243/season/11' + p1 = SiteList.get_site_by_id_type(IdType.TMDB_TVSeason) + self.assertIsNotNone(p1) + self.assertEqual(p1.validate_url(t_url), True) + self.assertEqual(p1.validate_url(t_url_unique), True) + p2 = SiteList.get_site_by_url(t_url) + self.assertEqual(p1.id_to_url(t_id), t_url_unique) + self.assertEqual(p2.url_to_id(t_url), t_id) + + @use_local_response + def test_scrape(self): + t_url = 'https://www.themoviedb.org/tv/57243-doctor-who/season/4' + site = SiteList.get_site_by_url(t_url) + self.assertEqual(site.ready, False) + self.assertEqual(site.id_value, '57243-4') + site.get_resource_ready() + self.assertEqual(site.ready, True) + self.assertEqual(site.resource.metadata['title'], '第 4 季') + self.assertEqual(site.resource.item.primary_lookup_id_type, IdType.IMDB) + self.assertEqual(site.resource.item.__class__.__name__, 'TVSeason') + self.assertEqual(site.resource.item.imdb, 'tt1159991') + self.assertIsNotNone(site.resource.item.show) + self.assertEqual(site.resource.item.show.imdb, 'tt0436992') + + +class DoubanMovieTVTestCase(TestCase): + @use_local_response + def test_scrape(self): + url3 = 'https://movie.douban.com/subject/3627919/' + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p3.item.__class__.__name__, 'TVSeason') + self.assertIsNotNone(p3.item.show) + self.assertEqual(p3.item.show.imdb, 'tt0436992') + + @use_local_response + def test_scrape_singleseason(self): + url3 = 'https://movie.douban.com/subject/26895436/' + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p3.item.__class__.__name__, 'TVShow') + + +class MultiTVSitesTestCase(TestCase): + @use_local_response + def test_tvshows(self): + url1 = 'https://www.themoviedb.org/tv/57243-doctor-who' + url2 = 'https://www.imdb.com/title/tt0436992/' + # url3 = 'https://movie.douban.com/subject/3541415/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + # p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p1.item.id, p2.item.id) + # self.assertEqual(p2.item.id, p3.item.id) + + @use_local_response + def test_tvseasons(self): + url1 = 'https://www.themoviedb.org/tv/57243-doctor-who/season/4' + url2 = 'https://www.imdb.com/title/tt1159991/' + url3 = 'https://movie.douban.com/subject/3627919/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p1.item.imdb, p2.item.imdb) + self.assertEqual(p2.item.imdb, p3.item.imdb) + self.assertEqual(p1.item.id, p2.item.id) + self.assertEqual(p2.item.id, p3.item.id) + + @use_local_response + def test_miniseries(self): + url1 = 'https://www.themoviedb.org/tv/86941-the-north-water' + url3 = 'https://movie.douban.com/subject/26895436/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p3.item.__class__.__name__, 'TVShow') + self.assertEqual(p1.item.id, p3.item.id) + + @use_local_response + def test_tvspecial(self): + url1 = 'https://www.themoviedb.org/movie/282758-doctor-who-the-runaway-bride' + url2 = 'hhttps://www.imdb.com/title/tt0827573/' + url3 = 'https://movie.douban.com/subject/4296866/' + p1 = SiteList.get_site_by_url(url1).get_resource_ready() + p2 = SiteList.get_site_by_url(url2).get_resource_ready() + p3 = SiteList.get_site_by_url(url3).get_resource_ready() + self.assertEqual(p1.item.imdb, p2.item.imdb) + self.assertEqual(p2.item.imdb, p3.item.imdb) + self.assertEqual(p1.item.id, p2.item.id) + self.assertEqual(p2.item.id, p3.item.id) diff --git a/catalog/urls.py b/catalog/urls.py new file mode 100644 index 00000000..6a2855b5 --- /dev/null +++ b/catalog/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from .api import api + +urlpatterns = [ + path("", api.urls), +] diff --git a/catalog/views.py b/catalog/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/catalog/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/common/scrapers/douban.py b/common/scrapers/douban.py index 6dcaff69..1cfd7310 100644 --- a/common/scrapers/douban.py +++ b/common/scrapers/douban.py @@ -497,7 +497,7 @@ class DoubanMovieScraper(DoubanScrapperMixin, AbstractScraper): episodes_elem = content.xpath( "//div[@id='info']//span[text()='集数:']/following-sibling::text()[1]") - episodes = int(episodes_elem[0].strip()) if episodes_elem and episodes_elem[0].isdigit() else None + episodes = int(episodes_elem[0].strip()) if episodes_elem and episodes_elem[0].strip().isdigit() else None single_episode_length_elem = content.xpath( "//div[@id='info']//span[text()='单集片长:']/following-sibling::text()[1]") diff --git a/common/scrapers/igdb.py b/common/scrapers/igdb.py index eb635f5c..2456497e 100644 --- a/common/scrapers/igdb.py +++ b/common/scrapers/igdb.py @@ -8,9 +8,22 @@ from common.scraper import * from igdb.wrapper import IGDBWrapper import json import datetime +import logging -wrapper = IGDBWrapper(settings.IGDB_CLIENT_ID, settings.IGDB_ACCESS_TOKEN) +_logger = logging.getLogger(__name__) + + +def _igdb_access_token(): + try: + token = requests.post(f'https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials').json()['access_token'] + except Exception: + _logger.error('unable to obtain IGDB token') + token = '' + return token + + +wrapper = IGDBWrapper(settings.IGDB_CLIENT_ID, _igdb_access_token()) class IgdbGameScraper(AbstractScraper): diff --git a/common/static/css/boofilsic.css b/common/static/css/boofilsic.css index 1307c064..b158c475 100644 --- a/common/static/css/boofilsic.css +++ b/common/static/css/boofilsic.css @@ -461,6 +461,11 @@ select::placeholder { color: #606c76; } +.navbar .current { + color: #00a1cc; + font-weight: bold; +} + .navbar .navbar__search-box { margin: 0 12% 0 15px; display: inline-flex; diff --git a/common/static/css/boofilsic.min.css b/common/static/css/boofilsic.min.css index 6b163092..461b5feb 100644 --- a/common/static/css/boofilsic.min.css +++ b/common/static/css/boofilsic.min.css @@ -1 +1 @@ -@import url(https://cdn.jsdelivr.net/npm/skeleton-css@2.0.4/css/normalize.css);.button,button,input[type='button'],input[type='reset'],input[type='submit']{background-color:#00a1cc;border:0.1rem solid #00a1cc;border-radius:.4rem;color:#fff;cursor:pointer;display:inline-block;font-size:1.1rem;font-weight:700;height:3.4rem;letter-spacing:.1rem;line-height:3.4rem;padding:0 2.8rem;text-align:center;text-decoration:none;text-transform:uppercase;white-space:nowrap}.button:focus,.button:hover,button:focus,button:hover,input[type='button']:focus,input[type='button']:hover,input[type='reset']:focus,input[type='reset']:hover,input[type='submit']:focus,input[type='submit']:hover{background-color:#606c76;border-color:#606c76;color:#fff;outline:0}.button[disabled],button[disabled],input[type='button'][disabled],input[type='reset'][disabled],input[type='submit'][disabled]{cursor:default;opacity:.5}.button[disabled]:focus,.button[disabled]:hover,button[disabled]:focus,button[disabled]:hover,input[type='button'][disabled]:focus,input[type='button'][disabled]:hover,input[type='reset'][disabled]:focus,input[type='reset'][disabled]:hover,input[type='submit'][disabled]:focus,input[type='submit'][disabled]:hover{background-color:#00a1cc;border-color:#00a1cc}.button.button-outline,button.button-outline,input[type='button'].button-outline,input[type='reset'].button-outline,input[type='submit'].button-outline{background-color:transparent;color:#00a1cc}.button.button-outline:focus,.button.button-outline:hover,button.button-outline:focus,button.button-outline:hover,input[type='button'].button-outline:focus,input[type='button'].button-outline:hover,input[type='reset'].button-outline:focus,input[type='reset'].button-outline:hover,input[type='submit'].button-outline:focus,input[type='submit'].button-outline:hover{background-color:transparent;border-color:#606c76;color:#606c76}.button.button-outline[disabled]:focus,.button.button-outline[disabled]:hover,button.button-outline[disabled]:focus,button.button-outline[disabled]:hover,input[type='button'].button-outline[disabled]:focus,input[type='button'].button-outline[disabled]:hover,input[type='reset'].button-outline[disabled]:focus,input[type='reset'].button-outline[disabled]:hover,input[type='submit'].button-outline[disabled]:focus,input[type='submit'].button-outline[disabled]:hover{border-color:inherit;color:#00a1cc}.button.button-clear,button.button-clear,input[type='button'].button-clear,input[type='reset'].button-clear,input[type='submit'].button-clear{background-color:transparent;border-color:transparent;color:#00a1cc}.button.button-clear:focus,.button.button-clear:hover,button.button-clear:focus,button.button-clear:hover,input[type='button'].button-clear:focus,input[type='button'].button-clear:hover,input[type='reset'].button-clear:focus,input[type='reset'].button-clear:hover,input[type='submit'].button-clear:focus,input[type='submit'].button-clear:hover{background-color:transparent;border-color:transparent;color:#606c76}.button.button-clear[disabled]:focus,.button.button-clear[disabled]:hover,button.button-clear[disabled]:focus,button.button-clear[disabled]:hover,input[type='button'].button-clear[disabled]:focus,input[type='button'].button-clear[disabled]:hover,input[type='reset'].button-clear[disabled]:focus,input[type='reset'].button-clear[disabled]:hover,input[type='submit'].button-clear[disabled]:focus,input[type='submit'].button-clear[disabled]:hover{color:#00a1cc}select{background:url('data:image/svg+xml;utf8,') center right no-repeat;padding-right:3.0rem}select:focus{background-image:url('data:image/svg+xml;utf8,')}textarea{min-height:6.5rem;width:100%}select{width:100%}label,legend{display:block;margin-bottom:.5rem}fieldset{border-width:0;padding:0}input[type='checkbox'],input[type='radio']{display:inline}.label-inline{display:inline-block;font-weight:normal;margin-left:.5rem}dl,ol,ul{list-style:none;margin-top:0;padding-left:0}ol{list-style:decimal inside}ul{list-style:circle inside}.button,button,dd,dt,li{margin-bottom:1.0rem}fieldset,input,select,textarea{margin-bottom:1.5rem}blockquote,dl,figure,form,ol,p,pre,table,ul{margin-bottom:1rem}b,strong{font-weight:bold}p{margin-top:0}h1,h2,h3,h4,h5,h6{font-weight:300;letter-spacing:-.1rem;margin-bottom:2.0rem;margin-top:0}h1{font-size:4.6rem;line-height:1.2}h2{font-size:3.6rem;line-height:1.25}h3{font-size:2.8rem;line-height:1.3}h4{font-size:2.2rem;letter-spacing:-.08rem;line-height:1.35}h5{font-size:1.8rem;letter-spacing:-.05rem;line-height:1.5}h6{font-size:1.6rem;letter-spacing:0;line-height:1.4}img{max-width:100%;object-fit:contain}img.emoji{height:14px;box-sizing:border-box;object-fit:contain;position:relative;top:3px}img.emoji--large{height:20px;position:relative;top:2px}.clearfix:after{clear:both;content:' ';display:table}.float-left{float:left}.float-right{float:right}.highlight{font-weight:bold}:root{font-size:10px}*,*:after,*:before{box-sizing:inherit}html{box-sizing:border-box;height:100%}body{color:#606c76;font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif;font-size:1.3rem;font-weight:300;letter-spacing:.05rem;line-height:1.6;margin:0;height:100%}textarea{font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif}a{color:#00a1cc;text-decoration:none}a:active,a:hover,a:hover:visited{color:#606c76}li{list-style:none}input[type=text]::-ms-clear,input[type=text]::-ms-reveal{display:none;width:0;height:0}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-results-button,input[type="search"]::-webkit-search-results-decoration{display:none}input[type='email'],input[type='number'],input[type='password'],input[type='search'],input[type='tel'],input[type='text'],input[type='url'],input[type='date'],input[type='time'],input[type='color'],textarea,select{appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem}input[type='email']:focus,input[type='number']:focus,input[type='password']:focus,input[type='search']:focus,input[type='tel']:focus,input[type='text']:focus,input[type='url']:focus,input[type='date']:focus,input[type='time']:focus,input[type='color']:focus,textarea:focus,select:focus{border-color:#00a1cc;outline:0}input[type='email']::placeholder,input[type='number']::placeholder,input[type='password']::placeholder,input[type='search']::placeholder,input[type='tel']::placeholder,input[type='text']::placeholder,input[type='url']::placeholder,input[type='date']::placeholder,input[type='time']::placeholder,input[type='color']::placeholder,textarea::placeholder,select::placeholder{color:#ccc}::selection{color:white;background-color:#00a1cc}.navbar{background-color:#f7f7f7;box-sizing:border-box;padding:10px 0;margin-bottom:50px;border-bottom:#ccc 0.5px solid}.navbar .navbar__wrapper{display:flex;justify-content:space-between;align-items:center;position:relative}.navbar .navbar__logo{flex-basis:100px}.navbar .navbar__logo-link{display:inline-block}.navbar .navbar__link-list{margin:0;display:flex;justify-content:space-around}.navbar .navbar__link{margin:9px;color:#606c76}.navbar .navbar__link:active,.navbar .navbar__link:hover,.navbar .navbar__link:hover:visited{color:#00a1cc}.navbar .navbar__link:visited{color:#606c76}.navbar .navbar__search-box{margin:0 12% 0 15px;display:inline-flex;flex:1}.navbar .navbar__search-box>input[type="search"]{border-top-right-radius:0;border-bottom-right-radius:0;margin:0;height:32px;background-color:white !important;width:100%}.navbar .navbar__search-box .navbar__search-dropdown{margin:0;margin-left:-1px;padding:0;padding-left:10px;color:#606c76;appearance:auto;background-color:white;height:32px;width:80px;border-top-left-radius:0;border-bottom-left-radius:0}.navbar .navbar__dropdown-btn{display:none;padding:0;margin:0;border:none;background-color:transparent;color:#00a1cc}.navbar .navbar__dropdown-btn:focus,.navbar .navbar__dropdown-btn:hover{background-color:transparent;color:#606c76}@media (max-width: 575.98px){.navbar{padding:2px 0}.navbar .navbar__wrapper{display:block}.navbar .navbar__logo-img{width:72px;margin-right:10px;position:relative;top:7px}.navbar .navbar__link-list{margin-top:7px;max-height:0;transition:max-height 0.6s ease-out;overflow:hidden}.navbar .navbar__dropdown-btn{display:block;position:absolute;right:5px;top:3px;transform:scale(0.7)}.navbar .navbar__dropdown-btn:hover+.navbar__link-list{max-height:500px;transition:max-height 0.6s ease-in}.navbar .navbar__search-box{margin:0;width:46vw}.navbar .navbar__search-box>input[type="search"]{height:26px;padding:4px 6px;width:32vw}.navbar .navbar__search-box .navbar__search-dropdown{cursor:pointer;height:26px;width:80px;padding-left:5px}}@media (max-width: 991.98px){.navbar{margin-bottom:20px}}.grid{margin:0 auto;position:relative;max-width:110rem;padding:0 2.0rem;width:100%}.grid .grid__main{width:70%;float:left;position:relative}.grid .grid__aside{width:26%;float:right;position:relative;display:flex;flex-direction:column;justify-content:space-around}.grid::after{content:' ';clear:both;display:table}@media (max-width: 575.98px){.grid .grid__aside{flex-direction:column !important}}@media (max-width: 991.98px){.grid .grid__main{width:100%;float:none}.grid .grid__aside{width:100%;float:none;flex-direction:row}.grid .grid__aside--tablet-column{flex-direction:column}.grid--reverse-order{transform:scaleY(-1)}.grid .grid__main--reverse-order{transform:scaleY(-1)}.grid .grid__aside--reverse-order{transform:scaleY(-1)}}.pagination{text-align:center;width:100%}.pagination .pagination__page-link{font-weight:normal;margin:0 5px}.pagination .pagination__page-link--current{font-weight:bold;font-size:1.2em;color:#606c76}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:18px}.pagination .pagination__nav-link--left-margin{margin-left:18px}.pagination .pagination__nav-link--hidden{display:none}@media (max-width: 575.98px){.pagination .pagination__page-link{margin:0 3px}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:10px}.pagination .pagination__nav-link--left-margin{margin-left:10px}}#page-wrapper{position:relative;min-height:100vh;z-index:0}#content-wrapper{padding-bottom:160px}.footer{padding-top:0.4em !important;text-align:center;margin-bottom:4px !important;position:absolute !important;left:50%;transform:translateX(-50%);bottom:0;width:100%}.footer__border{padding-top:4px;border-top:#f7f7f7 solid 2px}.footer__link{margin:0 12px;white-space:nowrap}@media (max-width: 575.98px){#content-wrapper{padding-bottom:120px}}.icon-lock svg{fill:#ccc;height:12px;position:relative;top:1px;margin-left:3px}.icon-edit svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-save svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-cross svg{fill:#ccc;height:10px;position:relative}.icon-arrow svg{fill:#606c76;height:15px;position:relative;top:3px}.spinner{display:inline-block;position:relative;left:50%;transform:translateX(-50%) scale(0.4);width:80px;height:80px}.spinner div{transform-origin:40px 40px;animation:spinner 1.2s linear infinite}.spinner div::after{content:" ";display:block;position:absolute;top:3px;left:37px;width:6px;height:18px;border-radius:20%;background:#606c76}.spinner div:nth-child(1){transform:rotate(0deg);animation-delay:-1.1s}.spinner div:nth-child(2){transform:rotate(30deg);animation-delay:-1s}.spinner div:nth-child(3){transform:rotate(60deg);animation-delay:-.9s}.spinner div:nth-child(4){transform:rotate(90deg);animation-delay:-.8s}.spinner div:nth-child(5){transform:rotate(120deg);animation-delay:-.7s}.spinner div:nth-child(6){transform:rotate(150deg);animation-delay:-.6s}.spinner div:nth-child(7){transform:rotate(180deg);animation-delay:-.5s}.spinner div:nth-child(8){transform:rotate(210deg);animation-delay:-.4s}.spinner div:nth-child(9){transform:rotate(240deg);animation-delay:-.3s}.spinner div:nth-child(10){transform:rotate(270deg);animation-delay:-.2s}.spinner div:nth-child(11){transform:rotate(300deg);animation-delay:-.1s}.spinner div:nth-child(12){transform:rotate(330deg);animation-delay:0s}@keyframes spinner{0%{opacity:1}100%{opacity:0}}.bg-mask{background-color:black;z-index:1;filter:opacity(20%);position:fixed;width:100%;height:100%;left:0;top:0;display:none}.mark-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.mark-modal .mark-modal__head{margin-bottom:20px}.mark-modal .mark-modal__head::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__title{font-weight:bold;font-size:1.2em;float:left}.mark-modal .mark-modal__close-button{float:right;cursor:pointer}.mark-modal .mark-modal__confirm-button{float:right}.mark-modal input[type="radio"]{margin-right:0}.mark-modal .mark-modal__rating-star{display:inline;float:left;position:relative;left:-3px}.mark-modal .mark-modal__status-radio{float:right}.mark-modal .mark-modal__status-radio ul{margin-bottom:0}.mark-modal .mark-modal__status-radio li,.mark-modal .mark-modal__status-radio label{display:inline}.mark-modal .mark-modal__status-radio input[type="radio"]{position:relative;top:1px}.mark-modal .mark-modal__clear{content:' ';clear:both;display:table}.mark-modal .mark-modal__content-input,.mark-modal form textarea{height:200px;width:100%;margin-top:5px;margin-bottom:5px;resize:vertical}.mark-modal .mark-modal__tag{margin-bottom:20px}.mark-modal .mark-modal__option{margin-bottom:24px}.mark-modal .mark-modal__option::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__visibility-radio{float:left}.mark-modal .mark-modal__visibility-radio ul,.mark-modal .mark-modal__visibility-radio li,.mark-modal .mark-modal__visibility-radio label{display:inline}.mark-modal .mark-modal__visibility-radio label{font-size:normal}.mark-modal .mark-modal__visibility-radio input[type="radio"]{position:relative;top:2px}.mark-modal .mark-modal__share-checkbox{float:right}.mark-modal .mark-modal__share-checkbox input[type="checkbox"]{position:relative;top:2px}.confirm-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.confirm-modal .confirm-modal__head{margin-bottom:20px}.confirm-modal .confirm-modal__head::after{content:' ';clear:both;display:table}.confirm-modal .confirm-modal__title{font-weight:bold;font-size:1.2em;float:left}.confirm-modal .confirm-modal__close-button{float:right;cursor:pointer}.confirm-modal .confirm-modal__confirm-button{float:right}.announcement-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.announcement-modal .announcement-modal__head{margin-bottom:20px}.announcement-modal .announcement-modal__head::after{content:' ';clear:both;display:table}.announcement-modal .announcement-modal__title{font-weight:bold;font-size:1.2em;float:left}.announcement-modal .announcement-modal__close-button{float:right;cursor:pointer}.announcement-modal .announcement-modal__confirm-button{float:right}.announcement-modal .announcement-modal__body{overflow-y:auto;max-height:64vh}.announcement-modal .announcement-modal__body .announcement__title{display:inline-block}.announcement-modal .announcement-modal__body .announcement__datetime{color:#ccc;margin-left:10px}.announcement-modal .announcement-modal__body .announcement__content{word-break:break-all}.add-to-list-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.add-to-list-modal .add-to-list-modal__head{margin-bottom:20px}.add-to-list-modal .add-to-list-modal__head::after{content:' ';clear:both;display:table}.add-to-list-modal .add-to-list-modal__title{font-weight:bold;font-size:1.2em;float:left}.add-to-list-modal .add-to-list-modal__close-button{float:right;cursor:pointer}.add-to-list-modal .add-to-list-modal__confirm-button{float:right}@media (max-width: 575.98px){.mark-modal,.confirm-modal,.announcement-modal .add-to-list-modal{width:100%}}.source-label{display:inline;background:transparent;border-radius:.3rem;border-style:solid;border-width:.1rem;line-height:1.2rem;font-size:1.1rem;margin:3px;padding:1px 3px;padding-top:2px;font-weight:lighter;letter-spacing:0.1rem;word-break:keep-all;opacity:1;position:relative;top:-1px}.source-label.source-label__in-site{border-color:#00a1cc;color:#00a1cc}.source-label.source-label__douban{border:none;color:#fff;background-color:#319840}.source-label.source-label__spotify{background-color:#1ed760;color:#000;border:none;font-weight:bold}.source-label.source-label__imdb{background-color:#F5C518;color:#121212;border:none;font-weight:bold}.source-label.source-label__igdb{background-color:#323A44;color:#DFE1E2;border:none;font-weight:bold}.source-label.source-label__steam{background:linear-gradient(30deg, #1387b8, #111d2e);color:white;border:none;font-weight:600;padding-top:2px}.source-label.source-label__bangumi{background:#FCFCFC;color:#F09199;font-style:italic;font-weight:600}.source-label.source-label__goodreads{background:#F4F1EA;color:#372213;font-weight:lighter}.source-label.source-label__tmdb{background:linear-gradient(90deg, #91CCA3, #1FB4E2);color:white;border:none;font-weight:lighter;padding-top:2px}.source-label.source-label__googlebooks{color:white;background-color:#4285F4;border-color:#4285F4}.source-label.source-label__bandcamp{color:#fff;background-color:#28A0C1;display:inline-block}.source-label.source-label__bandcamp span{display:inline-block;margin:0 4px}.main-section-wrapper{padding:32px 48px 32px 36px;background-color:#f7f7f7;overflow:auto}.main-section-wrapper input,.main-section-wrapper select{width:100%}.entity-list .entity-list__title{margin-bottom:20px}.entity-list .entity-list__entity{display:flex;margin-bottom:36px}.entity-list .entity-list__entity::after{content:' ';clear:both;display:table}.entity-list .entity-list__entity-img{object-fit:contain;min-width:130px;max-width:130px}.entity-list .entity-list__entity-text{margin-left:20px;overflow:hidden;width:100%}.entity-list .entity-list__entity-text .tag-collection{margin-left:-3px}.entity-list .entity-list__entity-link{font-size:1.2em}.entity-list .entity-list__entity-title{display:block}.entity-list .entity-list__entity-category{color:#bbb;margin-left:5px;position:relative;top:-1px}.entity-list .entity-list__entity-info{max-width:73%;white-space:nowrap;overflow:hidden;display:inline-block;text-overflow:ellipsis;position:relative;top:0.52em}.entity-list .entity-list__entity-info--full-length{max-width:100%}.entity-list .entity-list__entity-brief{margin-top:8px;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:4;overflow:hidden;margin-bottom:0}.entity-list .entity-list__rating{display:inline-block;margin:0}.entity-list .entity-list__rating--empty{margin-right:5px}.entity-list .entity-list__rating-score{margin-right:5px;position:relative;top:1px}.entity-list .entity-list__rating-star{display:inline;position:relative;top:0.3em;left:-0.3em}.entity-detail .entity-detail__img{height:210px;float:left;object-fit:contain;max-width:150px;object-position:top}.entity-detail .entity-detail__img-origin{cursor:zoom-in}.entity-detail .entity-detail__info{float:left;margin-left:20px;overflow:hidden;text-overflow:ellipsis;width:70%}.entity-detail .entity-detail__title{font-weight:bold}.entity-detail .entity-detail__title--secondary{color:#bbb}.entity-detail .entity-detail__fields{display:inline-block;vertical-align:top;width:46%;margin-left:2%}.entity-detail .entity-detail__fields div,.entity-detail .entity-detail__fields span{margin:1px 0}.entity-detail .entity-detail__fields+.tag-collection{margin-top:5px;margin-left:6px}.entity-detail .entity-detail__rating{position:relative;top:-5px}.entity-detail .entity-detail__rating-star{position:relative;left:-4px;top:3px}.entity-detail .entity-detail__rating-score{font-weight:bold}.entity-detail::after{content:' ';clear:both;display:table}.entity-desc{margin-bottom:28px}.entity-desc .entity-desc__title{display:inline-block;margin-bottom:8px}.entity-desc .entity-desc__content{overflow:hidden}.entity-desc .entity-desc__content--folded{max-height:202px}.entity-desc .entity-desc__unfold-button{display:flex;color:#00a1cc;background-color:transparent;justify-content:center;text-align:center}.entity-desc .entity-desc__unfold-button--hidden{display:none}.entity-marks{margin-bottom:28px}.entity-marks .entity-marks__title{margin-bottom:8px;display:inline-block}.entity-marks .entity-marks__title>a{margin-right:5px}.entity-marks .entity-marks__title--stand-alone{margin-bottom:20px}.entity-marks .entity-marks__more-link{margin-left:5px}.entity-marks .entity-marks__mark{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-marks .entity-marks__mark:last-child{border:none}.entity-marks .entity-marks__mark--wider{padding:6px 0}.entity-marks .entity-marks__mark-content{margin-bottom:0}.entity-marks .entity-marks__mark-time{color:#ccc;margin-left:2px}.entity-marks .entity-marks__rating-star{position:relative;top:4px}.entity-reviews:first-child{margin-bottom:28px}.entity-reviews .entity-reviews__title{display:inline-block;margin-bottom:8px}.entity-reviews .entity-reviews__title>a{margin-right:5px}.entity-reviews .entity-reviews__title--stand-alone{margin-bottom:20px}.entity-reviews .entity-reviews__more-link{margin-left:5px}.entity-reviews .entity-reviews__review{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-reviews .entity-reviews__review:last-child{border:none}.entity-reviews .entity-reviews__review--wider{padding:6px 0}.entity-reviews .entity-reviews__review-time{color:#ccc;margin-left:2px}.dividing-line{height:0;width:100%;margin:40px 0 24px 0;border-top:solid 1px #ccc}.dividing-line.dividing-line--dashed{margin:0;margin-top:10px;margin-bottom:2px;border-top:1px dashed #e5e5e5}.entity-sort{position:relative;margin-bottom:30px}.entity-sort .entity-sort__label{font-size:large;display:inline-block;margin-bottom:20px}.entity-sort .entity-sort__more-link{margin-left:8px}.entity-sort .entity-sort__count{color:#bbb}.entity-sort .entity-sort__count::before{content:'('}.entity-sort .entity-sort__count::after{content:')'}.entity-sort .entity-sort__entity-list{display:flex;justify-content:flex-start;flex-wrap:wrap}.entity-sort .entity-sort__entity{padding:0 10px;flex-basis:20%;text-align:center;display:inline-block;color:#606c76}.entity-sort .entity-sort__entity:hover{color:#00a1cc}.entity-sort .entity-sort__entity>a{color:inherit}.entity-sort .entity-sort__entity-img{height:110px}.entity-sort .entity-sort__entity-name{text-overflow:ellipsis;overflow:hidden;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:2}.entity-sort--placeholder{border:dashed #bbb 4px}.entity-sort--hover{padding:10px;border:dashed #00a1cc 2px !important;border-radius:3px}.entity-sort--sortable{padding:10px;margin:10px 0;border:dashed #bbb 2px;cursor:all-scroll}.entity-sort--hidden{opacity:0.4}.entity-sort-control{display:flex;justify-content:flex-end}.entity-sort-control__button{margin-top:5px;margin-left:12px;padding:0 2px;cursor:pointer;color:#bbb}.entity-sort-control__button:hover{color:#00a1cc}.entity-sort-control__button:hover>.icon-save svg,.entity-sort-control__button:hover>.icon-edit svg{fill:#00a1cc}.entity-sort-control__button--float-right{position:absolute;top:4px;right:10px}.related-user-list .related-user-list__title{margin-bottom:20px}.related-user-list .related-user-list__user{display:flex;justify-content:flex-start;margin-bottom:20px}.related-user-list .related-user-list__user-info{margin-left:15px;overflow:auto}.related-user-list .related-user-list__user-avatar{max-height:72px;min-width:72px}.review-head .review-head__title{display:inline-block;font-weight:bold}.review-head .review-head__body{margin-bottom:10px}.review-head .review-head__body::after{content:' ';clear:both;display:table}.review-head .review-head__info{float:left}.review-head .review-head__owner-link{color:#ccc}.review-head .review-head__owner-link:hover{color:#00a1cc}.review-head .review-head__time{color:#ccc}.review-head .review-head__rating-star{position:relative;top:3px;left:-1px}.review-head .review-head__actions{float:right}.review-head .review-head__action-link:not(:first-child){margin-left:5px}.tag-collection{margin-left:-9px}.tag-collection .tag-collection__tag{position:relative;display:block;float:left;color:white;background:#ccc;padding:5px;border-radius:.3rem;line-height:1.2em;font-size:80%;margin:3px}.tag-collection .tag-collection__tag a{color:white}.tag-collection .tag-collection__tag a:hover{color:#00a1cc}.track-carousel{position:relative;margin-top:5px}.track-carousel__content{overflow:auto;scroll-behavior:smooth;scrollbar-width:none;display:flex;margin:auto;box-sizing:border-box;padding-bottom:10px}.track-carousel__content::-webkit-scrollbar{height:3px;width:1px;background-color:#e5e5e5}.track-carousel__content::-webkit-scrollbar-thumb{background-color:#bbb}.track-carousel__track{text-align:center;overflow:hidden;text-overflow:ellipsis;min-width:18%;max-width:18%;margin-right:2.5%}.track-carousel__track img{object-fit:contain}.track-carousel__track-title{white-space:nowrap}.track-carousel__button{display:flex;justify-content:center;align-content:center;background:white;border:none;padding:8px;border-radius:50%;outline:0;cursor:pointer;position:absolute;top:50%}.track-carousel__button--prev{left:0;transform:translate(50%, -50%)}.track-carousel__button--next{right:0;transform:translate(-50%, -50%)}@media (max-width: 575.98px){.entity-list .entity-list__entity{flex-direction:column;margin-bottom:30px}.entity-list .entity-list__entity-text{margin-left:0}.entity-list .entity-list__entity-img-wrapper{margin-bottom:8px}.entity-list .entity-list__entity-info{max-width:unset}.entity-list .entity-list__rating--empty+.entity-list__entity-info{max-width:70%}.entity-list .entity-list__entity-brief{-webkit-line-clamp:5}.entity-detail{flex-direction:column}.entity-detail .entity-detail__title{margin-bottom:5px}.entity-detail .entity-detail__info{margin-left:0;float:none;display:flex;flex-direction:column;width:100%}.entity-detail .entity-detail__img{margin-bottom:24px;float:none;height:unset;max-width:170px}.entity-detail .entity-detail__fields{width:unset;margin-left:unset}.entity-detail .entity-detail__fields+.tag-collection{margin-left:-3px}.dividing-line{margin-top:24px}.entity-sort .entity-sort__entity{flex-basis:50%}.entity-sort .entity-sort__entity-img{height:130px}.review-head .review-head__info{float:unset}.review-head .review-head__actions{float:unset}.track-carousel__content{padding-bottom:10px}.track-carousel__track{min-width:31%;max-width:31%;margin-right:4.5%}}@media (max-width: 991.98px){.main-section-wrapper{padding:32px 28px 28px 28px}.entity-detail{display:flex}}.aside-section-wrapper{display:flex;flex:1;flex-direction:column;width:100%;padding:28px 25px 12px 25px;background-color:#f7f7f7;margin-bottom:30px;overflow:auto}.aside-section-wrapper--transparent{background-color:unset}.aside-section-wrapper--collapse{padding:unset}.add-entity-entries .add-entity-entries__entry{margin-bottom:10px}.add-entity-entries .add-entity-entries__label{font-size:1.2em;margin-bottom:8px}.add-entity-entries .add-entity-entries__button{line-height:unset;height:unset;padding:4px 15px;margin:5px}.action-panel{margin-bottom:20px}.action-panel .action-panel__label{font-weight:bold;margin-bottom:12px}.action-panel .action-panel__button-group{display:flex;justify-content:space-between}.action-panel .action-panel__button-group--center{justify-content:center}.action-panel .action-panel__button{line-height:unset;height:unset;padding:4px 15px;margin:0 5px}.mark-panel{margin-bottom:20px}.mark-panel .mark-panel__status{font-weight:bold}.mark-panel .mark-panel__rating-star{position:relative;top:2px}.mark-panel .mark-panel__actions{float:right}.mark-panel .mark-panel__actions form{display:inline}.mark-panel .mark-panel__time{color:#ccc;margin-bottom:10px}.mark-panel .mark-panel__clear{content:' ';clear:both;display:table}.review-panel .review-panel__label{font-weight:bold}.review-panel .review-panel__actions{float:right}.review-panel .review-panel__time{color:#ccc;margin-bottom:10px}.review-panel .review-panel__review-title{display:block;margin-bottom:15px;font-weight:bold}.review-panel .review-panel__clear{content:' ';clear:both;display:table}.user-profile .user-profile__header{display:flex;align-items:flex-start;margin-bottom:15px}.user-profile .user-profile__avatar{width:72px}.user-profile .user-profile__username{font-size:large;margin-left:10px;margin-bottom:0}.user-profile .user-profile__report-link{color:#ccc}.user-relation .user-relation__label{display:inline-block;font-size:large;margin-bottom:10px}.user-relation .user-relation__more-link{margin-left:5px}.user-relation .user-relation__related-user-list{display:flex;justify-content:flex-start}.user-relation .user-relation__related-user-list:last-of-type{margin-bottom:0}.user-relation .user-relation__related-user{flex-basis:25%;padding:0px 3px;text-align:center;display:inline-block;overflow:hidden}.user-relation .user-relation__related-user>a:hover{color:#606c76}.user-relation .user-relation__related-user-avatar{background-image:url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");width:48px;height:48px}@media (min-width: 575.98px) and (max-width: 991.98px){.user-relation .user-relation__related-user-avatar{height:unset;width:60%;max-width:96px}}.user-relation .user-relation__related-user-name{color:inherit;overflow:hidden;text-overflow:ellipsis;-webkit-box-orient:vertical;-webkit-line-clamp:2}.report-panel .report-panel__label{display:inline-block;margin-bottom:10px}.report-panel .report-panel__body{padding-left:0}.report-panel .report-panel__report{margin:2px 0}.report-panel .report-panel__user-link{margin:0 2px}.report-panel .report-panel__all-link{margin-left:5px}.import-panel{overflow-x:hidden}.import-panel .import-panel__label{display:inline-block;margin-bottom:10px}.import-panel .import-panel__body{padding-left:0;border:2px dashed #00a1cc;padding:6px 9px}.import-panel .import-panel__body form{margin:0}@media (max-width: 991.98px){.import-panel .import-panel__body{border:unset;padding-left:0}}.import-panel .import-panel__help{background-color:#e5e5e5;border-radius:100000px;display:inline-block;width:16px;height:16px;text-align:center;font-size:12px;cursor:help}.import-panel .import-panel__checkbox{display:inline-block;margin-right:10px}.import-panel .import-panel__checkbox label{display:inline}.import-panel .import-panel__checkbox input[type="checkbox"]{margin:0;position:relative;top:2px}.import-panel .import-panel__checkbox--last{margin-right:0}.import-panel .import-panel__file-input{margin-top:10px}.import-panel .import-panel__button{line-height:unset;height:unset;padding:4px 15px}.import-panel .import-panel__progress{padding-top:10px}.import-panel .import-panel__progress:not(:first-child){border-top:#bbb 1px dashed}.import-panel .import-panel__progress label{display:inline}.import-panel .import-panel__progress progress{background-color:#d5d5d5;border-radius:0;height:10px;width:54%}.import-panel .import-panel__progress progress::-webkit-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__progress progress::-webkit-progress-value{background-color:#00a1cc}.import-panel .import-panel__progress progress::-moz-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__last-task:not(:first-child){padding-top:4px;border-top:#bbb 1px dashed}.import-panel .import-panel__last-task .index:not(:last-of-type){margin-right:8px}.import-panel .import-panel__fail-urls{margin-top:10px}.import-panel .import-panel__fail-urls li{word-break:break-all}.import-panel .import-panel__fail-urls ul{max-height:100px;overflow-y:auto}.relation-dropdown .relation-dropdown__button{display:none}.entity-card{display:flex;margin-bottom:10px;flex-direction:column}.entity-card--horizontal{flex-direction:row}.entity-card .entity-card__img{height:150px}.entity-card .entity-card__rating-star{position:relative;top:4px;left:-3px}.entity-card .entity-card__rating-score{position:relative;top:1px;margin-left:2px}.entity-card .entity-card__title{margin-bottom:10px;margin-top:5px}.entity-card .entity-card__info-wrapper--horizontal{margin-left:20px}.entity-card .entity-card__img-wrapper{flex-basis:100px}@media (max-width: 575.98px){.add-entity-entries{display:block !important}.add-entity-entries .add-entity-entries__button{width:100%;margin:5px 0 5px 0}.aside-section-wrapper:first-child{margin-right:0 !important;margin-bottom:0 !important}.aside-section-wrapper--singular:first-child{margin-bottom:20px !important}.action-panel{flex-direction:column !important}.entity-card--horizontal{flex-direction:column !important}.entity-card .entity-card__info-wrapper{margin-left:10px !important}.entity-card .entity-card__info-wrapper--horizontal{margin-left:0 !important}}@media (max-width: 991.98px){.add-entity-entries{display:flex;justify-content:space-around}.aside-section-wrapper{padding:24px 25px 10px 25px;margin-top:20px}.aside-section-wrapper:not(:last-child){margin-right:20px}.aside-section-wrapper--collapse{padding:24px 25px 10px 25px !important;margin-top:0;margin-bottom:0}.aside-section-wrapper--collapse:first-child{margin-right:0}.aside-section-wrapper--no-margin{margin:0}.action-panel{flex-direction:row}.action-panel .action-panel__button-group{justify-content:space-evenly}.relation-dropdown{margin-bottom:20px}.relation-dropdown .relation-dropdown__button{padding-bottom:10px;background-color:#f7f7f7;width:100%;display:flex;justify-content:center;align-items:center;cursor:pointer;transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:focus{background-color:red}.relation-dropdown .relation-dropdown__button>.icon-arrow{transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:hover>.icon-arrow>svg{fill:#00a1cc}.relation-dropdown .relation-dropdown__button>.icon-arrow--expand{transform:rotate(-180deg)}.relation-dropdown .relation-dropdown__button+.relation-dropdown__body--expand{max-height:2000px;transition:max-height 1s ease-in}.relation-dropdown .relation-dropdown__body{background-color:#f7f7f7;max-height:0;transition:max-height 1s ease-out;overflow:hidden}.entity-card{flex-direction:row}.entity-card .entity-card__info-wrapper{margin-left:30px}}.single-section-wrapper{padding:32px 36px;background-color:#f7f7f7;overflow:auto}.single-section-wrapper .single-section-wrapper__link--secondary{display:inline-block;color:#ccc;margin-bottom:20px}.single-section-wrapper .single-section-wrapper__link--secondary:hover{color:#00a1cc}.entity-form,.review-form{overflow:auto}.entity-form>input[type='email'],.entity-form>input[type='number'],.entity-form>input[type='password'],.entity-form>input[type='search'],.entity-form>input[type='tel'],.entity-form>input[type='text'],.entity-form>input[type='url'],.entity-form textarea,.review-form>input[type='email'],.review-form>input[type='number'],.review-form>input[type='password'],.review-form>input[type='search'],.review-form>input[type='tel'],.review-form>input[type='text'],.review-form>input[type='url'],.review-form textarea{width:100%}.entity-form img,.review-form img{display:block}.review-form .review-form__preview-button{color:#00a1cc;font-weight:bold;cursor:pointer}.review-form .review-form__fyi{color:#ccc}.review-form .review-form__main-content,.review-form textarea{margin-bottom:5px;resize:vertical;height:400px}.review-form .review-form__option{margin-top:24px;margin-bottom:10px}.review-form .review-form__option::after{content:' ';clear:both;display:table}.review-form .review-form__visibility-radio{float:left}.review-form .review-form__visibility-radio ul,.review-form .review-form__visibility-radio li,.review-form .review-form__visibility-radio label{display:inline}.review-form .review-form__visibility-radio label{font-size:normal}.review-form .review-form__visibility-radio input[type="radio"]{position:relative;top:2px}.review-form .review-form__share-checkbox{float:right}.review-form .review-form__share-checkbox input[type="checkbox"]{position:relative;top:2px}.report-form input,.report-form select{width:100%}@media (max-width: 575.98px){.review-form .review-form__visibility-radio{float:unset}.review-form .review-form__share-checkbox{float:unset;position:relative;left:-3px}}.markdownx-preview{min-height:100px}.markdownx-preview ul li{list-style:circle inside}.markdownx-preview h1{font-size:2.5em}.markdownx-preview h2{font-size:2.0em}.markdownx-preview blockquote{border-left:lightgray solid 0.4em;padding-left:0.4em}.rating-star .jq-star{cursor:unset !important}.ms-parent>.ms-choice{margin-bottom:1.5rem;appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem;width:100%;height:30.126px}.ms-parent>.ms-choice:focus{border-color:#00a1cc}.ms-parent>.ms-choice>.icon-caret{top:15.5px}.ms-parent>.ms-choice>span{color:black;font-weight:initial;font-size:13.3333px;top:2.5px;left:2px}.ms-parent>.ms-choice>span:hover,.ms-parent>.ms-choice>span:focus{color:black}.ms-parent>.ms-drop>ul>li>label>span{margin-left:10px}.ms-parent>.ms-drop>ul>li>label>input{width:unset}.tippy-box{border:#606c76 1px solid;background-color:#f7f7f7;padding:3px 5px}.tag-input input{flex-grow:1}.tools-section-wrapper input,.tools-section-wrapper select{width:unset} +@import url(https://cdn.jsdelivr.net/npm/skeleton-css@2.0.4/css/normalize.css);.button,button,input[type='button'],input[type='reset'],input[type='submit']{background-color:#00a1cc;border:0.1rem solid #00a1cc;border-radius:.4rem;color:#fff;cursor:pointer;display:inline-block;font-size:1.1rem;font-weight:700;height:3.4rem;letter-spacing:.1rem;line-height:3.4rem;padding:0 2.8rem;text-align:center;text-decoration:none;text-transform:uppercase;white-space:nowrap}.button:focus,.button:hover,button:focus,button:hover,input[type='button']:focus,input[type='button']:hover,input[type='reset']:focus,input[type='reset']:hover,input[type='submit']:focus,input[type='submit']:hover{background-color:#606c76;border-color:#606c76;color:#fff;outline:0}.button[disabled],button[disabled],input[type='button'][disabled],input[type='reset'][disabled],input[type='submit'][disabled]{cursor:default;opacity:.5}.button[disabled]:focus,.button[disabled]:hover,button[disabled]:focus,button[disabled]:hover,input[type='button'][disabled]:focus,input[type='button'][disabled]:hover,input[type='reset'][disabled]:focus,input[type='reset'][disabled]:hover,input[type='submit'][disabled]:focus,input[type='submit'][disabled]:hover{background-color:#00a1cc;border-color:#00a1cc}.button.button-outline,button.button-outline,input[type='button'].button-outline,input[type='reset'].button-outline,input[type='submit'].button-outline{background-color:transparent;color:#00a1cc}.button.button-outline:focus,.button.button-outline:hover,button.button-outline:focus,button.button-outline:hover,input[type='button'].button-outline:focus,input[type='button'].button-outline:hover,input[type='reset'].button-outline:focus,input[type='reset'].button-outline:hover,input[type='submit'].button-outline:focus,input[type='submit'].button-outline:hover{background-color:transparent;border-color:#606c76;color:#606c76}.button.button-outline[disabled]:focus,.button.button-outline[disabled]:hover,button.button-outline[disabled]:focus,button.button-outline[disabled]:hover,input[type='button'].button-outline[disabled]:focus,input[type='button'].button-outline[disabled]:hover,input[type='reset'].button-outline[disabled]:focus,input[type='reset'].button-outline[disabled]:hover,input[type='submit'].button-outline[disabled]:focus,input[type='submit'].button-outline[disabled]:hover{border-color:inherit;color:#00a1cc}.button.button-clear,button.button-clear,input[type='button'].button-clear,input[type='reset'].button-clear,input[type='submit'].button-clear{background-color:transparent;border-color:transparent;color:#00a1cc}.button.button-clear:focus,.button.button-clear:hover,button.button-clear:focus,button.button-clear:hover,input[type='button'].button-clear:focus,input[type='button'].button-clear:hover,input[type='reset'].button-clear:focus,input[type='reset'].button-clear:hover,input[type='submit'].button-clear:focus,input[type='submit'].button-clear:hover{background-color:transparent;border-color:transparent;color:#606c76}.button.button-clear[disabled]:focus,.button.button-clear[disabled]:hover,button.button-clear[disabled]:focus,button.button-clear[disabled]:hover,input[type='button'].button-clear[disabled]:focus,input[type='button'].button-clear[disabled]:hover,input[type='reset'].button-clear[disabled]:focus,input[type='reset'].button-clear[disabled]:hover,input[type='submit'].button-clear[disabled]:focus,input[type='submit'].button-clear[disabled]:hover{color:#00a1cc}select{background:url('data:image/svg+xml;utf8,') center right no-repeat;padding-right:3.0rem}select:focus{background-image:url('data:image/svg+xml;utf8,')}textarea{min-height:6.5rem;width:100%}select{width:100%}label,legend{display:block;margin-bottom:.5rem}fieldset{border-width:0;padding:0}input[type='checkbox'],input[type='radio']{display:inline}.label-inline{display:inline-block;font-weight:normal;margin-left:.5rem}dl,ol,ul{list-style:none;margin-top:0;padding-left:0}ol{list-style:decimal inside}ul{list-style:circle inside}.button,button,dd,dt,li{margin-bottom:1.0rem}fieldset,input,select,textarea{margin-bottom:1.5rem}blockquote,dl,figure,form,ol,p,pre,table,ul{margin-bottom:1rem}b,strong{font-weight:bold}p{margin-top:0}h1,h2,h3,h4,h5,h6{font-weight:300;letter-spacing:-.1rem;margin-bottom:2.0rem;margin-top:0}h1{font-size:4.6rem;line-height:1.2}h2{font-size:3.6rem;line-height:1.25}h3{font-size:2.8rem;line-height:1.3}h4{font-size:2.2rem;letter-spacing:-.08rem;line-height:1.35}h5{font-size:1.8rem;letter-spacing:-.05rem;line-height:1.5}h6{font-size:1.6rem;letter-spacing:0;line-height:1.4}img{max-width:100%;object-fit:contain}img.emoji{height:14px;box-sizing:border-box;object-fit:contain;position:relative;top:3px}img.emoji--large{height:20px;position:relative;top:2px}.clearfix:after{clear:both;content:' ';display:table}.float-left{float:left}.float-right{float:right}.highlight{font-weight:bold}:root{font-size:10px}*,*:after,*:before{box-sizing:inherit}html{box-sizing:border-box;height:100%}body{color:#606c76;font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif;font-size:1.3rem;font-weight:300;letter-spacing:.05rem;line-height:1.6;margin:0;height:100%}textarea{font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif}a{color:#00a1cc;text-decoration:none}a:active,a:hover,a:hover:visited{color:#606c76}li{list-style:none}input[type=text]::-ms-clear,input[type=text]::-ms-reveal{display:none;width:0;height:0}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-results-button,input[type="search"]::-webkit-search-results-decoration{display:none}input[type='email'],input[type='number'],input[type='password'],input[type='search'],input[type='tel'],input[type='text'],input[type='url'],input[type='date'],input[type='time'],input[type='color'],textarea,select{appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem}input[type='email']:focus,input[type='number']:focus,input[type='password']:focus,input[type='search']:focus,input[type='tel']:focus,input[type='text']:focus,input[type='url']:focus,input[type='date']:focus,input[type='time']:focus,input[type='color']:focus,textarea:focus,select:focus{border-color:#00a1cc;outline:0}input[type='email']::placeholder,input[type='number']::placeholder,input[type='password']::placeholder,input[type='search']::placeholder,input[type='tel']::placeholder,input[type='text']::placeholder,input[type='url']::placeholder,input[type='date']::placeholder,input[type='time']::placeholder,input[type='color']::placeholder,textarea::placeholder,select::placeholder{color:#ccc}::selection{color:white;background-color:#00a1cc}.navbar{background-color:#f7f7f7;box-sizing:border-box;padding:10px 0;margin-bottom:50px;border-bottom:#ccc 0.5px solid}.navbar .navbar__wrapper{display:flex;justify-content:space-between;align-items:center;position:relative}.navbar .navbar__logo{flex-basis:100px}.navbar .navbar__logo-link{display:inline-block}.navbar .navbar__link-list{margin:0;display:flex;justify-content:space-around}.navbar .navbar__link{margin:9px;color:#606c76}.navbar .navbar__link:active,.navbar .navbar__link:hover,.navbar .navbar__link:hover:visited{color:#00a1cc}.navbar .navbar__link:visited{color:#606c76}.navbar .current{color:#00a1cc;font-weight:bold}.navbar .navbar__search-box{margin:0 12% 0 15px;display:inline-flex;flex:1}.navbar .navbar__search-box>input[type="search"]{border-top-right-radius:0;border-bottom-right-radius:0;margin:0;height:32px;background-color:white !important;width:100%}.navbar .navbar__search-box .navbar__search-dropdown{margin:0;margin-left:-1px;padding:0;padding-left:10px;color:#606c76;appearance:auto;background-color:white;height:32px;width:80px;border-top-left-radius:0;border-bottom-left-radius:0}.navbar .navbar__dropdown-btn{display:none;padding:0;margin:0;border:none;background-color:transparent;color:#00a1cc}.navbar .navbar__dropdown-btn:focus,.navbar .navbar__dropdown-btn:hover{background-color:transparent;color:#606c76}@media (max-width: 575.98px){.navbar{padding:2px 0}.navbar .navbar__wrapper{display:block}.navbar .navbar__logo-img{width:72px;margin-right:10px;position:relative;top:7px}.navbar .navbar__link-list{margin-top:7px;max-height:0;transition:max-height 0.6s ease-out;overflow:hidden}.navbar .navbar__dropdown-btn{display:block;position:absolute;right:5px;top:3px;transform:scale(0.7)}.navbar .navbar__dropdown-btn:hover+.navbar__link-list{max-height:500px;transition:max-height 0.6s ease-in}.navbar .navbar__search-box{margin:0;width:46vw}.navbar .navbar__search-box>input[type="search"]{height:26px;padding:4px 6px;width:32vw}.navbar .navbar__search-box .navbar__search-dropdown{cursor:pointer;height:26px;width:80px;padding-left:5px}}@media (max-width: 991.98px){.navbar{margin-bottom:20px}}.grid{margin:0 auto;position:relative;max-width:110rem;padding:0 2.0rem;width:100%}.grid .grid__main{width:70%;float:left;position:relative}.grid .grid__aside{width:26%;float:right;position:relative;display:flex;flex-direction:column;justify-content:space-around}.grid::after{content:' ';clear:both;display:table}@media (max-width: 575.98px){.grid .grid__aside{flex-direction:column !important}}@media (max-width: 991.98px){.grid .grid__main{width:100%;float:none}.grid .grid__aside{width:100%;float:none;flex-direction:row}.grid .grid__aside--tablet-column{flex-direction:column}.grid--reverse-order{transform:scaleY(-1)}.grid .grid__main--reverse-order{transform:scaleY(-1)}.grid .grid__aside--reverse-order{transform:scaleY(-1)}}.pagination{text-align:center;width:100%}.pagination .pagination__page-link{font-weight:normal;margin:0 5px}.pagination .pagination__page-link--current{font-weight:bold;font-size:1.2em;color:#606c76}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:18px}.pagination .pagination__nav-link--left-margin{margin-left:18px}.pagination .pagination__nav-link--hidden{display:none}@media (max-width: 575.98px){.pagination .pagination__page-link{margin:0 3px}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:10px}.pagination .pagination__nav-link--left-margin{margin-left:10px}}#page-wrapper{position:relative;min-height:100vh;z-index:0}#content-wrapper{padding-bottom:160px}.footer{padding-top:0.4em !important;text-align:center;margin-bottom:4px !important;position:absolute !important;left:50%;transform:translateX(-50%);bottom:0;width:100%}.footer__border{padding-top:4px;border-top:#f7f7f7 solid 2px}.footer__link{margin:0 12px;white-space:nowrap}@media (max-width: 575.98px){#content-wrapper{padding-bottom:120px}}.icon-lock svg{fill:#ccc;height:12px;position:relative;top:1px;margin-left:3px}.icon-edit svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-save svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-cross svg{fill:#ccc;height:10px;position:relative}.icon-arrow svg{fill:#606c76;height:15px;position:relative;top:3px}.spinner{display:inline-block;position:relative;left:50%;transform:translateX(-50%) scale(0.4);width:80px;height:80px}.spinner div{transform-origin:40px 40px;animation:spinner 1.2s linear infinite}.spinner div::after{content:" ";display:block;position:absolute;top:3px;left:37px;width:6px;height:18px;border-radius:20%;background:#606c76}.spinner div:nth-child(1){transform:rotate(0deg);animation-delay:-1.1s}.spinner div:nth-child(2){transform:rotate(30deg);animation-delay:-1s}.spinner div:nth-child(3){transform:rotate(60deg);animation-delay:-.9s}.spinner div:nth-child(4){transform:rotate(90deg);animation-delay:-.8s}.spinner div:nth-child(5){transform:rotate(120deg);animation-delay:-.7s}.spinner div:nth-child(6){transform:rotate(150deg);animation-delay:-.6s}.spinner div:nth-child(7){transform:rotate(180deg);animation-delay:-.5s}.spinner div:nth-child(8){transform:rotate(210deg);animation-delay:-.4s}.spinner div:nth-child(9){transform:rotate(240deg);animation-delay:-.3s}.spinner div:nth-child(10){transform:rotate(270deg);animation-delay:-.2s}.spinner div:nth-child(11){transform:rotate(300deg);animation-delay:-.1s}.spinner div:nth-child(12){transform:rotate(330deg);animation-delay:0s}@keyframes spinner{0%{opacity:1}100%{opacity:0}}.bg-mask{background-color:black;z-index:1;filter:opacity(20%);position:fixed;width:100%;height:100%;left:0;top:0;display:none}.mark-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.mark-modal .mark-modal__head{margin-bottom:20px}.mark-modal .mark-modal__head::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__title{font-weight:bold;font-size:1.2em;float:left}.mark-modal .mark-modal__close-button{float:right;cursor:pointer}.mark-modal .mark-modal__confirm-button{float:right}.mark-modal input[type="radio"]{margin-right:0}.mark-modal .mark-modal__rating-star{display:inline;float:left;position:relative;left:-3px}.mark-modal .mark-modal__status-radio{float:right}.mark-modal .mark-modal__status-radio ul{margin-bottom:0}.mark-modal .mark-modal__status-radio li,.mark-modal .mark-modal__status-radio label{display:inline}.mark-modal .mark-modal__status-radio input[type="radio"]{position:relative;top:1px}.mark-modal .mark-modal__clear{content:' ';clear:both;display:table}.mark-modal .mark-modal__content-input,.mark-modal form textarea{height:200px;width:100%;margin-top:5px;margin-bottom:5px;resize:vertical}.mark-modal .mark-modal__tag{margin-bottom:20px}.mark-modal .mark-modal__option{margin-bottom:24px}.mark-modal .mark-modal__option::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__visibility-radio{float:left}.mark-modal .mark-modal__visibility-radio ul,.mark-modal .mark-modal__visibility-radio li,.mark-modal .mark-modal__visibility-radio label{display:inline}.mark-modal .mark-modal__visibility-radio label{font-size:normal}.mark-modal .mark-modal__visibility-radio input[type="radio"]{position:relative;top:2px}.mark-modal .mark-modal__share-checkbox{float:right}.mark-modal .mark-modal__share-checkbox input[type="checkbox"]{position:relative;top:2px}.confirm-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.confirm-modal .confirm-modal__head{margin-bottom:20px}.confirm-modal .confirm-modal__head::after{content:' ';clear:both;display:table}.confirm-modal .confirm-modal__title{font-weight:bold;font-size:1.2em;float:left}.confirm-modal .confirm-modal__close-button{float:right;cursor:pointer}.confirm-modal .confirm-modal__confirm-button{float:right}.announcement-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.announcement-modal .announcement-modal__head{margin-bottom:20px}.announcement-modal .announcement-modal__head::after{content:' ';clear:both;display:table}.announcement-modal .announcement-modal__title{font-weight:bold;font-size:1.2em;float:left}.announcement-modal .announcement-modal__close-button{float:right;cursor:pointer}.announcement-modal .announcement-modal__confirm-button{float:right}.announcement-modal .announcement-modal__body{overflow-y:auto;max-height:64vh}.announcement-modal .announcement-modal__body .announcement__title{display:inline-block}.announcement-modal .announcement-modal__body .announcement__datetime{color:#ccc;margin-left:10px}.announcement-modal .announcement-modal__body .announcement__content{word-break:break-all}.add-to-list-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.add-to-list-modal .add-to-list-modal__head{margin-bottom:20px}.add-to-list-modal .add-to-list-modal__head::after{content:' ';clear:both;display:table}.add-to-list-modal .add-to-list-modal__title{font-weight:bold;font-size:1.2em;float:left}.add-to-list-modal .add-to-list-modal__close-button{float:right;cursor:pointer}.add-to-list-modal .add-to-list-modal__confirm-button{float:right}@media (max-width: 575.98px){.mark-modal,.confirm-modal,.announcement-modal .add-to-list-modal{width:100%}}.source-label{display:inline;background:transparent;border-radius:.3rem;border-style:solid;border-width:.1rem;line-height:1.2rem;font-size:1.1rem;margin:3px;padding:1px 3px;padding-top:2px;font-weight:lighter;letter-spacing:0.1rem;word-break:keep-all;opacity:1;position:relative;top:-1px}.source-label.source-label__in-site{border-color:#00a1cc;color:#00a1cc}.source-label.source-label__douban{border:none;color:#fff;background-color:#319840}.source-label.source-label__spotify{background-color:#1ed760;color:#000;border:none;font-weight:bold}.source-label.source-label__imdb{background-color:#F5C518;color:#121212;border:none;font-weight:bold}.source-label.source-label__igdb{background-color:#323A44;color:#DFE1E2;border:none;font-weight:bold}.source-label.source-label__steam{background:linear-gradient(30deg, #1387b8, #111d2e);color:white;border:none;font-weight:600;padding-top:2px}.source-label.source-label__bangumi{background:#FCFCFC;color:#F09199;font-style:italic;font-weight:600}.source-label.source-label__goodreads{background:#F4F1EA;color:#372213;font-weight:lighter}.source-label.source-label__tmdb{background:linear-gradient(90deg, #91CCA3, #1FB4E2);color:white;border:none;font-weight:lighter;padding-top:2px}.source-label.source-label__googlebooks{color:white;background-color:#4285F4;border-color:#4285F4}.source-label.source-label__bandcamp{color:#fff;background-color:#28A0C1;display:inline-block}.source-label.source-label__bandcamp span{display:inline-block;margin:0 4px}.main-section-wrapper{padding:32px 48px 32px 36px;background-color:#f7f7f7;overflow:auto}.main-section-wrapper input,.main-section-wrapper select{width:100%}.entity-list .entity-list__title{margin-bottom:20px}.entity-list .entity-list__entity{display:flex;margin-bottom:36px}.entity-list .entity-list__entity::after{content:' ';clear:both;display:table}.entity-list .entity-list__entity-img{object-fit:contain;min-width:130px;max-width:130px}.entity-list .entity-list__entity-text{margin-left:20px;overflow:hidden;width:100%}.entity-list .entity-list__entity-text .tag-collection{margin-left:-3px}.entity-list .entity-list__entity-link{font-size:1.2em}.entity-list .entity-list__entity-title{display:block}.entity-list .entity-list__entity-category{color:#bbb;margin-left:5px;position:relative;top:-1px}.entity-list .entity-list__entity-info{max-width:73%;white-space:nowrap;overflow:hidden;display:inline-block;text-overflow:ellipsis;position:relative;top:0.52em}.entity-list .entity-list__entity-info--full-length{max-width:100%}.entity-list .entity-list__entity-brief{margin-top:8px;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:4;overflow:hidden;margin-bottom:0}.entity-list .entity-list__rating{display:inline-block;margin:0}.entity-list .entity-list__rating--empty{margin-right:5px}.entity-list .entity-list__rating-score{margin-right:5px;position:relative;top:1px}.entity-list .entity-list__rating-star{display:inline;position:relative;top:0.3em;left:-0.3em}.entity-detail .entity-detail__img{height:210px;float:left;object-fit:contain;max-width:150px;object-position:top}.entity-detail .entity-detail__img-origin{cursor:zoom-in}.entity-detail .entity-detail__info{float:left;margin-left:20px;overflow:hidden;text-overflow:ellipsis;width:70%}.entity-detail .entity-detail__title{font-weight:bold}.entity-detail .entity-detail__title--secondary{color:#bbb}.entity-detail .entity-detail__fields{display:inline-block;vertical-align:top;width:46%;margin-left:2%}.entity-detail .entity-detail__fields div,.entity-detail .entity-detail__fields span{margin:1px 0}.entity-detail .entity-detail__fields+.tag-collection{margin-top:5px;margin-left:6px}.entity-detail .entity-detail__rating{position:relative;top:-5px}.entity-detail .entity-detail__rating-star{position:relative;left:-4px;top:3px}.entity-detail .entity-detail__rating-score{font-weight:bold}.entity-detail::after{content:' ';clear:both;display:table}.entity-desc{margin-bottom:28px}.entity-desc .entity-desc__title{display:inline-block;margin-bottom:8px}.entity-desc .entity-desc__content{overflow:hidden}.entity-desc .entity-desc__content--folded{max-height:202px}.entity-desc .entity-desc__unfold-button{display:flex;color:#00a1cc;background-color:transparent;justify-content:center;text-align:center}.entity-desc .entity-desc__unfold-button--hidden{display:none}.entity-marks{margin-bottom:28px}.entity-marks .entity-marks__title{margin-bottom:8px;display:inline-block}.entity-marks .entity-marks__title>a{margin-right:5px}.entity-marks .entity-marks__title--stand-alone{margin-bottom:20px}.entity-marks .entity-marks__more-link{margin-left:5px}.entity-marks .entity-marks__mark{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-marks .entity-marks__mark:last-child{border:none}.entity-marks .entity-marks__mark--wider{padding:6px 0}.entity-marks .entity-marks__mark-content{margin-bottom:0}.entity-marks .entity-marks__mark-time{color:#ccc;margin-left:2px}.entity-marks .entity-marks__rating-star{position:relative;top:4px}.entity-reviews:first-child{margin-bottom:28px}.entity-reviews .entity-reviews__title{display:inline-block;margin-bottom:8px}.entity-reviews .entity-reviews__title>a{margin-right:5px}.entity-reviews .entity-reviews__title--stand-alone{margin-bottom:20px}.entity-reviews .entity-reviews__more-link{margin-left:5px}.entity-reviews .entity-reviews__review{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-reviews .entity-reviews__review:last-child{border:none}.entity-reviews .entity-reviews__review--wider{padding:6px 0}.entity-reviews .entity-reviews__review-time{color:#ccc;margin-left:2px}.dividing-line{height:0;width:100%;margin:40px 0 24px 0;border-top:solid 1px #ccc}.dividing-line.dividing-line--dashed{margin:0;margin-top:10px;margin-bottom:2px;border-top:1px dashed #e5e5e5}.entity-sort{position:relative;margin-bottom:30px}.entity-sort .entity-sort__label{font-size:large;display:inline-block;margin-bottom:20px}.entity-sort .entity-sort__more-link{margin-left:8px}.entity-sort .entity-sort__count{color:#bbb}.entity-sort .entity-sort__count::before{content:'('}.entity-sort .entity-sort__count::after{content:')'}.entity-sort .entity-sort__entity-list{display:flex;justify-content:flex-start;flex-wrap:wrap}.entity-sort .entity-sort__entity{padding:0 10px;flex-basis:20%;text-align:center;display:inline-block;color:#606c76}.entity-sort .entity-sort__entity:hover{color:#00a1cc}.entity-sort .entity-sort__entity>a{color:inherit}.entity-sort .entity-sort__entity-img{height:110px}.entity-sort .entity-sort__entity-name{text-overflow:ellipsis;overflow:hidden;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:2}.entity-sort--placeholder{border:dashed #bbb 4px}.entity-sort--hover{padding:10px;border:dashed #00a1cc 2px !important;border-radius:3px}.entity-sort--sortable{padding:10px;margin:10px 0;border:dashed #bbb 2px;cursor:all-scroll}.entity-sort--hidden{opacity:0.4}.entity-sort-control{display:flex;justify-content:flex-end}.entity-sort-control__button{margin-top:5px;margin-left:12px;padding:0 2px;cursor:pointer;color:#bbb}.entity-sort-control__button:hover{color:#00a1cc}.entity-sort-control__button:hover>.icon-save svg,.entity-sort-control__button:hover>.icon-edit svg{fill:#00a1cc}.entity-sort-control__button--float-right{position:absolute;top:4px;right:10px}.related-user-list .related-user-list__title{margin-bottom:20px}.related-user-list .related-user-list__user{display:flex;justify-content:flex-start;margin-bottom:20px}.related-user-list .related-user-list__user-info{margin-left:15px;overflow:auto}.related-user-list .related-user-list__user-avatar{max-height:72px;min-width:72px}.review-head .review-head__title{display:inline-block;font-weight:bold}.review-head .review-head__body{margin-bottom:10px}.review-head .review-head__body::after{content:' ';clear:both;display:table}.review-head .review-head__info{float:left}.review-head .review-head__owner-link{color:#ccc}.review-head .review-head__owner-link:hover{color:#00a1cc}.review-head .review-head__time{color:#ccc}.review-head .review-head__rating-star{position:relative;top:3px;left:-1px}.review-head .review-head__actions{float:right}.review-head .review-head__action-link:not(:first-child){margin-left:5px}.tag-collection{margin-left:-9px}.tag-collection .tag-collection__tag{position:relative;display:block;float:left;color:white;background:#ccc;padding:5px;border-radius:.3rem;line-height:1.2em;font-size:80%;margin:3px}.tag-collection .tag-collection__tag a{color:white}.tag-collection .tag-collection__tag a:hover{color:#00a1cc}.track-carousel{position:relative;margin-top:5px}.track-carousel__content{overflow:auto;scroll-behavior:smooth;scrollbar-width:none;display:flex;margin:auto;box-sizing:border-box;padding-bottom:10px}.track-carousel__content::-webkit-scrollbar{height:3px;width:1px;background-color:#e5e5e5}.track-carousel__content::-webkit-scrollbar-thumb{background-color:#bbb}.track-carousel__track{text-align:center;overflow:hidden;text-overflow:ellipsis;min-width:18%;max-width:18%;margin-right:2.5%}.track-carousel__track img{object-fit:contain}.track-carousel__track-title{white-space:nowrap}.track-carousel__button{display:flex;justify-content:center;align-content:center;background:white;border:none;padding:8px;border-radius:50%;outline:0;cursor:pointer;position:absolute;top:50%}.track-carousel__button--prev{left:0;transform:translate(50%, -50%)}.track-carousel__button--next{right:0;transform:translate(-50%, -50%)}@media (max-width: 575.98px){.entity-list .entity-list__entity{flex-direction:column;margin-bottom:30px}.entity-list .entity-list__entity-text{margin-left:0}.entity-list .entity-list__entity-img-wrapper{margin-bottom:8px}.entity-list .entity-list__entity-info{max-width:unset}.entity-list .entity-list__rating--empty+.entity-list__entity-info{max-width:70%}.entity-list .entity-list__entity-brief{-webkit-line-clamp:5}.entity-detail{flex-direction:column}.entity-detail .entity-detail__title{margin-bottom:5px}.entity-detail .entity-detail__info{margin-left:0;float:none;display:flex;flex-direction:column;width:100%}.entity-detail .entity-detail__img{margin-bottom:24px;float:none;height:unset;max-width:170px}.entity-detail .entity-detail__fields{width:unset;margin-left:unset}.entity-detail .entity-detail__fields+.tag-collection{margin-left:-3px}.dividing-line{margin-top:24px}.entity-sort .entity-sort__entity{flex-basis:50%}.entity-sort .entity-sort__entity-img{height:130px}.review-head .review-head__info{float:unset}.review-head .review-head__actions{float:unset}.track-carousel__content{padding-bottom:10px}.track-carousel__track{min-width:31%;max-width:31%;margin-right:4.5%}}@media (max-width: 991.98px){.main-section-wrapper{padding:32px 28px 28px 28px}.entity-detail{display:flex}}.aside-section-wrapper{display:flex;flex:1;flex-direction:column;width:100%;padding:28px 25px 12px 25px;background-color:#f7f7f7;margin-bottom:30px;overflow:auto}.aside-section-wrapper--transparent{background-color:unset}.aside-section-wrapper--collapse{padding:unset}.add-entity-entries .add-entity-entries__entry{margin-bottom:10px}.add-entity-entries .add-entity-entries__label{font-size:1.2em;margin-bottom:8px}.add-entity-entries .add-entity-entries__button{line-height:unset;height:unset;padding:4px 15px;margin:5px}.action-panel{margin-bottom:20px}.action-panel .action-panel__label{font-weight:bold;margin-bottom:12px}.action-panel .action-panel__button-group{display:flex;justify-content:space-between}.action-panel .action-panel__button-group--center{justify-content:center}.action-panel .action-panel__button{line-height:unset;height:unset;padding:4px 15px;margin:0 5px}.mark-panel{margin-bottom:20px}.mark-panel .mark-panel__status{font-weight:bold}.mark-panel .mark-panel__rating-star{position:relative;top:2px}.mark-panel .mark-panel__actions{float:right}.mark-panel .mark-panel__actions form{display:inline}.mark-panel .mark-panel__time{color:#ccc;margin-bottom:10px}.mark-panel .mark-panel__clear{content:' ';clear:both;display:table}.review-panel .review-panel__label{font-weight:bold}.review-panel .review-panel__actions{float:right}.review-panel .review-panel__time{color:#ccc;margin-bottom:10px}.review-panel .review-panel__review-title{display:block;margin-bottom:15px;font-weight:bold}.review-panel .review-panel__clear{content:' ';clear:both;display:table}.user-profile .user-profile__header{display:flex;align-items:flex-start;margin-bottom:15px}.user-profile .user-profile__avatar{width:72px}.user-profile .user-profile__username{font-size:large;margin-left:10px;margin-bottom:0}.user-profile .user-profile__report-link{color:#ccc}.user-relation .user-relation__label{display:inline-block;font-size:large;margin-bottom:10px}.user-relation .user-relation__more-link{margin-left:5px}.user-relation .user-relation__related-user-list{display:flex;justify-content:flex-start}.user-relation .user-relation__related-user-list:last-of-type{margin-bottom:0}.user-relation .user-relation__related-user{flex-basis:25%;padding:0px 3px;text-align:center;display:inline-block;overflow:hidden}.user-relation .user-relation__related-user>a:hover{color:#606c76}.user-relation .user-relation__related-user-avatar{background-image:url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");width:48px;height:48px}@media (min-width: 575.98px) and (max-width: 991.98px){.user-relation .user-relation__related-user-avatar{height:unset;width:60%;max-width:96px}}.user-relation .user-relation__related-user-name{color:inherit;overflow:hidden;text-overflow:ellipsis;-webkit-box-orient:vertical;-webkit-line-clamp:2}.report-panel .report-panel__label{display:inline-block;margin-bottom:10px}.report-panel .report-panel__body{padding-left:0}.report-panel .report-panel__report{margin:2px 0}.report-panel .report-panel__user-link{margin:0 2px}.report-panel .report-panel__all-link{margin-left:5px}.import-panel{overflow-x:hidden}.import-panel .import-panel__label{display:inline-block;margin-bottom:10px}.import-panel .import-panel__body{padding-left:0;border:2px dashed #00a1cc;padding:6px 9px}.import-panel .import-panel__body form{margin:0}@media (max-width: 991.98px){.import-panel .import-panel__body{border:unset;padding-left:0}}.import-panel .import-panel__help{background-color:#e5e5e5;border-radius:100000px;display:inline-block;width:16px;height:16px;text-align:center;font-size:12px;cursor:help}.import-panel .import-panel__checkbox{display:inline-block;margin-right:10px}.import-panel .import-panel__checkbox label{display:inline}.import-panel .import-panel__checkbox input[type="checkbox"]{margin:0;position:relative;top:2px}.import-panel .import-panel__checkbox--last{margin-right:0}.import-panel .import-panel__file-input{margin-top:10px}.import-panel .import-panel__button{line-height:unset;height:unset;padding:4px 15px}.import-panel .import-panel__progress{padding-top:10px}.import-panel .import-panel__progress:not(:first-child){border-top:#bbb 1px dashed}.import-panel .import-panel__progress label{display:inline}.import-panel .import-panel__progress progress{background-color:#d5d5d5;border-radius:0;height:10px;width:54%}.import-panel .import-panel__progress progress::-webkit-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__progress progress::-webkit-progress-value{background-color:#00a1cc}.import-panel .import-panel__progress progress::-moz-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__last-task:not(:first-child){padding-top:4px;border-top:#bbb 1px dashed}.import-panel .import-panel__last-task .index:not(:last-of-type){margin-right:8px}.import-panel .import-panel__fail-urls{margin-top:10px}.import-panel .import-panel__fail-urls li{word-break:break-all}.import-panel .import-panel__fail-urls ul{max-height:100px;overflow-y:auto}.relation-dropdown .relation-dropdown__button{display:none}.entity-card{display:flex;margin-bottom:10px;flex-direction:column}.entity-card--horizontal{flex-direction:row}.entity-card .entity-card__img{height:150px}.entity-card .entity-card__rating-star{position:relative;top:4px;left:-3px}.entity-card .entity-card__rating-score{position:relative;top:1px;margin-left:2px}.entity-card .entity-card__title{margin-bottom:10px;margin-top:5px}.entity-card .entity-card__info-wrapper--horizontal{margin-left:20px}.entity-card .entity-card__img-wrapper{flex-basis:100px}@media (max-width: 575.98px){.add-entity-entries{display:block !important}.add-entity-entries .add-entity-entries__button{width:100%;margin:5px 0 5px 0}.aside-section-wrapper:first-child{margin-right:0 !important;margin-bottom:0 !important}.aside-section-wrapper--singular:first-child{margin-bottom:20px !important}.action-panel{flex-direction:column !important}.entity-card--horizontal{flex-direction:column !important}.entity-card .entity-card__info-wrapper{margin-left:10px !important}.entity-card .entity-card__info-wrapper--horizontal{margin-left:0 !important}}@media (max-width: 991.98px){.add-entity-entries{display:flex;justify-content:space-around}.aside-section-wrapper{padding:24px 25px 10px 25px;margin-top:20px}.aside-section-wrapper:not(:last-child){margin-right:20px}.aside-section-wrapper--collapse{padding:24px 25px 10px 25px !important;margin-top:0;margin-bottom:0}.aside-section-wrapper--collapse:first-child{margin-right:0}.aside-section-wrapper--no-margin{margin:0}.action-panel{flex-direction:row}.action-panel .action-panel__button-group{justify-content:space-evenly}.relation-dropdown{margin-bottom:20px}.relation-dropdown .relation-dropdown__button{padding-bottom:10px;background-color:#f7f7f7;width:100%;display:flex;justify-content:center;align-items:center;cursor:pointer;transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:focus{background-color:red}.relation-dropdown .relation-dropdown__button>.icon-arrow{transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:hover>.icon-arrow>svg{fill:#00a1cc}.relation-dropdown .relation-dropdown__button>.icon-arrow--expand{transform:rotate(-180deg)}.relation-dropdown .relation-dropdown__button+.relation-dropdown__body--expand{max-height:2000px;transition:max-height 1s ease-in}.relation-dropdown .relation-dropdown__body{background-color:#f7f7f7;max-height:0;transition:max-height 1s ease-out;overflow:hidden}.entity-card{flex-direction:row}.entity-card .entity-card__info-wrapper{margin-left:30px}}.single-section-wrapper{padding:32px 36px;background-color:#f7f7f7;overflow:auto}.single-section-wrapper .single-section-wrapper__link--secondary{display:inline-block;color:#ccc;margin-bottom:20px}.single-section-wrapper .single-section-wrapper__link--secondary:hover{color:#00a1cc}.entity-form,.review-form{overflow:auto}.entity-form>input[type='email'],.entity-form>input[type='number'],.entity-form>input[type='password'],.entity-form>input[type='search'],.entity-form>input[type='tel'],.entity-form>input[type='text'],.entity-form>input[type='url'],.entity-form textarea,.review-form>input[type='email'],.review-form>input[type='number'],.review-form>input[type='password'],.review-form>input[type='search'],.review-form>input[type='tel'],.review-form>input[type='text'],.review-form>input[type='url'],.review-form textarea{width:100%}.entity-form img,.review-form img{display:block}.review-form .review-form__preview-button{color:#00a1cc;font-weight:bold;cursor:pointer}.review-form .review-form__fyi{color:#ccc}.review-form .review-form__main-content,.review-form textarea{margin-bottom:5px;resize:vertical;height:400px}.review-form .review-form__option{margin-top:24px;margin-bottom:10px}.review-form .review-form__option::after{content:' ';clear:both;display:table}.review-form .review-form__visibility-radio{float:left}.review-form .review-form__visibility-radio ul,.review-form .review-form__visibility-radio li,.review-form .review-form__visibility-radio label{display:inline}.review-form .review-form__visibility-radio label{font-size:normal}.review-form .review-form__visibility-radio input[type="radio"]{position:relative;top:2px}.review-form .review-form__share-checkbox{float:right}.review-form .review-form__share-checkbox input[type="checkbox"]{position:relative;top:2px}.report-form input,.report-form select{width:100%}@media (max-width: 575.98px){.review-form .review-form__visibility-radio{float:unset}.review-form .review-form__share-checkbox{float:unset;position:relative;left:-3px}}.markdownx-preview{min-height:100px}.markdownx-preview ul li{list-style:circle inside}.markdownx-preview h1{font-size:2.5em}.markdownx-preview h2{font-size:2.0em}.markdownx-preview blockquote{border-left:lightgray solid 0.4em;padding-left:0.1em;margin-left:0}.markdownx-preview code{border-left:#00a1cc solid 0.3em;padding-left:0.1em}.rating-star .jq-star{cursor:unset !important}.ms-parent>.ms-choice{margin-bottom:1.5rem;appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem;width:100%;height:30.126px}.ms-parent>.ms-choice:focus{border-color:#00a1cc}.ms-parent>.ms-choice>.icon-caret{top:15.5px}.ms-parent>.ms-choice>span{color:black;font-weight:initial;font-size:13.3333px;top:2.5px;left:2px}.ms-parent>.ms-choice>span:hover,.ms-parent>.ms-choice>span:focus{color:black}.ms-parent>.ms-drop>ul>li>label>span{margin-left:10px}.ms-parent>.ms-drop>ul>li>label>input{width:unset}.tippy-box{border:#606c76 1px solid;background-color:#f7f7f7;padding:3px 5px}.tag-input input{flex-grow:1}.tools-section-wrapper input,.tools-section-wrapper select{width:unset} diff --git a/common/static/sass/_Navbar.sass b/common/static/sass/_Navbar.sass index 2b007ffa..f03a12a6 100644 --- a/common/static/sass/_Navbar.sass +++ b/common/static/sass/_Navbar.sass @@ -35,6 +35,10 @@ &:visited color: $color-secondary + .current + color: $color-primary + font-weight: bold + & &__search-box margin: 0 12% 0 15px display: inline-flex diff --git a/common/templates/partial/_navbar.html b/common/templates/partial/_navbar.html index 90d7a459..a2e9c989 100644 --- a/common/templates/partial/_navbar.html +++ b/common/templates/partial/_navbar.html @@ -23,13 +23,13 @@