diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..514df728 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "neo" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "neo" ] + schedule: + - cron: '35 0 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/pysa.yml b/.github/workflows/pysa.yml new file mode 100644 index 00000000..e4e20af3 --- /dev/null +++ b/.github/workflows/pysa.yml @@ -0,0 +1,50 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# This workflow integrates Python Static Analyzer (Pysa) with +# GitHub's Code Scanning feature. +# +# Python Static Analyzer (Pysa) is a security-focused static +# analysis tool that tracks flows of data from where they +# originate to where they terminate in a dangerous location. +# +# See https://pyre-check.org/docs/pysa-basics/ + +name: Pysa + +on: + workflow_dispatch: + push: + branches: [ "neo" ] + pull_request: + branches: [ "neo" ] + schedule: + - cron: '45 12 * * 4' + +permissions: + contents: read + +jobs: + pysa: + permissions: + actions: read + contents: read + security-events: write + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Run Pysa + uses: facebook/pysa-action@f46a63777e59268613bd6e2ff4e29f144ca9e88b + with: + # To customize these inputs: + # See https://github.com/facebook/pysa-action#inputs + repo-directory: './' + requirements-path: 'requirements.txt' + infer-types: true + include-default-sapp-filters: true diff --git a/.gitignore b/.gitignore index 086f9d04..d1edae82 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ migrations/ # debug log file /log -log \ No newline at end of file +log + +# conf folder for neodb +/neodb diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..cc9bf6e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1 +FROM python:3.8-slim +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential libpq-dev git \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm -rf /tmp/requirements.txt \ + && useradd -U app_user \ + && install -d -m 0755 -o app_user -g app_user /app/static + +ENV DJANGO_SETTINGS_MODULE=neodb.dev +WORKDIR /app +USER app_user:app_user +COPY --chown=app_user:app_user . . +RUN chmod +x docker/*.sh + +# Section 6- Docker Run Checks and Configurations +ENTRYPOINT [ "docker/entrypoint.sh" ] + +CMD [ "docker/start.sh", "server" ] \ No newline at end of file diff --git a/README.md b/README.md index 2cfde8ea..d9f46dbc 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,13 @@ An application allows you to mark any books, movies and more things you love. Depends on Mastodon. +## Install +Please see [doc/GUIDE.md](doc/GUIDE.md) + +## Bug Report + - to file a bug for NiceDB, please create an issue [here](https://github.com/doubaniux/boofilsic/issues/new) + - to file a bug or request new features for NeoDB, please contact NeoDB on [Fediverse](https://mastodon.social/@neodb) or [Twitter](https://twitter.com/NeoDBsocial) + ## Contribution The project is based on Django. If you are familiar with this technique and willing to read through the terrible code😝, your contribution would be the most welcome! @@ -11,8 +18,6 @@ Currently looking for someone to help with: - Explaining the structure of code - Refactoring (this is something big) -This project is still in its early stage, so you are not encouraged to deploy it on your own. If you do want to give it a try, please check the [fork of *alphatownsman*](https://github.com/alphatownsman/boofilsic), which is more friendly. - ## Sponsor -If you like this project, please consider sponsoring us on [Patreon](https://patreon.com/tertius). +If you like this project, please consider sponsoring NiceDB on [Patreon](https://patreon.com/tertius). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..cafb3b8b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +Please DM [us on Fediverse](https://mastodon.social/@neodb) or send email to `dev`@`neodb.social` to report a vulnerability. Please do not post publicly or create pr/issues directly. Thank you. diff --git a/boofilsic/context_processors.py b/boofilsic/context_processors.py new file mode 100644 index 00000000..6fd333b3 --- /dev/null +++ b/boofilsic/context_processors.py @@ -0,0 +1,5 @@ +from django.conf import settings + + +def site_info(request): + return settings.SITE_INFO diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 86d4abf2..6393f4a0 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -12,10 +12,13 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ import os import psycopg2.extensions +from git import Repo # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ @@ -38,6 +41,8 @@ INTERNAL_IPS = [ INSTALLED_APPS = [ 'django.contrib.admin', + 'hijack', + 'hijack.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', @@ -45,6 +50,9 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django.contrib.humanize', 'django.contrib.postgres', + 'django_sass', + 'django_rq', + 'simple_history', 'markdownx', 'management.apps.ManagementConfig', 'mastodon.apps.MastodonConfig', @@ -54,7 +62,12 @@ INSTALLED_APPS = [ 'movies.apps.MoviesConfig', 'music.apps.MusicConfig', 'games.apps.GamesConfig', + 'sync.apps.SyncConfig', + 'collection.apps.CollectionConfig', + 'timeline.apps.TimelineConfig', 'easy_thumbnails', + 'user_messages', + 'django_slack', ] MIDDLEWARE = [ @@ -65,6 +78,8 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'hijack.middleware.HijackUserMiddleware', + 'simple_history.middleware.HistoryRequestMiddleware', ] ROOT_URLCONF = 'boofilsic.urls' @@ -79,7 +94,9 @@ TEMPLATES = [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + # 'django.contrib.messages.context_processors.messages', + "user_messages.context_processors.messages", + 'boofilsic.context_processors.site_info', ], }, }, @@ -95,10 +112,10 @@ if DEBUG: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'test', - 'USER': 'donotban', - 'PASSWORD': 'donotbansilvousplait', - 'HOST': '172.18.116.29', + 'NAME': os.environ.get('DB_NAME', 'test'), + 'USER': os.environ.get('DB_USER', 'donotban'), + 'PASSWORD': os.environ.get('DB_PASSWORD', 'donotbansilvousplait'), + 'HOST': os.environ.get('DB_HOST', '172.18.116.29'), 'OPTIONS': { 'client_encoding': 'UTF8', # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, @@ -184,13 +201,29 @@ STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesSto AUTH_USER_MODEL = 'users.User' +SILENCED_SYSTEM_CHECKS = [ + "auth.W004", # User.username is non-unique + "admin.E404" # Required by django-user-messages +] + MEDIA_URL = '/media/' MEDIA_ROOT = os.path.join(BASE_DIR, 'media/') +PROJECT_ROOT = os.path.abspath(os.path.dirname(__name__)) +SITE_INFO = { + 'site_name': 'NiceDB', + 'support_link': 'https://github.com/doubaniux/boofilsic/issues', + 'version_hash': None, + 'settings_module': os.getenv('DJANGO_SETTINGS_MODULE'), + 'sentry_dsn': None, +} + # Mastodon configs -CLIENT_NAME = 'NiceDB' -APP_WEBSITE = 'https://nicedb.org' -REDIRECT_URIS = "https://nicedb.org/users/OAuth2_login/\nhttps://www.nicedb.org/users/OAuth2_login/" +CLIENT_NAME = os.environ.get('APP_NAME', 'NiceDB') +SITE_INFO['site_name'] = os.environ.get('APP_NAME', 'NiceDB') +APP_WEBSITE = os.environ.get('APP_URL', 'https://nicedb.org') +REDIRECT_URIS = APP_WEBSITE + "/users/OAuth2_login/" + # Path to save report related images, ends with slash REPORT_MEDIA_PATH_ROOT = 'report/' @@ -205,10 +238,23 @@ ALBUM_MEDIA_PATH_ROOT = 'album/' DEFAULT_ALBUM_IMAGE = os.path.join(ALBUM_MEDIA_PATH_ROOT, 'default.svg') GAME_MEDIA_PATH_ROOT = 'game/' DEFAULT_GAME_IMAGE = os.path.join(GAME_MEDIA_PATH_ROOT, 'default.svg') +COLLECTION_MEDIA_PATH_ROOT = 'collection/' +DEFAULT_COLLECTION_IMAGE = os.path.join(COLLECTION_MEDIA_PATH_ROOT, 'default.svg') +SYNC_FILE_PATH_ROOT = 'sync/' +EXPORT_FILE_PATH_ROOT = 'export/' + +# Allow user to login via any Mastodon/Pleroma sites +MASTODON_ALLOW_ANY_SITE = False # Timeout of requests to Mastodon, in seconds MASTODON_TIMEOUT = 30 +MASTODON_CLIENT_SCOPE = 'read write follow' +#use the following if it's a new site +#MASTODON_CLIENT_SCOPE = 'read:accounts read:follows read:search read:blocks read:mutes write:statuses write:media' + +MASTODON_LEGACY_CLIENT_SCOPE = 'read write follow' + # Tags for toots posted from this site MASTODON_TAGS = '#NiceDB #NiceDB%(category)s #NiceDB%(category)s%(type)s' @@ -217,7 +263,7 @@ STAR_SOLID = ':star_solid:' STAR_HALF = ':star_half:' STAR_EMPTY = ':star_empty:' -# Default password for each user. since assword is not used any way, +# Default password for each user. since password is not used any way, # any string that is not empty is ok DEFAULT_PASSWORD = 'ab7nsm8didusbaqPgq' @@ -231,8 +277,12 @@ ADMIN_URL = 'tertqX7256n7ej8nbv5cwvsegdse6w7ne5rHd' LUMINATI_USERNAME = 'lum-customer-hl_nw4tbv78-zone-static' LUMINATI_PASSWORD = 'nsb7te9bw0ney' +SCRAPING_TIMEOUT = 90 + # ScraperAPI api key SCRAPERAPI_KEY = 'wnb3794v675b8w475h0e8hr7tyge' +PROXYCRAWL_KEY = None +SCRAPESTACK_KEY = None # Spotify credentials SPOTIFY_CREDENTIAL = "NzYzNkYTE6MGQ0ODY0NTY2Y2b3n645sdfgAyY2I1ljYjg3Nzc0MjIwODQ0ZWE=" @@ -240,6 +290,17 @@ SPOTIFY_CREDENTIAL = "NzYzNkYTE6MGQ0ODY0NTY2Y2b3n645sdfgAyY2I1ljYjg3Nzc0MjIwODQ0 # IMDb API service https://imdb-api.com/ IMDB_API_KEY = "k23fwewff23" +# The Movie Database (TMDB) API Keys +TMDB_API3_KEY = "deadbeef" +TMDB_API4_KEY = "deadbeef.deadbeef.deadbeef" + +# Google Books API Key +GOOGLE_API_KEY = 'deadbeef-deadbeef-deadbeef' + +# IGDB +IGDB_CLIENT_ID = 'deadbeef' +IGDB_ACCESS_TOKEN = 'deadbeef' + # Thumbnail setting # It is possible to optimize the image size even more: https://easy-thumbnails.readthedocs.io/en/latest/ref/optimize/ THUMBNAIL_ALIASES = { @@ -257,3 +318,47 @@ if DEBUG: # https://django-debug-toolbar.readthedocs.io/en/latest/ # maybe benchmarking before deployment + +REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1') + +RQ_QUEUES = { + 'mastodon': { + 'HOST': REDIS_HOST, + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': -1, + }, + 'export': { + 'HOST': REDIS_HOST, + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': -1, + }, + 'doufen': { + 'HOST': REDIS_HOST, + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': -1, + } +} + +RQ_SHOW_ADMIN_LINK = True + +SEARCH_INDEX_NEW_ONLY = False + +SEARCH_BACKEND = None + +# SEARCH_BACKEND = 'MEILISEARCH' +# MEILISEARCH_SERVER = 'http://127.0.0.1:7700' +# MEILISEARCH_KEY = 'deadbeef' + +# SEARCH_BACKEND = 'TYPESENSE' +# TYPESENSE_CONNECTION = { +# 'api_key': 'deadbeef', +# 'nodes': [{ +# 'host': 'localhost', +# 'port': '8108', +# 'protocol': 'http' +# }], +# 'connection_timeout_seconds': 2 +# } diff --git a/boofilsic/urls.py b/boofilsic/urls.py index dd52087a..38d74a5a 100644 --- a/boofilsic/urls.py +++ b/boofilsic/urls.py @@ -27,10 +27,16 @@ urlpatterns = [ path('movies/', include('movies.urls')), path('music/', include('music.urls')), path('games/', include('games.urls')), + path('collections/', include('collection.urls')), + path('timeline/', include('timeline.urls')), path('sync/', include('sync.urls')), path('announcement/', include('management.urls')), + path('hijack/', include('hijack.urls')), path('', include('common.urls')), +] +urlpatterns += [ + path(settings.ADMIN_URL + '-rq/', include('django_rq.urls')) ] if settings.DEBUG: diff --git a/books/admin.py b/books/admin.py index 942dccb4..75df663b 100644 --- a/books/admin.py +++ b/books/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin from .models import * +from simple_history.admin import SimpleHistoryAdmin -admin.site.register(Book) +admin.site.register(Book, SimpleHistoryAdmin) admin.site.register(BookMark) admin.site.register(BookReview) admin.site.register(BookTag) diff --git a/books/apps.py b/books/apps.py index f716137a..b03e2d23 100644 --- a/books/apps.py +++ b/books/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class BooksConfig(AppConfig): name = 'books' + + def ready(self): + from common.index import Indexer + from .models import Book + Indexer.update_model_indexable(Book) diff --git a/books/forms.py b/books/forms.py index da7ecee6..27abda07 100644 --- a/books/forms.py +++ b/books/forms.py @@ -1,17 +1,12 @@ from django import forms from django.utils.translation import gettext_lazy as _ -from .models import Book, BookMark, BookReview +from .models import Book, BookMark, BookReview, BookMarkStatusTranslation from common.models import MarkStatusEnum from common.forms import * def BookMarkStatusTranslator(status): - trans_dict = { - MarkStatusEnum.DO.value: _("在读"), - MarkStatusEnum.WISH.value: _("想读"), - MarkStatusEnum.COLLECT.value: _("读过") - } - return trans_dict[status] + return BookMarkStatusTranslation[status] class BookForm(forms.ModelForm): @@ -96,11 +91,8 @@ class BookMarkForm(MarkForm): 'status', 'rating', 'text', - 'is_private', - ] - labels = { - 'rating': _("评分"), - } + 'visibility', + ] widgets = { 'book': forms.TextInput(attrs={"hidden": ""}), } @@ -115,14 +107,8 @@ class BookReviewForm(ReviewForm): 'book', 'title', 'content', - 'is_private' + 'visibility' ] - labels = { - 'book': "", - 'title': _("标题"), - 'content': _("正文"), - 'share_to_mastodon': _("分享到长毛象") - } widgets = { 'book': forms.TextInput(attrs={"hidden": ""}), } diff --git a/books/management/commands/fix-book-cover.py b/books/management/commands/fix-book-cover.py new file mode 100644 index 00000000..ae9227b5 --- /dev/null +++ b/books/management/commands/fix-book-cover.py @@ -0,0 +1,200 @@ +from django.core.management.base import BaseCommand +from django.core.files.uploadedfile import SimpleUploadedFile +from django.conf import settings +from common.scraper import * +from books.models import Book +from books.forms import BookForm +import requests +import re +import filetype +from lxml import html +from PIL import Image +from io import BytesIO + + +class DoubanPatcherMixin: + @classmethod + def download_page(cls, url, headers): + url = cls.get_effective_url(url) + r = None + error = 'DoubanScrapper: error occured when downloading ' + url + content = None + + def get(url, timeout): + nonlocal r + # print('Douban GET ' + url) + try: + r = requests.get(url, timeout=timeout) + except Exception as e: + r = requests.Response() + r.status_code = f"Exception when GET {url} {e}" + url + # print('Douban CODE ' + str(r.status_code)) + return r + + def check_content(): + nonlocal r, error, content + content = None + if r.status_code == 200: + content = r.content.decode('utf-8') + if content.find('关于豆瓣') == -1: + # with open('/tmp/temp.html', 'w', encoding='utf-8') as fp: + # fp.write(content) + content = None + error = error + 'Content not authentic' # response is garbage + elif re.search('不存在[^<]+', content, re.MULTILINE): + content = None + error = error + 'Not found or hidden by Douban' + else: + error = error + str(r.status_code) + + def fix_wayback_links(): + nonlocal content + # fix links + content = re.sub(r'href="http[^"]+http', r'href="http', content) + # https://img9.doubanio.com/view/subject/{l|m|s}/public/s1234.jpg + content = re.sub(r'src="[^"]+/(s\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/subject/m/public/\1"', content) + # https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2681329386.jpg + # https://img9.doubanio.com/view/photo/{l|m|s}/public/p1234.webp + content = re.sub(r'src="[^"]+/(p\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/photo/m/public/\1"', content) + + # Wayback Machine: get latest available + def wayback(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://archive.org/wayback/available?url=' + url, 10) + if r.status_code == 200: + w = r.json() + if w['archived_snapshots'] and w['archived_snapshots']['closest']: + get(w['archived_snapshots']['closest']['url'], 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + # Wayback Machine: guess via CDX API + def wayback_cdx(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://web.archive.org/cdx/search/cdx?url=' + url, 10) + if r.status_code == 200: + dates = re.findall(r'[^\s]+\s+(\d+)\s+[^\s]+\s+[^\s]+\s+\d+\s+[^\s]+\s+\d{5,}', + r.content.decode('utf-8')) + # assume snapshots whose size >9999 contain real content, use the latest one of them + if len(dates) > 0: + get('http://web.archive.org/web/' + dates[-1] + '/' + url, 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + def latest(): + nonlocal r, error, content + if settings.SCRAPESTACK_KEY is None: + error = error + '\nDirect: ' + get(url, 60) + else: + error = error + '\nScrapeStack: ' + get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}', 60) + check_content() + + wayback_cdx() + if content is None: + latest() + + if content is None: + logger.error(error) + content = '' + return html.fromstring(content) + + @classmethod + def download_image(cls, url, item_url=None): + if url is None: + logger.error(f"Douban: no image url for {item_url}") + return None, None + raw_img = None + ext = None + + dl_url = url + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + # raise RuntimeError(f"Douban: download image failed {img_response.status_code} {dl_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + if raw_img is None and settings.SCRAPESTACK_KEY is not None: + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + return raw_img, ext + + +class DoubanBookPatcher(DoubanPatcherMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'book.douban.com' + data_class = Book + form_class = BookForm + + regex = re.compile(r"https://book\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + img_url_elem = content.xpath("//*[@id='mainpic']/a/img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + return raw_img, ext + + +class Command(BaseCommand): + help = 'fix cover image' + + def add_arguments(self, parser): + parser.add_argument('threadId', type=int, help='% 8') + + def handle(self, *args, **options): + t = int(options['threadId']) + for m in Book.objects.filter(cover='book/default.svg', source_site='douban'): + if m.id % 8 == t: + self.stdout.write(f'Re-fetching {m.source_url}') + try: + raw_img, img_ext = DoubanBookPatcher.scrape(m.source_url) + if img_ext is not None: + m.cover = SimpleUploadedFile('temp.' + img_ext, raw_img) + m.save() + self.stdout.write(self.style.SUCCESS(f'Saved {m.source_url}')) + else: + self.stdout.write(self.style.ERROR(f'Skipped {m.source_url}')) + except Exception as e: + print(e) diff --git a/books/models.py b/books/models.py index 4f898709..8b23e9a6 100644 --- a/books/models.py +++ b/books/models.py @@ -1,98 +1,184 @@ -import uuid import django.contrib.postgres.fields as postgres -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.db import models -from django.core.serializers.json import DjangoJSONEncoder from django.shortcuts import reverse -from common.models import Entity, Mark, Review, Tag +from common.models import Entity, Mark, Review, Tag, MarkStatusEnum from common.utils import GenerateDateUUIDMediaFilePath -from boofilsic.settings import BOOK_MEDIA_PATH_ROOT, DEFAULT_BOOK_IMAGE -from django.utils import timezone +from django.conf import settings +from django.db.models import Q +from simple_history.models import HistoricalRecords + + +BookMarkStatusTranslation = { + MarkStatusEnum.DO.value: _("在读"), + MarkStatusEnum.WISH.value: _("想读"), + MarkStatusEnum.COLLECT.value: _("读过") +} def book_cover_path(instance, filename): - return GenerateDateUUIDMediaFilePath(instance, filename, BOOK_MEDIA_PATH_ROOT) + return GenerateDateUUIDMediaFilePath(instance, filename, settings.BOOK_MEDIA_PATH_ROOT) class Book(Entity): # widely recognized name, usually in Chinese - title = models.CharField(_("title"), max_length=200) - subtitle = models.CharField(_("subtitle"), blank=True, default='', max_length=200) + title = models.CharField(_("title"), max_length=500) + subtitle = models.CharField( + _("subtitle"), blank=True, default='', max_length=500) # original name, for books in foreign language - orig_title = models.CharField(_("original title"), blank=True, default='', max_length=200) + orig_title = models.CharField( + _("original title"), blank=True, default='', max_length=500) author = postgres.ArrayField( - models.CharField(_("author"), blank=True, default='', max_length=100), + models.CharField(_("author"), blank=True, default='', max_length=200), null=True, blank=True, default=list, ) translator = postgres.ArrayField( - models.CharField(_("translator"), blank=True, default='', max_length=100), + models.CharField(_("translator"), blank=True, + default='', max_length=200), null=True, blank=True, default=list, ) - language = models.CharField(_("language"), blank=True, default='', max_length=10) - pub_house = models.CharField(_("publishing house"), blank=True, default='', max_length=200) + language = models.CharField( + _("language"), blank=True, default='', max_length=50) + pub_house = models.CharField( + _("publishing house"), blank=True, default='', max_length=200) pub_year = models.IntegerField(_("published year"), null=True, blank=True) - pub_month = models.IntegerField(_("published month"), null=True, blank=True) - binding = models.CharField(_("binding"), blank=True, default='', max_length=50) + pub_month = models.IntegerField( + _("published month"), null=True, blank=True) + binding = models.CharField( + _("binding"), blank=True, default='', max_length=200) # since data origin is not formatted and might be CNY USD or other currency, use char instead - price = models.CharField(_("pricing"), blank=True, default='', max_length=50) + price = models.CharField(_("pricing"), blank=True, + default='', max_length=50) pages = models.PositiveIntegerField(_("pages"), null=True, blank=True) - isbn = models.CharField(_("ISBN"), blank=True, null=False, max_length=20, db_index=True, default='') - # to store previously scrapped data - cover = models.ImageField(_("cover picture"), upload_to=book_cover_path, default=DEFAULT_BOOK_IMAGE, blank=True) + isbn = models.CharField(_("ISBN"), blank=True, null=False, + max_length=20, db_index=True, default='') + # to store previously scrapped data + cover = models.ImageField(_("cover picture"), upload_to=book_cover_path, + default=settings.DEFAULT_BOOK_IMAGE, blank=True) contents = models.TextField(blank=True, default="") + history = HistoricalRecords() class Meta: - # more info: https://docs.djangoproject.com/en/2.2/ref/models/options/ - # set managed=False if the model represents an existing table or - # a database view that has been created by some other means. - # check the link above for further info - # managed = True - # db_table = 'book' constraints = [ - models.CheckConstraint(check=models.Q(pub_year__gte=0), name='pub_year_lowerbound'), - models.CheckConstraint(check=models.Q(pub_month__lte=12), name='pub_month_upperbound'), - models.CheckConstraint(check=models.Q(pub_month__gte=1), name='pub_month_lowerbound'), + models.CheckConstraint(check=models.Q( + pub_year__gte=0), name='pub_year_lowerbound'), + models.CheckConstraint(check=models.Q( + pub_month__lte=12), name='pub_month_upperbound'), + models.CheckConstraint(check=models.Q( + pub_month__gte=1), name='pub_month_lowerbound'), ] def __str__(self): return self.title - + + def get_json(self): + r = { + 'subtitle': self.subtitle, + 'original_title': self.orig_title, + 'author': self.author, + 'translator': self.translator, + 'publisher': self.pub_house, + 'publish_year': self.pub_year, + 'publish_month': self.pub_month, + 'language': self.language, + 'isbn': self.isbn, + } + r.update(super().get_json()) + return r + def get_absolute_url(self): return reverse("books:retrieve", args=[self.id]) + @property + def wish_url(self): + return reverse("books:wish", args=[self.id]) + def get_tags_manager(self): return self.book_tags + def get_related_books(self): + qs = Q(orig_title=self.title) + if self.isbn: + qs = qs | Q(isbn=self.isbn) + if self.orig_title: + qs = qs | Q(title=self.orig_title) + qs = qs | Q(orig_title=self.orig_title) + qs = qs & ~Q(id=self.id) + return Book.objects.filter(qs) + + def get_identicals(self): + qs = Q(orig_title=self.title) + if self.isbn: + qs = Q(isbn=self.isbn) + # qs = qs & ~Q(id=self.id) + return Book.objects.filter(qs) + else: + return [self] # Book.objects.filter(id=self.id) + @property def verbose_category_name(self): return _("书籍") + @property + def mark_class(self): + return BookMark + + @property + def tag_class(self): + return BookTag + class BookMark(Mark): - book = models.ForeignKey(Book, on_delete=models.CASCADE, related_name='book_marks', null=True) + book = models.ForeignKey( + Book, on_delete=models.CASCADE, related_name='book_marks', null=True) + class Meta: constraints = [ - models.UniqueConstraint(fields=['owner', 'book'], name="unique_book_mark") + models.UniqueConstraint( + fields=['owner', 'book'], name="unique_book_mark") ] + @property + def translated_status(self): + return BookMarkStatusTranslation[self.status] + class BookReview(Review): - book = models.ForeignKey(Book, on_delete=models.CASCADE, related_name='book_reviews', null=True) + book = models.ForeignKey( + Book, on_delete=models.CASCADE, related_name='book_reviews', null=True) + class Meta: constraints = [ - models.UniqueConstraint(fields=['owner', 'book'], name="unique_book_review") - ] + models.UniqueConstraint( + fields=['owner', 'book'], name="unique_book_review") + ] + + @property + def url(self): + return settings.APP_WEBSITE + reverse("books:retrieve_review", args=[self.id]) + + @property + def item(self): + return self.book class BookTag(Tag): - book = models.ForeignKey(Book, on_delete=models.CASCADE, related_name='book_tags', null=True) - mark = models.ForeignKey(BookMark, on_delete=models.CASCADE, related_name='bookmark_tags', null=True) + book = models.ForeignKey( + Book, on_delete=models.CASCADE, related_name='book_tags', null=True) + mark = models.ForeignKey( + BookMark, on_delete=models.CASCADE, related_name='bookmark_tags', null=True) + class Meta: constraints = [ - models.UniqueConstraint(fields=['content', 'mark'], name="unique_bookmark_tag") + models.UniqueConstraint( + fields=['content', 'mark'], name="unique_bookmark_tag") ] + + @property + def item(self): + return self.book diff --git a/books/templates/books/create_update.html b/books/templates/books/create_update.html index fdd7ec61..de4b8ca7 100644 --- a/books/templates/books/create_update.html +++ b/books/templates/books/create_update.html @@ -10,8 +10,8 @@ - {% trans 'NiceDB - ' %}{{ title }} - + {{ site_name }} - {{ title }} + @@ -22,8 +22,24 @@
+ {% if is_update and form.source_site.value != 'in-site' %} +
+
+
+
{% trans '源网站' %}: {{ form.source_site.value }}
+
+
+ {% csrf_token %} + +
+
+
+
+
+ {% endif %} +
- {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
{% csrf_token %} {{ form.media }} @@ -38,12 +54,6 @@
- {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -80,7 +80,7 @@
- {{ form.is_private.label }}{{ form.is_private }} + {{ form.visibility.label }}{{ form.visibility }}
- {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除图书' %} + @@ -55,7 +55,7 @@ {% if book.last_editor %}
{% trans '最近编辑者：' %} - + {{ book.last_editor | default:"" }}
@@ -89,12 +89,6 @@
- {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除评论' %} + @@ -35,7 +35,7 @@
{{ review.title }}
- {% if review.is_private %} + {% if review.visibility > 0 %} @@ -47,7 +47,7 @@
- {{ review.owner.username }} {% if mark %} @@ -90,12 +90,6 @@
- {% comment %} - - - - - {% endcomment %} - - @@ -57,11 +60,12 @@
- {% if book.rating %} + {% if book.rating and book.rating_number >= 5 %} {{ book.rating }} + ({{ book.rating_number }}人评分) {% else %} - {% trans '评分：暂无评分' %} + {% trans '评分：评分人数不足' %} {% endif %}
{% if book.isbn %}{% trans 'ISBN：' %}{{ book.isbn }}{% endif %}
@@ -96,7 +100,7 @@ {% if book.last_editor %} -
{% trans '最近编辑者：' %}{{ book.last_editor | default:"" }}
+
{% trans '最近编辑者：' %}{{ book.last_editor | default:"" }}
{% endif %}
@@ -148,46 +152,27 @@
{% trans '这本书的标记' %}
- {% if mark_list_more %} - {% trans '更多' %} - {% endif %} - {% if mark_list %} -
    - {% for others_mark in mark_list %} -
  • - {{ others_mark.owner.username }} - {{ others_mark.get_status_display }} - {% if others_mark.rating %} - - {% endif %} - {% if others_mark.is_private %} - - {% endif %} - {{ others_mark.edited_time }} - {% if others_mark.text %} -

    {{ others_mark.text }}

    - {% endif %} -
    • - {% endfor %} -
- {% else %} -
{% trans '暂无标记' %}
- {% endif %} + {% trans '全部标记' %} + 关注的人的标记 + {% include "partial/mark_list.html" with mark_list=mark_list current_item=book %}
{% trans '这本书的评论' %}
{% if review_list_more %} - {% trans '更多' %} + {% trans '全部评论' %} {% endif %} {% if review_list %}
    {% for others_review in review_list %}
  • - {{ others_review.owner.username }} - {% if others_review.is_private %} + {{ others_review.owner.username }} + {% if others_review.visibility > 0 %} {% endif %} {{ others_review.edited_time }} + {% if others_review.book != book %} + {{ others_review.book.get_source_site_display }} + {% endif %} {{ others_review.title }} {{ others_review.get_plain_content | truncate:100 }}
    • @@ -202,7 +187,6 @@
    - {% if mark %}
    @@ -212,7 +196,7 @@ {% endif %} {% endif %} - {% if mark.is_private %} + {% if mark.visibility > 0 %} {% endif %} @@ -224,7 +208,7 @@
    -
    {{ mark.edited_time }}
    +
    {{ mark.created_time }}
    {% if mark.text %}

    {{ mark.text }}

    @@ -245,9 +229,8 @@
    -
    +
    {% endif %} -
@@ -255,7 +238,7 @@
{% trans '我的评论' %} - {% if review.is_private %} + {% if review.visibility > 0 %} {% endif %} @@ -284,7 +267,53 @@ {% endif %}
- + + {% if book.get_related_books.count > 0 %} +
+
+
{% trans '相关书目' %}
+
+ {% for b in book.get_related_books %} +

+ {{ b.title }} + ({{ b.pub_house }} {{ b.pub_year }}) + {{ b.get_source_site_display }} +

+ {% endfor %} +
+
+
+ {% endif %} + + {% if book.isbn %} +
+
+
{% trans '借阅或购买' %}
+
+ {% trans 'WorldCat' %} + {% trans 'Open Library' %} +
+
+
+ {% endif %} + + {% if collection_list %} +
+
+
{% trans '相关收藏单' %}
+
+ {% for c in collection_list %} +

+ {{ c.title }} +

+ {% endfor %} +
+ +
+
+
+
+ {% endif %}
@@ -296,7 +325,6 @@
- {% if not mark %} + + + +
+ {% include "partial/_navbar.html" %} +
+
+
+
+ + {% csrf_token %} + {{ form }} + + + {{ form.media }} +
+
+
+ {% include "partial/_footer.html" %} +
+ + + + + + + \ No newline at end of file diff --git a/collection/templates/delete.html b/collection/templates/delete.html new file mode 100644 index 00000000..ee6d440d --- /dev/null +++ b/collection/templates/delete.html @@ -0,0 +1,117 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} + + + + + + + + + + + + + {{ site_name }} {% trans '收藏单' %} - {{ collection.title }} + + + + + + + + + +
+
+ {% include "partial/_navbar.html" %} + +
+
+
+
+
+
+ 确认删除收藏单「{{ collection.title }}」吗？ +
+ {% if collection.visibility > 0 %} + + + + {% endif %} +
+
+ + {{ collection.owner.mastodon_username }} + + + {{ collection.edited_time }} + +
+
+
+
+
+ {{ form.description }} +
+ {{ form.media }} +
+
+
+ {% csrf_token %} + +
+ +
+ + +
+
+
+
+
+
+
+ + + +
+
+
+ + {{ collection.title }} + +
+
+
+
+
+
+
+
+ {% include "partial/_footer.html" %} +
+ + + + + + + + diff --git a/collection/templates/detail.html b/collection/templates/detail.html new file mode 100644 index 00000000..d78513b9 --- /dev/null +++ b/collection/templates/detail.html @@ -0,0 +1,147 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} + + + + + + + + + + + + + + {{ site_name }} {% trans '收藏单' %} - {{ collection.title }} + + {% include "partial/_common_libs.html" with jquery=1 %} + + + + + + +
+
+ {% include "partial/_navbar.html" %} + +
+
+
+
+
+
+ {{ collection.title }} +
+ {% if collection.visibility > 0 %} + + + + {% endif %} +
+
+ + {{ collection.owner.mastodon_username }} + + + {{ collection.edited_time }} + +
+
+ {% if request.user == collection.owner %} + {% trans '编辑' %} + {% trans '删除' %} + {% elif editable %} + 可协作整理 + {% endif %} +
+
+ + +
+ {{ form.description }} +
+ {{ form.media }} +
+
+
+
+
+
+
+
+
+ + + +
+
+
+ + {{ collection.title }} + +
+
+
+
+ + {% if request.user != collection.owner %} +
+
+
+ {% if following %} +
+ {% csrf_token %} + +
+ {% else %} +
+ {% csrf_token %} + +
+ {% endif %} +
+
+
+ {% endif %} + +
+
+
+
+ +
+
+
+
+
+
+
+
+ {% include "partial/_footer.html" %} +
+ + + + + + + + diff --git a/collection/templates/edit_item_comment.html b/collection/templates/edit_item_comment.html new file mode 100644 index 00000000..674985e3 --- /dev/null +++ b/collection/templates/edit_item_comment.html @@ -0,0 +1,5 @@ +
+ + + +
\ No newline at end of file diff --git a/collection/templates/entity_list.html b/collection/templates/entity_list.html new file mode 100644 index 00000000..4cdc9ee2 --- /dev/null +++ b/collection/templates/entity_list.html @@ -0,0 +1,21 @@ +{% load thumb %} +{% load i18n %} +{% load l10n %} +
    + {% for collectionitem in collection.collectionitem_list %} + {% if collectionitem.item is not None %} + {% include "partial/list_item.html" with item=collectionitem.item %} + {% endif %} + {% empty %} + {% endfor %} + {% if editable %} +
  • +
    + {% csrf_token %} + + + +
    +
    • + {% endif %} +
diff --git a/collection/templates/list.html b/collection/templates/list.html new file mode 100644 index 00000000..0c9c75c3 --- /dev/null +++ b/collection/templates/list.html @@ -0,0 +1,99 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load highlight %} +{% load thumb %} + + + + + + + {{ site_name }} - {{ title }} + + + + + + + + +
+
+ {% include "partial/_navbar.html" %} + +
+
+
+
+
+
+ {{ title }} +
+
    + + {% for collection in collections %} + +
  • + + {{ collection.title }} + {{ collection.edited_time }} + {% if collection.visibility > 0 %} + + {% endif %} +
    • + {% empty %} +
    {% trans '无结果' %}
    + {% endfor %} + +
+
+ +
+
+ + +
+
+
+ {% include "partial/_footer.html" %} +
+ + + + + + + + diff --git a/collection/templates/share_collection.html b/collection/templates/share_collection.html new file mode 100644 index 00000000..dc8e3beb --- /dev/null +++ b/collection/templates/share_collection.html @@ -0,0 +1,56 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load highlight %} +{% load thumb %} + + diff --git a/collection/templates/show_item_comment.html b/collection/templates/show_item_comment.html new file mode 100644 index 00000000..f9add2bd --- /dev/null +++ b/collection/templates/show_item_comment.html @@ -0,0 +1,4 @@ +{{ collectionitem.comment }} +{% if editable %} + +{% endif %} \ No newline at end of file diff --git a/collection/tests.py b/collection/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/collection/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/collection/urls.py b/collection/urls.py new file mode 100644 index 00000000..f3c62663 --- /dev/null +++ b/collection/urls.py @@ -0,0 +1,27 @@ +from django.urls import path, re_path +from .views import * + + +app_name = 'collection' +urlpatterns = [ + path('mine/', list, name='list'), + path('create/', create, name='create'), + path('/', retrieve, name='retrieve'), + path('/entity_list', retrieve_entity_list, name='retrieve_entity_list'), + path('update//', update, name='update'), + path('delete//', delete, name='delete'), + path('follow//', follow, name='follow'), + path('unfollow//', unfollow, name='unfollow'), + path('/append_item/', append_item, name='append_item'), + path('/delete_item/', delete_item, name='delete_item'), + path('/move_up_item/', move_up_item, name='move_up_item'), + path('/move_down_item/', move_down_item, name='move_down_item'), + path('/update_item_comment/', update_item_comment, name='update_item_comment'), + path('/show_item_comment/', show_item_comment, name='show_item_comment'), + path('with///', list_with, name='list_with'), + path('add_to_list///', add_to_list, name='add_to_list'), + path('share//', share, name='share'), + path('follow2//', wish, name='wish'), + + # TODO: tag +] diff --git a/collection/views.py b/collection/views.py new file mode 100644 index 00000000..659cb586 --- /dev/null +++ b/collection/views.py @@ -0,0 +1,442 @@ +import logging +from django.shortcuts import render, get_object_or_404, redirect, reverse +from django.contrib.auth.decorators import login_required, permission_required +from django.utils.translation import gettext_lazy as _ +from django.http import HttpResponseBadRequest, HttpResponseServerError, HttpResponse +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied +from django.db import IntegrityError, transaction +from django.db.models import Count +from django.utils import timezone +from django.core.paginator import Paginator +from mastodon import mastodon_request_included +from mastodon.models import MastodonApplication +from mastodon.api import post_toot, TootVisibilityEnum, share_collection +from common.utils import PageLinksGenerator +from common.views import PAGE_LINK_NUMBER, jump_or_scrape, go_relogin +from common.models import SourceSiteEnum +from .models import * +from .forms import * +from django.conf import settings +import re +from users.models import User +from django.http import HttpResponseRedirect + + +logger = logging.getLogger(__name__) +mastodon_logger = logging.getLogger("django.mastodon") + + +# how many marks showed on the detail page +MARK_NUMBER = 5 +# how many marks at the mark page +MARK_PER_PAGE = 20 +# how many reviews showed on the detail page +REVIEW_NUMBER = 5 +# how many reviews at the mark page +REVIEW_PER_PAGE = 20 +# max tags on detail page +TAG_NUMBER = 10 + + +class HTTPResponseHXRedirect(HttpResponseRedirect): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self['HX-Redirect'] = self['Location'] + status_code = 200 + + +# public data +########################### +@login_required +def create(request): + if request.method == 'GET': + form = CollectionForm() + return render( + request, + 'create_update.html', + { + 'form': form, + 'title': _('添加收藏单'), + 'submit_url': reverse("collection:create"), + # provided for frontend js + 'this_site_enum_value': SourceSiteEnum.IN_SITE.value, + } + ) + elif request.method == 'POST': + if request.user.is_authenticated: + # only local user can alter public data + form = CollectionForm(request.POST, request.FILES) + form.instance.owner = request.user + if form.is_valid(): + form.instance.last_editor = request.user + try: + with transaction.atomic(): + form.save() + except IntegrityError as e: + logger.error(e.__str__()) + return HttpResponseServerError("integrity error") + return redirect(reverse("collection:retrieve", args=[form.instance.id])) + else: + return render( + request, + 'create_update.html', + { + 'form': form, + 'title': _('添加收藏单'), + 'submit_url': reverse("collection:create"), + # provided for frontend js + 'this_site_enum_value': SourceSiteEnum.IN_SITE.value, + } + ) + else: + return redirect(reverse("users:login")) + else: + return HttpResponseBadRequest() + + +@login_required +def update(request, id): + page_title = _("修改收藏单") + collection = get_object_or_404(Collection, pk=id) + if not collection.is_visible_to(request.user): + raise PermissionDenied() + if request.method == 'GET': + form = CollectionForm(instance=collection) + return render( + request, + 'create_update.html', + { + 'form': form, + 'is_update': True, + 'title': page_title, + 'submit_url': reverse("collection:update", args=[collection.id]), + # provided for frontend js + 'this_site_enum_value': SourceSiteEnum.IN_SITE.value, + } + ) + elif request.method == 'POST': + form = CollectionForm(request.POST, request.FILES, instance=collection) + if form.is_valid(): + form.instance.last_editor = request.user + form.instance.edited_time = timezone.now() + try: + with transaction.atomic(): + form.save() + except IntegrityError as e: + logger.error(e.__str__()) + return HttpResponseServerError("integrity error") + else: + return render( + request, + 'create_update.html', + { + 'form': form, + 'is_update': True, + 'title': page_title, + 'submit_url': reverse("collection:update", args=[collection.id]), + # provided for frontend js + 'this_site_enum_value': SourceSiteEnum.IN_SITE.value, + } + ) + return redirect(reverse("collection:retrieve", args=[form.instance.id])) + + else: + return HttpResponseBadRequest() + + +@mastodon_request_included +# @login_required +def retrieve(request, id): + if request.method == 'GET': + collection = get_object_or_404(Collection, pk=id) + if not collection.is_visible_to(request.user): + raise PermissionDenied() + form = CollectionForm(instance=collection) + + if request.user.is_authenticated: + following = True if CollectionMark.objects.filter(owner=request.user, collection=collection).first() is not None else False + followers = [] + else: + following = False + followers = [] + + return render( + request, + 'detail.html', + { + 'collection': collection, + 'form': form, + 'editable': request.user.is_authenticated and collection.is_editable_by(request.user), + 'followers': followers, + 'following': following, + } + ) + else: + logger.warning('non-GET method at /collections/') + return HttpResponseBadRequest() + + +@mastodon_request_included +# @login_required +def retrieve_entity_list(request, id): + collection = get_object_or_404(Collection, pk=id) + if not collection.is_visible_to(request.user): + raise PermissionDenied() + form = CollectionForm(instance=collection) + + followers = [] + if request.user.is_authenticated: + followers = [] + + return render( + request, + 'entity_list.html', + { + 'collection': collection, + 'form': form, + 'editable': request.user.is_authenticated and collection.is_editable_by(request.user), + 'followers': followers, + + } + ) + + +@login_required +def delete(request, id): + collection = get_object_or_404(Collection, pk=id) + if request.user.is_staff or request.user == collection.owner: + if request.method == 'GET': + return render( + request, + 'delete.html', + { + 'collection': collection, + 'form': CollectionForm(instance=collection) + } + ) + elif request.method == 'POST': + collection.delete() + return redirect(reverse("common:home")) + else: + raise PermissionDenied() + + +@login_required +def wish(request, id): + try: + CollectionMark.objects.create(owner=request.user, collection=Collection.objects.get(id=id)) + except Exception: + pass + return HttpResponse("✔️") + + +@login_required +def follow(request, id): + CollectionMark.objects.create(owner=request.user, collection=Collection.objects.get(id=id)) + return redirect(reverse("collection:retrieve", args=[id])) + + +@login_required +def unfollow(request, id): + CollectionMark.objects.filter(owner=request.user, collection=Collection.objects.get(id=id)).delete() + return redirect(reverse("collection:retrieve", args=[id])) + + +@login_required +def list(request, user_id=None, marked=False): + if request.method == 'GET': + user = request.user if user_id is None else User.objects.get(id=user_id) + if marked: + title = user.mastodon_username + _('关注的收藏单') + queryset = Collection.objects.filter(pk__in=CollectionMark.objects.filter(owner=user).values_list('collection', flat=True)) + else: + title = user.mastodon_username + _('创建的收藏单') + queryset = Collection.objects.filter(owner=user) + paginator = Paginator(queryset, REVIEW_PER_PAGE) + page_number = request.GET.get('page', default=1) + collections = paginator.get_page(page_number) + collections.pagination = PageLinksGenerator( + PAGE_LINK_NUMBER, page_number, paginator.num_pages) + return render( + request, + 'list.html', + { + 'collections': collections, + 'title': title, + } + ) + else: + return HttpResponseBadRequest() + + +def get_entity_by_url(url): + m = re.findall(r'^/?(movies|books|games|music/album|music/song)/(\d+)/?', url.strip().lower().replace(settings.APP_WEBSITE.lower(), '')) + if len(m) > 0: + mapping = { + 'movies': Movie, + 'books': Book, + 'games': Game, + 'music/album': Album, + 'music/song': Song, + } + cls = mapping.get(m[0][0]) + id = int(m[0][1]) + if cls is not None: + return cls.objects.get(id=id) + return None + + +@login_required +def append_item(request, id): + collection = get_object_or_404(Collection, pk=id) + if request.method == 'POST' and collection.is_editable_by(request.user): + url = request.POST.get('url') + comment = request.POST.get('comment') + item = get_entity_by_url(url) + collection.append_item(item, comment) + collection.save() + # return redirect(reverse("collection:retrieve", args=[id])) + return retrieve_entity_list(request, id) + else: + return HttpResponseBadRequest() + + +@login_required +def delete_item(request, id, item_id): + collection = get_object_or_404(Collection, pk=id) + if request.method == 'POST' and collection.is_editable_by(request.user): + # item_id = int(request.POST.get('item_id')) + item = CollectionItem.objects.get(id=item_id) + if item is not None and item.collection == collection: + item.delete() + # collection.save() + # return HTTPResponseHXRedirect(redirect_to=reverse("collection:retrieve", args=[id])) + return retrieve_entity_list(request, id) + return HttpResponseBadRequest() + + +@login_required +def move_up_item(request, id, item_id): + collection = get_object_or_404(Collection, pk=id) + if request.method == 'POST' and collection.is_editable_by(request.user): + # item_id = int(request.POST.get('item_id')) + item = CollectionItem.objects.get(id=item_id) + if item is not None and item.collection == collection: + items = collection.collectionitem_list + idx = items.index(item) + if idx > 0: + o = items[idx - 1] + p = o.position + o.position = item.position + item.position = p + o.save() + item.save() + # collection.save() + # return HTTPResponseHXRedirect(redirect_to=reverse("collection:retrieve", args=[id])) + return retrieve_entity_list(request, id) + return HttpResponseBadRequest() + + +@login_required +def move_down_item(request, id, item_id): + collection = get_object_or_404(Collection, pk=id) + if request.method == 'POST' and collection.is_editable_by(request.user): + # item_id = int(request.POST.get('item_id')) + item = CollectionItem.objects.get(id=item_id) + if item is not None and item.collection == collection: + items = collection.collectionitem_list + idx = items.index(item) + if idx + 1 < len(items): + o = items[idx + 1] + p = o.position + o.position = item.position + item.position = p + o.save() + item.save() + # collection.save() + # return HTTPResponseHXRedirect(redirect_to=reverse("collection:retrieve", args=[id])) + return retrieve_entity_list(request, id) + return HttpResponseBadRequest() + + +def show_item_comment(request, id, item_id): + collection = get_object_or_404(Collection, pk=id) + item = CollectionItem.objects.get(id=item_id) + editable = request.user.is_authenticated and collection.is_editable_by(request.user) + return render(request, 'show_item_comment.html', {'collection': collection, 'collectionitem': item, 'editable': editable}) + + +@login_required +def update_item_comment(request, id, item_id): + collection = get_object_or_404(Collection, pk=id) + if collection.is_editable_by(request.user): + # item_id = int(request.POST.get('item_id')) + item = CollectionItem.objects.get(id=item_id) + if item is not None and item.collection == collection: + if request.method == 'POST': + item.comment = request.POST.get('comment', default='') + item.save() + return render(request, 'show_item_comment.html', {'collection': collection, 'collectionitem': item, 'editable': True}) + else: + return render(request, 'edit_item_comment.html', {'collection': collection, 'collectionitem': item}) + return retrieve_entity_list(request, id) + return HttpResponseBadRequest() + + +@login_required +def list_with(request, type, id): + pass + + +def get_entity_by_type_id(type, id): + mapping = { + 'movie': Movie, + 'book': Book, + 'game': Game, + 'album': Album, + 'song': Song, + } + cls = mapping.get(type) + if cls is not None: + return cls.objects.get(id=id) + return None + + +@login_required +def add_to_list(request, type, id): + item = get_entity_by_type_id(type, id) + if request.method == 'GET': + queryset = Collection.objects.filter(owner=request.user) + return render( + request, + 'add_to_list.html', + { + 'type': type, + 'id': id, + 'item': item, + 'collections': queryset, + } + ) + else: + cid = int(request.POST.get('collection_id', default=0)) + if not cid: + cid = Collection.objects.create(owner=request.user, title=f'{request.user.username}的收藏单').id + collection = Collection.objects.filter(owner=request.user, id=cid).first() + collection.append_item(item, request.POST.get('comment')) + return HttpResponseRedirect(request.META.get('HTTP_REFERER')) + + +@login_required +def share(request, id): + collection = Collection.objects.filter(id=id).first() + if not collection: + return HttpResponseBadRequest() + if request.method == 'GET': + return render(request, 'share_collection.html', {'id': id, 'visibility': request.user.get_preference().default_visibility}) + else: + visibility = int(request.POST.get('visibility', default=0)) + comment = request.POST.get('comment') + if share_collection(collection, comment, request.user, visibility): + return HttpResponseRedirect(request.META.get('HTTP_REFERER')) + else: + return go_relogin(request) diff --git a/common/forms.py b/common/forms.py index 5372ce12..849b31c4 100644 --- a/common/forms.py +++ b/common/forms.py @@ -1,4 +1,5 @@ from django import forms +from markdownx.fields import MarkdownxFormField import django.contrib.postgres.forms as postgres from django.utils import formats from django.core.exceptions import ValidationError @@ -45,7 +46,7 @@ class HstoreInput(forms.Widget): js = ('js/key_value_input.js',) -class JSONField(postgres.JSONField): +class JSONField(forms.fields.JSONField): widget = KeyValueInput def to_python(self, value): if not value: @@ -88,7 +89,7 @@ class RatingValidator: _('%(value)s is not an integer'), params={'value': value}, ) - if not str(value) in [str(i) for i in range(1, 11)]: + if not str(value) in [str(i) for i in range(0, 11)]: raise ValidationError( _('%(value)s is not an integer in range 1-10'), params={'value': value}, @@ -154,9 +155,9 @@ class MultiSelect(forms.SelectMultiple): class Media: css = { - 'all': ('lib/css/multiple-select.min.css',) + 'all': ('https://cdn.jsdelivr.net/npm/multiple-select@1.5.2/dist/multiple-select.min.css',) } - js = ('lib/js/multiple-select.min.js',) + js = ('https://cdn.jsdelivr.net/npm/multiple-select@1.5.2/dist/multiple-select.min.js',) class HstoreField(forms.CharField): @@ -223,22 +224,25 @@ class DurationField(forms.TimeField): ############################# # Form ############################# +VISIBILITY_CHOICES = [ + (0, _("公开")), + (1, _("仅关注者")), + (2, _("仅自己")), +] -class MarkForm(forms.ModelForm): - IS_PRIVATE_CHOICES = [ - (True, _("仅关注者")), - (False, _("公开")), - ] - + +class MarkForm(forms.ModelForm): id = forms.IntegerField(required=False, widget=forms.HiddenInput()) share_to_mastodon = forms.BooleanField( - label=_("分享到长毛象"), initial=True, required=False) + label=_("分享到联邦网络"), initial=True, required=False) rating = forms.IntegerField( - validators=[RatingValidator()], widget=forms.HiddenInput(), required=False) - is_private = RadioBooleanField( + label=_("评分"), validators=[RatingValidator()], widget=forms.HiddenInput(), required=False) + visibility = forms.TypedChoiceField( label=_("可见性"), - initial=True, - choices=IS_PRIVATE_CHOICES + initial=0, + coerce=int, + choices=VISIBILITY_CHOICES, + widget=forms.RadioSelect ) tags = TagField( required=False, @@ -259,15 +263,15 @@ class MarkForm(forms.ModelForm): class ReviewForm(forms.ModelForm): - IS_PRIVATE_CHOICES = [ - (True, _("仅关注者")), - (False, _("公开")), - ] + title = forms.CharField(label=_("标题")) + content = MarkdownxFormField(label=_("正文 (Markdown)")) share_to_mastodon = forms.BooleanField( - label=_("分享到长毛象"), initial=True, required=False) + label=_("分享到联邦网络"), initial=True, required=False) id = forms.IntegerField(required=False, widget=forms.HiddenInput()) - is_private = RadioBooleanField( + visibility = forms.TypedChoiceField( label=_("可见性"), - initial=True, - choices=IS_PRIVATE_CHOICES + initial=0, + coerce=int, + choices=VISIBILITY_CHOICES, + widget=forms.RadioSelect ) diff --git a/common/importers/douban.py b/common/importers/douban.py new file mode 100644 index 00000000..321cea8f --- /dev/null +++ b/common/importers/douban.py @@ -0,0 +1,270 @@ +import openpyxl +import requests +import re +from lxml import html +from markdownify import markdownify as md +from datetime import datetime +from common.scraper import get_scraper_by_url +import logging +import pytz +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from user_messages import api as msg +import django_rq +from common.utils import GenerateDateUUIDMediaFilePath +import os +from books.models import BookReview, Book, BookMark, BookTag +from movies.models import MovieReview, Movie, MovieMark, MovieTag +from music.models import AlbumReview, Album, AlbumMark, AlbumTag +from games.models import GameReview, Game, GameMark, GameTag +from common.scraper import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper +from PIL import Image +from io import BytesIO +import filetype +from common.models import MarkStatusEnum + + +logger = logging.getLogger(__name__) +tz_sh = pytz.timezone('Asia/Shanghai') + + +def fetch_remote_image(url): + try: + print(f'fetching remote image {url}') + raw_img = None + ext = None + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + elif settings.SCRAPERAPI_KEY is not None: + dl_url = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}' + else: + dl_url = url + img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT) + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + f = GenerateDateUUIDMediaFilePath(None, "x." + ext, settings.MARKDOWNX_MEDIA_PATH) + file = settings.MEDIA_ROOT + f + local_url = settings.MEDIA_URL + f + os.makedirs(os.path.dirname(file), exist_ok=True) + img.save(file) + # print(f'remote image saved as {local_url}') + return local_url + except Exception: + print(f'unable to fetch remote image {url}') + return url + + +class DoubanImporter: + total = 0 + processed = 0 + skipped = 0 + imported = 0 + failed = [] + user = None + visibility = 0 + file = None + + def __init__(self, user, visibility): + self.user = user + self.visibility = visibility + + def update_user_import_status(self, status): + self.user.preference.import_status['douban_pending'] = status + self.user.preference.import_status['douban_file'] = self.file + self.user.preference.import_status['douban_visibility'] = self.visibility + self.user.preference.import_status['douban_total'] = self.total + self.user.preference.import_status['douban_processed'] = self.processed + self.user.preference.import_status['douban_skipped'] = self.skipped + self.user.preference.import_status['douban_imported'] = self.imported + self.user.preference.import_status['douban_failed'] = self.failed + self.user.preference.save(update_fields=['import_status']) + + def import_from_file(self, uploaded_file): + try: + wb = openpyxl.open(uploaded_file, read_only=True, data_only=True, keep_links=False) + wb.close() + file = settings.MEDIA_ROOT + GenerateDateUUIDMediaFilePath(None, "x.xlsx", settings.SYNC_FILE_PATH_ROOT) + os.makedirs(os.path.dirname(file), exist_ok=True) + with open(file, 'wb') as destination: + for chunk in uploaded_file.chunks(): + destination.write(chunk) + self.file = file + self.update_user_import_status(2) + jid = f'Douban_{self.user.id}_{os.path.basename(self.file)}' + django_rq.get_queue('doufen').enqueue(self.import_from_file_task, job_id=jid) + except Exception: + return False + # self.import_from_file_task(file, user, visibility) + return True + + mark_sheet_config = { + '想读': [MarkStatusEnum.WISH, DoubanBookScraper, Book, BookMark, BookTag], + '在读': [MarkStatusEnum.DO, DoubanBookScraper, Book, BookMark, BookTag], + '读过': [MarkStatusEnum.COLLECT, DoubanBookScraper, Book, BookMark, BookTag], + '想看': [MarkStatusEnum.WISH, DoubanMovieScraper, Movie, MovieMark, MovieTag], + '在看': [MarkStatusEnum.DO, DoubanMovieScraper, Movie, MovieMark, MovieTag], + '想看': [MarkStatusEnum.COLLECT, DoubanMovieScraper, Movie, MovieMark, MovieTag], + '想听': [MarkStatusEnum.WISH, DoubanAlbumScraper, Album, AlbumMark, AlbumTag], + '在听': [MarkStatusEnum.DO, DoubanAlbumScraper, Album, AlbumMark, AlbumTag], + '听过': [MarkStatusEnum.COLLECT, DoubanAlbumScraper, Album, AlbumMark, AlbumTag], + '想玩': [MarkStatusEnum.WISH, DoubanGameScraper, Game, GameMark, GameTag], + '在玩': [MarkStatusEnum.DO, DoubanGameScraper, Game, GameMark, GameTag], + '玩过': [MarkStatusEnum.COLLECT, DoubanGameScraper, Game, GameMark, GameTag], + } + review_sheet_config = { + '书评': [DoubanBookScraper, Book, BookReview], + '影评': [DoubanMovieScraper, Movie, MovieReview], + '乐评': [DoubanAlbumScraper, Album, AlbumReview], + '游戏评论&攻略': [DoubanGameScraper, Game, GameReview], + } + mark_data = {} + review_data = {} + entity_lookup = {} + + def load_sheets(self): + f = open(self.file, 'rb') + wb = openpyxl.load_workbook(f, read_only=True, data_only=True, keep_links=False) + for data, config in [(self.mark_data, self.mark_sheet_config), (self.review_data, self.review_sheet_config)]: + for name in config: + data[name] = [] + if name in wb: + print(f'{self.user} parsing {name}') + for row in wb[name].iter_rows(min_row=2, values_only=True): + cells = [cell for cell in row] + if len(cells) > 6: + data[name].append(cells) + for sheet in self.mark_data.values(): + for cells in sheet: + # entity_lookup["title|rating"] = [(url, time), ...] + k = f'{cells[0]}|{cells[5]}' + v = (cells[3], cells[4]) + if k in self.entity_lookup: + self.entity_lookup[k].append(v) + else: + self.entity_lookup[k] = [v] + self.total = sum(map(lambda a: len(a), self.review_data.values())) + + def guess_entity_url(self, title, rating, timestamp): + k = f'{title}|{rating}' + if k not in self.entity_lookup: + return None + v = self.entity_lookup[k] + if len(v) > 1: + v.sort(key=lambda c: abs(timestamp - (datetime.strptime(c[1], "%Y-%m-%d %H:%M:%S") if type(c[1])==str else c[1]).replace(tzinfo=tz_sh))) + return v[0][0] + # for sheet in self.mark_data.values(): + # for cells in sheet: + # if cells[0] == title and cells[5] == rating: + # return cells[3] + + def import_from_file_task(self): + print(f'{self.user} import start') + msg.info(self.user, f'开始导入豆瓣评论') + self.update_user_import_status(1) + self.load_sheets() + print(f'{self.user} sheet loaded, {self.total} lines total') + self.update_user_import_status(1) + for name, param in self.review_sheet_config.items(): + self.import_review_sheet(self.review_data[name], param[0], param[1], param[2]) + self.update_user_import_status(0) + msg.success(self.user, f'豆瓣评论导入完成，共处理{self.total}篇，已存在{self.skipped}篇，新增{self.imported}篇。') + if len(self.failed): + msg.error(self.user, f'豆瓣评论导入时未能处理以下网址：\n{" , ".join(self.failed)}') + + def import_review_sheet(self, worksheet, scraper, entity_class, review_class): + prefix = f'{self.user} |' + if worksheet is None: # or worksheet.max_row < 2: + print(f'{prefix} {review_class.__name__} empty sheet') + return + for cells in worksheet: + if len(cells) < 6: + continue + title = cells[0] + entity_title = re.sub('^《', '', re.sub('》$', '', cells[1])) + review_url = cells[2] + time = cells[3] + rating = cells[4] + content = cells[6] + self.processed += 1 + if time: + if type(time) == str: + time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") + time = time.replace(tzinfo=tz_sh) + else: + time = None + if not content: + content = "" + if not title: + title = "" + r = self.import_review(entity_title, rating, title, review_url, content, time, scraper, entity_class, review_class) + if r == 1: + self.imported += 1 + elif r == 2: + self.skipped += 1 + else: + self.failed.append(review_url) + self.update_user_import_status(1) + + def import_review(self, entity_title, rating, title, review_url, content, time, scraper, entity_class, review_class): + # return 1: done / 2: skipped / None: failed + prefix = f'{self.user} |' + url = self.guess_entity_url(entity_title, rating, time) + if url is None: + print(f'{prefix} fetching {review_url}') + try: + if settings.SCRAPESTACK_KEY is not None: + _review_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={review_url}' + else: + _review_url = review_url + r = requests.get(_review_url, timeout=settings.SCRAPING_TIMEOUT) + if r.status_code != 200: + print(f'{prefix} fetching error {review_url} {r.status_code}') + return + h = html.fromstring(r.content.decode('utf-8')) + for u in h.xpath("//header[@class='main-hd']/a/@href"): + if '.douban.com/subject/' in u: + url = u + if not url: + print(f'{prefix} fetching error {review_url} unable to locate entity url') + return + except Exception: + print(f'{prefix} fetching exception {review_url}') + return + try: + entity = entity_class.objects.get(source_url=url) + print(f'{prefix} matched {url}') + except ObjectDoesNotExist: + try: + print(f'{prefix} scraping {url}') + scraper.scrape(url) + form = scraper.save(request_user=self.user) + entity = form.instance + except Exception as e: + print(f"{prefix} scrape failed: {url} {e}") + logger.error(f"{prefix} scrape failed: {url}", exc_info=e) + return + params = { + 'owner': self.user, + entity_class.__name__.lower(): entity + } + if review_class.objects.filter(**params).exists(): + return 2 + content = re.sub(r'([^<]+)', r'\1', content) + content = re.sub(r'(]+>)', r'\1
', content) + content = re.sub(r'
([^<]+)
', r'
\1
', content) + content = md(content) + content = re.sub(r'(?<=!\[\]\()([^)]+)(?=\))', lambda x: fetch_remote_image(x[1]), content) + params = { + 'owner': self.user, + 'created_time': time, + 'edited_time': time, + 'title': title, + 'content': content, + 'visibility': self.visibility, + entity_class.__name__.lower(): entity, + } + review_class.objects.create(**params) + return 1 diff --git a/common/importers/goodreads.py b/common/importers/goodreads.py new file mode 100644 index 00000000..4e2fec58 --- /dev/null +++ b/common/importers/goodreads.py @@ -0,0 +1,202 @@ +import re +import requests +from lxml import html +from datetime import datetime +# from common.scrapers.goodreads import GoodreadsScraper +from common.scraper import get_scraper_by_url +from books.models import Book, BookMark +from collection.models import Collection +from common.models import MarkStatusEnum +from django.conf import settings +from user_messages import api as msg +import django_rq +from django.utils.timezone import make_aware + + +re_list = r'^https://www.goodreads.com/list/show/\d+' +re_shelf = r'^https://www.goodreads.com/review/list/\d+[^?]*\?shelf=[^&]+' +re_profile = r'^https://www.goodreads.com/user/show/(\d+)' +gr_rating = { + 'did not like it': 2, + 'it was ok': 4, + 'liked it': 6, + 'really liked it': 8, + 'it was amazing': 10 +} + + +class GoodreadsImporter: + @classmethod + def import_from_url(self, raw_url, user): + match_list = re.match(re_list, raw_url) + match_shelf = re.match(re_shelf, raw_url) + match_profile = re.match(re_profile, raw_url) + if match_profile or match_shelf or match_list: + django_rq.get_queue('doufen').enqueue(self.import_from_url_task, raw_url, user) + return True + else: + return False + + @classmethod + def import_from_url_task(cls, url, user): + match_list = re.match(re_list, url) + match_shelf = re.match(re_shelf, url) + match_profile = re.match(re_profile, url) + total = 0 + if match_list or match_shelf: + shelf = cls.parse_shelf(match_shelf[0], user) if match_shelf else cls.parse_list(match_list[0], user) + if shelf['title'] and shelf['books']: + collection = Collection.objects.create(title=shelf['title'], + description=shelf['description'] + '\n\nImported from [Goodreads](' + url + ')', + owner=user) + for book in shelf['books']: + collection.append_item(book['book'], book['review']) + total += 1 + collection.save() + msg.success(user, f'成功从Goodreads导入包含{total}本书的收藏单{shelf["title"]}。') + elif match_profile: + uid = match_profile[1] + shelves = { + MarkStatusEnum.WISH: f'https://www.goodreads.com/review/list/{uid}?shelf=to-read', + MarkStatusEnum.DO: f'https://www.goodreads.com/review/list/{uid}?shelf=currently-reading', + MarkStatusEnum.COLLECT: f'https://www.goodreads.com/review/list/{uid}?shelf=read', + } + for status in shelves: + shelf_url = shelves.get(status) + shelf = cls.parse_shelf(shelf_url, user) + for book in shelf['books']: + params = { + 'owner': user, + 'rating': book['rating'], + 'text': book['review'], + 'status': status, + 'visibility': 0, + 'book': book['book'], + } + if book['last_updated']: + params['created_time'] = book['last_updated'] + params['edited_time'] = book['last_updated'] + try: + mark = BookMark.objects.create(**params) + mark.book.update_rating(None, mark.rating) + except Exception: + print(f'Skip mark for {book["book"]}') + pass + total += 1 + msg.success(user, f'成功从Goodreads用户主页导入{total}个标记。') + + @classmethod + def parse_shelf(cls, url, user): # return {'title': 'abc', books: [{'book': obj, 'rating': 10, 'review': 'txt'}, ...]} + title = None + books = [] + url_shelf = url + '&view=table' + while url_shelf: + print(f'Shelf loading {url_shelf}') + r = requests.get(url_shelf, timeout=settings.SCRAPING_TIMEOUT) + if r.status_code != 200: + print(f'Shelf loading error {url_shelf}') + break + url_shelf = None + content = html.fromstring(r.content.decode('utf-8')) + title_elem = content.xpath("//span[@class='h1Shelf']/text()") + if not title_elem: + print(f'Shelf parsing error {url_shelf}') + break + title = title_elem[0].strip() + print("Shelf title: " + title) + for cell in content.xpath("//tbody[@id='booksBody']/tr"): + url_book = 'https://www.goodreads.com' + \ + cell.xpath( + ".//td[@class='field title']//a/@href")[0].strip() + # has_review = cell.xpath( + # ".//td[@class='field actions']//a/text()")[0].strip() == 'view (with text)' + rating_elem = cell.xpath( + ".//td[@class='field rating']//span/@title") + rating = gr_rating.get( + rating_elem[0].strip()) if rating_elem else None + url_review = 'https://www.goodreads.com' + \ + cell.xpath( + ".//td[@class='field actions']//a/@href")[0].strip() + review = '' + last_updated = None + try: + r2 = requests.get( + url_review, timeout=settings.SCRAPING_TIMEOUT) + if r2.status_code == 200: + c2 = html.fromstring(r2.content.decode('utf-8')) + review_elem = c2.xpath( + "//div[@itemprop='reviewBody']/text()") + review = '\n'.join( + p.strip() for p in review_elem) if review_elem else '' + date_elem = c2.xpath( + "//div[@class='readingTimeline__text']/text()") + for d in date_elem: + date_matched = re.search(r'(\w+)\s+(\d+),\s+(\d+)', d) + if date_matched: + last_updated = make_aware(datetime.strptime(date_matched[1] + ' ' + date_matched[2] + ' ' + date_matched[3], '%B %d %Y')) + else: + print(f"Error loading review{url_review}, ignored") + scraper = get_scraper_by_url(url_book) + url_book = scraper.get_effective_url(url_book) + book = Book.objects.filter(source_url=url_book).first() + if not book: + print("add new book " + url_book) + scraper.scrape(url_book) + form = scraper.save(request_user=user) + book = form.instance + books.append({ + 'url': url_book, + 'book': book, + 'rating': rating, + 'review': review, + 'last_updated': last_updated + }) + except Exception: + print("Error adding " + url_book) + pass # likely just download error + next_elem = content.xpath("//a[@class='next_page']/@href") + url_shelf = ('https://www.goodreads.com' + next_elem[0].strip()) if next_elem else None + return {'title': title, 'description': '', 'books': books} + + @classmethod + def parse_list(cls, url, user): # return {'title': 'abc', books: [{'book': obj, 'rating': 10, 'review': 'txt'}, ...]} + title = None + books = [] + url_shelf = url + while url_shelf: + print(f'List loading {url_shelf}') + r = requests.get(url_shelf, timeout=settings.SCRAPING_TIMEOUT) + if r.status_code != 200: + print(f'List loading error {url_shelf}') + break + url_shelf = None + content = html.fromstring(r.content.decode('utf-8')) + title_elem = content.xpath('//h1[@class="gr-h1 gr-h1--serif"]/text()') + if not title_elem: + print(f'List parsing error {url_shelf}') + break + title = title_elem[0].strip() + description = content.xpath('//div[@class="mediumText"]/text()')[0].strip() + print("List title: " + title) + for link in content.xpath('//a[@class="bookTitle"]/@href'): + url_book = 'https://www.goodreads.com' + link + try: + scraper = get_scraper_by_url(url_book) + url_book = scraper.get_effective_url(url_book) + book = Book.objects.filter(source_url=url_book).first() + if not book: + print("add new book " + url_book) + scraper.scrape(url_book) + form = scraper.save(request_user=user) + book = form.instance + books.append({ + 'url': url_book, + 'book': book, + 'review': '', + }) + except Exception: + print("Error adding " + url_book) + pass # likely just download error + next_elem = content.xpath("//a[@class='next_page']/@href") + url_shelf = ('https://www.goodreads.com' + next_elem[0].strip()) if next_elem else None + return {'title': title, 'description': description, 'books': books} diff --git a/common/index.py b/common/index.py new file mode 100644 index 00000000..42227b60 --- /dev/null +++ b/common/index.py @@ -0,0 +1,12 @@ +from django.conf import settings + + +if settings.SEARCH_BACKEND == 'MEILISEARCH': + from .search.meilisearch import Indexer +elif settings.SEARCH_BACKEND == 'TYPESENSE': + from .search.typesense import Indexer +else: + class Indexer: + @classmethod + def update_model_indexable(self, cls): + pass diff --git a/common/management/commands/delete_job.py b/common/management/commands/delete_job.py new file mode 100644 index 00000000..1c1d6c86 --- /dev/null +++ b/common/management/commands/delete_job.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand +import pprint +from redis import Redis +from rq.job import Job +from rq import Queue + + +class Command(BaseCommand): + help = 'Delete a job' + + def add_arguments(self, parser): + parser.add_argument('job_id', type=str, help='Job ID') + + def handle(self, *args, **options): + redis = Redis() + job_id = str(options['job_id']) + job = Job.fetch(job_id, connection=redis) + job.delete() + self.stdout.write(self.style.SUCCESS(f'Deleted {job}')) diff --git a/common/management/commands/index_stats.py b/common/management/commands/index_stats.py new file mode 100644 index 00000000..28a9f07e --- /dev/null +++ b/common/management/commands/index_stats.py @@ -0,0 +1,40 @@ +from django.core.management.base import BaseCommand +from common.index import Indexer +from django.conf import settings +from movies.models import Movie +from books.models import Book +from games.models import Game +from music.models import Album, Song +from django.core.paginator import Paginator +from tqdm import tqdm +from time import sleep +from datetime import timedelta +from django.utils import timezone + + +class Command(BaseCommand): + help = 'Check search index' + + def handle(self, *args, **options): + print(f'Connecting to search server') + stats = Indexer.get_stats() + print(stats) + st = Indexer.instance().get_all_update_status() + cnt = {"enqueued": [0, 0], "processing": [0, 0], "processed": [0, 0], "failed": [0, 0]} + lastEnq = {"enqueuedAt": ""} + lastProc = {"enqueuedAt": ""} + for s in st: + n = s["type"].get("number") + cnt[s["status"]][0] += 1 + cnt[s["status"]][1] += n if n else 0 + if s["status"] == "processing": + print(s) + elif s["status"] == "enqueued": + if s["enqueuedAt"] > lastEnq["enqueuedAt"]: + lastEnq = s + elif s["status"] == "processed": + if s["enqueuedAt"] > lastProc["enqueuedAt"]: + lastProc = s + print(lastEnq) + print(lastProc) + print(cnt) diff --git a/common/management/commands/init_index.py b/common/management/commands/init_index.py new file mode 100644 index 00000000..797d32e4 --- /dev/null +++ b/common/management/commands/init_index.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand +from common.index import Indexer +from django.conf import settings + + +class Command(BaseCommand): + help = 'Initialize the search index' + + def handle(self, *args, **options): + print(f'Connecting to search server') + Indexer.init() + self.stdout.write(self.style.SUCCESS('Index created.')) + # try: + # Indexer.init() + # self.stdout.write(self.style.SUCCESS('Index created.')) + # except Exception: + # Indexer.update_settings() + # self.stdout.write(self.style.SUCCESS('Index settings updated.')) diff --git a/common/management/commands/list_jobs.py b/common/management/commands/list_jobs.py new file mode 100644 index 00000000..e843f527 --- /dev/null +++ b/common/management/commands/list_jobs.py @@ -0,0 +1,24 @@ +from django.core.management.base import BaseCommand +import pprint +from redis import Redis +from rq.job import Job +from rq import Queue + + +class Command(BaseCommand): + help = 'Show jobs in queue' + + def add_arguments(self, parser): + parser.add_argument('queue', type=str, help='Queue') + + def handle(self, *args, **options): + redis = Redis() + queue = Queue(str(options['queue']), connection=redis) + for registry in [queue.started_job_registry, queue.deferred_job_registry, queue.finished_job_registry, queue.failed_job_registry, queue.scheduled_job_registry]: + self.stdout.write(self.style.SUCCESS(f'Registry {registry}')) + for job_id in registry.get_job_ids(): + try: + job = Job.fetch(job_id, connection=redis) + pprint.pp(job) + except Exception as e: + print(f'Error fetching {job_id}') diff --git a/common/management/commands/reindex.py b/common/management/commands/reindex.py new file mode 100644 index 00000000..5dcc766f --- /dev/null +++ b/common/management/commands/reindex.py @@ -0,0 +1,40 @@ +from django.core.management.base import BaseCommand +from common.index import Indexer +from django.conf import settings +from movies.models import Movie +from books.models import Book +from games.models import Game +from music.models import Album, Song +from django.core.paginator import Paginator +from tqdm import tqdm +from time import sleep +from datetime import timedelta +from django.utils import timezone + + +BATCH_SIZE = 1000 + + +class Command(BaseCommand): + help = 'Regenerate the search index' + + # def add_arguments(self, parser): + # parser.add_argument('hours', type=int, help='Re-index items modified in last N hours, 0 to reindex all') + + def handle(self, *args, **options): + # h = int(options['hours']) + print(f'Connecting to search server') + if Indexer.busy(): + print('Please wait for previous updates') + # Indexer.update_settings() + # self.stdout.write(self.style.SUCCESS('Index settings updated.')) + for c in [Book, Song, Album, Game, Movie]: + print(f'Re-indexing {c}') + qs = c.objects.all() # if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h)) + pg = Paginator(qs.order_by('id'), BATCH_SIZE) + for p in tqdm(pg.page_range): + items = list(map(lambda o: Indexer.obj_to_dict(o), pg.get_page(p).object_list)) + if items: + Indexer.replace_batch(items) + while Indexer.busy(): + sleep(0.5) diff --git a/common/management/commands/restart_sync.py b/common/management/commands/restart_sync.py new file mode 100644 index 00000000..93903c57 --- /dev/null +++ b/common/management/commands/restart_sync.py @@ -0,0 +1,28 @@ +from django.core.management.base import BaseCommand +from redis import Redis +from rq.job import Job +from sync.models import SyncTask +from sync.jobs import import_doufen_task +from django.utils import timezone +import django_rq + + +class Command(BaseCommand): + help = 'Restart a sync task' + + def add_arguments(self, parser): + parser.add_argument('synctask_id', type=int, help='Sync Task ID') + + def handle(self, *args, **options): + task = SyncTask.objects.get(id=options['synctask_id']) + task.finished_items = 0 + task.failed_urls = [] + task.success_items = 0 + task.total_items = 0 + task.is_finished = False + task.is_failed = False + task.break_point = '' + task.started_time = timezone.now() + task.save() + django_rq.get_queue('doufen').enqueue(import_doufen_task, task, job_id=f'SyncTask_{task.id}') + self.stdout.write(self.style.SUCCESS(f'Queued {task}')) diff --git a/common/management/commands/scrape.py b/common/management/commands/scrape.py new file mode 100644 index 00000000..d48898e6 --- /dev/null +++ b/common/management/commands/scrape.py @@ -0,0 +1,25 @@ +from django.core.management.base import BaseCommand +from common.scraper import get_scraper_by_url, get_normalized_url +import pprint + + +class Command(BaseCommand): + help = 'Scrape an item from URL (but not save it)' + + def add_arguments(self, parser): + parser.add_argument('url', type=str, help='URL to scrape') + + def handle(self, *args, **options): + url = str(options['url']) + url = get_normalized_url(url) + scraper = get_scraper_by_url(url) + + if scraper is None: + self.stdout.write(self.style.ERROR(f'Unable to match a scraper for {url}')) + return + + effective_url = scraper.get_effective_url(url) + self.stdout.write(f'Fetching {effective_url} via {scraper.__name__}') + data, img = scraper.scrape(effective_url) + self.stdout.write(self.style.SUCCESS(f'Done.')) + pprint.pp(data) diff --git a/common/models.py b/common/models.py index c6bf6a70..5d1b55b5 100644 --- a/common/models.py +++ b/common/models.py @@ -1,29 +1,34 @@ import re from decimal import * from markdown import markdown -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.db import models, IntegrityError from django.core.serializers.json import DjangoJSONEncoder -from django.db.models import Q +from django.db.models import Q, Count, Sum from markdownx.models import MarkdownxField from users.models import User -from mastodon.api import get_relationships, get_cross_site_id -from boofilsic.settings import CLIENT_NAME from django.utils import timezone +from django.conf import settings RE_HTML_TAG = re.compile(r"<[^>]*>") +MAX_TOP_TAGS = 5 # abstract base classes ################################### class SourceSiteEnum(models.TextChoices): - IN_SITE = "in-site", CLIENT_NAME - DOUBAN = "douban", _("豆瓣") + IN_SITE = "in-site", settings.CLIENT_NAME + DOUBAN = "douban", _("豆瓣") SPOTIFY = "spotify", _("Spotify") IMDB = "imdb", _("IMDb") STEAM = "steam", _("STEAM") BANGUMI = 'bangumi', _("bangumi") + GOODREADS = "goodreads", _("goodreads") + TMDB = "tmdb", _("The Movie Database") + GOOGLEBOOKS = "googlebooks", _("Google Books") + BANDCAMP = "bandcamp", _("BandCamp") + IGDB = "igdb", _("IGDB") class Entity(models.Model): @@ -52,10 +57,25 @@ class Entity(models.Model): rating__lte=10), name='%(class)s_rating_upperbound'), ] - def get_absolute_url(self): raise NotImplementedError("Subclass should implement this method") + @property + def url(self): + return settings.APP_WEBSITE + self.get_absolute_url() + + def get_json(self): + return { + 'title': self.title, + 'brief': self.brief, + 'rating': self.rating, + 'url': self.url, + 'cover_url': settings.APP_WEBSITE + self.cover.url, + 'top_tags': self.tags[:5], + 'category_name': self.verbose_category_name, + 'other_info': self.other_info, + } + def save(self, *args, **kwargs): """ update rating and strip source url scheme & querystring before save to db """ if self.rating_number and self.rating_total_score: @@ -108,6 +128,15 @@ class Entity(models.Model): self.calculate_rating(old_rating, new_rating) self.save() + def refresh_rating(self): # TODO: replace update_rating() + a = self.marks.filter(rating__gt=0).aggregate(Sum('rating'), Count('rating')) + if self.rating_total_score != a['rating__sum'] or self.rating_number != a['rating__count']: + self.rating_total_score = a['rating__sum'] + self.rating_number = a['rating__count'] + self.rating = a['rating__sum'] / a['rating__count'] if a['rating__count'] > 0 else None + self.save() + return self.rating + def get_tags_manager(self): """ Since relation between tag and entity is foreign key, and related name has to be unique, @@ -115,9 +144,13 @@ class Entity(models.Model): """ raise NotImplementedError("Subclass should implement this method.") + @property + def top_tags(self): + return self.get_tags_manager().values('content').annotate(tag_frequency=Count('content')).order_by('-tag_frequency')[:MAX_TOP_TAGS] + def get_marks_manager(self): """ - Normally this won't be used. + Normally this won't be used. There is no ocassion where visitor can simply view all the marks. """ raise NotImplementedError("Subclass should implement this method.") @@ -129,6 +162,19 @@ class Entity(models.Model): """ raise NotImplementedError("Subclass should implement this method.") + @property + def all_tag_list(self): + return self.get_tags_manager().values('content').annotate(frequency=Count('content')).order_by('-frequency') + + @property + def tags(self): + return list(map(lambda t: t['content'], self.all_tag_list)) + + @property + def marks(self): + params = {self.__class__.__name__.lower() + '_id': self.id} + return self.mark_class.objects.filter(**params) + @classmethod def get_category_mapping_dict(cls): category_mapping_dict = {} @@ -144,75 +190,64 @@ class Entity(models.Model): def verbose_category_name(self): raise NotImplementedError("Subclass should implement this.") + @property + def mark_class(self): + raise NotImplementedError("Subclass should implement this.") + + @property + def tag_class(self): + raise NotImplementedError("Subclass should implement this.") + class UserOwnedEntity(models.Model): - is_private = models.BooleanField() - owner = models.ForeignKey( - User, on_delete=models.CASCADE, related_name='user_%(class)ss') + is_private = models.BooleanField(default=False, null=True) # first set allow null, then migration, finally (in a few days) remove for good + visibility = models.PositiveSmallIntegerField(default=0) # 0: Public / 1: Follower only / 2: Self only + owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name='user_%(class)ss') created_time = models.DateTimeField(default=timezone.now) edited_time = models.DateTimeField(default=timezone.now) class Meta: abstract = True + def is_visible_to(self, viewer): + if not viewer.is_authenticated: + return self.visibility == 0 + owner = self.owner + if owner == viewer: + return True + if not owner.is_active: + return False + if self.visibility == 2: + return False + if viewer.is_blocking(owner) or owner.is_blocking(viewer) or viewer.is_muting(owner): + return False + if self.visibility == 1: + return viewer.is_following(owner) + else: + return True + + def is_editable_by(self, viewer): + return True if viewer.is_staff or viewer.is_superuser or viewer == self.owner else False + @classmethod - def get_available(cls, entity, request_user, token): - # TODO add amount limit for once query - """ - Returns all avaliable user-owned entities related to given entity. - This method handles mute/block relationships and private/public visibilities. - """ - # the foreign key field that points to entity - # has to be named as the lower case name of that entity + def get_available(cls, entity, request_user, following_only=False): + # e.g. SongMark.get_available(song, request.user) query_kwargs = {entity.__class__.__name__.lower(): entity} - user_owned_entities = cls.objects.filter( - **query_kwargs).order_by("-edited_time") - - # every user should only be abled to have one user owned entity for each entity - # this is guaranteed by models - id_list = [] - - # none_index tracks those failed cross site id query - none_index = [] - - for (i, entity) in enumerate(user_owned_entities): - if entity.owner.mastodon_site == request_user.mastodon_site: - id_list.append(entity.owner.mastodon_id) - else: - # TODO there could be many requests therefore make the pulling asynchronized - cross_site_id = get_cross_site_id( - entity.owner, request_user.mastodon_site, token) - if not cross_site_id is None: - id_list.append(cross_site_id) - else: - none_index.append(i) - # populate those query-failed None postions - # to ensure the consistency of the orders of - # the three(id_list, user_owned_entities, relationships) - id_list.append(request_user.mastodon_id) - - # Mastodon request - relationships = get_relationships( - request_user.mastodon_site, id_list, token) - mute_block_blocked_index = [] - following_index = [] - for i, r in enumerate(relationships): - # the order of relationships is corresponding to the id_list, - # and the order of id_list is the same as user_owned_entiies - if r['blocking'] or r['blocked_by'] or r['muting']: - mute_block_blocked_index.append(i) - if r['following']: - following_index.append(i) - available_entities = [ - e for i, e in enumerate(user_owned_entities) - if ((e.is_private == True and i in following_index) or e.is_private == False or e.owner == request_user) - and not i in mute_block_blocked_index and not i in none_index - ] - return available_entities + all_entities = cls.objects.filter(**query_kwargs).order_by("-created_time") # get all marks for song + visible_entities = list(filter(lambda _entity: _entity.is_visible_to(request_user) and (_entity.owner.mastodon_username in request_user.mastodon_following if following_only else True), all_entities)) + return visible_entities @classmethod - def get_available_by_user(cls, owner, is_following): - """ + def get_available_for_identicals(cls, entity, request_user, following_only=False): + # e.g. SongMark.get_available(song, request.user) + query_kwargs = {entity.__class__.__name__.lower() + '__in': entity.get_identicals()} + all_entities = cls.objects.filter(**query_kwargs).order_by("-created_time") # get all marks for song + visible_entities = list(filter(lambda _entity: _entity.is_visible_to(request_user) and (_entity.owner.mastodon_username in request_user.mastodon_following if following_only else True), all_entities)) + return visible_entities + + @classmethod + def get_available_by_user(cls, owner, is_following): # FIXME + """ Returns all avaliable owner's entities. Mute/Block relation is not handled in this method. @@ -220,10 +255,17 @@ class UserOwnedEntity(models.Model): :param is_following: if the current user is following the owner """ user_owned_entities = cls.objects.filter(owner=owner) - if not is_following: - user_owned_entities = user_owned_entities.exclude(is_private=True) + if is_following: + user_owned_entities = user_owned_entities.exclude(visibility=2) + else: + user_owned_entities = user_owned_entities.filter(visibility=0) return user_owned_entities + @property + def item(self): + attr = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))', self.__class__.__name__)[0].lower() + return getattr(self, attr) + # commonly used entity classes ################################### @@ -236,10 +278,20 @@ class MarkStatusEnum(models.TextChoices): class Mark(UserOwnedEntity): status = models.CharField(choices=MarkStatusEnum.choices, max_length=20) rating = models.PositiveSmallIntegerField(blank=True, null=True) - text = models.CharField(max_length=500, blank=True, default='') + text = models.CharField(max_length=5000, blank=True, default='') + shared_link = models.CharField(max_length=5000, blank=True, default='') def __str__(self): - return f"({self.id}) {self.owner} {self.status.upper()}" + return f"Mark({self.id} {self.owner} {self.status.upper()})" + + @property + def translated_status(self): + raise NotImplementedError("Subclass should implement this.") + + @property + def tags(self): + tags = self.item.tag_class.objects.filter(mark_id=self.id) + return tags class Meta: abstract = True @@ -249,7 +301,7 @@ class Mark(UserOwnedEntity): models.CheckConstraint(check=models.Q( rating__lte=10), name='mark_rating_upperbound'), ] - + # TODO update entity rating when save # TODO update tags @@ -257,6 +309,7 @@ class Mark(UserOwnedEntity): class Review(UserOwnedEntity): title = models.CharField(max_length=120) content = MarkdownxField() + shared_link = models.CharField(max_length=5000, blank=True, default='') def __str__(self): return self.title @@ -271,6 +324,10 @@ class Review(UserOwnedEntity): class Meta: abstract = True + @property + def translated_status(self): + return '评论了' + class Tag(models.Model): content = models.CharField(max_length=50) @@ -278,5 +335,28 @@ class Tag(models.Model): def __str__(self): return self.content + @property + def edited_time(self): + return self.mark.edited_time + + @property + def created_time(self): + return self.mark.created_time + + @property + def text(self): + return self.mark.text + + @classmethod + def find_by_user(cls, tag, owner, viewer): + qs = cls.objects.filter(content=tag, mark__owner=owner) + if owner != viewer: + qs = qs.filter(mark__visibility__lte=owner.get_max_visibility(viewer)) + return qs + + @classmethod + def all_by_user(cls, owner): + return cls.objects.filter(mark__owner=owner).values('content').annotate(total=Count('content')).order_by('-total') + class Meta: abstract = True diff --git a/common/scraper.py b/common/scraper.py index 33bef7e4..defe6f38 100644 --- a/common/scraper.py +++ b/common/scraper.py @@ -7,23 +7,17 @@ import dateparser import datetime import time import filetype +import dns.resolver +import urllib.parse from lxml import html from threading import Thread -from boofilsic.settings import LUMINATI_USERNAME, LUMINATI_PASSWORD, DEBUG, IMDB_API_KEY, SCRAPERAPI_KEY -from boofilsic.settings import SPOTIFY_CREDENTIAL from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from common.models import SourceSiteEnum -from movies.models import Movie, MovieGenreEnum -from movies.forms import MovieForm -from books.models import Book -from books.forms import BookForm -from music.models import Album, Song -from music.forms import AlbumForm, SongForm -from games.models import Game -from games.forms import GameForm +from django.conf import settings +from django.core.exceptions import ValidationError RE_NUMBERS = re.compile(r"\d+\d*") @@ -43,12 +37,11 @@ DEFAULT_REQUEST_HEADERS = { 'Cache-Control': 'no-cache', } -# in seconds -TIMEOUT = 60 # luminati account credentials PORT = 22225 + logger = logging.getLogger(__name__) @@ -56,6 +49,12 @@ logger = logging.getLogger(__name__) scraper_registry = {} +def get_normalized_url(raw_url): + url = re.sub(r'//m.douban.com/(\w+)/', r'//\1.douban.com/', raw_url) + url = re.sub(r'//www.google.com/books/edition/_/([A-Za-z0-9_\-]+)[\?]*', r'//books.google.com/books?id=\1&', url) + return url + + def log_url(func): """ Catch exceptions and log then pass the exceptions. @@ -67,20 +66,23 @@ def log_url(func): return func(*args, **kwargs) except Exception as e: # log the url and trace stack - logger.error(f"Scrape Failed URL: {args[1]}") - logger.error("Expections during scraping:", exc_info=e) + logger.error(f"Scrape Failed URL: {args[1]}\n{e}") + if settings.DEBUG: + logger.error("Expections during scraping:", exc_info=e) raise e return wrapper + def parse_date(raw_str): return dateparser.parse( - raw_str, + raw_str, settings={ - "RELATIVE_BASE": datetime.datetime(1900, 1, 1) + "RELATIVE_BASE": datetime.datetime(1900, 1, 1) } ) + class AbstractScraper: """ Scrape entities. The entities means those defined in the models.py file, @@ -119,7 +121,7 @@ class AbstractScraper: # decorate the scrape method cls.scrape = classmethod(log_url(cls.scrape)) - + # register scraper if isinstance(cls.host, list): for host in cls.host: @@ -141,26 +143,30 @@ class AbstractScraper: """ The return value should be identical with that saved in DB as `source_url` """ - url = cls.regex.findall(raw_url) + url = cls.regex.findall(raw_url.replace('http:', 'https:')) # force all http to be https if not url: - raise ValueError("not valid url") + raise ValueError(f"not valid url: {raw_url}") return url[0] @classmethod def download_page(cls, url, headers): url = cls.get_effective_url(url) - session_id = random.random() - proxy_url = ('http://%s-country-cn-session-%s:%s@zproxy.lum-superproxy.io:%d' % - (LUMINATI_USERNAME, session_id, LUMINATI_PASSWORD, PORT)) - proxies = { - 'http': proxy_url, - 'https': proxy_url, - } - # if DEBUG: - # proxies = None + if settings.LUMINATI_USERNAME is None: + proxies = None + if settings.SCRAPESTACK_KEY is not None: + url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + else: + session_id = random.random() + proxy_url = ('http://%s-country-cn-session-%s:%s@zproxy.lum-superproxy.io:%d' % + (settings.LUMINATI_USERNAME, session_id, settings.LUMINATI_PASSWORD, PORT)) + proxies = { + 'http': proxy_url, + 'https': proxy_url, + } + r = requests.get(url, proxies=proxies, - headers=headers, timeout=TIMEOUT) + headers=headers, timeout=settings.SCRAPING_TIMEOUT) if r.status_code != 200: raise RuntimeError(f"download page failed, status code {r.status_code}") @@ -169,19 +175,19 @@ class AbstractScraper: return html.fromstring(r.content.decode('utf-8')) @classmethod - def download_image(cls, url): + def download_image(cls, url, item_url=None): if url is None: - return + return None, None raw_img = None session_id = random.random() proxy_url = ('http://%s-country-cn-session-%s:%s@zproxy.lum-superproxy.io:%d' % - (LUMINATI_USERNAME, session_id, LUMINATI_PASSWORD, PORT)) + (settings.LUMINATI_USERNAME, session_id, settings.LUMINATI_PASSWORD, PORT)) proxies = { 'http': proxy_url, 'https': proxy_url, } - # if DEBUG: - # proxies = None + if settings.LUMINATI_USERNAME is None: + proxies = None if url: img_response = requests.get( url, @@ -194,7 +200,7 @@ class AbstractScraper: 'dnt': '1', }, proxies=proxies, - timeout=TIMEOUT, + timeout=settings.SCRAPING_TIMEOUT, ) if img_response.status_code == 200: raw_img = img_response.content @@ -205,13 +211,14 @@ class AbstractScraper: return raw_img, ext @classmethod - def save(cls, request_user): + def save(cls, request_user, instance=None): entity_cover = { 'cover': SimpleUploadedFile('temp.' + cls.img_ext, cls.raw_img) - } - form = cls.form_class(cls.raw_data, entity_cover) + } if cls.img_ext is not None else None + form = cls.form_class(data=cls.raw_data, files=entity_cover, instance=instance) if form.is_valid(): form.instance.last_editor = request_user + form.instance._change_reason = 'scrape' form.save() cls.instance = form.instance else: @@ -220,1145 +227,37 @@ class AbstractScraper: return form -class DoubanScrapperMixin: - @classmethod - def download_page(cls, url, headers): - url = cls.get_effective_url(url) - - scraper_api_endpoint = f'http://api.scraperapi.com?api_key={SCRAPERAPI_KEY}&url={url}' - - r = requests.get(scraper_api_endpoint, timeout=TIMEOUT) - - if r.status_code != 200: - raise RuntimeError(f"download page failed, status code {r.status_code}") - # with open('temp.html', 'w', encoding='utf-8') as fp: - # fp.write(r.content.decode('utf-8')) - return html.fromstring(r.content.decode('utf-8')) - - -class DoubanBookScraper(DoubanScrapperMixin, AbstractScraper): - site_name = SourceSiteEnum.DOUBAN.value - host = "book.douban.com" - data_class = Book - form_class = BookForm - - regex = re.compile(r"https://book\.douban\.com/subject/\d+/{0,1}") - - def scrape(self, url): - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = self.host - content = self.download_page(url, headers) - - # parsing starts here - try: - title = content.xpath("/html/body//h1/span/text()")[0].strip() - except IndexError: - raise ValueError("given url contains no book info") - - subtitle_elem = content.xpath( - "//div[@id='info']//span[text()='副标题:']/following::text()") - subtitle = subtitle_elem[0].strip() if subtitle_elem else None - - orig_title_elem = content.xpath( - "//div[@id='info']//span[text()='原作名:']/following::text()") - orig_title = orig_title_elem[0].strip() if orig_title_elem else None - - language_elem = content.xpath( - "//div[@id='info']//span[text()='语言:']/following::text()") - language = language_elem[0].strip() if language_elem else None - - pub_house_elem = content.xpath( - "//div[@id='info']//span[text()='出版社:']/following::text()") - pub_house = pub_house_elem[0].strip() if pub_house_elem else None - - pub_date_elem = content.xpath( - "//div[@id='info']//span[text()='出版年:']/following::text()") - pub_date = pub_date_elem[0].strip() if pub_date_elem else '' - year_month_day = RE_NUMBERS.findall(pub_date) - if len(year_month_day) in (2, 3): - pub_year = int(year_month_day[0]) - pub_month = int(year_month_day[1]) - elif len(year_month_day) == 1: - pub_year = int(year_month_day[0]) - pub_month = None - else: - pub_year = None - pub_month = None - if pub_year and pub_month and pub_year < pub_month: - pub_year, pub_month = pub_month, pub_year - pub_year = None if pub_year is not None and not pub_year in range( - 0, 3000) else pub_year - pub_month = None if pub_month is not None and not pub_month in range( - 1, 12) else pub_month - - binding_elem = content.xpath( - "//div[@id='info']//span[text()='装帧:']/following::text()") - binding = binding_elem[0].strip() if binding_elem else None - - price_elem = content.xpath( - "//div[@id='info']//span[text()='定价:']/following::text()") - price = price_elem[0].strip() if price_elem else None - - pages_elem = content.xpath( - "//div[@id='info']//span[text()='页数:']/following::text()") - pages = pages_elem[0].strip() if pages_elem else None - if pages is not None: - pages = int(RE_NUMBERS.findall(pages)[ - 0]) if RE_NUMBERS.findall(pages) else None - - isbn_elem = content.xpath( - "//div[@id='info']//span[text()='ISBN:']/following::text()") - isbn = isbn_elem[0].strip() if isbn_elem else None - - brief_elem = content.xpath( - "//h2/span[text()='内容简介']/../following-sibling::div[1]//div[@class='intro'][not(ancestor::span[@class='short'])]/p/text()") - brief = '\n'.join(p.strip() - for p in brief_elem) if brief_elem else None - - contents = None - try: - contents_elem = content.xpath( - "//h2/span[text()='目录']/../following-sibling::div[1]")[0] - # if next the id of next sibling contains `dir`, that would be the full contents - if "dir" in contents_elem.getnext().xpath("@id")[0]: - contents_elem = contents_elem.getnext() - contents = '\n'.join(p.strip() for p in contents_elem.xpath( - "text()")[:-2]) if contents_elem else None - else: - contents = '\n'.join(p.strip() for p in contents_elem.xpath( - "text()")) if contents_elem else None - except: - pass - - img_url_elem = content.xpath("//*[@id='mainpic']/a/img/@src") - img_url = img_url_elem[0].strip() if img_url_elem else None - raw_img, ext = self.download_image(img_url) - - # there are two html formats for authors and translators - authors_elem = content.xpath("""//div[@id='info']//span[text()='作者:']/following-sibling::br[1]/ - preceding-sibling::a[preceding-sibling::span[text()='作者:']]/text()""") - if not authors_elem: - authors_elem = content.xpath( - """//div[@id='info']//span[text()=' 作者']/following-sibling::a/text()""") - if authors_elem: - authors = [] - for author in authors_elem: - authors.append(RE_WHITESPACES.sub(' ', author.strip())) - else: - authors = None - - translators_elem = content.xpath("""//div[@id='info']//span[text()='译者:']/following-sibling::br[1]/ - preceding-sibling::a[preceding-sibling::span[text()='译者:']]/text()""") - if not translators_elem: - translators_elem = content.xpath( - """//div[@id='info']//span[text()=' 译者']/following-sibling::a/text()""") - if translators_elem: - translators = [] - for translator in translators_elem: - translators.append(RE_WHITESPACES.sub(' ', translator.strip())) - else: - translators = None - - other = {} - cncode_elem = content.xpath( - "//div[@id='info']//span[text()='统一书号:']/following::text()") - if cncode_elem: - other['统一书号'] = cncode_elem[0].strip() - series_elem = content.xpath( - "//div[@id='info']//span[text()='丛书:']/following-sibling::a[1]/text()") - if series_elem: - other['丛书'] = series_elem[0].strip() - imprint_elem = content.xpath( - "//div[@id='info']//span[text()='出品方:']/following-sibling::a[1]/text()") - if imprint_elem: - other['出品方'] = imprint_elem[0].strip() - - data = { - 'title': title, - 'subtitle': subtitle, - 'orig_title': orig_title, - 'author': authors, - 'translator': translators, - 'language': language, - 'pub_house': pub_house, - 'pub_year': pub_year, - 'pub_month': pub_month, - 'binding': binding, - 'price': price, - 'pages': pages, - 'isbn': isbn, - 'brief': brief, - 'contents': contents, - 'other_info': other, - 'source_site': self.site_name, - 'source_url': self.get_effective_url(url), - } - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - -class DoubanMovieScraper(DoubanScrapperMixin, AbstractScraper): - site_name = SourceSiteEnum.DOUBAN.value - host = 'movie.douban.com' - data_class = Movie - form_class = MovieForm - - regex = re.compile(r"https://movie\.douban\.com/subject/\d+/{0,1}") - - def scrape(self, url): - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = self.host - content = self.download_page(url, headers) - - # parsing starts here - try: - raw_title = content.xpath( - "//span[@property='v:itemreviewed']/text()")[0].strip() - except IndexError: - raise ValueError("given url contains no movie info") - - orig_title = content.xpath( - "//img[@rel='v:image']/@alt")[0].strip() - title = raw_title.split(orig_title)[0].strip() - # if has no chinese title - if title == '': - title = orig_title - - if title == orig_title: - orig_title = None - - # there are two html formats for authors and translators - other_title_elem = content.xpath( - "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") - other_title = other_title_elem[0].strip().split( - ' / ') if other_title_elem else None - - imdb_elem = content.xpath( - "//div[@id='info']//span[text()='IMDb链接:']/following-sibling::a[1]/text()") - if not imdb_elem: - imdb_elem = content.xpath( - "//div[@id='info']//span[text()='IMDb:']/following-sibling::text()[1]") - imdb_code = imdb_elem[0].strip() if imdb_elem else None - - director_elem = content.xpath( - "//div[@id='info']//span[text()='导演']/following-sibling::span[1]/a/text()") - director = director_elem if director_elem else None - - playwright_elem = content.xpath( - "//div[@id='info']//span[text()='编剧']/following-sibling::span[1]/a/text()") - playwright = playwright_elem if playwright_elem else None - - actor_elem = content.xpath( - "//div[@id='info']//span[text()='主演']/following-sibling::span[1]/a/text()") - actor = actor_elem if actor_elem else None - - # construct genre translator - genre_translator = {} - attrs = [attr for attr in dir(MovieGenreEnum) if not '__' in attr] - for attr in attrs: - genre_translator[getattr(MovieGenreEnum, attr).label] = getattr( - MovieGenreEnum, attr).value - - genre_elem = content.xpath("//span[@property='v:genre']/text()") - if genre_elem: - genre = [] - for g in genre_elem: - genre.append(genre_translator[g]) - else: - genre = None - - showtime_elem = content.xpath( - "//span[@property='v:initialReleaseDate']/text()") - if showtime_elem: - showtime = [] - for st in showtime_elem: - parts = st.split('(') - if len(parts) == 1: - time = st.split('(')[0] - region = '' - else: - time = st.split('(')[0] - region = st.split('(')[1][0:-1] - showtime.append({time: region}) - else: - showtime = None - - site_elem = content.xpath( - "//div[@id='info']//span[text()='官方网站:']/following-sibling::a[1]/@href") - site = site_elem[0].strip() if site_elem else None - - area_elem = content.xpath( - "//div[@id='info']//span[text()='制片国家/地区:']/following-sibling::text()[1]") - if area_elem: - area = [a.strip() for a in area_elem[0].split(' / ')] - else: - area = None - - language_elem = content.xpath( - "//div[@id='info']//span[text()='语言:']/following-sibling::text()[1]") - if language_elem: - language = [a.strip() for a in language_elem[0].split(' / ')] - else: - language = None - - year_elem = content.xpath("//span[@class='year']/text()") - year = int(year_elem[0][1:-1]) if year_elem else None - - duration_elem = content.xpath("//span[@property='v:runtime']/text()") - other_duration_elem = content.xpath( - "//span[@property='v:runtime']/following-sibling::text()[1]") - if duration_elem: - duration = duration_elem[0].strip() - if other_duration_elem: - duration += other_duration_elem[0].rstrip() - else: - duration = None - - season_elem = content.xpath( - - "//*[@id='season']/option[@selected='selected']/text()") - if not season_elem: - season_elem = content.xpath( - "//div[@id='info']//span[text()='季数:']/following-sibling::text()[1]") - season = int(season_elem[0].strip()) if season_elem else None - else: - season = int(season_elem[0].strip()) - - episodes_elem = content.xpath( - "//div[@id='info']//span[text()='集数:']/following-sibling::text()[1]") - episodes = int(episodes_elem[0].strip()) if episodes_elem else None - - single_episode_length_elem = content.xpath( - "//div[@id='info']//span[text()='单集片长:']/following-sibling::text()[1]") - single_episode_length = single_episode_length_elem[0].strip( - ) if single_episode_length_elem else None - - # if has field `episodes` not none then must be series - is_series = True if episodes else False - - brief_elem = content.xpath("//span[@class='all hidden']") - if not brief_elem: - brief_elem = content.xpath("//span[@property='v:summary']") - brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( - './text()')]) if brief_elem else None - - img_url_elem = content.xpath("//img[@rel='v:image']/@src") - img_url = img_url_elem[0].strip() if img_url_elem else None - raw_img, ext = self.download_image(img_url) - - data = { - 'title': title, - 'orig_title': orig_title, - 'other_title': other_title, - 'imdb_code': imdb_code, - 'director': director, - 'playwright': playwright, - 'actor': actor, - 'genre': genre, - 'showtime': showtime, - 'site': site, - 'area': area, - 'language': language, - 'year': year, - 'duration': duration, - 'season': season, - 'episodes': episodes, - 'single_episode_length': single_episode_length, - 'brief': brief, - 'is_series': is_series, - 'source_site': self.site_name, - 'source_url': self.get_effective_url(url), - } - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - -class DoubanAlbumScraper(DoubanScrapperMixin, AbstractScraper): - site_name = SourceSiteEnum.DOUBAN.value - host = 'music.douban.com' - data_class = Album - form_class = AlbumForm - - regex = re.compile(r"https://music\.douban\.com/subject/\d+/{0,1}") - - def scrape(self, url): - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = self.host - content = self.download_page(url, headers) - - # parsing starts here - try: - title = content.xpath("//h1/span/text()")[0].strip() - except IndexError: - raise ValueError("given url contains no album info") - if not title: - raise ValueError("given url contains no album info") - - - artists_elem = content.xpath("""//div[@id='info']/span/span[@class='pl']/a/text()""") - artist = None if not artists_elem else artists_elem - - genre_elem = content.xpath( - "//div[@id='info']//span[text()='流派:']/following::text()[1]") - genre = genre_elem[0].strip() if genre_elem else None - - date_elem = content.xpath( - "//div[@id='info']//span[text()='发行时间:']/following::text()[1]") - release_date = parse_date(date_elem[0].strip()) if date_elem else None - - company_elem = content.xpath( - "//div[@id='info']//span[text()='出版者:']/following::text()[1]") - company = company_elem[0].strip() if company_elem else None - - track_list_elem = content.xpath( - "//div[@class='track-list']/div[@class='indent']/div/text()" - ) - if track_list_elem: - track_list = '\n'.join([track.strip() for track in track_list_elem]) - else: - track_list = None - - brief_elem = content.xpath("//span[@class='all hidden']") - if not brief_elem: - brief_elem = content.xpath("//span[@property='v:summary']") - brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( - './text()')]) if brief_elem else None - - other_info = {} - other_elem = content.xpath( - "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") - if other_elem: - other_info['又名'] = other_elem[0].strip() - other_elem = content.xpath( - "//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]") - if other_elem: - other_info['专辑类型'] = other_elem[0].strip() - other_elem = content.xpath( - "//div[@id='info']//span[text()='介质:']/following-sibling::text()[1]") - if other_elem: - other_info['介质'] = other_elem[0].strip() - other_elem = content.xpath( - "//div[@id='info']//span[text()='ISRC:']/following-sibling::text()[1]") - if other_elem: - other_info['ISRC'] = other_elem[0].strip() - other_elem = content.xpath( - "//div[@id='info']//span[text()='条形码:']/following-sibling::text()[1]") - if other_elem: - other_info['条形码'] = other_elem[0].strip() - other_elem = content.xpath( - "//div[@id='info']//span[text()='碟片数:']/following-sibling::text()[1]") - if other_elem: - other_info['碟片数'] = other_elem[0].strip() - - img_url_elem = content.xpath("//div[@id='mainpic']//img/@src") - img_url = img_url_elem[0].strip() if img_url_elem else None - raw_img, ext = self.download_image(img_url) - - data = { - 'title': title, - 'artist': artist, - 'genre': genre, - 'release_date': release_date, - 'duration': None, - 'company': company, - 'track_list': track_list, - 'brief': brief, - 'other_info': other_info, - 'source_site': self.site_name, - 'source_url': self.get_effective_url(url), - } - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - -spotify_token = None -spotify_token_expire_time = time.time() - -class SpotifyTrackScraper(AbstractScraper): - site_name = SourceSiteEnum.SPOTIFY.value - host = 'https://open.spotify.com/track/' - data_class = Song - form_class = SongForm - - regex = re.compile(r"(?<=https://open\.spotify\.com/track/)[a-zA-Z0-9]+") - - def scrape(self, url): - """ - Request from API, not really scraping - """ - global spotify_token, spotify_token_expire_time - - if spotify_token is None or is_spotify_token_expired(): - invoke_spotify_token() - effective_url = self.get_effective_url(url) - if effective_url is None: - raise ValueError("not valid url") - - api_url = self.get_api_url(effective_url) - headers = { - 'Authorization': f"Bearer {spotify_token}" - } - r = requests.get(api_url, headers=headers) - res_data = r.json() - - artist = [] - for artist_dict in res_data['artists']: - artist.append(artist_dict['name']) - if not artist: - artist = None - - title = res_data['name'] - - release_date = parse_date(res_data['album']['release_date']) - - duration = res_data['duration_ms'] - - if res_data['external_ids'].get('isrc'): - isrc = res_data['external_ids']['isrc'] - else: - isrc = None - - raw_img, ext = self.download_image(res_data['album']['images'][0]['url']) - - data = { - 'title': title, - 'artist': artist, - 'genre': None, - 'release_date': release_date, - 'duration': duration, - 'isrc': isrc, - 'album': None, - 'brief': None, - 'other_info': None, - 'source_site': self.site_name, - 'source_url': effective_url, - } - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - @classmethod - def get_effective_url(cls, raw_url): - code = cls.regex.findall(raw_url) - if code: - return f"https://open.spotify.com/track/{code[0]}" - else: - return None - - @classmethod - def get_api_url(cls, url): - return "https://api.spotify.com/v1/tracks/" + cls.regex.findall(url)[0] - - -class SpotifyAlbumScraper(AbstractScraper): - site_name = SourceSiteEnum.SPOTIFY.value - # API URL - host = 'https://open.spotify.com/album/' - data_class = Album - form_class = AlbumForm - - regex = re.compile(r"(?<=https://open\.spotify\.com/album/)[a-zA-Z0-9]+") - - def scrape(self, url): - """ - Request from API, not really scraping - """ - global spotify_token, spotify_token_expire_time - - if spotify_token is None or is_spotify_token_expired(): - invoke_spotify_token() - effective_url = self.get_effective_url(url) - if effective_url is None: - raise ValueError("not valid url") - - api_url = self.get_api_url(effective_url) - headers = { - 'Authorization': f"Bearer {spotify_token}" - } - r = requests.get(api_url, headers=headers) - res_data = r.json() - - artist = [] - for artist_dict in res_data['artists']: - artist.append(artist_dict['name']) - - title = res_data['name'] - - genre = ', '.join(res_data['genres']) - - company = [] - for com in res_data['copyrights']: - company.append(com['text']) - - duration = 0 - track_list = [] - track_urls = [] - for track in res_data['tracks']['items']: - track_urls.append(track['external_urls']['spotify']) - duration += track['duration_ms'] - if res_data['tracks']['items'][-1]['disc_number'] > 1: - # more than one disc - track_list.append(str( - track['disc_number']) + '-' + str(track['track_number']) + '. ' + track['name']) - else: - track_list.append(str(track['track_number']) + '. ' + track['name']) - track_list = '\n'.join(track_list) - - release_date = parse_date(res_data['release_date']) - - other_info = {} - if res_data['external_ids'].get('upc'): - # bar code - other_info['UPC'] = res_data['external_ids']['upc'] - - raw_img, ext = self.download_image(res_data['images'][0]['url']) - - data = { - 'title': title, - 'artist': artist, - 'genre': genre, - 'track_list': track_list, - 'release_date': release_date, - 'duration': duration, - 'company': company, - 'brief': None, - 'other_info': other_info, - 'source_site': self.site_name, - 'source_url': effective_url, - } - - # set tracks_data, used for adding tracks - self.track_urls = track_urls - - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - @classmethod - def get_effective_url(cls, raw_url): - code = cls.regex.findall(raw_url) - if code: - return f"https://open.spotify.com/album/{code[0]}" - else: - return None - - @classmethod - def save(cls, request_user): - form = super().save(request_user) - task = Thread( - target=cls.add_tracks, - args=(form.instance, request_user), - daemon=True - ) - task.start() - return form - - @classmethod - def get_api_url(cls, url): - return "https://api.spotify.com/v1/albums/" + cls.regex.findall(url)[0] - - @classmethod - def add_tracks(cls, album: Album, request_user): - to_be_updated_tracks = [] - for track_url in cls.track_urls: - track = cls.get_track_or_none(track_url) - # seems lik if fire too many requests at the same time - # spotify would limit access - if track is None: - task = Thread( - target=cls.scrape_and_save_track, - args=(track_url, album, request_user), - daemon=True - ) - task.start() - task.join() - else: - to_be_updated_tracks.append(track) - cls.bulk_update_track_album(to_be_updated_tracks, album, request_user) - - @classmethod - def get_track_or_none(cls, track_url: str): - try: - instance = Song.objects.get(source_url=track_url) - return instance - except ObjectDoesNotExist: - return None - - @classmethod - def scrape_and_save_track(cls, url: str, album: Album, request_user): - data, img = SpotifyTrackScraper.scrape(url) - SpotifyTrackScraper.raw_data['album'] = album - SpotifyTrackScraper.save(request_user) - - @classmethod - def bulk_update_track_album(cls, tracks, album, request_user): - for track in tracks: - track.last_editor = request_user - track.edited_time = timezone.now() - track.album = album - Song.objects.bulk_update(tracks, [ - 'last_editor', - 'edited_time', - 'album' - ]) - - -def is_spotify_token_expired(): - global spotify_token_expire_time - return True if spotify_token_expire_time <= time.time() else False - - -def invoke_spotify_token(): - global spotify_token, spotify_token_expire_time - r = requests.post( - "https://accounts.spotify.com/api/token", - data={ - "grant_type": "client_credentials" - }, - headers={ - "Authorization": f"Basic {SPOTIFY_CREDENTIAL}" - } - ) - data = r.json() - if r.status_code == 401: - # token expired, try one more time - # this maybe caused by external operations, - # for example debugging using a http client - r = requests.post( - "https://accounts.spotify.com/api/token", - data={ - "grant_type": "client_credentials" - }, - headers={ - "Authorization": f"Basic {SPOTIFY_CREDENTIAL}" - } - ) - data = r.json() - elif r.status_code != 200: - raise Exception(f"Request to spotify API fails. Reason: {r.reason}") - # minus 2 for execution time error - spotify_token_expire_time = int(data['expires_in']) + time.time() - 2 - spotify_token = data['access_token'] - - -class ImdbMovieScraper(AbstractScraper): - site_name = SourceSiteEnum.IMDB.value - host = 'https://www.imdb.com/title/' - data_class = Movie - form_class = MovieForm - - regex = re.compile(r"(?<=https://www\.imdb\.com/title/)[a-zA-Z0-9]+") - - def scrape(self, url): - - effective_url = self.get_effective_url(url) - if effective_url is None: - raise ValueError("not valid url") - - api_url = self.get_api_url(effective_url) - r = requests.get(api_url) - res_data = r.json() - - if not res_data['type'] in ['Movie', 'TVSeries']: - raise ValueError("not movie/series item") - - if res_data['type'] == 'Movie': - is_series = False - elif res_data['type'] == 'TVSeries': - is_series = True - - title = res_data['title'] - orig_title = res_data['originalTitle'] - imdb_code = self.regex.findall(effective_url)[0] - director = [] - for direct_dict in res_data['directorList']: - director.append(direct_dict['name']) - playwright = [] - for writer_dict in res_data['writerList']: - playwright.append(writer_dict['name']) - actor = [] - for actor_dict in res_data['actorList']: - actor.append(actor_dict['name']) - genre = res_data['genres'].split(', ') - area = res_data['countries'].split(', ') - language = res_data['languages'].split(', ') - year = int(res_data['year']) - duration = res_data['runtimeStr'] - brief = res_data['plotLocal'] if res_data['plotLocal'] else res_data['plot'] - if res_data['releaseDate']: - showtime = [{res_data['releaseDate']: "发布日期"}] - else: - showtime = None - - other_info = {} - if res_data['contentRating']: - other_info['分级'] = res_data['contentRating'] - if res_data['imDbRating']: - other_info['IMDb评分'] = res_data['imDbRating'] - if res_data['metacriticRating']: - other_info['Metacritic评分'] = res_data['metacriticRating'] - if res_data['awards']: - other_info['奖项'] = res_data['awards'] - - raw_img, ext = self.download_image(res_data['image']) - - data = { - 'title': title, - 'orig_title': orig_title, - 'other_title': None, - 'imdb_code': imdb_code, - 'director': director, - 'playwright': playwright, - 'actor': actor, - 'genre': genre, - 'showtime': showtime, - 'site': None, - 'area': area, - 'language': language, - 'year': year, - 'duration': duration, - 'season': None, - 'episodes': None, - 'single_episode_length': None, - 'brief': brief, - 'is_series': is_series, - 'other_info': other_info, - 'source_site': self.site_name, - 'source_url': effective_url, - } - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - @classmethod - def get_effective_url(cls, raw_url): - code = cls.regex.findall(raw_url) - if code: - return f"https://www.imdb.com/title/{code[0]}/" - else: - return None - - @classmethod - def get_api_url(cls, url): - return f"https://imdb-api.com/zh/API/Title/{IMDB_API_KEY}/{cls.regex.findall(url)[0]}/FullActor," - - -class DoubanGameScraper(DoubanScrapperMixin, AbstractScraper): - site_name = SourceSiteEnum.DOUBAN.value - host = 'www.douban.com/game/' - data_class = Game - form_class = GameForm - - regex = re.compile(r"https://www\.douban\.com/game/\d+/{0,1}") - - def scrape(self, url): - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = 'www.douban.com' - content = self.download_page(url, headers) - - try: - raw_title = content.xpath( - "//div[@id='content']/h1/text()")[0].strip() - except IndexError: - raise ValueError("given url contains no game info") - - title = raw_title - - other_title_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()") - other_title = other_title_elem[0].strip().split(' / ') if other_title_elem else None - - developer_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()") - developer = developer_elem[0].strip().split(' / ') if developer_elem else None - - publisher_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()") - publisher = publisher_elem[0].strip().split(' / ') if publisher_elem else None - - platform_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()") - platform = platform_elem if platform_elem else None - - genre_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()") - genre = None - if genre_elem: - genre = [g for g in genre_elem if g != '游戏'] - - date_elem = content.xpath( - "//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()") - release_date = parse_date(date_elem[0].strip()) if date_elem else None - - brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()") - brief = '\n'.join(brief_elem) if brief_elem else None - - img_url_elem = content.xpath( - "//div[@class='item-subject-info']/div[@class='pic']//img/@src") - img_url = img_url_elem[0].strip() if img_url_elem else None - raw_img, ext = self.download_image(img_url) - - data = { - 'title': title, - 'other_title': other_title, - 'developer': developer, - 'publisher': publisher, - 'release_date': release_date, - 'genre': genre, - 'platform': platform, - 'brief': brief, - 'other_info': None, - 'source_site': self.site_name, - 'source_url': self.get_effective_url(url), - } - - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - -class SteamGameScraper(AbstractScraper): - site_name = SourceSiteEnum.STEAM.value - host = 'store.steampowered.com' - data_class = Game - form_class = GameForm - - regex = re.compile(r"https://store\.steampowered\.com/app/\d+/{0,1}") - - def scrape(self, url): - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = self.host - headers['Cookie'] = "wants_mature_content=1; birthtime=754700401;" - content = self.download_page(url, headers) - - title = content.xpath("//div[@class='apphub_AppName']/text()")[0] - developer = content.xpath("//div[@id='developers_list']/a/text()") - publisher = content.xpath("//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()") - release_date = parse_date( - content.xpath( - "//div[@class='release_date']/div[@class='date']/text()")[0] - ) - - genre = content.xpath( - "//div[@class='details_block']/b[2]/following-sibling::a/text()") - - platform = ['PC'] - - brief = content.xpath( - "//div[@class='game_description_snippet']/text()")[0].strip() - - img_url = content.xpath( - "//img[@class='game_header_image_full']/@src" - )[0].replace("header.jpg", "library_600x900.jpg") - raw_img, ext = self.download_image(img_url) - - # no 600x900 picture - if raw_img is None: - img_url = content.xpath("//img[@class='game_header_image_full']/@src")[0] - raw_img, ext = self.download_image(img_url) - - data = { - 'title': title, - 'other_title': None, - 'developer': developer, - 'publisher': publisher, - 'release_date': release_date, - 'genre': genre, - 'platform': platform, - 'brief': brief, - 'other_info': None, - 'source_site': self.site_name, - 'source_url': self.get_effective_url(url), - } - - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - -def find_entity(source_url): - """ - for bangumi - """ - # to be added when new scrape method is implemented - result = Game.objects.filter(source_url=source_url) - if result: - return result[0] - else: - raise ObjectDoesNotExist - -class BangumiScraper(AbstractScraper): - site_name = SourceSiteEnum.BANGUMI.value - host = 'bgm.tv' - - # for interface coherence - data_class = type("FakeDataClass", (object,), {})() - data_class.objects = type("FakeObjectsClass", (object,), {})() - data_class.objects.get = find_entity - # should be set at scrape_* method - form_class = '' - - - regex = re.compile(r"https{0,1}://bgm\.tv/subject/\d+") - - def scrape(self, url): - """ - This is the scraping portal - """ - headers = DEFAULT_REQUEST_HEADERS.copy() - headers['Host'] = self.host - content = self.download_page(url, headers) - - # download image - img_url = 'http:' + content.xpath("//div[@class='infobox']//img[1]/@src")[0] - raw_img, ext = self.download_image(img_url) - - # Test category - category_code = content.xpath("//div[@id='headerSearch']//option[@selected]/@value")[0] - handler_map = { - '1': self.scrape_book, - '2': self.scrape_movie, - '3': self.scrape_album, - '4': self.scrape_game - } - data = handler_map[category_code](self, content) - data['source_url'] = self.get_effective_url(url) - - self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext - return data, raw_img - - - def scrape_game(self, content): - self.data_class = Game - self.form_class = GameForm - - title_elem = content.xpath("//a[@property='v:itemreviewed']/text()") - if not title_elem: - raise ValueError("no game info found on this page") - title = None - else: - title = title_elem[0].strip() - - other_title_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/text()") - if not other_title_elem: - other_title_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/a/text()") - other_title = other_title_elem if other_title_elem else [] - - chinese_name_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/text()") - if not chinese_name_elem: - chinese_name_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/a/text()") - if chinese_name_elem: - chinese_name = chinese_name_elem[0] - # switch chinese name with original name - title, chinese_name = chinese_name, title - # actually the name appended is original - other_title.append(chinese_name) - - developer_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/text()") - if not developer_elem: - developer_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/a/text()") - developer = developer_elem if developer_elem else None - - publisher_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/text()") - if not publisher_elem: - publisher_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/a/text()") - publisher = publisher_elem if publisher_elem else None - - platform_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/text()") - if not platform_elem: - platform_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/a/text()") - platform = platform_elem if platform_elem else None - - genre_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/text()") - if not genre_elem: - genre_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/a/text()") - genre = genre_elem if genre_elem else None - - date_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/text()") - if not date_elem: - date_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/a/text()") - release_date = parse_date(date_elem[0]) if date_elem else None - - brief = ''.join(content.xpath("//div[@property='v:summary']/text()")) - - other_info = {} - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'人数')]]/text()") - if other_elem: - other_info['游玩人数'] = other_elem[0] - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'引擎')]]/text()") - if other_elem: - other_info['引擎'] = ' '.join(other_elem) - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'售价')]]/text()") - if other_elem: - other_info['售价'] = ' '.join(other_elem) - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'官方网站')]]/text()") - if other_elem: - other_info['网站'] = other_elem[0] - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/a/text()") or content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/text()") - if other_elem: - other_info['剧本'] = ' '.join(other_elem) - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/a/text()") or content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/text()") - if other_elem: - other_info['编剧'] = ' '.join(other_elem) - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/a/text()") or content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/text()") - if other_elem: - other_info['音乐'] = ' '.join(other_elem) - other_elem = content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/a/text()") or content.xpath( - "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/text()") - if other_elem: - other_info['美术'] = ' '.join(other_elem) - - data = { - 'title': title, - 'other_title': None, - 'developer': developer, - 'publisher': publisher, - 'release_date': release_date, - 'genre': genre, - 'platform': platform, - 'brief': brief, - 'other_info': other_info, - 'source_site': self.site_name, - } - - return data - - def scrape_movie(self, content): - self.data_class = Movie - self.form_class = MovieForm - raise NotImplementedError - - def scrape_book(self, content): - self.data_class = Book - self.form_class = BookForm - raise NotImplementedError - - def scrape_album(self, content): - self.data_class = Album - self.form_class = AlbumForm - raise NotImplementedError - - -# https://developers.google.com/youtube/v3/docs/?apix=true -# https://developers.google.com/books/docs/v1/using - +from common.scrapers.bandcamp import BandcampAlbumScraper +from common.scrapers.goodreads import GoodreadsScraper +from common.scrapers.google import GoogleBooksScraper +from common.scrapers.tmdb import TmdbMovieScraper +from common.scrapers.steam import SteamGameScraper +from common.scrapers.imdb import ImdbMovieScraper +from common.scrapers.igdb import IgdbGameScraper +from common.scrapers.spotify import SpotifyAlbumScraper, SpotifyTrackScraper +from common.scrapers.douban import DoubanAlbumScraper, DoubanBookScraper, DoubanGameScraper, DoubanMovieScraper +from common.scrapers.bangumi import BangumiScraper + + +def get_scraper_by_url(url): + parsed_url = urllib.parse.urlparse(url) + hostname = parsed_url.netloc + for host in scraper_registry: + if host in url: + return scraper_registry[host] + # TODO move this logic to scraper class + try: + answers = dns.resolver.query(hostname, 'CNAME') + for rdata in answers: + if str(rdata.target) == 'dom.bandcamp.com.': + return BandcampAlbumScraper + except Exception as e: + pass + try: + answers = dns.resolver.query(hostname, 'A') + for rdata in answers: + if str(rdata.address) == '35.241.62.186': + return BandcampAlbumScraper + except Exception as e: + pass + return None diff --git a/common/scrapers/bandcamp.py b/common/scrapers/bandcamp.py new file mode 100644 index 00000000..5f939da6 --- /dev/null +++ b/common/scrapers/bandcamp.py @@ -0,0 +1,71 @@ +import re +import dateparser +import json +from lxml import html +from common.models import SourceSiteEnum +from common.scraper import AbstractScraper +from music.models import Album +from music.forms import AlbumForm + + +class BandcampAlbumScraper(AbstractScraper): + site_name = SourceSiteEnum.BANDCAMP.value + # API URL + host = '.bandcamp.com/' + data_class = Album + form_class = AlbumForm + + regex = re.compile(r"https://[a-zA-Z0-9\-\.]+/album/[^?#]+") + + def scrape(self, url, response=None): + effective_url = self.get_effective_url(url) + if effective_url is None: + raise ValueError("not valid url") + if response is not None: + content = html.fromstring(response.content.decode('utf-8')) + else: + content = self.download_page(url, {}) + try: + title = content.xpath("//h2[@class='trackTitle']/text()")[0].strip() + artist = [content.xpath("//div[@id='name-section']/h3/span/a/text()")[0].strip()] + except IndexError: + raise ValueError("given url contains no valid info") + + genre = [] # TODO: parse tags + track_list = [] + release_nodes = content.xpath("//div[@class='tralbumData tralbum-credits']/text()") + release_date = dateparser.parse(re.sub(r'releas\w+ ', '', release_nodes[0].strip())) if release_nodes else None + duration = None + company = None + brief_nodes = content.xpath("//div[@class='tralbumData tralbum-about']/text()") + brief = "".join(brief_nodes) if brief_nodes else None + cover_url = content.xpath("//div[@id='tralbumArt']/a/@href")[0].strip() + bandcamp_page_data = json.loads(content.xpath( + "//meta[@name='bc-page-properties']/@content")[0].strip()) + other_info = {} + other_info['bandcamp_album_id'] = bandcamp_page_data['item_id'] + + raw_img, ext = self.download_image(cover_url, url) + + data = { + 'title': title, + 'artist': artist, + 'genre': genre, + 'track_list': track_list, + 'release_date': release_date, + 'duration': duration, + 'company': company, + 'brief': brief, + 'other_info': other_info, + 'source_site': self.site_name, + 'source_url': effective_url, + 'cover_url': cover_url, + } + + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + @classmethod + def get_effective_url(cls, raw_url): + url = cls.regex.findall(raw_url) + return url[0] if len(url) > 0 else None diff --git a/common/scrapers/bangumi.py b/common/scrapers/bangumi.py new file mode 100644 index 00000000..498ba849 --- /dev/null +++ b/common/scrapers/bangumi.py @@ -0,0 +1,199 @@ +import re +from common.models import SourceSiteEnum +from movies.models import Movie, MovieGenreEnum +from movies.forms import MovieForm +from books.models import Book +from books.forms import BookForm +from music.models import Album, Song +from music.forms import AlbumForm, SongForm +from games.models import Game +from games.forms import GameForm +from common.scraper import * +from django.core.exceptions import ObjectDoesNotExist + + +def find_entity(source_url): + """ + for bangumi + """ + # to be added when new scrape method is implemented + result = Game.objects.filter(source_url=source_url) + if result: + return result[0] + else: + raise ObjectDoesNotExist + + +class BangumiScraper(AbstractScraper): + site_name = SourceSiteEnum.BANGUMI.value + host = 'bgm.tv' + + # for interface coherence + data_class = type("FakeDataClass", (object,), {})() + data_class.objects = type("FakeObjectsClass", (object,), {})() + data_class.objects.get = find_entity + # should be set at scrape_* method + form_class = '' + + regex = re.compile(r"https{0,1}://bgm\.tv/subject/\d+") + + def scrape(self, url): + """ + This is the scraping portal + """ + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + + # download image + img_url = 'http:' + content.xpath("//div[@class='infobox']//img[1]/@src")[0] + raw_img, ext = self.download_image(img_url, url) + + # Test category + category_code = content.xpath("//div[@id='headerSearch']//option[@selected]/@value")[0] + handler_map = { + '1': self.scrape_book, + '2': self.scrape_movie, + '3': self.scrape_album, + '4': self.scrape_game + } + data = handler_map[category_code](self, content) + data['source_url'] = self.get_effective_url(url) + + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + def scrape_game(self, content): + self.data_class = Game + self.form_class = GameForm + + title_elem = content.xpath("//a[@property='v:itemreviewed']/text()") + if not title_elem: + raise ValueError("no game info found on this page") + title = None + else: + title = title_elem[0].strip() + + other_title_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/text()") + if not other_title_elem: + other_title_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'别名')]]/a/text()") + other_title = other_title_elem if other_title_elem else [] + + chinese_name_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/text()") + if not chinese_name_elem: + chinese_name_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'中文')]]/a/text()") + if chinese_name_elem: + chinese_name = chinese_name_elem[0] + # switch chinese name with original name + title, chinese_name = chinese_name, title + # actually the name appended is original + other_title.append(chinese_name) + + developer_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/text()") + if not developer_elem: + developer_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'开发')]]/a/text()") + developer = developer_elem if developer_elem else None + + publisher_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/text()") + if not publisher_elem: + publisher_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'发行:')]]/a/text()") + publisher = publisher_elem if publisher_elem else None + + platform_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/text()") + if not platform_elem: + platform_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'平台')]]/a/text()") + platform = platform_elem if platform_elem else None + + genre_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/text()") + if not genre_elem: + genre_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'类型')]]/a/text()") + genre = genre_elem if genre_elem else None + + date_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/text()") + if not date_elem: + date_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'发行日期')]]/a/text()") + release_date = parse_date(date_elem[0]) if date_elem else None + + brief = ''.join(content.xpath("//div[@property='v:summary']/text()")) + + other_info = {} + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'人数')]]/text()") + if other_elem: + other_info['游玩人数'] = other_elem[0] + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'引擎')]]/text()") + if other_elem: + other_info['引擎'] = ' '.join(other_elem) + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'售价')]]/text()") + if other_elem: + other_info['售价'] = ' '.join(other_elem) + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'官方网站')]]/text()") + if other_elem: + other_info['网站'] = other_elem[0] + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/a/text()") or content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'剧本')]]/text()") + if other_elem: + other_info['剧本'] = ' '.join(other_elem) + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/a/text()") or content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'编剧')]]/text()") + if other_elem: + other_info['编剧'] = ' '.join(other_elem) + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/a/text()") or content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'音乐')]]/text()") + if other_elem: + other_info['音乐'] = ' '.join(other_elem) + other_elem = content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/a/text()") or content.xpath( + "//ul[@id='infobox']/li[child::span[contains(text(),'美术')]]/text()") + if other_elem: + other_info['美术'] = ' '.join(other_elem) + + data = { + 'title': title, + 'other_title': None, + 'developer': developer, + 'publisher': publisher, + 'release_date': release_date, + 'genre': genre, + 'platform': platform, + 'brief': brief, + 'other_info': other_info, + 'source_site': self.site_name, + } + + return data + + def scrape_movie(self, content): + self.data_class = Movie + self.form_class = MovieForm + raise NotImplementedError + + def scrape_book(self, content): + self.data_class = Book + self.form_class = BookForm + raise NotImplementedError + + def scrape_album(self, content): + self.data_class = Album + self.form_class = AlbumForm + raise NotImplementedError diff --git a/common/scrapers/douban.py b/common/scrapers/douban.py new file mode 100644 index 00000000..6dcaff69 --- /dev/null +++ b/common/scrapers/douban.py @@ -0,0 +1,714 @@ +import requests +import re +import filetype +from lxml import html +from common.models import SourceSiteEnum +from movies.models import Movie, MovieGenreEnum +from movies.forms import MovieForm +from books.models import Book +from books.forms import BookForm +from music.models import Album +from music.forms import AlbumForm +from games.models import Game +from games.forms import GameForm +from django.core.validators import URLValidator +from django.conf import settings +from PIL import Image +from io import BytesIO +from common.scraper import * + + +class DoubanScrapperMixin: + @classmethod + def download_page(cls, url, headers): + url = cls.get_effective_url(url) + r = None + error = 'DoubanScrapper: error occured when downloading ' + url + content = None + last_error = None + + def get(url): + nonlocal r + # print('Douban GET ' + url) + try: + r = requests.get(url, timeout=settings.SCRAPING_TIMEOUT) + except Exception as e: + r = requests.Response() + r.status_code = f"Exception when GET {url} {e}" + url + # print('Douban CODE ' + str(r.status_code)) + return r + + def check_content(): + nonlocal r, error, content, last_error + content = None + last_error = None + if r.status_code == 200: + content = r.content.decode('utf-8') + if content.find('关于豆瓣') == -1: + if content.find('你的 IP 发出') == -1: + error = error + 'Content not authentic' # response is garbage + else: + error = error + 'IP banned' + content = None + last_error = 'network' + elif content.find('页面不存在') != -1 or content.find('呃... 你想访问的条目豆瓣不收录。') != -1: # re.search('不存在[^<]+', content, re.MULTILINE): + content = None + last_error = 'censorship' + error = error + 'Not found or hidden by Douban' + elif r.status_code == 204: + content = None + last_error = 'censorship' + error = error + 'Not found or hidden by Douban' + else: + content = None + last_error = 'network' + error = error + str(r.status_code) + + def fix_wayback_links(): + nonlocal content + # fix links + content = re.sub(r'href="http[^"]+http', r'href="http', content) + # https://img9.doubanio.com/view/subject/{l|m|s}/public/s1234.jpg + content = re.sub(r'src="[^"]+/(s\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/subject/m/public/\1"', content) + # https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2681329386.jpg + # https://img9.doubanio.com/view/photo/{l|m|s}/public/p1234.webp + content = re.sub(r'src="[^"]+/(p\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/photo/m/public/\1"', content) + + # Wayback Machine: get latest available + def wayback(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://archive.org/wayback/available?url=' + url) + if r.status_code == 200: + w = r.json() + if w['archived_snapshots'] and w['archived_snapshots']['closest']: + get(w['archived_snapshots']['closest']['url']) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + # Wayback Machine: guess via CDX API + def wayback_cdx(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://web.archive.org/cdx/search/cdx?url=' + url) + if r.status_code == 200: + dates = re.findall(r'[^\s]+\s+(\d+)\s+[^\s]+\s+[^\s]+\s+\d+\s+[^\s]+\s+\d{5,}', + r.content.decode('utf-8')) + # assume snapshots whose size >9999 contain real content, use the latest one of them + if len(dates) > 0: + get('http://web.archive.org/web/' + dates[-1] + '/' + url) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + def latest(): + nonlocal r, error, content + if settings.SCRAPESTACK_KEY is not None: + error = error + '\nScrapeStack: ' + get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}') + elif settings.SCRAPERAPI_KEY is not None: + error = error + '\nScraperAPI: ' + get(f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}') + else: + error = error + '\nDirect: ' + get(url) + check_content() + if last_error == 'network' and settings.PROXYCRAWL_KEY is not None: + error = error + '\nProxyCrawl: ' + get(f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}') + check_content() + if last_error == 'censorship' and settings.LOCAL_PROXY is not None: + error = error + '\nLocal: ' + get(f'{settings.LOCAL_PROXY}?url={url}') + check_content() + + latest() + if content is None: + wayback_cdx() + + if content is None: + raise RuntimeError(error) + # with open('/tmp/temp.html', 'w', encoding='utf-8') as fp: + # fp.write(content) + return html.fromstring(content) + + @classmethod + def download_image(cls, url, item_url=None): + raw_img = None + ext = None + + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + elif settings.SCRAPERAPI_KEY is not None: + dl_url = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}' + else: + dl_url = url + + try: + img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + # raise RuntimeError(f"Douban: download image failed {img_response.status_code} {dl_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + + if raw_img is None and settings.PROXYCRAWL_KEY is not None: + try: + dl_url = f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}' + img_response = requests.get(dl_url, timeout=settings.SCRAPING_TIMEOUT) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + return raw_img, ext + + +class DoubanBookScraper(DoubanScrapperMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = "book.douban.com" + data_class = Book + form_class = BookForm + + regex = re.compile(r"https://book\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + + isbn_elem = content.xpath("//div[@id='info']//span[text()='ISBN:']/following::text()") + isbn = isbn_elem[0].strip() if isbn_elem else None + title_elem = content.xpath("/html/body//h1/span/text()") + title = title_elem[0].strip() if title_elem else None + if not title: + if isbn: + title = 'isbn: ' + isbn + else: + raise ValueError("given url contains no book title or isbn") + + subtitle_elem = content.xpath( + "//div[@id='info']//span[text()='副标题:']/following::text()") + subtitle = subtitle_elem[0].strip()[:500] if subtitle_elem else None + + orig_title_elem = content.xpath( + "//div[@id='info']//span[text()='原作名:']/following::text()") + orig_title = orig_title_elem[0].strip()[:500] if orig_title_elem else None + + language_elem = content.xpath( + "//div[@id='info']//span[text()='语言:']/following::text()") + language = language_elem[0].strip() if language_elem else None + + pub_house_elem = content.xpath( + "//div[@id='info']//span[text()='出版社:']/following::text()") + pub_house = pub_house_elem[0].strip() if pub_house_elem else None + + pub_date_elem = content.xpath( + "//div[@id='info']//span[text()='出版年:']/following::text()") + pub_date = pub_date_elem[0].strip() if pub_date_elem else '' + year_month_day = RE_NUMBERS.findall(pub_date) + if len(year_month_day) in (2, 3): + pub_year = int(year_month_day[0]) + pub_month = int(year_month_day[1]) + elif len(year_month_day) == 1: + pub_year = int(year_month_day[0]) + pub_month = None + else: + pub_year = None + pub_month = None + if pub_year and pub_month and pub_year < pub_month: + pub_year, pub_month = pub_month, pub_year + pub_year = None if pub_year is not None and pub_year not in range( + 0, 3000) else pub_year + pub_month = None if pub_month is not None and pub_month not in range( + 1, 12) else pub_month + + binding_elem = content.xpath( + "//div[@id='info']//span[text()='装帧:']/following::text()") + binding = binding_elem[0].strip() if binding_elem else None + + price_elem = content.xpath( + "//div[@id='info']//span[text()='定价:']/following::text()") + price = price_elem[0].strip() if price_elem else None + + pages_elem = content.xpath( + "//div[@id='info']//span[text()='页数:']/following::text()") + pages = pages_elem[0].strip() if pages_elem else None + if pages is not None: + pages = int(RE_NUMBERS.findall(pages)[ + 0]) if RE_NUMBERS.findall(pages) else None + if pages and (pages > 999999 or pages < 1): + pages = None + + brief_elem = content.xpath( + "//h2/span[text()='内容简介']/../following-sibling::div[1]//div[@class='intro'][not(ancestor::span[@class='short'])]/p/text()") + brief = '\n'.join(p.strip() + for p in brief_elem) if brief_elem else None + + contents = None + try: + contents_elem = content.xpath( + "//h2/span[text()='目录']/../following-sibling::div[1]")[0] + # if next the id of next sibling contains `dir`, that would be the full contents + if "dir" in contents_elem.getnext().xpath("@id")[0]: + contents_elem = contents_elem.getnext() + contents = '\n'.join(p.strip() for p in contents_elem.xpath( + "text()")[:-2]) if contents_elem else None + else: + contents = '\n'.join(p.strip() for p in contents_elem.xpath( + "text()")) if contents_elem else None + except Exception: + pass + + img_url_elem = content.xpath("//*[@id='mainpic']/a/img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + + # there are two html formats for authors and translators + authors_elem = content.xpath("""//div[@id='info']//span[text()='作者:']/following-sibling::br[1]/ + preceding-sibling::a[preceding-sibling::span[text()='作者:']]/text()""") + if not authors_elem: + authors_elem = content.xpath( + """//div[@id='info']//span[text()=' 作者']/following-sibling::a/text()""") + if authors_elem: + authors = [] + for author in authors_elem: + authors.append(RE_WHITESPACES.sub(' ', author.strip())[:200]) + else: + authors = None + + translators_elem = content.xpath("""//div[@id='info']//span[text()='译者:']/following-sibling::br[1]/ + preceding-sibling::a[preceding-sibling::span[text()='译者:']]/text()""") + if not translators_elem: + translators_elem = content.xpath( + """//div[@id='info']//span[text()=' 译者']/following-sibling::a/text()""") + if translators_elem: + translators = [] + for translator in translators_elem: + translators.append(RE_WHITESPACES.sub(' ', translator.strip())) + else: + translators = None + + other = {} + cncode_elem = content.xpath( + "//div[@id='info']//span[text()='统一书号:']/following::text()") + if cncode_elem: + other['统一书号'] = cncode_elem[0].strip() + series_elem = content.xpath( + "//div[@id='info']//span[text()='丛书:']/following-sibling::a[1]/text()") + if series_elem: + other['丛书'] = series_elem[0].strip() + imprint_elem = content.xpath( + "//div[@id='info']//span[text()='出品方:']/following-sibling::a[1]/text()") + if imprint_elem: + other['出品方'] = imprint_elem[0].strip() + + data = { + 'title': title, + 'subtitle': subtitle, + 'orig_title': orig_title, + 'author': authors, + 'translator': translators, + 'language': language, + 'pub_house': pub_house, + 'pub_year': pub_year, + 'pub_month': pub_month, + 'binding': binding, + 'price': price, + 'pages': pages, + 'isbn': isbn, + 'brief': brief, + 'contents': contents, + 'other_info': other, + 'source_site': self.site_name, + 'source_url': self.get_effective_url(url), + } + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + +class DoubanMovieScraper(DoubanScrapperMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'movie.douban.com' + data_class = Movie + form_class = MovieForm + + regex = re.compile(r"https://movie\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + + # parsing starts here + try: + raw_title = content.xpath( + "//span[@property='v:itemreviewed']/text()")[0].strip() + except IndexError: + raise ValueError("given url contains no movie info") + + orig_title = content.xpath( + "//img[@rel='v:image']/@alt")[0].strip() + title = raw_title.split(orig_title)[0].strip() + # if has no chinese title + if title == '': + title = orig_title + + if title == orig_title: + orig_title = None + + # there are two html formats for authors and translators + other_title_elem = content.xpath( + "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") + other_title = other_title_elem[0].strip().split( + ' / ') if other_title_elem else None + + imdb_elem = content.xpath( + "//div[@id='info']//span[text()='IMDb链接:']/following-sibling::a[1]/text()") + if not imdb_elem: + imdb_elem = content.xpath( + "//div[@id='info']//span[text()='IMDb:']/following-sibling::text()[1]") + imdb_code = imdb_elem[0].strip() if imdb_elem else None + + director_elem = content.xpath( + "//div[@id='info']//span[text()='导演']/following-sibling::span[1]/a/text()") + director = director_elem if director_elem else None + + playwright_elem = content.xpath( + "//div[@id='info']//span[text()='编剧']/following-sibling::span[1]/a/text()") + playwright = list(map(lambda a: a[:200], playwright_elem)) if playwright_elem else None + + actor_elem = content.xpath( + "//div[@id='info']//span[text()='主演']/following-sibling::span[1]/a/text()") + actor = list(map(lambda a: a[:200], actor_elem)) if actor_elem else None + + # construct genre translator + genre_translator = {} + attrs = [attr for attr in dir(MovieGenreEnum) if '__' not in attr] + for attr in attrs: + genre_translator[getattr(MovieGenreEnum, attr).label] = getattr( + MovieGenreEnum, attr).value + + genre_elem = content.xpath("//span[@property='v:genre']/text()") + if genre_elem: + genre = [] + for g in genre_elem: + g = g.split(' ')[0] + if g == '紀錄片': # likely some original data on douban was corrupted + g = '纪录片' + elif g == '鬼怪': + g = '惊悚' + if g in genre_translator: + genre.append(genre_translator[g]) + elif g in genre_translator.values(): + genre.append(g) + else: + logger.error(f'unable to map genre {g}') + else: + genre = None + + showtime_elem = content.xpath( + "//span[@property='v:initialReleaseDate']/text()") + if showtime_elem: + showtime = [] + for st in showtime_elem: + parts = st.split('(') + if len(parts) == 1: + time = st.split('(')[0] + region = '' + else: + time = st.split('(')[0] + region = st.split('(')[1][0:-1] + showtime.append({time: region}) + else: + showtime = None + + site_elem = content.xpath( + "//div[@id='info']//span[text()='官方网站:']/following-sibling::a[1]/@href") + site = site_elem[0].strip()[:200] if site_elem else None + try: + validator = URLValidator() + validator(site) + except ValidationError: + site = None + + area_elem = content.xpath( + "//div[@id='info']//span[text()='制片国家/地区:']/following-sibling::text()[1]") + if area_elem: + area = [a.strip()[:100] for a in area_elem[0].split('/')] + else: + area = None + + language_elem = content.xpath( + "//div[@id='info']//span[text()='语言:']/following-sibling::text()[1]") + if language_elem: + language = [a.strip() for a in language_elem[0].split(' / ')] + else: + language = None + + year_elem = content.xpath("//span[@class='year']/text()") + year = int(re.search(r'\d+', year_elem[0])[0]) if year_elem and re.search(r'\d+', year_elem[0]) else None + + duration_elem = content.xpath("//span[@property='v:runtime']/text()") + other_duration_elem = content.xpath( + "//span[@property='v:runtime']/following-sibling::text()[1]") + if duration_elem: + duration = duration_elem[0].strip() + if other_duration_elem: + duration += other_duration_elem[0].rstrip() + duration = duration.split('/')[0].strip() + else: + duration = None + + season_elem = content.xpath( + "//*[@id='season']/option[@selected='selected']/text()") + if not season_elem: + season_elem = content.xpath( + "//div[@id='info']//span[text()='季数:']/following-sibling::text()[1]") + season = int(season_elem[0].strip()) if season_elem else None + else: + season = int(season_elem[0].strip()) + + episodes_elem = content.xpath( + "//div[@id='info']//span[text()='集数:']/following-sibling::text()[1]") + episodes = int(episodes_elem[0].strip()) if episodes_elem and episodes_elem[0].isdigit() else None + + single_episode_length_elem = content.xpath( + "//div[@id='info']//span[text()='单集片长:']/following-sibling::text()[1]") + single_episode_length = single_episode_length_elem[0].strip( + )[:100] if single_episode_length_elem else None + + # if has field `episodes` not none then must be series + is_series = True if episodes else False + + brief_elem = content.xpath("//span[@class='all hidden']") + if not brief_elem: + brief_elem = content.xpath("//span[@property='v:summary']") + brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( + './text()')]) if brief_elem else None + + img_url_elem = content.xpath("//img[@rel='v:image']/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + + data = { + 'title': title, + 'orig_title': orig_title, + 'other_title': other_title, + 'imdb_code': imdb_code, + 'director': director, + 'playwright': playwright, + 'actor': actor, + 'genre': genre, + 'showtime': showtime, + 'site': site, + 'area': area, + 'language': language, + 'year': year, + 'duration': duration, + 'season': season, + 'episodes': episodes, + 'single_episode_length': single_episode_length, + 'brief': brief, + 'is_series': is_series, + 'source_site': self.site_name, + 'source_url': self.get_effective_url(url), + } + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + +class DoubanAlbumScraper(DoubanScrapperMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'music.douban.com' + data_class = Album + form_class = AlbumForm + + regex = re.compile(r"https://music\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + + # parsing starts here + try: + title = content.xpath("//h1/span/text()")[0].strip() + except IndexError: + raise ValueError("given url contains no album info") + if not title: + raise ValueError("given url contains no album info") + + artists_elem = content.xpath("//div[@id='info']/span/span[@class='pl']/a/text()") + artist = None if not artists_elem else list(map(lambda a: a[:200], artists_elem)) + + genre_elem = content.xpath( + "//div[@id='info']//span[text()='流派:']/following::text()[1]") + genre = genre_elem[0].strip() if genre_elem else None + + date_elem = content.xpath( + "//div[@id='info']//span[text()='发行时间:']/following::text()[1]") + release_date = parse_date(date_elem[0].strip()) if date_elem else None + + company_elem = content.xpath( + "//div[@id='info']//span[text()='出版者:']/following::text()[1]") + company = company_elem[0].strip() if company_elem else None + + track_list_elem = content.xpath( + "//div[@class='track-list']/div[@class='indent']/div/text()" + ) + if track_list_elem: + track_list = '\n'.join([track.strip() for track in track_list_elem]) + else: + track_list = None + + brief_elem = content.xpath("//span[@class='all hidden']") + if not brief_elem: + brief_elem = content.xpath("//span[@property='v:summary']") + brief = '\n'.join([e.strip() for e in brief_elem[0].xpath( + './text()')]) if brief_elem else None + + other_info = {} + other_elem = content.xpath( + "//div[@id='info']//span[text()='又名:']/following-sibling::text()[1]") + if other_elem: + other_info['又名'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='专辑类型:']/following-sibling::text()[1]") + if other_elem: + other_info['专辑类型'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='介质:']/following-sibling::text()[1]") + if other_elem: + other_info['介质'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='ISRC:']/following-sibling::text()[1]") + if other_elem: + other_info['ISRC'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='条形码:']/following-sibling::text()[1]") + if other_elem: + other_info['条形码'] = other_elem[0].strip() + other_elem = content.xpath( + "//div[@id='info']//span[text()='碟片数:']/following-sibling::text()[1]") + if other_elem: + other_info['碟片数'] = other_elem[0].strip() + + img_url_elem = content.xpath("//div[@id='mainpic']//img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + + data = { + 'title': title, + 'artist': artist, + 'genre': genre, + 'release_date': release_date, + 'duration': None, + 'company': company, + 'track_list': track_list, + 'brief': brief, + 'other_info': other_info, + 'source_site': self.site_name, + 'source_url': self.get_effective_url(url), + } + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + +class DoubanGameScraper(DoubanScrapperMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'www.douban.com/game/' + data_class = Game + form_class = GameForm + + regex = re.compile(r"https://www\.douban\.com/game/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = 'www.douban.com' + content = self.download_page(url, headers) + + try: + raw_title = content.xpath( + "//div[@id='content']/h1/text()")[0].strip() + except IndexError: + raise ValueError("given url contains no game info") + + title = raw_title + + other_title_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='别名:']/following-sibling::dd[1]/text()") + other_title = other_title_elem[0].strip().split(' / ') if other_title_elem else None + + developer_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='开发商:']/following-sibling::dd[1]/text()") + developer = developer_elem[0].strip().split(' / ') if developer_elem else None + + publisher_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='发行商:']/following-sibling::dd[1]/text()") + publisher = publisher_elem[0].strip().split(' / ') if publisher_elem else None + + platform_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='平台:']/following-sibling::dd[1]/a/text()") + platform = platform_elem if platform_elem else None + + genre_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='类型:']/following-sibling::dd[1]/a/text()") + genre = None + if genre_elem: + genre = [g for g in genre_elem if g != '游戏'] + + date_elem = content.xpath( + "//dl[@class='game-attr']//dt[text()='发行日期:']/following-sibling::dd[1]/text()") + release_date = parse_date(date_elem[0].strip()) if date_elem else None + + brief_elem = content.xpath("//div[@class='mod item-desc']/p/text()") + brief = '\n'.join(brief_elem) if brief_elem else None + + img_url_elem = content.xpath( + "//div[@class='item-subject-info']/div[@class='pic']//img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + + data = { + 'title': title, + 'other_title': other_title, + 'developer': developer, + 'publisher': publisher, + 'release_date': release_date, + 'genre': genre, + 'platform': platform, + 'brief': brief, + 'other_info': None, + 'source_site': self.site_name, + 'source_url': self.get_effective_url(url), + } + + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img diff --git a/common/scrapers/goodreads.py b/common/scrapers/goodreads.py new file mode 100644 index 00000000..813fd063 --- /dev/null +++ b/common/scrapers/goodreads.py @@ -0,0 +1,157 @@ +import requests +import re +import filetype +from lxml import html +from common.models import SourceSiteEnum +from movies.models import Movie, MovieGenreEnum +from movies.forms import MovieForm +from books.models import Book +from books.forms import BookForm +from music.models import Album, Song +from music.forms import AlbumForm, SongForm +from games.models import Game +from games.forms import GameForm +from django.conf import settings +from PIL import Image +from io import BytesIO +from common.scraper import * + + +class GoodreadsScraper(AbstractScraper): + site_name = SourceSiteEnum.GOODREADS.value + host = "www.goodreads.com" + data_class = Book + form_class = BookForm + regex = re.compile(r"https://www\.goodreads\.com/book/show/\d+") + + @classmethod + def get_effective_url(cls, raw_url): + u = re.match(r".+/book/show/(\d+)", raw_url) + if not u: + u = re.match(r".+book/(\d+)", raw_url) + return "https://www.goodreads.com/book/show/" + u[1] if u else None + + def scrape(self, url, response=None): + """ + This is the scraping portal + """ + if response is not None: + content = html.fromstring(response.content.decode('utf-8')) + else: + headers = None # DEFAULT_REQUEST_HEADERS.copy() + content = self.download_page(url, headers) + + try: + title = content.xpath("//h1[@id='bookTitle']/text()")[0].strip() + except IndexError: + raise ValueError("given url contains no book info") + + subtitle = None + + orig_title_elem = content.xpath("//div[@id='bookDataBox']//div[text()='Original Title']/following-sibling::div/text()") + orig_title = orig_title_elem[0].strip() if orig_title_elem else None + + language_elem = content.xpath('//div[@itemprop="inLanguage"]/text()') + language = language_elem[0].strip() if language_elem else None + + pub_house_elem = content.xpath("//div[contains(text(), 'Published') and @class='row']/text()") + try: + months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + r = re.compile('.*Published.*(' + '|'.join(months) + ').*(\\d\\d\\d\\d).+by\\s*(.+)\\s*', re.DOTALL) + pub = r.match(pub_house_elem[0]) + pub_year = pub[2] + pub_month = months.index(pub[1]) + 1 + pub_house = pub[3].strip() + except Exception: + pub_year = None + pub_month = None + pub_house = None + + pub_house_elem = content.xpath("//nobr[contains(text(), 'first published')]/text()") + try: + pub = re.match(r'.*first published\s+(.+\d\d\d\d).*', pub_house_elem[0], re.DOTALL) + first_pub = pub[1] + except Exception: + first_pub = None + + binding_elem = content.xpath('//span[@itemprop="bookFormat"]/text()') + binding = binding_elem[0].strip() if binding_elem else None + + pages_elem = content.xpath('//span[@itemprop="numberOfPages"]/text()') + pages = pages_elem[0].strip() if pages_elem else None + if pages is not None: + pages = int(RE_NUMBERS.findall(pages)[ + 0]) if RE_NUMBERS.findall(pages) else None + + isbn_elem = content.xpath('//span[@itemprop="isbn"]/text()') + if not isbn_elem: + isbn_elem = content.xpath('//div[@itemprop="isbn"]/text()') # this is likely ASIN + isbn = isbn_elem[0].strip() if isbn_elem else None + + brief_elem = content.xpath('//div[@id="description"]/span[@style="display:none"]/text()') + if brief_elem: + brief = '\n'.join(p.strip() for p in brief_elem) + else: + brief_elem = content.xpath('//div[@id="description"]/span/text()') + brief = '\n'.join(p.strip() for p in brief_elem) if brief_elem else None + + genre = content.xpath('//div[@class="bigBoxBody"]/div/div/div/a/text()') + genre = genre[0] if genre else None + book_title = re.sub('\n', '', content.xpath('//h1[@id="bookTitle"]/text()')[0]).strip() + author = content.xpath('//a[@class="authorName"]/span/text()')[0] + contents = None + + img_url_elem = content.xpath("//img[@id='coverImage']/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + + authors_elem = content.xpath("//a[@class='authorName'][not(../span[@class='authorName greyText smallText role'])]/span/text()") + if authors_elem: + authors = [] + for author in authors_elem: + authors.append(RE_WHITESPACES.sub(' ', author.strip())) + else: + authors = None + + translators = None + authors_elem = content.xpath("//a[@class='authorName'][../span/text()='(Translator)']/span/text()") + if authors_elem: + translators = [] + for translator in authors_elem: + translators.append(RE_WHITESPACES.sub(' ', translator.strip())) + else: + translators = None + + other = {} + if first_pub: + other['首版时间'] = first_pub + if genre: + other['分类'] = genre + series_elem = content.xpath("//h2[@id='bookSeries']/a/text()") + if series_elem: + other['丛书'] = re.sub(r'\(\s*(.+[^\s])\s*#.*\)', '\\1', series_elem[0].strip()) + + data = { + 'title': title, + 'subtitle': subtitle, + 'orig_title': orig_title, + 'author': authors, + 'translator': translators, + 'language': language, + 'pub_house': pub_house, + 'pub_year': pub_year, + 'pub_month': pub_month, + 'binding': binding, + 'pages': pages, + 'isbn': isbn, + 'brief': brief, + 'contents': contents, + 'other_info': other, + 'cover_url': img_url, + 'source_site': self.site_name, + 'source_url': self.get_effective_url(url), + } + data['source_url'] = self.get_effective_url(url) + + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img diff --git a/common/scrapers/google.py b/common/scrapers/google.py new file mode 100644 index 00000000..0082fb3e --- /dev/null +++ b/common/scrapers/google.py @@ -0,0 +1,102 @@ +import requests +import re +import filetype +from lxml import html +from common.models import SourceSiteEnum +from movies.models import Movie, MovieGenreEnum +from movies.forms import MovieForm +from books.models import Book +from books.forms import BookForm +from music.models import Album, Song +from music.forms import AlbumForm, SongForm +from games.models import Game +from games.forms import GameForm +from django.conf import settings +from PIL import Image +from io import BytesIO +from common.scraper import * + + +# https://developers.google.com/youtube/v3/docs/?apix=true +# https://developers.google.com/books/docs/v1/using +class GoogleBooksScraper(AbstractScraper): + site_name = SourceSiteEnum.GOOGLEBOOKS.value + host = ["books.google.com", "www.google.com/books"] + data_class = Book + form_class = BookForm + regex = re.compile(r"https://books\.google\.com/books\?id=([^&#]+)") + + @classmethod + def get_effective_url(cls, raw_url): + # https://books.google.com/books?id=wUHxzgEACAAJ + # https://books.google.com/books/about/%E7%8F%BE%E5%A0%B4%E6%AD%B7%E5%8F%B2.html?id=nvNoAAAAIAAJ + # https://www.google.com/books/edition/_/nvNoAAAAIAAJ?hl=en&gbpv=1 + u = re.match(r"https://books\.google\.com/books.*id=([^&#]+)", raw_url) + if not u: + u = re.match(r"https://www\.google\.com/books/edition/[^/]+/([^&#?]+)", raw_url) + return 'https://books.google.com/books?id=' + u[1] if u else None + + def scrape(self, url, response=None): + url = self.get_effective_url(url) + m = self.regex.match(url) + if m: + api_url = f'https://www.googleapis.com/books/v1/volumes/{m[1]}' + else: + raise ValueError("not valid url") + b = requests.get(api_url).json() + other = {} + title = b['volumeInfo']['title'] + subtitle = b['volumeInfo']['subtitle'] if 'subtitle' in b['volumeInfo'] else None + pub_year = None + pub_month = None + if 'publishedDate' in b['volumeInfo']: + pub_date = b['volumeInfo']['publishedDate'].split('-') + pub_year = pub_date[0] + pub_month = pub_date[1] if len(pub_date) > 1 else None + pub_house = b['volumeInfo']['publisher'] if 'publisher' in b['volumeInfo'] else None + language = b['volumeInfo']['language'] if 'language' in b['volumeInfo'] else None + pages = b['volumeInfo']['pageCount'] if 'pageCount' in b['volumeInfo'] else None + if 'mainCategory' in b['volumeInfo']: + other['分类'] = b['volumeInfo']['mainCategory'] + authors = b['volumeInfo']['authors'] if 'authors' in b['volumeInfo'] else None + if 'description' in b['volumeInfo']: + brief = b['volumeInfo']['description'] + elif 'textSnippet' in b['volumeInfo']: + brief = b["volumeInfo"]["textSnippet"]["searchInfo"] + else: + brief = '' + brief = re.sub(r'<.*?>', '', brief.replace(' 1: + # more than one disc + track_list.append(str( + track['disc_number']) + '-' + str(track['track_number']) + '. ' + track['name']) + else: + track_list.append(str(track['track_number']) + '. ' + track['name']) + track_list = '\n'.join(track_list) + + release_date = parse_date(res_data['release_date']) + + other_info = {} + if res_data['external_ids'].get('upc'): + # bar code + other_info['UPC'] = res_data['external_ids']['upc'] + + raw_img, ext = self.download_image(res_data['images'][0]['url'], url) + + data = { + 'title': title, + 'artist': artist, + 'genre': genre, + 'track_list': track_list, + 'release_date': release_date, + 'duration': duration, + 'company': company, + 'brief': None, + 'other_info': other_info, + 'source_site': self.site_name, + 'source_url': effective_url, + } + + # set tracks_data, used for adding tracks + self.track_urls = track_urls + + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + @classmethod + def get_effective_url(cls, raw_url): + code = cls.regex.findall(raw_url) + if code: + return f"https://open.spotify.com/album/{code[0]}" + else: + return None + + # @classmethod + # def save(cls, request_user): + # form = super().save(request_user) + # task = Thread( + # target=cls.add_tracks, + # args=(form.instance, request_user), + # daemon=True + # ) + # task.start() + # return form + + @classmethod + def get_api_url(cls, url): + return "https://api.spotify.com/v1/albums/" + cls.regex.findall(url)[0] + + @classmethod + def add_tracks(cls, album: Album, request_user): + to_be_updated_tracks = [] + for track_url in cls.track_urls: + track = cls.get_track_or_none(track_url) + # seems lik if fire too many requests at the same time + # spotify would limit access + if track is None: + task = Thread( + target=cls.scrape_and_save_track, + args=(track_url, album, request_user), + daemon=True + ) + task.start() + task.join() + else: + to_be_updated_tracks.append(track) + cls.bulk_update_track_album(to_be_updated_tracks, album, request_user) + + @classmethod + def get_track_or_none(cls, track_url: str): + try: + instance = Song.objects.get(source_url=track_url) + return instance + except ObjectDoesNotExist: + return None + + @classmethod + def scrape_and_save_track(cls, url: str, album: Album, request_user): + data, img = SpotifyTrackScraper.scrape(url) + SpotifyTrackScraper.raw_data['album'] = album + SpotifyTrackScraper.save(request_user) + + @classmethod + def bulk_update_track_album(cls, tracks, album, request_user): + for track in tracks: + track.last_editor = request_user + track.edited_time = timezone.now() + track.album = album + Song.objects.bulk_update(tracks, [ + 'last_editor', + 'edited_time', + 'album' + ]) + + +def get_spotify_token(): + global spotify_token, spotify_token_expire_time + if spotify_token is None or is_spotify_token_expired(): + invoke_spotify_token() + return spotify_token + + +def is_spotify_token_expired(): + global spotify_token_expire_time + return True if spotify_token_expire_time <= time.time() else False + + +def invoke_spotify_token(): + global spotify_token, spotify_token_expire_time + r = requests.post( + "https://accounts.spotify.com/api/token", + data={ + "grant_type": "client_credentials" + }, + headers={ + "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}" + } + ) + data = r.json() + if r.status_code == 401: + # token expired, try one more time + # this maybe caused by external operations, + # for example debugging using a http client + r = requests.post( + "https://accounts.spotify.com/api/token", + data={ + "grant_type": "client_credentials" + }, + headers={ + "Authorization": f"Basic {settings.SPOTIFY_CREDENTIAL}" + } + ) + data = r.json() + elif r.status_code != 200: + raise Exception(f"Request to spotify API fails. Reason: {r.reason}") + # minus 2 for execution time error + spotify_token_expire_time = int(data['expires_in']) + time.time() - 2 + spotify_token = data['access_token'] diff --git a/common/scrapers/steam.py b/common/scrapers/steam.py new file mode 100644 index 00000000..43f1c76b --- /dev/null +++ b/common/scrapers/steam.py @@ -0,0 +1,92 @@ +import re +from common.models import SourceSiteEnum +from games.models import Game +from games.forms import GameForm +from common.scraper import * +from common.scrapers.igdb import IgdbGameScraper + + +class SteamGameScraper(AbstractScraper): + site_name = SourceSiteEnum.STEAM.value + host = 'store.steampowered.com' + data_class = Game + form_class = GameForm + + regex = re.compile(r"https://store\.steampowered\.com/app/\d+") + + def scrape(self, url): + m = self.regex.match(url) + if m: + effective_url = m[0] + else: + raise ValueError("not valid url") + try: + s = IgdbGameScraper() + s.scrape_steam(effective_url) + self.raw_data = s.raw_data + self.raw_img = s.raw_img + self.img_ext = s.img_ext + self.raw_data['source_site'] = self.site_name + self.raw_data['source_url'] = effective_url + # return self.raw_data, self.raw_img + except: + self.raw_img = None + self.raw_data = {} + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + headers['Cookie'] = "wants_mature_content=1; birthtime=754700401;" + content = self.download_page(url, headers) + + title = content.xpath("//div[@class='apphub_AppName']/text()")[0] + developer = content.xpath("//div[@id='developers_list']/a/text()") + publisher = content.xpath("//div[@class='glance_ctn']//div[@class='dev_row'][2]//a/text()") + release_date = parse_date( + content.xpath( + "//div[@class='release_date']/div[@class='date']/text()")[0] + ) + + genre = content.xpath( + "//div[@class='details_block']/b[2]/following-sibling::a/text()") + + platform = ['PC'] + + brief = content.xpath( + "//div[@class='game_description_snippet']/text()")[0].strip() + + img_url = content.xpath( + "//img[@class='game_header_image_full']/@src" + )[0].replace("header.jpg", "library_600x900.jpg") + raw_img, img_ext = self.download_image(img_url, url) + + # no 600x900 picture + if raw_img is None: + img_url = content.xpath("//img[@class='game_header_image_full']/@src")[0] + raw_img, img_ext = self.download_image(img_url, url) + + if raw_img is not None: + self.raw_img = raw_img + self.img_ext = img_ext + + data = { + 'title': title if title else self.raw_data['title'], + 'other_title': None, + 'developer': developer if 'developer' not in self.raw_data else self.raw_data['developer'], + 'publisher': publisher if 'publisher' not in self.raw_data else self.raw_data['publisher'], + 'release_date': release_date if 'release_date' not in self.raw_data else self.raw_data['release_date'], + 'genre': genre if 'genre' not in self.raw_data else self.raw_data['genre'], + 'platform': platform if 'platform' not in self.raw_data else self.raw_data['platform'], + 'brief': brief if brief else self.raw_data['brief'], + 'other_info': None if 'other_info' not in self.raw_data else self.raw_data['other_info'], + 'source_site': self.site_name, + 'source_url': effective_url + } + self.raw_data = data + return self.raw_data, self.raw_img + + @classmethod + def get_effective_url(cls, raw_url): + m = cls.regex.match(raw_url) + if m: + return m[0] + else: + return None diff --git a/common/scrapers/tmdb.py b/common/scrapers/tmdb.py new file mode 100644 index 00000000..15072add --- /dev/null +++ b/common/scrapers/tmdb.py @@ -0,0 +1,150 @@ +import requests +import re +from common.models import SourceSiteEnum +from movies.models import Movie +from movies.forms import MovieForm +from django.conf import settings +from common.scraper import * + + +class TmdbMovieScraper(AbstractScraper): + site_name = SourceSiteEnum.TMDB.value + host = 'https://www.themoviedb.org/' + data_class = Movie + form_class = MovieForm + regex = re.compile(r"https://www\.themoviedb\.org/(movie|tv)/([a-zA-Z0-9]+)") + # http://api.themoviedb.org/3/genre/movie/list?api_key=&language=zh + # http://api.themoviedb.org/3/genre/tv/list?api_key=&language=zh + genre_map = { + 'Sci-Fi & Fantasy': 'Sci-Fi', + 'War & Politics': 'War', + '儿童': 'Kids', + '冒险': 'Adventure', + '剧情': 'Drama', + '动作': 'Action', + '动作冒险': 'Action', + '动画': 'Animation', + '历史': 'History', + '喜剧': 'Comedy', + '奇幻': 'Fantasy', + '家庭': 'Family', + '恐怖': 'Horror', + '悬疑': 'Mystery', + '惊悚': 'Thriller', + '战争': 'War', + '新闻': 'News', + '爱情': 'Romance', + '犯罪': 'Crime', + '电视电影': 'TV Movie', + '真人秀': 'Reality-TV', + '科幻': 'Sci-Fi', + '纪录': 'Documentary', + '肥皂剧': 'Soap', + '脱口秀': 'Talk-Show', + '西部': 'Western', + '音乐': 'Music', + } + + def scrape_imdb(self, imdb_code): + api_url = f"https://api.themoviedb.org/3/find/{imdb_code}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&external_source=imdb_id" + r = requests.get(api_url) + res_data = r.json() + if 'movie_results' in res_data and len(res_data['movie_results']) > 0: + url = f"https://www.themoviedb.org/movie/{res_data['movie_results'][0]['id']}" + elif 'tv_results' in res_data and len(res_data['tv_results']) > 0: + url = f"https://www.themoviedb.org/tv/{res_data['tv_results'][0]['id']}" + else: + raise ValueError("Cannot find IMDb ID in TMDB") + return self.scrape(url) + + def scrape(self, url): + m = self.regex.match(url) + if m: + effective_url = m[0] + else: + raise ValueError("not valid url") + effective_url = m[0] + is_series = m[1] == 'tv' + id = m[2] + if is_series: + api_url = f"https://api.themoviedb.org/3/tv/{id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + else: + api_url = f"https://api.themoviedb.org/3/movie/{id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits" + r = requests.get(api_url) + res_data = r.json() + + if is_series: + title = res_data['name'] + orig_title = res_data['original_name'] + year = int(res_data['first_air_date'].split('-')[0]) if res_data['first_air_date'] else None + imdb_code = res_data['external_ids']['imdb_id'] + showtime = [{res_data['first_air_date']: "首播日期"}] if res_data['first_air_date'] else None + duration = None + else: + title = res_data['title'] + orig_title = res_data['original_title'] + year = int(res_data['release_date'].split('-')[0]) if res_data['release_date'] else None + showtime = [{res_data['release_date']: "发布日期"}] if res_data['release_date'] else None + imdb_code = res_data['imdb_id'] + duration = res_data['runtime'] if res_data['runtime'] else None # in minutes + + genre = list(map(lambda x: self.genre_map[x['name']] if x['name'] in self.genre_map else 'Other', res_data['genres'])) + language = list(map(lambda x: x['name'], res_data['spoken_languages'])) + brief = res_data['overview'] + + if is_series: + director = list(map(lambda x: x['name'], res_data['created_by'])) + else: + director = list(map(lambda x: x['name'], filter(lambda c: c['job'] == 'Director', res_data['credits']['crew']))) + playwright = list(map(lambda x: x['name'], filter(lambda c: c['job'] == 'Screenplay', res_data['credits']['crew']))) + actor = list(map(lambda x: x['name'], res_data['credits']['cast'])) + area = [] + + other_info = {} + other_info['TMDB评分'] = res_data['vote_average'] + # other_info['分级'] = res_data['contentRating'] + # other_info['Metacritic评分'] = res_data['metacriticRating'] + # other_info['奖项'] = res_data['awards'] + other_info['TMDB_ID'] = id + if is_series: + other_info['Seasons'] = res_data['number_of_seasons'] + other_info['Episodes'] = res_data['number_of_episodes'] + + img_url = ('https://image.tmdb.org/t/p/original/' + res_data['poster_path']) if res_data['poster_path'] is not None else None + # TODO: use GET /configuration to get base url + raw_img, ext = self.download_image(img_url, url) + + data = { + 'title': title, + 'orig_title': orig_title, + 'other_title': None, + 'imdb_code': imdb_code, + 'director': director, + 'playwright': playwright, + 'actor': actor, + 'genre': genre, + 'showtime': showtime, + 'site': None, + 'area': area, + 'language': language, + 'year': year, + 'duration': duration, + 'season': None, + 'episodes': None, + 'single_episode_length': None, + 'brief': brief, + 'is_series': is_series, + 'other_info': other_info, + 'source_site': self.site_name, + 'source_url': effective_url, + } + self.raw_data, self.raw_img, self.img_ext = data, raw_img, ext + return data, raw_img + + @classmethod + def get_effective_url(cls, raw_url): + m = cls.regex.match(raw_url) + if raw_url: + return m[0] + else: + return None diff --git a/common/search/meilisearch.py b/common/search/meilisearch.py new file mode 100644 index 00000000..d7a13a2f --- /dev/null +++ b/common/search/meilisearch.py @@ -0,0 +1,183 @@ +import logging +import meilisearch +from django.conf import settings +from django.db.models.signals import post_save, post_delete +import types + + +INDEX_NAME = 'items' +SEARCHABLE_ATTRIBUTES = ['title', 'orig_title', 'other_title', 'subtitle', 'artist', 'author', 'translator', 'developer', 'director', 'actor', 'playwright', 'pub_house', 'company', 'publisher', 'isbn', 'imdb_code'] +INDEXABLE_DIRECT_TYPES = ['BigAutoField', 'BooleanField', 'CharField', 'PositiveIntegerField', 'PositiveSmallIntegerField', 'TextField', 'ArrayField'] +INDEXABLE_TIME_TYPES = ['DateTimeField'] +INDEXABLE_DICT_TYPES = ['JSONField'] +INDEXABLE_FLOAT_TYPES = ['DecimalField'] +# NONINDEXABLE_TYPES = ['ForeignKey', 'FileField',] +SEARCH_PAGE_SIZE = 20 + + +logger = logging.getLogger(__name__) + + +def item_post_save_handler(sender, instance, created, **kwargs): + if not created and settings.SEARCH_INDEX_NEW_ONLY: + return + Indexer.replace_item(instance) + + +def item_post_delete_handler(sender, instance, **kwargs): + Indexer.delete_item(instance) + + +def tag_post_save_handler(sender, instance, **kwargs): + pass + + +def tag_post_delete_handler(sender, instance, **kwargs): + pass + + +class Indexer: + class_map = {} + _instance = None + + @classmethod + def instance(self): + if self._instance is None: + self._instance = meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).index(INDEX_NAME) + return self._instance + + @classmethod + def init(self): + meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).create_index(INDEX_NAME, {'primaryKey': '_id'}) + self.update_settings() + + @classmethod + def update_settings(self): + self.instance().update_searchable_attributes(SEARCHABLE_ATTRIBUTES) + self.instance().update_filterable_attributes(['_class', 'tags', 'source_site']) + self.instance().update_settings({'displayedAttributes': ['_id', '_class', 'id', 'title', 'tags']}) + + @classmethod + def get_stats(self): + return self.instance().get_stats() + + @classmethod + def busy(self): + return self.instance().get_stats()['isIndexing'] + + @classmethod + def update_model_indexable(self, model): + if settings.SEARCH_BACKEND is None: + return + self.class_map[model.__name__] = model + model.indexable_fields = ['tags'] + model.indexable_fields_time = [] + model.indexable_fields_dict = [] + model.indexable_fields_float = [] + for field in model._meta.get_fields(): + type = field.get_internal_type() + if type in INDEXABLE_DIRECT_TYPES: + model.indexable_fields.append(field.name) + elif type in INDEXABLE_TIME_TYPES: + model.indexable_fields_time.append(field.name) + elif type in INDEXABLE_DICT_TYPES: + model.indexable_fields_dict.append(field.name) + elif type in INDEXABLE_FLOAT_TYPES: + model.indexable_fields_float.append(field.name) + post_save.connect(item_post_save_handler, sender=model) + post_delete.connect(item_post_delete_handler, sender=model) + + @classmethod + def obj_to_dict(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + item = { + '_id': pk, + '_class': obj.__class__.__name__, + # 'id': obj.id + } + for field in obj.__class__.indexable_fields: + item[field] = getattr(obj, field) + for field in obj.__class__.indexable_fields_time: + item[field] = getattr(obj, field).timestamp() + for field in obj.__class__.indexable_fields_float: + item[field] = float(getattr(obj, field)) if getattr(obj, field) else None + for field in obj.__class__.indexable_fields_dict: + d = getattr(obj, field) + if d.__class__ is dict: + item.update(d) + item = {k: v for k, v in item.items() if v} + return item + + @classmethod + def replace_item(self, obj): + try: + self.instance().add_documents([self.obj_to_dict(obj)]) + except Exception as e: + logger.error(f"replace item error: \n{e}") + + @classmethod + def replace_batch(self, objects): + try: + self.instance().update_documents(documents=objects) + except Exception as e: + logger.error(f"replace batch error: \n{e}") + + @classmethod + def delete_item(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + try: + self.instance().delete_document(pk) + except Exception as e: + logger.error(f"delete item error: \n{e}") + + @classmethod + def patch_item(self, obj, fields): + pk = f'{obj.__class__.__name__}-{obj.id}' + data = {} + for f in fields: + data[f] = getattr(obj, f) + try: + self.instance().update_documents(documents=[data], primary_key=[pk]) + except Exception as e: + logger.error(f"patch item error: \n{e}") + + @classmethod + def search(self, q, page=1, category=None, tag=None, sort=None): + if category or tag: + f = [] + if category == 'music': + f.append("(_class = 'Album' OR _class = 'Song')") + elif category: + f.append(f"_class = '{category}'") + if tag: + t = tag.replace("'", "\'") + f.append(f"tags = '{t}'") + filter = ' AND '.join(f) + else: + filter = None + options = { + 'offset': (page - 1) * SEARCH_PAGE_SIZE, + 'limit': SEARCH_PAGE_SIZE, + 'filter': filter, + 'facetsDistribution': ['_class'], + 'sort': None + } + try: + r = self.instance().search(q, options) + except Exception as e: + logger.error(f"MeiliSearch error: \n{e}") + r = {'nbHits': 0, 'hits': []} + # print(r) + results = types.SimpleNamespace() + results.items = list([x for x in map(lambda i: self.item_to_obj(i), r['hits']) if x is not None]) + results.num_pages = (r['nbHits'] + SEARCH_PAGE_SIZE - 1) // SEARCH_PAGE_SIZE + # print(results) + return results + + @classmethod + def item_to_obj(self, item): + try: + return self.class_map[item['_class']].objects.get(id=item['id']) + except Exception as e: + logger.error(f"unable to load search result item from db:\n{item}") + return None diff --git a/common/search/typesense.py b/common/search/typesense.py new file mode 100644 index 00000000..523fb370 --- /dev/null +++ b/common/search/typesense.py @@ -0,0 +1,215 @@ +import logging +import typesense +from django.conf import settings +from django.db.models.signals import post_save, post_delete + + +INDEX_NAME = 'items' +SEARCHABLE_ATTRIBUTES = ['title', 'orig_title', 'other_title', 'subtitle', 'artist', 'author', 'translator', + 'developer', 'director', 'actor', 'playwright', 'pub_house', 'company', 'publisher', 'isbn', 'imdb_code'] +FILTERABLE_ATTRIBUTES = ['_class', 'tags', 'source_site'] +INDEXABLE_DIRECT_TYPES = ['BigAutoField', 'BooleanField', 'CharField', + 'PositiveIntegerField', 'PositiveSmallIntegerField', 'TextField', 'ArrayField'] +INDEXABLE_TIME_TYPES = ['DateTimeField'] +INDEXABLE_DICT_TYPES = ['JSONField'] +INDEXABLE_FLOAT_TYPES = ['DecimalField'] +SORTING_ATTRIBUTE = None +# NONINDEXABLE_TYPES = ['ForeignKey', 'FileField',] +SEARCH_PAGE_SIZE = 20 + + +logger = logging.getLogger(__name__) + + +def item_post_save_handler(sender, instance, created, **kwargs): + if not created and settings.SEARCH_INDEX_NEW_ONLY: + return + Indexer.replace_item(instance) + + +def item_post_delete_handler(sender, instance, **kwargs): + Indexer.delete_item(instance) + + +def tag_post_save_handler(sender, instance, **kwargs): + pass + + +def tag_post_delete_handler(sender, instance, **kwargs): + pass + + +class Indexer: + class_map = {} + _instance = None + + @classmethod + def instance(self): + if self._instance is None: + self._instance = typesense.Client(settings.TYPESENSE_CONNECTION) + return self._instance + + @classmethod + def init(self): + # self.instance().collections[INDEX_NAME].delete() + # fields = [ + # {"name": "_class", "type": "string", "facet": True}, + # {"name": "source_site", "type": "string", "facet": True}, + # {"name": ".*", "type": "auto", "locale": "zh"}, + # ] + # use dumb schema below before typesense fix a bug + fields = [ + {'name': 'id', 'type': 'string'}, + {'name': '_id', 'type': 'int64'}, + {'name': '_class', 'type': 'string', "facet": True}, + {'name': 'source_site', 'type': 'string', "facet": True}, + {'name': 'isbn', 'optional': True, 'type': 'string'}, + {'name': 'imdb_code', 'optional': True, 'type': 'string'}, + {'name': 'author', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'orig_title', 'optional': True, 'locale': 'zh', 'type': 'string'}, + {'name': 'pub_house', 'optional': True, 'locale': 'zh', 'type': 'string'}, + {'name': 'title', 'optional': True, 'locale': 'zh', 'type': 'string'}, + {'name': 'translator', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'subtitle', 'optional': True, 'locale': 'zh', 'type': 'string'}, + {'name': 'artist', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'company', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'developer', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'other_title', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'publisher', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'actor', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'director', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'playwright', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': 'tags', 'optional': True, 'locale': 'zh', 'type': 'string[]'}, + {'name': '.*', 'optional': True, 'locale': 'zh', 'type': 'auto'}, + ] + + self.instance().collections.create({ + "name": INDEX_NAME, + "fields": fields + }) + + @classmethod + def update_settings(self): + # https://github.com/typesense/typesense/issues/96 + print('not supported by typesense yet') + pass + + @classmethod + def get_stats(self): + return self.instance().collections[INDEX_NAME].retrieve() + + @classmethod + def busy(self): + return False + + @classmethod + def update_model_indexable(self, model): + if settings.SEARCH_BACKEND is None: + return + self.class_map[model.__name__] = model + model.indexable_fields = ['tags'] + model.indexable_fields_time = [] + model.indexable_fields_dict = [] + model.indexable_fields_float = [] + for field in model._meta.get_fields(): + type = field.get_internal_type() + if type in INDEXABLE_DIRECT_TYPES: + model.indexable_fields.append(field.name) + elif type in INDEXABLE_TIME_TYPES: + model.indexable_fields_time.append(field.name) + elif type in INDEXABLE_DICT_TYPES: + model.indexable_fields_dict.append(field.name) + elif type in INDEXABLE_FLOAT_TYPES: + model.indexable_fields_float.append(field.name) + post_save.connect(item_post_save_handler, sender=model) + post_delete.connect(item_post_delete_handler, sender=model) + + @classmethod + def obj_to_dict(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + item = { + '_class': obj.__class__.__name__, + } + for field in obj.__class__.indexable_fields: + item[field] = getattr(obj, field) + for field in obj.__class__.indexable_fields_time: + item[field] = getattr(obj, field).timestamp() + for field in obj.__class__.indexable_fields_float: + item[field] = float(getattr(obj, field)) if getattr( + obj, field) else None + for field in obj.__class__.indexable_fields_dict: + d = getattr(obj, field) + if d.__class__ is dict: + item.update(d) + item = {k: v for k, v in item.items() if v and ( + k in SEARCHABLE_ATTRIBUTES or k in FILTERABLE_ATTRIBUTES or k == 'id')} + item['_id'] = item['id'] + # typesense requires primary key to be named 'id', type string + item['id'] = pk + return item + + @classmethod + def replace_item(self, obj): + try: + self.instance().collections[INDEX_NAME].documents.upsert(self.obj_to_dict(obj), { + 'dirty_values': 'coerce_or_drop' + }) + except Exception as e: + logger.error(f"replace item error: \n{e}") + + @classmethod + def replace_batch(self, objects): + try: + self.instance().collections[INDEX_NAME].documents.import_( + objects, {'action': 'upsert'}) + except Exception as e: + logger.error(f"replace batch error: \n{e}") + + @classmethod + def delete_item(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + try: + self.instance().collections[INDEX_NAME].documents[pk].delete() + except Exception as e: + logger.error(f"delete item error: \n{e}") + + @classmethod + def search(self, q, page=1, category=None, tag=None, sort=None): + f = [] + if category == 'music': + f.append('_class:= [Album, Song]') + elif category: + f.append('_class:= ' + category) + else: + f.append('') + if tag: + f.append(f"tags:= '{tag}'") + filter = ' && '.join(f) + options = { + 'q': q, + 'page': page, + 'per_page': SEARCH_PAGE_SIZE, + 'query_by': ','.join(SEARCHABLE_ATTRIBUTES), + 'filter_by': filter, + # 'facetsDistribution': ['_class'], + # 'sort_by': None, + } + # print(q) + r = self.instance().collections[INDEX_NAME].documents.search(options) + # print(r) + import types + results = types.SimpleNamespace() + results.items = list([x for x in map(lambda i: self.item_to_obj( + i['document']), r['hits']) if x is not None]) + results.num_pages = ( + r['found'] + SEARCH_PAGE_SIZE - 1) // SEARCH_PAGE_SIZE + # print(results) + return results + + @classmethod + def item_to_obj(self, item): + try: + return self.class_map[item['_class']].objects.get(id=item['_id']) + except Exception as e: + logger.error(f"unable to load search result item from db:\n{item}") + return None diff --git a/common/searcher.py b/common/searcher.py new file mode 100644 index 00000000..48c43af9 --- /dev/null +++ b/common/searcher.py @@ -0,0 +1,209 @@ +from urllib.parse import quote_plus +from enum import Enum +from common.models import SourceSiteEnum +from django.conf import settings +from common.scrapers.goodreads import GoodreadsScraper +from common.scrapers.spotify import get_spotify_token +import requests +from lxml import html +import logging + +SEARCH_PAGE_SIZE = 5 # not all apis support page size +logger = logging.getLogger(__name__) + + +class Category(Enum): + Book = '书籍' + Movie = '电影' + Music = '音乐' + Game = '游戏' + TV = '剧集' + + +class SearchResultItem: + def __init__(self, category, source_site, source_url, title, subtitle, brief, cover_url): + self.category = category + self.source_site = source_site + self.source_url = source_url + self.title = title + self.subtitle = subtitle + self.brief = brief + self.cover_url = cover_url + + @property + def verbose_category_name(self): + return self.category.value + + @property + def link(self): + return f"/search?q={quote_plus(self.source_url)}" + + @property + def scraped(self): + return False + + +class ProxiedRequest: + @classmethod + def get(cls, url): + u = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={quote_plus(url)}' + return requests.get(u, timeout=10) + + +class Goodreads: + @classmethod + def search(self, q, page=1): + results = [] + try: + search_url = f'https://www.goodreads.com/search?page={page}&q={quote_plus(q)}' + r = requests.get(search_url) + if r.url.startswith('https://www.goodreads.com/book/show/'): + # Goodreads will 302 if only one result matches ISBN + data, img = GoodreadsScraper.scrape(r.url, r) + subtitle = f"{data['pub_year']} {', '.join(data['author'])} {', '.join(data['translator'] if data['translator'] else [])}" + results.append(SearchResultItem(Category.Book, SourceSiteEnum.GOODREADS, + data['source_url'], data['title'], subtitle, + data['brief'], data['cover_url'])) + else: + h = html.fromstring(r.content.decode('utf-8')) + for c in h.xpath('//tr[@itemtype="http://schema.org/Book"]'): + el_cover = c.xpath('.//img[@class="bookCover"]/@src') + cover = el_cover[0] if el_cover else None + el_title = c.xpath('.//a[@class="bookTitle"]//text()') + title = ''.join(el_title).strip() if el_title else None + el_url = c.xpath('.//a[@class="bookTitle"]/@href') + url = 'https://www.goodreads.com' + \ + el_url[0] if el_url else None + el_authors = c.xpath('.//a[@class="authorName"]//text()') + subtitle = ', '.join(el_authors) if el_authors else None + results.append(SearchResultItem( + Category.Book, SourceSiteEnum.GOODREADS, url, title, subtitle, '', cover)) + except Exception as e: + logger.error(f"Goodreads search '{q}' error: {e}") + return results + + +class GoogleBooks: + @classmethod + def search(self, q, page=1): + results = [] + try: + api_url = f'https://www.googleapis.com/books/v1/volumes?country=us&q={quote_plus(q)}&startIndex={SEARCH_PAGE_SIZE*(page-1)}&maxResults={SEARCH_PAGE_SIZE}&maxAllowedMaturityRating=MATURE' + j = requests.get(api_url).json() + if 'items' in j: + for b in j['items']: + if 'title' not in b['volumeInfo']: + continue + title = b['volumeInfo']['title'] + subtitle = '' + if 'publishedDate' in b['volumeInfo']: + subtitle += b['volumeInfo']['publishedDate'] + ' ' + if 'authors' in b['volumeInfo']: + subtitle += ', '.join(b['volumeInfo']['authors']) + if 'description' in b['volumeInfo']: + brief = b['volumeInfo']['description'] + elif 'textSnippet' in b['volumeInfo']: + brief = b["volumeInfo"]["textSnippet"]["searchInfo"] + else: + brief = '' + category = Category.Book + # b['volumeInfo']['infoLink'].replace('http:', 'https:') + url = 'https://books.google.com/books?id=' + b['id'] + cover = b['volumeInfo']['imageLinks']['thumbnail'] if 'imageLinks' in b['volumeInfo'] else None + results.append(SearchResultItem( + category, SourceSiteEnum.GOOGLEBOOKS, url, title, subtitle, brief, cover)) + except Exception as e: + logger.error(f"GoogleBooks search '{q}' error: {e}") + return results + + +class TheMovieDatabase: + @classmethod + def search(self, q, page=1): + results = [] + try: + api_url = f'https://api.themoviedb.org/3/search/multi?query={quote_plus(q)}&page={page}&api_key={settings.TMDB_API3_KEY}&language=zh-CN&include_adult=true' + j = requests.get(api_url).json() + for m in j['results']: + if m['media_type'] in ['tv', 'movie']: + url = f"https://www.themoviedb.org/{m['media_type']}/{m['id']}" + if m['media_type'] == 'tv': + cat = Category.TV + title = m['name'] + subtitle = f"{m.get('first_air_date')} {m.get('original_name')}" + else: + cat = Category.Movie + title = m['title'] + subtitle = f"{m.get('release_date')} {m.get('original_name')}" + cover = f"https://image.tmdb.org/t/p/w500/{m.get('poster_path')}" + results.append(SearchResultItem( + cat, SourceSiteEnum.TMDB, url, title, subtitle, m.get('overview'), cover)) + except Exception as e: + logger.error(f"TMDb search '{q}' error: {e}") + return results + + +class Spotify: + @classmethod + def search(self, q, page=1): + results = [] + try: + api_url = f"https://api.spotify.com/v1/search?q={q}&type=album&limit={SEARCH_PAGE_SIZE}&offset={page*SEARCH_PAGE_SIZE}" + headers = { + 'Authorization': f"Bearer {get_spotify_token()}" + } + j = requests.get(api_url, headers=headers).json() + for a in j['albums']['items']: + title = a['name'] + subtitle = a['release_date'] + for artist in a['artists']: + subtitle += ' ' + artist['name'] + url = a['external_urls']['spotify'] + cover = a['images'][0]['url'] + results.append(SearchResultItem( + Category.Music, SourceSiteEnum.SPOTIFY, url, title, subtitle, '', cover)) + except Exception as e: + logger.error(f"Spotify search '{q}' error: {e}") + return results + + +class Bandcamp: + @classmethod + def search(self, q, page=1): + results = [] + try: + search_url = f'https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}' + r = requests.get(search_url) + h = html.fromstring(r.content.decode('utf-8')) + for c in h.xpath('//li[@class="searchresult data-search"]'): + el_cover = c.xpath('.//div[@class="art"]/img/@src') + cover = el_cover[0] if el_cover else None + el_title = c.xpath('.//div[@class="heading"]//text()') + title = ''.join(el_title).strip() if el_title else None + el_url = c.xpath('..//div[@class="itemurl"]/a/@href') + url = el_url[0] if el_url else None + el_authors = c.xpath('.//div[@class="subhead"]//text()') + subtitle = ', '.join(el_authors) if el_authors else None + results.append(SearchResultItem(Category.Music, SourceSiteEnum.BANDCAMP, url, title, subtitle, '', cover)) + except Exception as e: + logger.error(f"Goodreads search '{q}' error: {e}") + return results + + +class ExternalSources: + @classmethod + def search(self, c, q, page=1): + if not q: + return [] + results = [] + if c == '' or c is None: + c = 'all' + if c == 'all' or c == 'movie': + results.extend(TheMovieDatabase.search(q, page)) + if c == 'all' or c == 'book': + results.extend(GoogleBooks.search(q, page)) + results.extend(Goodreads.search(q, page)) + if c == 'all' or c == 'music': + results.extend(Spotify.search(q, page)) + results.extend(Bandcamp.search(q, page)) + return results diff --git a/common/static/css/boofilsic.css b/common/static/css/boofilsic.css index 514f1aba..b93d83b6 100644 --- a/common/static/css/boofilsic.css +++ b/common/static/css/boofilsic.css @@ -270,16 +270,13 @@ h6 { img { max-width: 100%; - -o-object-fit: contain; - object-fit: contain; + object-fit: contain; } img.emoji { height: 14px; - -webkit-box-sizing: border-box; - box-sizing: border-box; - -o-object-fit: contain; - object-fit: contain; + box-sizing: border-box; + object-fit: contain; position: relative; top: 3px; } @@ -315,13 +312,11 @@ img.emoji--large { *, *:after, *:before { - -webkit-box-sizing: inherit; - box-sizing: inherit; + box-sizing: inherit; } html { - -webkit-box-sizing: border-box; - box-sizing: border-box; + box-sizing: border-box; height: 100%; } @@ -379,16 +374,12 @@ input[type='time'], input[type='color'], textarea, select { - -webkit-appearance: none; - -moz-appearance: none; - appearance: none; + appearance: none; background-color: transparent; border: 0.1rem solid #ccc; border-radius: .4rem; - -webkit-box-shadow: none; - box-shadow: none; - -webkit-box-sizing: inherit; - box-sizing: inherit; + box-shadow: none; + box-sizing: inherit; padding: .6rem 1.0rem; } @@ -408,51 +399,6 @@ select:focus { outline: 0; } -input[type='email']::-webkit-input-placeholder, -input[type='number']::-webkit-input-placeholder, -input[type='password']::-webkit-input-placeholder, -input[type='search']::-webkit-input-placeholder, -input[type='tel']::-webkit-input-placeholder, -input[type='text']::-webkit-input-placeholder, -input[type='url']::-webkit-input-placeholder, -input[type='date']::-webkit-input-placeholder, -input[type='time']::-webkit-input-placeholder, -input[type='color']::-webkit-input-placeholder, -textarea::-webkit-input-placeholder, -select::-webkit-input-placeholder { - color: #ccc; -} - -input[type='email']:-ms-input-placeholder, -input[type='number']:-ms-input-placeholder, -input[type='password']:-ms-input-placeholder, -input[type='search']:-ms-input-placeholder, -input[type='tel']:-ms-input-placeholder, -input[type='text']:-ms-input-placeholder, -input[type='url']:-ms-input-placeholder, -input[type='date']:-ms-input-placeholder, -input[type='time']:-ms-input-placeholder, -input[type='color']:-ms-input-placeholder, -textarea:-ms-input-placeholder, -select:-ms-input-placeholder { - color: #ccc; -} - -input[type='email']::-ms-input-placeholder, -input[type='number']::-ms-input-placeholder, -input[type='password']::-ms-input-placeholder, -input[type='search']::-ms-input-placeholder, -input[type='tel']::-ms-input-placeholder, -input[type='text']::-ms-input-placeholder, -input[type='url']::-ms-input-placeholder, -input[type='date']::-ms-input-placeholder, -input[type='time']::-ms-input-placeholder, -input[type='color']::-ms-input-placeholder, -textarea::-ms-input-placeholder, -select::-ms-input-placeholder { - color: #ccc; -} - input[type='email']::placeholder, input[type='number']::placeholder, input[type='password']::placeholder, @@ -468,11 +414,6 @@ select::placeholder { color: #ccc; } -::-moz-selection { - color: white; - background-color: #00a1cc; -} - ::selection { color: white; background-color: #00a1cc; @@ -480,29 +421,21 @@ select::placeholder { .navbar { background-color: #f7f7f7; - -webkit-box-sizing: border-box; - box-sizing: border-box; + box-sizing: border-box; padding: 10px 0; margin-bottom: 50px; border-bottom: #ccc 0.5px solid; } .navbar .navbar__wrapper { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: justify; - -ms-flex-pack: justify; - justify-content: space-between; - -webkit-box-align: center; - -ms-flex-align: center; - align-items: center; + justify-content: space-between; + align-items: center; position: relative; } .navbar .navbar__logo { - -ms-flex-preferred-size: 100px; - flex-basis: 100px; + flex-basis: 100px; } .navbar .navbar__logo-link { @@ -511,11 +444,8 @@ select::placeholder { .navbar .navbar__link-list { margin: 0; - display: -webkit-box; - display: -ms-flexbox; display: flex; - -ms-flex-pack: distribute; - justify-content: space-around; + justify-content: space-around; } .navbar .navbar__link { @@ -533,12 +463,8 @@ select::placeholder { .navbar .navbar__search-box { margin: 0 12% 0 15px; - display: -webkit-inline-box; - display: -ms-inline-flexbox; display: inline-flex; - -webkit-box-flex: 1; - -ms-flex: 1; - flex: 1; + flex: 1; } .navbar .navbar__search-box > input[type="search"] { @@ -556,9 +482,7 @@ select::placeholder { padding: 0; padding-left: 10px; color: #606c76; - -webkit-appearance: auto; - -moz-appearance: auto; - appearance: auto; + appearance: auto; background-color: white; height: 32px; width: 80px; @@ -596,7 +520,6 @@ select::placeholder { .navbar .navbar__link-list { margin-top: 7px; max-height: 0; - -webkit-transition: max-height 0.6s ease-out; transition: max-height 0.6s ease-out; overflow: hidden; } @@ -605,12 +528,10 @@ select::placeholder { position: absolute; right: 5px; top: 3px; - -webkit-transform: scale(0.7); - transform: scale(0.7); + transform: scale(0.7); } .navbar .navbar__dropdown-btn:hover + .navbar__link-list { max-height: 500px; - -webkit-transition: max-height 0.6s ease-in; transition: max-height 0.6s ease-in; } .navbar .navbar__search-box { @@ -654,15 +575,9 @@ select::placeholder { width: 26%; float: right; position: relative; - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; - -ms-flex-pack: distribute; - justify-content: space-around; + flex-direction: column; + justify-content: space-around; } .grid::after { @@ -673,10 +588,7 @@ select::placeholder { @media (max-width: 575.98px) { .grid .grid__aside { - -webkit-box-orient: vertical !important; - -webkit-box-direction: normal !important; - -ms-flex-direction: column !important; - flex-direction: column !important; + flex-direction: column !important; } } @@ -688,28 +600,19 @@ select::placeholder { .grid .grid__aside { width: 100%; float: none; - -webkit-box-orient: horizontal; - -webkit-box-direction: normal; - -ms-flex-direction: row; - flex-direction: row; + flex-direction: row; } .grid .grid__aside--tablet-column { - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex-direction: column; } .grid--reverse-order { - -webkit-transform: scaleY(-1); - transform: scaleY(-1); + transform: scaleY(-1); } .grid .grid__main--reverse-order { - -webkit-transform: scaleY(-1); - transform: scaleY(-1); + transform: scaleY(-1); } .grid .grid__aside--reverse-order { - -webkit-transform: scaleY(-1); - transform: scaleY(-1); + transform: scaleY(-1); } } @@ -778,8 +681,7 @@ select::placeholder { margin-bottom: 4px !important; position: absolute !important; left: 50%; - -webkit-transform: translateX(-50%); - transform: translateX(-50%); + transform: translateX(-50%); bottom: 0; width: 100%; } @@ -839,17 +741,14 @@ select::placeholder { display: inline-block; position: relative; left: 50%; - -webkit-transform: translateX(-50%) scale(0.4); - transform: translateX(-50%) scale(0.4); + transform: translateX(-50%) scale(0.4); width: 80px; height: 80px; } .spinner div { - -webkit-transform-origin: 40px 40px; - transform-origin: 40px 40px; - -webkit-animation: spinner 1.2s linear infinite; - animation: spinner 1.2s linear infinite; + transform-origin: 40px 40px; + animation: spinner 1.2s linear infinite; } .spinner div::after { @@ -865,96 +764,63 @@ select::placeholder { } .spinner div:nth-child(1) { - -webkit-transform: rotate(0deg); - transform: rotate(0deg); - -webkit-animation-delay: -1.1s; - animation-delay: -1.1s; + transform: rotate(0deg); + animation-delay: -1.1s; } .spinner div:nth-child(2) { - -webkit-transform: rotate(30deg); - transform: rotate(30deg); - -webkit-animation-delay: -1s; - animation-delay: -1s; + transform: rotate(30deg); + animation-delay: -1s; } .spinner div:nth-child(3) { - -webkit-transform: rotate(60deg); - transform: rotate(60deg); - -webkit-animation-delay: -0.9s; - animation-delay: -0.9s; + transform: rotate(60deg); + animation-delay: -0.9s; } .spinner div:nth-child(4) { - -webkit-transform: rotate(90deg); - transform: rotate(90deg); - -webkit-animation-delay: -0.8s; - animation-delay: -0.8s; + transform: rotate(90deg); + animation-delay: -0.8s; } .spinner div:nth-child(5) { - -webkit-transform: rotate(120deg); - transform: rotate(120deg); - -webkit-animation-delay: -0.7s; - animation-delay: -0.7s; + transform: rotate(120deg); + animation-delay: -0.7s; } .spinner div:nth-child(6) { - -webkit-transform: rotate(150deg); - transform: rotate(150deg); - -webkit-animation-delay: -0.6s; - animation-delay: -0.6s; + transform: rotate(150deg); + animation-delay: -0.6s; } .spinner div:nth-child(7) { - -webkit-transform: rotate(180deg); - transform: rotate(180deg); - -webkit-animation-delay: -0.5s; - animation-delay: -0.5s; + transform: rotate(180deg); + animation-delay: -0.5s; } .spinner div:nth-child(8) { - -webkit-transform: rotate(210deg); - transform: rotate(210deg); - -webkit-animation-delay: -0.4s; - animation-delay: -0.4s; + transform: rotate(210deg); + animation-delay: -0.4s; } .spinner div:nth-child(9) { - -webkit-transform: rotate(240deg); - transform: rotate(240deg); - -webkit-animation-delay: -0.3s; - animation-delay: -0.3s; + transform: rotate(240deg); + animation-delay: -0.3s; } .spinner div:nth-child(10) { - -webkit-transform: rotate(270deg); - transform: rotate(270deg); - -webkit-animation-delay: -0.2s; - animation-delay: -0.2s; + transform: rotate(270deg); + animation-delay: -0.2s; } .spinner div:nth-child(11) { - -webkit-transform: rotate(300deg); - transform: rotate(300deg); - -webkit-animation-delay: -0.1s; - animation-delay: -0.1s; + transform: rotate(300deg); + animation-delay: -0.1s; } .spinner div:nth-child(12) { - -webkit-transform: rotate(330deg); - transform: rotate(330deg); - -webkit-animation-delay: 0s; - animation-delay: 0s; -} - -@-webkit-keyframes spinner { - 0% { - opacity: 1; - } - 100% { - opacity: 0; - } + transform: rotate(330deg); + animation-delay: 0s; } @keyframes spinner { @@ -969,8 +835,7 @@ select::placeholder { .bg-mask { background-color: black; z-index: 1; - -webkit-filter: opacity(20%); - filter: opacity(20%); + filter: opacity(20%); position: fixed; width: 100%; height: 100%; @@ -986,8 +851,7 @@ select::placeholder { width: 500px; top: 50%; left: 50%; - -webkit-transform: translate(-50%, -50%); - transform: translate(-50%, -50%); + transform: translate(-50%, -50%); background-color: #f7f7f7; padding: 20px 20px 10px 20px; color: #606c76; @@ -1107,8 +971,7 @@ select::placeholder { width: 500px; top: 50%; left: 50%; - -webkit-transform: translate(-50%, -50%); - transform: translate(-50%, -50%); + transform: translate(-50%, -50%); background-color: #f7f7f7; padding: 20px 20px 10px 20px; color: #606c76; @@ -1146,8 +1009,7 @@ select::placeholder { width: 500px; top: 50%; left: 50%; - -webkit-transform: translate(-50%, -50%); - transform: translate(-50%, -50%); + transform: translate(-50%, -50%); background-color: #f7f7f7; padding: 20px 20px 10px 20px; color: #606c76; @@ -1196,8 +1058,46 @@ select::placeholder { word-break: break-all; } +.add-to-list-modal { + z-index: 2; + display: none; + position: fixed; + width: 500px; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + background-color: #f7f7f7; + padding: 20px 20px 10px 20px; + color: #606c76; +} + +.add-to-list-modal .add-to-list-modal__head { + margin-bottom: 20px; +} + +.add-to-list-modal .add-to-list-modal__head::after { + content: ' '; + clear: both; + display: table; +} + +.add-to-list-modal .add-to-list-modal__title { + font-weight: bold; + font-size: 1.2em; + float: left; +} + +.add-to-list-modal .add-to-list-modal__close-button { + float: right; + cursor: pointer; +} + +.add-to-list-modal .add-to-list-modal__confirm-button { + float: right; +} + @media (max-width: 575.98px) { - .mark-modal, .confirm-modal, .announcement-modal { + .mark-modal, .confirm-modal, .announcement-modal .add-to-list-modal { width: 100%; } } @@ -1246,6 +1146,13 @@ select::placeholder { font-weight: bold; } +.source-label.source-label__igdb { + background-color: #323A44; + color: #DFE1E2; + border: none; + font-weight: bold; +} + .source-label.source-label__steam { background: linear-gradient(30deg, #1387b8, #111d2e); color: white; @@ -1261,6 +1168,37 @@ select::placeholder { font-weight: 600; } +.source-label.source-label__goodreads { + background: #F4F1EA; + color: #372213; + font-weight: lighter; +} + +.source-label.source-label__tmdb { + background: linear-gradient(90deg, #91CCA3, #1FB4E2); + color: white; + border: none; + font-weight: lighter; + padding-top: 2px; +} + +.source-label.source-label__googlebooks { + color: white; + background-color: #4285F4; + border-color: #4285F4; +} + +.source-label.source-label__bandcamp { + color: white; + background-color: #28A0C1; + display: inline-block; +} + +.source-label.source-label__bandcamp span { + display: inline-block; + margin: 0 4px; +} + .main-section-wrapper { padding: 32px 48px 32px 36px; background-color: #f7f7f7; @@ -1276,8 +1214,6 @@ select::placeholder { } .entity-list .entity-list__entity { - display: -webkit-box; - display: -ms-flexbox; display: flex; margin-bottom: 36px; } @@ -1289,8 +1225,7 @@ select::placeholder { } .entity-list .entity-list__entity-img { - -o-object-fit: contain; - object-fit: contain; + object-fit: contain; min-width: 130px; max-width: 130px; } @@ -1368,15 +1303,12 @@ select::placeholder { .entity-detail .entity-detail__img { height: 210px; float: left; - -o-object-fit: contain; - object-fit: contain; + object-fit: contain; max-width: 150px; - -o-object-position: top; - object-position: top; + object-position: top; } .entity-detail .entity-detail__img-origin { - cursor: -webkit-zoom-in; cursor: zoom-in; } @@ -1451,14 +1383,10 @@ select::placeholder { } .entity-desc .entity-desc__unfold-button { - display: -webkit-box; - display: -ms-flexbox; display: flex; color: #00a1cc; background-color: transparent; - -webkit-box-pack: center; - -ms-flex-pack: center; - justify-content: center; + justify-content: center; text-align: center; } @@ -1597,20 +1525,14 @@ select::placeholder { } .entity-sort .entity-sort__entity-list { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: start; - -ms-flex-pack: start; - justify-content: flex-start; - -ms-flex-wrap: wrap; - flex-wrap: wrap; + justify-content: flex-start; + flex-wrap: wrap; } .entity-sort .entity-sort__entity { padding: 0 10px; - -ms-flex-preferred-size: 20%; - flex-basis: 20%; + flex-basis: 20%; text-align: center; display: inline-block; color: #606c76; @@ -1658,12 +1580,8 @@ select::placeholder { } .entity-sort-control { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: end; - -ms-flex-pack: end; - justify-content: flex-end; + justify-content: flex-end; } .entity-sort-control__button { @@ -1693,12 +1611,8 @@ select::placeholder { } .related-user-list .related-user-list__user { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: start; - -ms-flex-pack: start; - justify-content: flex-start; + justify-content: flex-start; margin-bottom: 20px; } @@ -1791,12 +1705,9 @@ select::placeholder { overflow: auto; scroll-behavior: smooth; scrollbar-width: none; - display: -webkit-box; - display: -ms-flexbox; display: flex; margin: auto; - -webkit-box-sizing: border-box; - box-sizing: border-box; + box-sizing: border-box; padding-bottom: 10px; } @@ -1820,8 +1731,7 @@ select::placeholder { } .track-carousel__track img { - -o-object-fit: contain; - object-fit: contain; + object-fit: contain; } .track-carousel__track-title { @@ -1829,14 +1739,9 @@ select::placeholder { } .track-carousel__button { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: center; - -ms-flex-pack: center; - justify-content: center; - -ms-flex-line-pack: center; - align-content: center; + justify-content: center; + align-content: center; background: white; border: none; padding: 8px; @@ -1849,22 +1754,17 @@ select::placeholder { .track-carousel__button--prev { left: 0; - -webkit-transform: translate(50%, -50%); - transform: translate(50%, -50%); + transform: translate(50%, -50%); } .track-carousel__button--next { right: 0; - -webkit-transform: translate(-50%, -50%); - transform: translate(-50%, -50%); + transform: translate(-50%, -50%); } @media (max-width: 575.98px) { .entity-list .entity-list__entity { - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex-direction: column; margin-bottom: 30px; } .entity-list .entity-list__entity-text { @@ -1883,10 +1783,7 @@ select::placeholder { -webkit-line-clamp: 5; } .entity-detail { - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex-direction: column; } .entity-detail .entity-detail__title { margin-bottom: 5px; @@ -1894,13 +1791,8 @@ select::placeholder { .entity-detail .entity-detail__info { margin-left: 0; float: none; - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex-direction: column; width: 100%; } .entity-detail .entity-detail__img { @@ -1920,8 +1812,7 @@ select::placeholder { margin-top: 24px; } .entity-sort .entity-sort__entity { - -ms-flex-preferred-size: 50%; - flex-basis: 50%; + flex-basis: 50%; } .entity-sort .entity-sort__entity-img { height: 130px; @@ -1947,23 +1838,14 @@ select::placeholder { padding: 32px 28px 28px 28px; } .entity-detail { - display: -webkit-box; - display: -ms-flexbox; display: flex; } } .aside-section-wrapper { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-flex: 1; - -ms-flex: 1; - flex: 1; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex: 1; + flex-direction: column; width: 100%; padding: 28px 25px 12px 25px; background-color: #f7f7f7; @@ -2005,18 +1887,12 @@ select::placeholder { } .action-panel .action-panel__button-group { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: justify; - -ms-flex-pack: justify; - justify-content: space-between; + justify-content: space-between; } .action-panel .action-panel__button-group--center { - -webkit-box-pack: center; - -ms-flex-pack: center; - justify-content: center; + justify-content: center; } .action-panel .action-panel__button { @@ -2084,12 +1960,8 @@ select::placeholder { } .user-profile .user-profile__header { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-align: start; - -ms-flex-align: start; - align-items: flex-start; + align-items: flex-start; margin-bottom: 15px; } @@ -2118,12 +1990,8 @@ select::placeholder { } .user-relation .user-relation__related-user-list { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: start; - -ms-flex-pack: start; - justify-content: flex-start; + justify-content: flex-start; } .user-relation .user-relation__related-user-list:last-of-type { @@ -2131,8 +1999,7 @@ select::placeholder { } .user-relation .user-relation__related-user { - -ms-flex-preferred-size: 25%; - flex-basis: 25%; + flex-basis: 25%; padding: 0px 3px; text-align: center; display: inline-block; @@ -2268,7 +2135,7 @@ select::placeholder { background-color: #d5d5d5; border-radius: 0; height: 10px; - width: 65%; + width: 54%; } .import-panel .import-panel__progress progress::-webkit-progress-bar { @@ -2310,21 +2177,13 @@ select::placeholder { } .entity-card { - display: -webkit-box; - display: -ms-flexbox; display: flex; margin-bottom: 10px; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -ms-flex-direction: column; - flex-direction: column; + flex-direction: column; } .entity-card--horizontal { - -webkit-box-orient: horizontal; - -webkit-box-direction: normal; - -ms-flex-direction: row; - flex-direction: row; + flex-direction: row; } .entity-card .entity-card__img { @@ -2353,8 +2212,7 @@ select::placeholder { } .entity-card .entity-card__img-wrapper { - -ms-flex-preferred-size: 100px; - flex-basis: 100px; + flex-basis: 100px; } @media (max-width: 575.98px) { @@ -2373,16 +2231,10 @@ select::placeholder { margin-bottom: 20px !important; } .action-panel { - -webkit-box-orient: vertical !important; - -webkit-box-direction: normal !important; - -ms-flex-direction: column !important; - flex-direction: column !important; + flex-direction: column !important; } .entity-card--horizontal { - -webkit-box-orient: vertical !important; - -webkit-box-direction: normal !important; - -ms-flex-direction: column !important; - flex-direction: column !important; + flex-direction: column !important; } .entity-card .entity-card__info-wrapper { margin-left: 10px !important; @@ -2394,11 +2246,8 @@ select::placeholder { @media (max-width: 991.98px) { .add-entity-entries { - display: -webkit-box; - display: -ms-flexbox; display: flex; - -ms-flex-pack: distribute; - justify-content: space-around; + justify-content: space-around; } .aside-section-wrapper { padding: 24px 25px 10px 25px; @@ -2419,15 +2268,10 @@ select::placeholder { margin: 0; } .action-panel { - -webkit-box-orient: horizontal; - -webkit-box-direction: normal; - -ms-flex-direction: row; - flex-direction: row; + flex-direction: row; } .action-panel .action-panel__button-group { - -webkit-box-pack: space-evenly; - -ms-flex-pack: space-evenly; - justify-content: space-evenly; + justify-content: space-evenly; } .relation-dropdown { margin-bottom: 20px; @@ -2436,54 +2280,36 @@ select::placeholder { padding-bottom: 10px; background-color: #f7f7f7; width: 100%; - display: -webkit-box; - display: -ms-flexbox; display: flex; - -webkit-box-pack: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -ms-flex-align: center; - align-items: center; + justify-content: center; + align-items: center; cursor: pointer; - -webkit-transition: -webkit-transform 0.3s; - transition: -webkit-transform 0.3s; transition: transform 0.3s; - transition: transform 0.3s, -webkit-transform 0.3s; } .relation-dropdown .relation-dropdown__button:focus { background-color: red; } .relation-dropdown .relation-dropdown__button > .icon-arrow { - -webkit-transition: -webkit-transform 0.3s; - transition: -webkit-transform 0.3s; transition: transform 0.3s; - transition: transform 0.3s, -webkit-transform 0.3s; } .relation-dropdown .relation-dropdown__button:hover > .icon-arrow > svg { fill: #00a1cc; } .relation-dropdown .relation-dropdown__button > .icon-arrow--expand { - -webkit-transform: rotate(-180deg); - transform: rotate(-180deg); + transform: rotate(-180deg); } .relation-dropdown .relation-dropdown__button + .relation-dropdown__body--expand { max-height: 2000px; - -webkit-transition: max-height 1s ease-in; transition: max-height 1s ease-in; } .relation-dropdown .relation-dropdown__body { background-color: #f7f7f7; max-height: 0; - -webkit-transition: max-height 1s ease-out; transition: max-height 1s ease-out; overflow: hidden; } .entity-card { - -webkit-box-orient: horizontal; - -webkit-box-direction: normal; - -ms-flex-direction: row; - flex-direction: row; + flex-direction: row; } .entity-card .entity-card__info-wrapper { margin-left: 30px; @@ -2510,21 +2336,7 @@ select::placeholder { overflow: auto; } -.entity-form > input[type='email'], -.entity-form > input[type='number'], -.entity-form > input[type='password'], -.entity-form > input[type='search'], -.entity-form > input[type='tel'], -.entity-form > input[type='text'], -.entity-form > input[type='url'], -.entity-form textarea, .review-form > input[type='email'], -.review-form > input[type='number'], -.review-form > input[type='password'], -.review-form > input[type='search'], -.review-form > input[type='tel'], -.review-form > input[type='text'], -.review-form > input[type='url'], -.review-form textarea { +.entity-form > input[type='email'], .entity-form > input[type='number'], .entity-form > input[type='password'], .entity-form > input[type='search'], .entity-form > input[type='tel'], .entity-form > input[type='text'], .entity-form > input[type='url'], .entity-form textarea, .review-form > input[type='email'], .review-form > input[type='number'], .review-form > input[type='password'], .review-form > input[type='search'], .review-form > input[type='tel'], .review-form > input[type='text'], .review-form > input[type='url'], .review-form textarea { width: 100%; } @@ -2614,16 +2426,12 @@ select::placeholder { .ms-parent > .ms-choice { margin-bottom: 1.5rem; - -webkit-appearance: none; - -moz-appearance: none; - appearance: none; + appearance: none; background-color: transparent; border: 0.1rem solid #ccc; border-radius: .4rem; - -webkit-box-shadow: none; - box-shadow: none; - -webkit-box-sizing: inherit; - box-sizing: inherit; + box-shadow: none; + box-sizing: inherit; padding: .6rem 1.0rem; width: 100%; height: 30.126px; @@ -2664,7 +2472,9 @@ select::placeholder { } .tag-input input { - -webkit-box-flex: 1; - -ms-flex-positive: 1; - flex-grow: 1; + flex-grow: 1; +} + +.tools-section-wrapper input, .tools-section-wrapper select { + width: unset; } diff --git a/common/static/css/boofilsic.min.css b/common/static/css/boofilsic.min.css index 19ea74b6..4a5c94d6 100644 --- a/common/static/css/boofilsic.min.css +++ b/common/static/css/boofilsic.min.css @@ -1 +1 @@ -@import url(https://cdn.jsdelivr.net/npm/skeleton-css@2.0.4/css/normalize.css);.button,button,input[type='button'],input[type='reset'],input[type='submit']{background-color:#00a1cc;border:0.1rem solid #00a1cc;border-radius:.4rem;color:#fff;cursor:pointer;display:inline-block;font-size:1.1rem;font-weight:700;height:3.4rem;letter-spacing:.1rem;line-height:3.4rem;padding:0 2.8rem;text-align:center;text-decoration:none;text-transform:uppercase;white-space:nowrap}.button:focus,.button:hover,button:focus,button:hover,input[type='button']:focus,input[type='button']:hover,input[type='reset']:focus,input[type='reset']:hover,input[type='submit']:focus,input[type='submit']:hover{background-color:#606c76;border-color:#606c76;color:#fff;outline:0}.button[disabled],button[disabled],input[type='button'][disabled],input[type='reset'][disabled],input[type='submit'][disabled]{cursor:default;opacity:.5}.button[disabled]:focus,.button[disabled]:hover,button[disabled]:focus,button[disabled]:hover,input[type='button'][disabled]:focus,input[type='button'][disabled]:hover,input[type='reset'][disabled]:focus,input[type='reset'][disabled]:hover,input[type='submit'][disabled]:focus,input[type='submit'][disabled]:hover{background-color:#00a1cc;border-color:#00a1cc}.button.button-outline,button.button-outline,input[type='button'].button-outline,input[type='reset'].button-outline,input[type='submit'].button-outline{background-color:transparent;color:#00a1cc}.button.button-outline:focus,.button.button-outline:hover,button.button-outline:focus,button.button-outline:hover,input[type='button'].button-outline:focus,input[type='button'].button-outline:hover,input[type='reset'].button-outline:focus,input[type='reset'].button-outline:hover,input[type='submit'].button-outline:focus,input[type='submit'].button-outline:hover{background-color:transparent;border-color:#606c76;color:#606c76}.button.button-outline[disabled]:focus,.button.button-outline[disabled]:hover,button.button-outline[disabled]:focus,button.button-outline[disabled]:hover,input[type='button'].button-outline[disabled]:focus,input[type='button'].button-outline[disabled]:hover,input[type='reset'].button-outline[disabled]:focus,input[type='reset'].button-outline[disabled]:hover,input[type='submit'].button-outline[disabled]:focus,input[type='submit'].button-outline[disabled]:hover{border-color:inherit;color:#00a1cc}.button.button-clear,button.button-clear,input[type='button'].button-clear,input[type='reset'].button-clear,input[type='submit'].button-clear{background-color:transparent;border-color:transparent;color:#00a1cc}.button.button-clear:focus,.button.button-clear:hover,button.button-clear:focus,button.button-clear:hover,input[type='button'].button-clear:focus,input[type='button'].button-clear:hover,input[type='reset'].button-clear:focus,input[type='reset'].button-clear:hover,input[type='submit'].button-clear:focus,input[type='submit'].button-clear:hover{background-color:transparent;border-color:transparent;color:#606c76}.button.button-clear[disabled]:focus,.button.button-clear[disabled]:hover,button.button-clear[disabled]:focus,button.button-clear[disabled]:hover,input[type='button'].button-clear[disabled]:focus,input[type='button'].button-clear[disabled]:hover,input[type='reset'].button-clear[disabled]:focus,input[type='reset'].button-clear[disabled]:hover,input[type='submit'].button-clear[disabled]:focus,input[type='submit'].button-clear[disabled]:hover{color:#00a1cc}select{background:url('data:image/svg+xml;utf8,') center right no-repeat;padding-right:3.0rem}select:focus{background-image:url('data:image/svg+xml;utf8,')}textarea{min-height:6.5rem;width:100%}select{width:100%}label,legend{display:block;margin-bottom:.5rem}fieldset{border-width:0;padding:0}input[type='checkbox'],input[type='radio']{display:inline}.label-inline{display:inline-block;font-weight:normal;margin-left:.5rem}dl,ol,ul{list-style:none;margin-top:0;padding-left:0}ol{list-style:decimal inside}ul{list-style:circle inside}.button,button,dd,dt,li{margin-bottom:1.0rem}fieldset,input,select,textarea{margin-bottom:1.5rem}blockquote,dl,figure,form,ol,p,pre,table,ul{margin-bottom:1rem}b,strong{font-weight:bold}p{margin-top:0}h1,h2,h3,h4,h5,h6{font-weight:300;letter-spacing:-.1rem;margin-bottom:2.0rem;margin-top:0}h1{font-size:4.6rem;line-height:1.2}h2{font-size:3.6rem;line-height:1.25}h3{font-size:2.8rem;line-height:1.3}h4{font-size:2.2rem;letter-spacing:-.08rem;line-height:1.35}h5{font-size:1.8rem;letter-spacing:-.05rem;line-height:1.5}h6{font-size:1.6rem;letter-spacing:0;line-height:1.4}img{max-width:100%;-o-object-fit:contain;object-fit:contain}img.emoji{height:14px;-webkit-box-sizing:border-box;box-sizing:border-box;-o-object-fit:contain;object-fit:contain;position:relative;top:3px}img.emoji--large{height:20px;position:relative;top:2px}.clearfix:after{clear:both;content:' ';display:table}.float-left{float:left}.float-right{float:right}.highlight{font-weight:bold}:root{font-size:10px}*,*:after,*:before{-webkit-box-sizing:inherit;box-sizing:inherit}html{-webkit-box-sizing:border-box;box-sizing:border-box;height:100%}body{color:#606c76;font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif;font-size:1.3rem;font-weight:300;letter-spacing:.05rem;line-height:1.6;margin:0;height:100%}textarea{font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif}a{color:#00a1cc;text-decoration:none}a:active,a:hover,a:hover:visited{color:#606c76}li{list-style:none}input[type=text]::-ms-clear,input[type=text]::-ms-reveal{display:none;width:0;height:0}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-results-button,input[type="search"]::-webkit-search-results-decoration{display:none}input[type='email'],input[type='number'],input[type='password'],input[type='search'],input[type='tel'],input[type='text'],input[type='url'],input[type='date'],input[type='time'],input[type='color'],textarea,select{-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;-webkit-box-shadow:none;box-shadow:none;-webkit-box-sizing:inherit;box-sizing:inherit;padding:.6rem 1.0rem}input[type='email']:focus,input[type='number']:focus,input[type='password']:focus,input[type='search']:focus,input[type='tel']:focus,input[type='text']:focus,input[type='url']:focus,input[type='date']:focus,input[type='time']:focus,input[type='color']:focus,textarea:focus,select:focus{border-color:#00a1cc;outline:0}input[type='email']::-webkit-input-placeholder,input[type='number']::-webkit-input-placeholder,input[type='password']::-webkit-input-placeholder,input[type='search']::-webkit-input-placeholder,input[type='tel']::-webkit-input-placeholder,input[type='text']::-webkit-input-placeholder,input[type='url']::-webkit-input-placeholder,input[type='date']::-webkit-input-placeholder,input[type='time']::-webkit-input-placeholder,input[type='color']::-webkit-input-placeholder,textarea::-webkit-input-placeholder,select::-webkit-input-placeholder{color:#ccc}input[type='email']:-ms-input-placeholder,input[type='number']:-ms-input-placeholder,input[type='password']:-ms-input-placeholder,input[type='search']:-ms-input-placeholder,input[type='tel']:-ms-input-placeholder,input[type='text']:-ms-input-placeholder,input[type='url']:-ms-input-placeholder,input[type='date']:-ms-input-placeholder,input[type='time']:-ms-input-placeholder,input[type='color']:-ms-input-placeholder,textarea:-ms-input-placeholder,select:-ms-input-placeholder{color:#ccc}input[type='email']::-ms-input-placeholder,input[type='number']::-ms-input-placeholder,input[type='password']::-ms-input-placeholder,input[type='search']::-ms-input-placeholder,input[type='tel']::-ms-input-placeholder,input[type='text']::-ms-input-placeholder,input[type='url']::-ms-input-placeholder,input[type='date']::-ms-input-placeholder,input[type='time']::-ms-input-placeholder,input[type='color']::-ms-input-placeholder,textarea::-ms-input-placeholder,select::-ms-input-placeholder{color:#ccc}input[type='email']::placeholder,input[type='number']::placeholder,input[type='password']::placeholder,input[type='search']::placeholder,input[type='tel']::placeholder,input[type='text']::placeholder,input[type='url']::placeholder,input[type='date']::placeholder,input[type='time']::placeholder,input[type='color']::placeholder,textarea::placeholder,select::placeholder{color:#ccc}::-moz-selection{color:white;background-color:#00a1cc}::selection{color:white;background-color:#00a1cc}.navbar{background-color:#f7f7f7;-webkit-box-sizing:border-box;box-sizing:border-box;padding:10px 0;margin-bottom:50px;border-bottom:#ccc 0.5px solid}.navbar .navbar__wrapper{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between;-webkit-box-align:center;-ms-flex-align:center;align-items:center;position:relative}.navbar .navbar__logo{-ms-flex-preferred-size:100px;flex-basis:100px}.navbar .navbar__logo-link{display:inline-block}.navbar .navbar__link-list{margin:0;display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-pack:distribute;justify-content:space-around}.navbar .navbar__link{margin:9px;color:#606c76}.navbar .navbar__link:active,.navbar .navbar__link:hover,.navbar .navbar__link:hover:visited{color:#00a1cc}.navbar .navbar__link:visited{color:#606c76}.navbar .navbar__search-box{margin:0 12% 0 15px;display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-flex:1;-ms-flex:1;flex:1}.navbar .navbar__search-box>input[type="search"]{border-top-right-radius:0;border-bottom-right-radius:0;margin:0;height:32px;background-color:white !important;width:100%}.navbar .navbar__search-box .navbar__search-dropdown{margin:0;margin-left:-1px;padding:0;padding-left:10px;color:#606c76;-webkit-appearance:auto;-moz-appearance:auto;appearance:auto;background-color:white;height:32px;width:80px;border-top-left-radius:0;border-bottom-left-radius:0}.navbar .navbar__dropdown-btn{display:none;padding:0;margin:0;border:none;background-color:transparent;color:#00a1cc}.navbar .navbar__dropdown-btn:focus,.navbar .navbar__dropdown-btn:hover{background-color:transparent;color:#606c76}@media (max-width: 575.98px){.navbar{padding:2px 0}.navbar .navbar__wrapper{display:block}.navbar .navbar__logo-img{width:72px;margin-right:10px;position:relative;top:7px}.navbar .navbar__link-list{margin-top:7px;max-height:0;-webkit-transition:max-height 0.6s ease-out;transition:max-height 0.6s ease-out;overflow:hidden}.navbar .navbar__dropdown-btn{display:block;position:absolute;right:5px;top:3px;-webkit-transform:scale(0.7);transform:scale(0.7)}.navbar .navbar__dropdown-btn:hover+.navbar__link-list{max-height:500px;-webkit-transition:max-height 0.6s ease-in;transition:max-height 0.6s ease-in}.navbar .navbar__search-box{margin:0;width:46vw}.navbar .navbar__search-box>input[type="search"]{height:26px;padding:4px 6px;width:32vw}.navbar .navbar__search-box .navbar__search-dropdown{cursor:pointer;height:26px;width:80px;padding-left:5px}}@media (max-width: 991.98px){.navbar{margin-bottom:20px}}.grid{margin:0 auto;position:relative;max-width:110rem;padding:0 2.0rem;width:100%}.grid .grid__main{width:70%;float:left;position:relative}.grid .grid__aside{width:26%;float:right;position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:distribute;justify-content:space-around}.grid::after{content:' ';clear:both;display:table}@media (max-width: 575.98px){.grid .grid__aside{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}}@media (max-width: 991.98px){.grid .grid__main{width:100%;float:none}.grid .grid__aside{width:100%;float:none;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.grid .grid__aside--tablet-column{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.grid--reverse-order{-webkit-transform:scaleY(-1);transform:scaleY(-1)}.grid .grid__main--reverse-order{-webkit-transform:scaleY(-1);transform:scaleY(-1)}.grid .grid__aside--reverse-order{-webkit-transform:scaleY(-1);transform:scaleY(-1)}}.pagination{text-align:center;width:100%}.pagination .pagination__page-link{font-weight:normal;margin:0 5px}.pagination .pagination__page-link--current{font-weight:bold;font-size:1.2em;color:#606c76}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:18px}.pagination .pagination__nav-link--left-margin{margin-left:18px}.pagination .pagination__nav-link--hidden{display:none}@media (max-width: 575.98px){.pagination .pagination__page-link{margin:0 3px}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:10px}.pagination .pagination__nav-link--left-margin{margin-left:10px}}#page-wrapper{position:relative;min-height:100vh;z-index:0}#content-wrapper{padding-bottom:160px}.footer{padding-top:0.4em !important;text-align:center;margin-bottom:4px !important;position:absolute !important;left:50%;-webkit-transform:translateX(-50%);transform:translateX(-50%);bottom:0;width:100%}.footer__border{padding-top:4px;border-top:#f7f7f7 solid 2px}.footer__link{margin:0 12px;white-space:nowrap}@media (max-width: 575.98px){#content-wrapper{padding-bottom:120px}}.icon-lock svg{fill:#ccc;height:12px;position:relative;top:1px;margin-left:3px}.icon-edit svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-save svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-cross svg{fill:#ccc;height:10px;position:relative}.icon-arrow svg{fill:#606c76;height:15px;position:relative;top:3px}.spinner{display:inline-block;position:relative;left:50%;-webkit-transform:translateX(-50%) scale(0.4);transform:translateX(-50%) scale(0.4);width:80px;height:80px}.spinner div{-webkit-transform-origin:40px 40px;transform-origin:40px 40px;-webkit-animation:spinner 1.2s linear infinite;animation:spinner 1.2s linear infinite}.spinner div::after{content:" ";display:block;position:absolute;top:3px;left:37px;width:6px;height:18px;border-radius:20%;background:#606c76}.spinner div:nth-child(1){-webkit-transform:rotate(0deg);transform:rotate(0deg);-webkit-animation-delay:-1.1s;animation-delay:-1.1s}.spinner div:nth-child(2){-webkit-transform:rotate(30deg);transform:rotate(30deg);-webkit-animation-delay:-1s;animation-delay:-1s}.spinner div:nth-child(3){-webkit-transform:rotate(60deg);transform:rotate(60deg);-webkit-animation-delay:-.9s;animation-delay:-.9s}.spinner div:nth-child(4){-webkit-transform:rotate(90deg);transform:rotate(90deg);-webkit-animation-delay:-.8s;animation-delay:-.8s}.spinner div:nth-child(5){-webkit-transform:rotate(120deg);transform:rotate(120deg);-webkit-animation-delay:-.7s;animation-delay:-.7s}.spinner div:nth-child(6){-webkit-transform:rotate(150deg);transform:rotate(150deg);-webkit-animation-delay:-.6s;animation-delay:-.6s}.spinner div:nth-child(7){-webkit-transform:rotate(180deg);transform:rotate(180deg);-webkit-animation-delay:-.5s;animation-delay:-.5s}.spinner div:nth-child(8){-webkit-transform:rotate(210deg);transform:rotate(210deg);-webkit-animation-delay:-.4s;animation-delay:-.4s}.spinner div:nth-child(9){-webkit-transform:rotate(240deg);transform:rotate(240deg);-webkit-animation-delay:-.3s;animation-delay:-.3s}.spinner div:nth-child(10){-webkit-transform:rotate(270deg);transform:rotate(270deg);-webkit-animation-delay:-.2s;animation-delay:-.2s}.spinner div:nth-child(11){-webkit-transform:rotate(300deg);transform:rotate(300deg);-webkit-animation-delay:-.1s;animation-delay:-.1s}.spinner div:nth-child(12){-webkit-transform:rotate(330deg);transform:rotate(330deg);-webkit-animation-delay:0s;animation-delay:0s}@-webkit-keyframes spinner{0%{opacity:1}100%{opacity:0}}@keyframes spinner{0%{opacity:1}100%{opacity:0}}.bg-mask{background-color:black;z-index:1;-webkit-filter:opacity(20%);filter:opacity(20%);position:fixed;width:100%;height:100%;left:0;top:0;display:none}.mark-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;-webkit-transform:translate(-50%, -50%);transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.mark-modal .mark-modal__head{margin-bottom:20px}.mark-modal .mark-modal__head::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__title{font-weight:bold;font-size:1.2em;float:left}.mark-modal .mark-modal__close-button{float:right;cursor:pointer}.mark-modal .mark-modal__confirm-button{float:right}.mark-modal input[type="radio"]{margin-right:0}.mark-modal .mark-modal__rating-star{display:inline;float:left;position:relative;left:-3px}.mark-modal .mark-modal__status-radio{float:right}.mark-modal .mark-modal__status-radio ul{margin-bottom:0}.mark-modal .mark-modal__status-radio li,.mark-modal .mark-modal__status-radio label{display:inline}.mark-modal .mark-modal__status-radio input[type="radio"]{position:relative;top:1px}.mark-modal .mark-modal__clear{content:' ';clear:both;display:table}.mark-modal .mark-modal__content-input,.mark-modal form textarea{height:200px;width:100%;margin-top:5px;margin-bottom:5px;resize:vertical}.mark-modal .mark-modal__tag{margin-bottom:20px}.mark-modal .mark-modal__option{margin-bottom:24px}.mark-modal .mark-modal__option::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__visibility-radio{float:left}.mark-modal .mark-modal__visibility-radio ul,.mark-modal .mark-modal__visibility-radio li,.mark-modal .mark-modal__visibility-radio label{display:inline}.mark-modal .mark-modal__visibility-radio label{font-size:normal}.mark-modal .mark-modal__visibility-radio input[type="radio"]{position:relative;top:2px}.mark-modal .mark-modal__share-checkbox{float:right}.mark-modal .mark-modal__share-checkbox input[type="checkbox"]{position:relative;top:2px}.confirm-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;-webkit-transform:translate(-50%, -50%);transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.confirm-modal .confirm-modal__head{margin-bottom:20px}.confirm-modal .confirm-modal__head::after{content:' ';clear:both;display:table}.confirm-modal .confirm-modal__title{font-weight:bold;font-size:1.2em;float:left}.confirm-modal .confirm-modal__close-button{float:right;cursor:pointer}.confirm-modal .confirm-modal__confirm-button{float:right}.announcement-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;-webkit-transform:translate(-50%, -50%);transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.announcement-modal .announcement-modal__head{margin-bottom:20px}.announcement-modal .announcement-modal__head::after{content:' ';clear:both;display:table}.announcement-modal .announcement-modal__title{font-weight:bold;font-size:1.2em;float:left}.announcement-modal .announcement-modal__close-button{float:right;cursor:pointer}.announcement-modal .announcement-modal__confirm-button{float:right}.announcement-modal .announcement-modal__body{overflow-y:auto;max-height:64vh}.announcement-modal .announcement-modal__body .announcement__title{display:inline-block}.announcement-modal .announcement-modal__body .announcement__datetime{color:#ccc;margin-left:10px}.announcement-modal .announcement-modal__body .announcement__content{word-break:break-all}@media (max-width: 575.98px){.mark-modal,.confirm-modal,.announcement-modal{width:100%}}.source-label{display:inline;background:transparent;border-radius:.3rem;border-style:solid;border-width:.1rem;line-height:1.2rem;font-size:1.1rem;margin:3px;padding:1px 3px;padding-top:2px;font-weight:lighter;letter-spacing:0.1rem;word-break:keep-all;opacity:1;position:relative;top:-1px}.source-label.source-label__in-site{border-color:#00a1cc;color:#00a1cc}.source-label.source-label__douban{border:none;color:#fff;background-color:#319840}.source-label.source-label__spotify{background-color:#1ed760;color:#000;border:none;font-weight:bold}.source-label.source-label__imdb{background-color:#F5C518;color:#121212;border:none;font-weight:bold}.source-label.source-label__steam{background:linear-gradient(30deg, #1387b8, #111d2e);color:white;border:none;font-weight:600;padding-top:2px}.source-label.source-label__bangumi{background:#FCFCFC;color:#F09199;font-style:italic;font-weight:600}.main-section-wrapper{padding:32px 48px 32px 36px;background-color:#f7f7f7;overflow:auto}.main-section-wrapper input,.main-section-wrapper select{width:100%}.entity-list .entity-list__title{margin-bottom:20px}.entity-list .entity-list__entity{display:-webkit-box;display:-ms-flexbox;display:flex;margin-bottom:36px}.entity-list .entity-list__entity::after{content:' ';clear:both;display:table}.entity-list .entity-list__entity-img{-o-object-fit:contain;object-fit:contain;min-width:130px;max-width:130px}.entity-list .entity-list__entity-text{margin-left:20px;overflow:hidden;width:100%}.entity-list .entity-list__entity-text .tag-collection{margin-left:-3px}.entity-list .entity-list__entity-link{font-size:1.2em}.entity-list .entity-list__entity-title{display:block}.entity-list .entity-list__entity-category{color:#bbb;margin-left:5px;position:relative;top:-1px}.entity-list .entity-list__entity-info{max-width:73%;white-space:nowrap;overflow:hidden;display:inline-block;text-overflow:ellipsis;position:relative;top:0.52em}.entity-list .entity-list__entity-info--full-length{max-width:100%}.entity-list .entity-list__entity-brief{margin-top:8px;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:4;overflow:hidden;margin-bottom:0}.entity-list .entity-list__rating{display:inline-block;margin:0}.entity-list .entity-list__rating--empty{margin-right:5px}.entity-list .entity-list__rating-score{margin-right:5px;position:relative;top:1px}.entity-list .entity-list__rating-star{display:inline;position:relative;top:0.3em;left:-0.3em}.entity-detail .entity-detail__img{height:210px;float:left;-o-object-fit:contain;object-fit:contain;max-width:150px;-o-object-position:top;object-position:top}.entity-detail .entity-detail__img-origin{cursor:-webkit-zoom-in;cursor:zoom-in}.entity-detail .entity-detail__info{float:left;margin-left:20px;overflow:hidden;text-overflow:ellipsis;width:70%}.entity-detail .entity-detail__title{font-weight:bold}.entity-detail .entity-detail__title--secondary{color:#bbb}.entity-detail .entity-detail__fields{display:inline-block;vertical-align:top;width:46%;margin-left:2%}.entity-detail .entity-detail__fields div,.entity-detail .entity-detail__fields span{margin:1px 0}.entity-detail .entity-detail__fields+.tag-collection{margin-top:5px;margin-left:6px}.entity-detail .entity-detail__rating{position:relative;top:-5px}.entity-detail .entity-detail__rating-star{position:relative;left:-4px;top:3px}.entity-detail .entity-detail__rating-score{font-weight:bold}.entity-detail::after{content:' ';clear:both;display:table}.entity-desc{margin-bottom:28px}.entity-desc .entity-desc__title{display:inline-block;margin-bottom:8px}.entity-desc .entity-desc__content{overflow:hidden}.entity-desc .entity-desc__content--folded{max-height:202px}.entity-desc .entity-desc__unfold-button{display:-webkit-box;display:-ms-flexbox;display:flex;color:#00a1cc;background-color:transparent;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;text-align:center}.entity-desc .entity-desc__unfold-button--hidden{display:none}.entity-marks{margin-bottom:28px}.entity-marks .entity-marks__title{margin-bottom:8px;display:inline-block}.entity-marks .entity-marks__title>a{margin-right:5px}.entity-marks .entity-marks__title--stand-alone{margin-bottom:20px}.entity-marks .entity-marks__more-link{margin-left:5px}.entity-marks .entity-marks__mark{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-marks .entity-marks__mark:last-child{border:none}.entity-marks .entity-marks__mark--wider{padding:6px 0}.entity-marks .entity-marks__mark-content{margin-bottom:0}.entity-marks .entity-marks__mark-time{color:#ccc;margin-left:2px}.entity-marks .entity-marks__rating-star{position:relative;top:4px}.entity-reviews:first-child{margin-bottom:28px}.entity-reviews .entity-reviews__title{display:inline-block;margin-bottom:8px}.entity-reviews .entity-reviews__title>a{margin-right:5px}.entity-reviews .entity-reviews__title--stand-alone{margin-bottom:20px}.entity-reviews .entity-reviews__more-link{margin-left:5px}.entity-reviews .entity-reviews__review{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-reviews .entity-reviews__review:last-child{border:none}.entity-reviews .entity-reviews__review--wider{padding:6px 0}.entity-reviews .entity-reviews__review-time{color:#ccc;margin-left:2px}.dividing-line{height:0;width:100%;margin:40px 0 24px 0;border-top:solid 1px #ccc}.dividing-line.dividing-line--dashed{margin:0;margin-top:10px;margin-bottom:2px;border-top:1px dashed #e5e5e5}.entity-sort{position:relative;margin-bottom:30px}.entity-sort .entity-sort__label{font-size:large;display:inline-block;margin-bottom:20px}.entity-sort .entity-sort__more-link{margin-left:8px}.entity-sort .entity-sort__count{color:#bbb}.entity-sort .entity-sort__count::before{content:'('}.entity-sort .entity-sort__count::after{content:')'}.entity-sort .entity-sort__entity-list{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start;-ms-flex-wrap:wrap;flex-wrap:wrap}.entity-sort .entity-sort__entity{padding:0 10px;-ms-flex-preferred-size:20%;flex-basis:20%;text-align:center;display:inline-block;color:#606c76}.entity-sort .entity-sort__entity:hover{color:#00a1cc}.entity-sort .entity-sort__entity>a{color:inherit}.entity-sort .entity-sort__entity-img{height:110px}.entity-sort .entity-sort__entity-name{text-overflow:ellipsis;overflow:hidden;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:2}.entity-sort--placeholder{border:dashed #bbb 4px}.entity-sort--hover{padding:10px;border:dashed #00a1cc 2px !important;border-radius:3px}.entity-sort--sortable{padding:10px;margin:10px 0;border:dashed #bbb 2px;cursor:all-scroll}.entity-sort--hidden{opacity:0.4}.entity-sort-control{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.entity-sort-control__button{margin-top:5px;margin-left:12px;padding:0 2px;cursor:pointer;color:#bbb}.entity-sort-control__button:hover{color:#00a1cc}.entity-sort-control__button:hover>.icon-save svg,.entity-sort-control__button:hover>.icon-edit svg{fill:#00a1cc}.entity-sort-control__button--float-right{position:absolute;top:4px;right:10px}.related-user-list .related-user-list__title{margin-bottom:20px}.related-user-list .related-user-list__user{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start;margin-bottom:20px}.related-user-list .related-user-list__user-info{margin-left:15px;overflow:auto}.related-user-list .related-user-list__user-avatar{max-height:72px;min-width:72px}.review-head .review-head__title{display:inline-block;font-weight:bold}.review-head .review-head__body{margin-bottom:10px}.review-head .review-head__body::after{content:' ';clear:both;display:table}.review-head .review-head__info{float:left}.review-head .review-head__owner-link{color:#ccc}.review-head .review-head__owner-link:hover{color:#00a1cc}.review-head .review-head__time{color:#ccc}.review-head .review-head__rating-star{position:relative;top:3px;left:-1px}.review-head .review-head__actions{float:right}.review-head .review-head__action-link:not(:first-child){margin-left:5px}.tag-collection{margin-left:-9px}.tag-collection .tag-collection__tag{position:relative;display:block;float:left;color:white;background:#ccc;padding:5px;border-radius:.3rem;line-height:1.2em;font-size:80%;margin:3px}.tag-collection .tag-collection__tag a{color:white}.tag-collection .tag-collection__tag a:hover{color:#00a1cc}.track-carousel{position:relative;margin-top:5px}.track-carousel__content{overflow:auto;scroll-behavior:smooth;scrollbar-width:none;display:-webkit-box;display:-ms-flexbox;display:flex;margin:auto;-webkit-box-sizing:border-box;box-sizing:border-box;padding-bottom:10px}.track-carousel__content::-webkit-scrollbar{height:3px;width:1px;background-color:#e5e5e5}.track-carousel__content::-webkit-scrollbar-thumb{background-color:#bbb}.track-carousel__track{text-align:center;overflow:hidden;text-overflow:ellipsis;min-width:18%;max-width:18%;margin-right:2.5%}.track-carousel__track img{-o-object-fit:contain;object-fit:contain}.track-carousel__track-title{white-space:nowrap}.track-carousel__button{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-ms-flex-line-pack:center;align-content:center;background:white;border:none;padding:8px;border-radius:50%;outline:0;cursor:pointer;position:absolute;top:50%}.track-carousel__button--prev{left:0;-webkit-transform:translate(50%, -50%);transform:translate(50%, -50%)}.track-carousel__button--next{right:0;-webkit-transform:translate(-50%, -50%);transform:translate(-50%, -50%)}@media (max-width: 575.98px){.entity-list .entity-list__entity{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin-bottom:30px}.entity-list .entity-list__entity-text{margin-left:0}.entity-list .entity-list__entity-img-wrapper{margin-bottom:8px}.entity-list .entity-list__entity-info{max-width:unset}.entity-list .entity-list__rating--empty+.entity-list__entity-info{max-width:70%}.entity-list .entity-list__entity-brief{-webkit-line-clamp:5}.entity-detail{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.entity-detail .entity-detail__title{margin-bottom:5px}.entity-detail .entity-detail__info{margin-left:0;float:none;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;width:100%}.entity-detail .entity-detail__img{margin-bottom:24px;float:none;height:unset;max-width:170px}.entity-detail .entity-detail__fields{width:unset;margin-left:unset}.entity-detail .entity-detail__fields+.tag-collection{margin-left:-3px}.dividing-line{margin-top:24px}.entity-sort .entity-sort__entity{-ms-flex-preferred-size:50%;flex-basis:50%}.entity-sort .entity-sort__entity-img{height:130px}.review-head .review-head__info{float:unset}.review-head .review-head__actions{float:unset}.track-carousel__content{padding-bottom:10px}.track-carousel__track{min-width:31%;max-width:31%;margin-right:4.5%}}@media (max-width: 991.98px){.main-section-wrapper{padding:32px 28px 28px 28px}.entity-detail{display:-webkit-box;display:-ms-flexbox;display:flex}}.aside-section-wrapper{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-ms-flex:1;flex:1;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;width:100%;padding:28px 25px 12px 25px;background-color:#f7f7f7;margin-bottom:30px;overflow:auto}.aside-section-wrapper--transparent{background-color:unset}.aside-section-wrapper--collapse{padding:unset}.add-entity-entries .add-entity-entries__entry{margin-bottom:10px}.add-entity-entries .add-entity-entries__label{font-size:1.2em;margin-bottom:8px}.add-entity-entries .add-entity-entries__button{line-height:unset;height:unset;padding:4px 15px;margin:5px}.action-panel{margin-bottom:20px}.action-panel .action-panel__label{font-weight:bold;margin-bottom:12px}.action-panel .action-panel__button-group{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.action-panel .action-panel__button-group--center{-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.action-panel .action-panel__button{line-height:unset;height:unset;padding:4px 15px;margin:0 5px}.mark-panel{margin-bottom:20px}.mark-panel .mark-panel__status{font-weight:bold}.mark-panel .mark-panel__rating-star{position:relative;top:2px}.mark-panel .mark-panel__actions{float:right}.mark-panel .mark-panel__actions form{display:inline}.mark-panel .mark-panel__time{color:#ccc;margin-bottom:10px}.mark-panel .mark-panel__clear{content:' ';clear:both;display:table}.review-panel .review-panel__label{font-weight:bold}.review-panel .review-panel__actions{float:right}.review-panel .review-panel__time{color:#ccc;margin-bottom:10px}.review-panel .review-panel__review-title{display:block;margin-bottom:15px;font-weight:bold}.review-panel .review-panel__clear{content:' ';clear:both;display:table}.user-profile .user-profile__header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start;margin-bottom:15px}.user-profile .user-profile__avatar{width:72px}.user-profile .user-profile__username{font-size:large;margin-left:10px;margin-bottom:0}.user-profile .user-profile__report-link{color:#ccc}.user-relation .user-relation__label{display:inline-block;font-size:large;margin-bottom:10px}.user-relation .user-relation__more-link{margin-left:5px}.user-relation .user-relation__related-user-list{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.user-relation .user-relation__related-user-list:last-of-type{margin-bottom:0}.user-relation .user-relation__related-user{-ms-flex-preferred-size:25%;flex-basis:25%;padding:0px 3px;text-align:center;display:inline-block;overflow:hidden}.user-relation .user-relation__related-user>a:hover{color:#606c76}.user-relation .user-relation__related-user-avatar{background-image:url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");width:48px;height:48px}@media (min-width: 575.98px) and (max-width: 991.98px){.user-relation .user-relation__related-user-avatar{height:unset;width:60%;max-width:96px}}.user-relation .user-relation__related-user-name{color:inherit;overflow:hidden;text-overflow:ellipsis;-webkit-box-orient:vertical;-webkit-line-clamp:2}.report-panel .report-panel__label{display:inline-block;margin-bottom:10px}.report-panel .report-panel__body{padding-left:0}.report-panel .report-panel__report{margin:2px 0}.report-panel .report-panel__user-link{margin:0 2px}.report-panel .report-panel__all-link{margin-left:5px}.import-panel{overflow-x:hidden}.import-panel .import-panel__label{display:inline-block;margin-bottom:10px}.import-panel .import-panel__body{padding-left:0;border:2px dashed #00a1cc;padding:6px 9px}.import-panel .import-panel__body form{margin:0}@media (max-width: 991.98px){.import-panel .import-panel__body{border:unset;padding-left:0}}.import-panel .import-panel__help{background-color:#e5e5e5;border-radius:100000px;display:inline-block;width:16px;height:16px;text-align:center;font-size:12px;cursor:help}.import-panel .import-panel__checkbox{display:inline-block;margin-right:10px}.import-panel .import-panel__checkbox label{display:inline}.import-panel .import-panel__checkbox input[type="checkbox"]{margin:0;position:relative;top:2px}.import-panel .import-panel__checkbox--last{margin-right:0}.import-panel .import-panel__file-input{margin-top:10px}.import-panel .import-panel__button{line-height:unset;height:unset;padding:4px 15px}.import-panel .import-panel__progress{padding-top:10px}.import-panel .import-panel__progress:not(:first-child){border-top:#bbb 1px dashed}.import-panel .import-panel__progress label{display:inline}.import-panel .import-panel__progress progress{background-color:#d5d5d5;border-radius:0;height:10px;width:65%}.import-panel .import-panel__progress progress::-webkit-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__progress progress::-webkit-progress-value{background-color:#00a1cc}.import-panel .import-panel__progress progress::-moz-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__last-task:not(:first-child){padding-top:4px;border-top:#bbb 1px dashed}.import-panel .import-panel__last-task .index:not(:last-of-type){margin-right:8px}.import-panel .import-panel__fail-urls{margin-top:10px}.import-panel .import-panel__fail-urls li{word-break:break-all}.import-panel .import-panel__fail-urls ul{max-height:100px;overflow-y:auto}.relation-dropdown .relation-dropdown__button{display:none}.entity-card{display:-webkit-box;display:-ms-flexbox;display:flex;margin-bottom:10px;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.entity-card--horizontal{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.entity-card .entity-card__img{height:150px}.entity-card .entity-card__rating-star{position:relative;top:4px;left:-3px}.entity-card .entity-card__rating-score{position:relative;top:1px;margin-left:2px}.entity-card .entity-card__title{margin-bottom:10px;margin-top:5px}.entity-card .entity-card__info-wrapper--horizontal{margin-left:20px}.entity-card .entity-card__img-wrapper{-ms-flex-preferred-size:100px;flex-basis:100px}@media (max-width: 575.98px){.add-entity-entries{display:block !important}.add-entity-entries .add-entity-entries__button{width:100%;margin:5px 0 5px 0}.aside-section-wrapper:first-child{margin-right:0 !important;margin-bottom:0 !important}.aside-section-wrapper--singular:first-child{margin-bottom:20px !important}.action-panel{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.entity-card--horizontal{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.entity-card .entity-card__info-wrapper{margin-left:10px !important}.entity-card .entity-card__info-wrapper--horizontal{margin-left:0 !important}}@media (max-width: 991.98px){.add-entity-entries{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-pack:distribute;justify-content:space-around}.aside-section-wrapper{padding:24px 25px 10px 25px;margin-top:20px}.aside-section-wrapper:not(:last-child){margin-right:20px}.aside-section-wrapper--collapse{padding:24px 25px 10px 25px !important;margin-top:0;margin-bottom:0}.aside-section-wrapper--collapse:first-child{margin-right:0}.aside-section-wrapper--no-margin{margin:0}.action-panel{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.action-panel .action-panel__button-group{-webkit-box-pack:space-evenly;-ms-flex-pack:space-evenly;justify-content:space-evenly}.relation-dropdown{margin-bottom:20px}.relation-dropdown .relation-dropdown__button{padding-bottom:10px;background-color:#f7f7f7;width:100%;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;cursor:pointer;-webkit-transition:-webkit-transform 0.3s;transition:-webkit-transform 0.3s;transition:transform 0.3s;transition:transform 0.3s, -webkit-transform 0.3s}.relation-dropdown .relation-dropdown__button:focus{background-color:red}.relation-dropdown .relation-dropdown__button>.icon-arrow{-webkit-transition:-webkit-transform 0.3s;transition:-webkit-transform 0.3s;transition:transform 0.3s;transition:transform 0.3s, -webkit-transform 0.3s}.relation-dropdown .relation-dropdown__button:hover>.icon-arrow>svg{fill:#00a1cc}.relation-dropdown .relation-dropdown__button>.icon-arrow--expand{-webkit-transform:rotate(-180deg);transform:rotate(-180deg)}.relation-dropdown .relation-dropdown__button+.relation-dropdown__body--expand{max-height:2000px;-webkit-transition:max-height 1s ease-in;transition:max-height 1s ease-in}.relation-dropdown .relation-dropdown__body{background-color:#f7f7f7;max-height:0;-webkit-transition:max-height 1s ease-out;transition:max-height 1s ease-out;overflow:hidden}.entity-card{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.entity-card .entity-card__info-wrapper{margin-left:30px}}.single-section-wrapper{padding:32px 36px;background-color:#f7f7f7;overflow:auto}.single-section-wrapper .single-section-wrapper__link--secondary{display:inline-block;color:#ccc;margin-bottom:20px}.single-section-wrapper .single-section-wrapper__link--secondary:hover{color:#00a1cc}.entity-form,.review-form{overflow:auto}.entity-form>input[type='email'],.entity-form>input[type='number'],.entity-form>input[type='password'],.entity-form>input[type='search'],.entity-form>input[type='tel'],.entity-form>input[type='text'],.entity-form>input[type='url'],.entity-form textarea,.review-form>input[type='email'],.review-form>input[type='number'],.review-form>input[type='password'],.review-form>input[type='search'],.review-form>input[type='tel'],.review-form>input[type='text'],.review-form>input[type='url'],.review-form textarea{width:100%}.entity-form img,.review-form img{display:block}.review-form .review-form__preview-button{color:#00a1cc;font-weight:bold;cursor:pointer}.review-form .review-form__fyi{color:#ccc}.review-form .review-form__main-content,.review-form textarea{margin-bottom:5px;resize:vertical;height:400px}.review-form .review-form__option{margin-top:24px;margin-bottom:10px}.review-form .review-form__option::after{content:' ';clear:both;display:table}.review-form .review-form__visibility-radio{float:left}.review-form .review-form__visibility-radio ul,.review-form .review-form__visibility-radio li,.review-form .review-form__visibility-radio label{display:inline}.review-form .review-form__visibility-radio label{font-size:normal}.review-form .review-form__visibility-radio input[type="radio"]{position:relative;top:2px}.review-form .review-form__share-checkbox{float:right}.review-form .review-form__share-checkbox input[type="checkbox"]{position:relative;top:2px}.report-form input,.report-form select{width:100%}@media (max-width: 575.98px){.review-form .review-form__visibility-radio{float:unset}.review-form .review-form__share-checkbox{float:unset;position:relative;left:-3px}}.markdownx-preview{min-height:100px}.markdownx-preview ul li{list-style:circle inside}.rating-star .jq-star{cursor:unset !important}.ms-parent>.ms-choice{margin-bottom:1.5rem;-webkit-appearance:none;-moz-appearance:none;appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;-webkit-box-shadow:none;box-shadow:none;-webkit-box-sizing:inherit;box-sizing:inherit;padding:.6rem 1.0rem;width:100%;height:30.126px}.ms-parent>.ms-choice:focus{border-color:#00a1cc}.ms-parent>.ms-choice>.icon-caret{top:15.5px}.ms-parent>.ms-choice>span{color:black;font-weight:initial;font-size:13.3333px;top:2.5px;left:2px}.ms-parent>.ms-choice>span:hover,.ms-parent>.ms-choice>span:focus{color:black}.ms-parent>.ms-drop>ul>li>label>span{margin-left:10px}.ms-parent>.ms-drop>ul>li>label>input{width:unset}.tippy-box{border:#606c76 1px solid;background-color:#f7f7f7;padding:3px 5px}.tag-input input{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1} +@import url(https://cdn.jsdelivr.net/npm/skeleton-css@2.0.4/css/normalize.css);.button,button,input[type='button'],input[type='reset'],input[type='submit']{background-color:#00a1cc;border:0.1rem solid #00a1cc;border-radius:.4rem;color:#fff;cursor:pointer;display:inline-block;font-size:1.1rem;font-weight:700;height:3.4rem;letter-spacing:.1rem;line-height:3.4rem;padding:0 2.8rem;text-align:center;text-decoration:none;text-transform:uppercase;white-space:nowrap}.button:focus,.button:hover,button:focus,button:hover,input[type='button']:focus,input[type='button']:hover,input[type='reset']:focus,input[type='reset']:hover,input[type='submit']:focus,input[type='submit']:hover{background-color:#606c76;border-color:#606c76;color:#fff;outline:0}.button[disabled],button[disabled],input[type='button'][disabled],input[type='reset'][disabled],input[type='submit'][disabled]{cursor:default;opacity:.5}.button[disabled]:focus,.button[disabled]:hover,button[disabled]:focus,button[disabled]:hover,input[type='button'][disabled]:focus,input[type='button'][disabled]:hover,input[type='reset'][disabled]:focus,input[type='reset'][disabled]:hover,input[type='submit'][disabled]:focus,input[type='submit'][disabled]:hover{background-color:#00a1cc;border-color:#00a1cc}.button.button-outline,button.button-outline,input[type='button'].button-outline,input[type='reset'].button-outline,input[type='submit'].button-outline{background-color:transparent;color:#00a1cc}.button.button-outline:focus,.button.button-outline:hover,button.button-outline:focus,button.button-outline:hover,input[type='button'].button-outline:focus,input[type='button'].button-outline:hover,input[type='reset'].button-outline:focus,input[type='reset'].button-outline:hover,input[type='submit'].button-outline:focus,input[type='submit'].button-outline:hover{background-color:transparent;border-color:#606c76;color:#606c76}.button.button-outline[disabled]:focus,.button.button-outline[disabled]:hover,button.button-outline[disabled]:focus,button.button-outline[disabled]:hover,input[type='button'].button-outline[disabled]:focus,input[type='button'].button-outline[disabled]:hover,input[type='reset'].button-outline[disabled]:focus,input[type='reset'].button-outline[disabled]:hover,input[type='submit'].button-outline[disabled]:focus,input[type='submit'].button-outline[disabled]:hover{border-color:inherit;color:#00a1cc}.button.button-clear,button.button-clear,input[type='button'].button-clear,input[type='reset'].button-clear,input[type='submit'].button-clear{background-color:transparent;border-color:transparent;color:#00a1cc}.button.button-clear:focus,.button.button-clear:hover,button.button-clear:focus,button.button-clear:hover,input[type='button'].button-clear:focus,input[type='button'].button-clear:hover,input[type='reset'].button-clear:focus,input[type='reset'].button-clear:hover,input[type='submit'].button-clear:focus,input[type='submit'].button-clear:hover{background-color:transparent;border-color:transparent;color:#606c76}.button.button-clear[disabled]:focus,.button.button-clear[disabled]:hover,button.button-clear[disabled]:focus,button.button-clear[disabled]:hover,input[type='button'].button-clear[disabled]:focus,input[type='button'].button-clear[disabled]:hover,input[type='reset'].button-clear[disabled]:focus,input[type='reset'].button-clear[disabled]:hover,input[type='submit'].button-clear[disabled]:focus,input[type='submit'].button-clear[disabled]:hover{color:#00a1cc}select{background:url('data:image/svg+xml;utf8,') center right no-repeat;padding-right:3.0rem}select:focus{background-image:url('data:image/svg+xml;utf8,')}textarea{min-height:6.5rem;width:100%}select{width:100%}label,legend{display:block;margin-bottom:.5rem}fieldset{border-width:0;padding:0}input[type='checkbox'],input[type='radio']{display:inline}.label-inline{display:inline-block;font-weight:normal;margin-left:.5rem}dl,ol,ul{list-style:none;margin-top:0;padding-left:0}ol{list-style:decimal inside}ul{list-style:circle inside}.button,button,dd,dt,li{margin-bottom:1.0rem}fieldset,input,select,textarea{margin-bottom:1.5rem}blockquote,dl,figure,form,ol,p,pre,table,ul{margin-bottom:1rem}b,strong{font-weight:bold}p{margin-top:0}h1,h2,h3,h4,h5,h6{font-weight:300;letter-spacing:-.1rem;margin-bottom:2.0rem;margin-top:0}h1{font-size:4.6rem;line-height:1.2}h2{font-size:3.6rem;line-height:1.25}h3{font-size:2.8rem;line-height:1.3}h4{font-size:2.2rem;letter-spacing:-.08rem;line-height:1.35}h5{font-size:1.8rem;letter-spacing:-.05rem;line-height:1.5}h6{font-size:1.6rem;letter-spacing:0;line-height:1.4}img{max-width:100%;object-fit:contain}img.emoji{height:14px;box-sizing:border-box;object-fit:contain;position:relative;top:3px}img.emoji--large{height:20px;position:relative;top:2px}.clearfix:after{clear:both;content:' ';display:table}.float-left{float:left}.float-right{float:right}.highlight{font-weight:bold}:root{font-size:10px}*,*:after,*:before{box-sizing:inherit}html{box-sizing:border-box;height:100%}body{color:#606c76;font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif;font-size:1.3rem;font-weight:300;letter-spacing:.05rem;line-height:1.6;margin:0;height:100%}textarea{font-family:'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', 'Microsoft YaHei Light', sans-serif}a{color:#00a1cc;text-decoration:none}a:active,a:hover,a:hover:visited{color:#606c76}li{list-style:none}input[type=text]::-ms-clear,input[type=text]::-ms-reveal{display:none;width:0;height:0}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-results-button,input[type="search"]::-webkit-search-results-decoration{display:none}input[type='email'],input[type='number'],input[type='password'],input[type='search'],input[type='tel'],input[type='text'],input[type='url'],input[type='date'],input[type='time'],input[type='color'],textarea,select{appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem}input[type='email']:focus,input[type='number']:focus,input[type='password']:focus,input[type='search']:focus,input[type='tel']:focus,input[type='text']:focus,input[type='url']:focus,input[type='date']:focus,input[type='time']:focus,input[type='color']:focus,textarea:focus,select:focus{border-color:#00a1cc;outline:0}input[type='email']::placeholder,input[type='number']::placeholder,input[type='password']::placeholder,input[type='search']::placeholder,input[type='tel']::placeholder,input[type='text']::placeholder,input[type='url']::placeholder,input[type='date']::placeholder,input[type='time']::placeholder,input[type='color']::placeholder,textarea::placeholder,select::placeholder{color:#ccc}::selection{color:white;background-color:#00a1cc}.navbar{background-color:#f7f7f7;box-sizing:border-box;padding:10px 0;margin-bottom:50px;border-bottom:#ccc 0.5px solid}.navbar .navbar__wrapper{display:flex;justify-content:space-between;align-items:center;position:relative}.navbar .navbar__logo{flex-basis:100px}.navbar .navbar__logo-link{display:inline-block}.navbar .navbar__link-list{margin:0;display:flex;justify-content:space-around}.navbar .navbar__link{margin:9px;color:#606c76}.navbar .navbar__link:active,.navbar .navbar__link:hover,.navbar .navbar__link:hover:visited{color:#00a1cc}.navbar .navbar__link:visited{color:#606c76}.navbar .navbar__search-box{margin:0 12% 0 15px;display:inline-flex;flex:1}.navbar .navbar__search-box>input[type="search"]{border-top-right-radius:0;border-bottom-right-radius:0;margin:0;height:32px;background-color:white !important;width:100%}.navbar .navbar__search-box .navbar__search-dropdown{margin:0;margin-left:-1px;padding:0;padding-left:10px;color:#606c76;appearance:auto;background-color:white;height:32px;width:80px;border-top-left-radius:0;border-bottom-left-radius:0}.navbar .navbar__dropdown-btn{display:none;padding:0;margin:0;border:none;background-color:transparent;color:#00a1cc}.navbar .navbar__dropdown-btn:focus,.navbar .navbar__dropdown-btn:hover{background-color:transparent;color:#606c76}@media (max-width: 575.98px){.navbar{padding:2px 0}.navbar .navbar__wrapper{display:block}.navbar .navbar__logo-img{width:72px;margin-right:10px;position:relative;top:7px}.navbar .navbar__link-list{margin-top:7px;max-height:0;transition:max-height 0.6s ease-out;overflow:hidden}.navbar .navbar__dropdown-btn{display:block;position:absolute;right:5px;top:3px;transform:scale(0.7)}.navbar .navbar__dropdown-btn:hover+.navbar__link-list{max-height:500px;transition:max-height 0.6s ease-in}.navbar .navbar__search-box{margin:0;width:46vw}.navbar .navbar__search-box>input[type="search"]{height:26px;padding:4px 6px;width:32vw}.navbar .navbar__search-box .navbar__search-dropdown{cursor:pointer;height:26px;width:80px;padding-left:5px}}@media (max-width: 991.98px){.navbar{margin-bottom:20px}}.grid{margin:0 auto;position:relative;max-width:110rem;padding:0 2.0rem;width:100%}.grid .grid__main{width:70%;float:left;position:relative}.grid .grid__aside{width:26%;float:right;position:relative;display:flex;flex-direction:column;justify-content:space-around}.grid::after{content:' ';clear:both;display:table}@media (max-width: 575.98px){.grid .grid__aside{flex-direction:column !important}}@media (max-width: 991.98px){.grid .grid__main{width:100%;float:none}.grid .grid__aside{width:100%;float:none;flex-direction:row}.grid .grid__aside--tablet-column{flex-direction:column}.grid--reverse-order{transform:scaleY(-1)}.grid .grid__main--reverse-order{transform:scaleY(-1)}.grid .grid__aside--reverse-order{transform:scaleY(-1)}}.pagination{text-align:center;width:100%}.pagination .pagination__page-link{font-weight:normal;margin:0 5px}.pagination .pagination__page-link--current{font-weight:bold;font-size:1.2em;color:#606c76}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:18px}.pagination .pagination__nav-link--left-margin{margin-left:18px}.pagination .pagination__nav-link--hidden{display:none}@media (max-width: 575.98px){.pagination .pagination__page-link{margin:0 3px}.pagination .pagination__nav-link{font-size:1.4em;margin:0 2px}.pagination .pagination__nav-link--right-margin{margin-right:10px}.pagination .pagination__nav-link--left-margin{margin-left:10px}}#page-wrapper{position:relative;min-height:100vh;z-index:0}#content-wrapper{padding-bottom:160px}.footer{padding-top:0.4em !important;text-align:center;margin-bottom:4px !important;position:absolute !important;left:50%;transform:translateX(-50%);bottom:0;width:100%}.footer__border{padding-top:4px;border-top:#f7f7f7 solid 2px}.footer__link{margin:0 12px;white-space:nowrap}@media (max-width: 575.98px){#content-wrapper{padding-bottom:120px}}.icon-lock svg{fill:#ccc;height:12px;position:relative;top:1px;margin-left:3px}.icon-edit svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-save svg{fill:#ccc;height:12px;position:relative;top:2px}.icon-cross svg{fill:#ccc;height:10px;position:relative}.icon-arrow svg{fill:#606c76;height:15px;position:relative;top:3px}.spinner{display:inline-block;position:relative;left:50%;transform:translateX(-50%) scale(0.4);width:80px;height:80px}.spinner div{transform-origin:40px 40px;animation:spinner 1.2s linear infinite}.spinner div::after{content:" ";display:block;position:absolute;top:3px;left:37px;width:6px;height:18px;border-radius:20%;background:#606c76}.spinner div:nth-child(1){transform:rotate(0deg);animation-delay:-1.1s}.spinner div:nth-child(2){transform:rotate(30deg);animation-delay:-1s}.spinner div:nth-child(3){transform:rotate(60deg);animation-delay:-.9s}.spinner div:nth-child(4){transform:rotate(90deg);animation-delay:-.8s}.spinner div:nth-child(5){transform:rotate(120deg);animation-delay:-.7s}.spinner div:nth-child(6){transform:rotate(150deg);animation-delay:-.6s}.spinner div:nth-child(7){transform:rotate(180deg);animation-delay:-.5s}.spinner div:nth-child(8){transform:rotate(210deg);animation-delay:-.4s}.spinner div:nth-child(9){transform:rotate(240deg);animation-delay:-.3s}.spinner div:nth-child(10){transform:rotate(270deg);animation-delay:-.2s}.spinner div:nth-child(11){transform:rotate(300deg);animation-delay:-.1s}.spinner div:nth-child(12){transform:rotate(330deg);animation-delay:0s}@keyframes spinner{0%{opacity:1}100%{opacity:0}}.bg-mask{background-color:black;z-index:1;filter:opacity(20%);position:fixed;width:100%;height:100%;left:0;top:0;display:none}.mark-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.mark-modal .mark-modal__head{margin-bottom:20px}.mark-modal .mark-modal__head::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__title{font-weight:bold;font-size:1.2em;float:left}.mark-modal .mark-modal__close-button{float:right;cursor:pointer}.mark-modal .mark-modal__confirm-button{float:right}.mark-modal input[type="radio"]{margin-right:0}.mark-modal .mark-modal__rating-star{display:inline;float:left;position:relative;left:-3px}.mark-modal .mark-modal__status-radio{float:right}.mark-modal .mark-modal__status-radio ul{margin-bottom:0}.mark-modal .mark-modal__status-radio li,.mark-modal .mark-modal__status-radio label{display:inline}.mark-modal .mark-modal__status-radio input[type="radio"]{position:relative;top:1px}.mark-modal .mark-modal__clear{content:' ';clear:both;display:table}.mark-modal .mark-modal__content-input,.mark-modal form textarea{height:200px;width:100%;margin-top:5px;margin-bottom:5px;resize:vertical}.mark-modal .mark-modal__tag{margin-bottom:20px}.mark-modal .mark-modal__option{margin-bottom:24px}.mark-modal .mark-modal__option::after{content:' ';clear:both;display:table}.mark-modal .mark-modal__visibility-radio{float:left}.mark-modal .mark-modal__visibility-radio ul,.mark-modal .mark-modal__visibility-radio li,.mark-modal .mark-modal__visibility-radio label{display:inline}.mark-modal .mark-modal__visibility-radio label{font-size:normal}.mark-modal .mark-modal__visibility-radio input[type="radio"]{position:relative;top:2px}.mark-modal .mark-modal__share-checkbox{float:right}.mark-modal .mark-modal__share-checkbox input[type="checkbox"]{position:relative;top:2px}.confirm-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.confirm-modal .confirm-modal__head{margin-bottom:20px}.confirm-modal .confirm-modal__head::after{content:' ';clear:both;display:table}.confirm-modal .confirm-modal__title{font-weight:bold;font-size:1.2em;float:left}.confirm-modal .confirm-modal__close-button{float:right;cursor:pointer}.confirm-modal .confirm-modal__confirm-button{float:right}.announcement-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.announcement-modal .announcement-modal__head{margin-bottom:20px}.announcement-modal .announcement-modal__head::after{content:' ';clear:both;display:table}.announcement-modal .announcement-modal__title{font-weight:bold;font-size:1.2em;float:left}.announcement-modal .announcement-modal__close-button{float:right;cursor:pointer}.announcement-modal .announcement-modal__confirm-button{float:right}.announcement-modal .announcement-modal__body{overflow-y:auto;max-height:64vh}.announcement-modal .announcement-modal__body .announcement__title{display:inline-block}.announcement-modal .announcement-modal__body .announcement__datetime{color:#ccc;margin-left:10px}.announcement-modal .announcement-modal__body .announcement__content{word-break:break-all}.add-to-list-modal{z-index:2;display:none;position:fixed;width:500px;top:50%;left:50%;transform:translate(-50%, -50%);background-color:#f7f7f7;padding:20px 20px 10px 20px;color:#606c76}.add-to-list-modal .add-to-list-modal__head{margin-bottom:20px}.add-to-list-modal .add-to-list-modal__head::after{content:' ';clear:both;display:table}.add-to-list-modal .add-to-list-modal__title{font-weight:bold;font-size:1.2em;float:left}.add-to-list-modal .add-to-list-modal__close-button{float:right;cursor:pointer}.add-to-list-modal .add-to-list-modal__confirm-button{float:right}@media (max-width: 575.98px){.mark-modal,.confirm-modal,.announcement-modal .add-to-list-modal{width:100%}}.source-label{display:inline;background:transparent;border-radius:.3rem;border-style:solid;border-width:.1rem;line-height:1.2rem;font-size:1.1rem;margin:3px;padding:1px 3px;padding-top:2px;font-weight:lighter;letter-spacing:0.1rem;word-break:keep-all;opacity:1;position:relative;top:-1px}.source-label.source-label__in-site{border-color:#00a1cc;color:#00a1cc}.source-label.source-label__douban{border:none;color:#fff;background-color:#319840}.source-label.source-label__spotify{background-color:#1ed760;color:#000;border:none;font-weight:bold}.source-label.source-label__imdb{background-color:#F5C518;color:#121212;border:none;font-weight:bold}.source-label.source-label__igdb{background-color:#323A44;color:#DFE1E2;border:none;font-weight:bold}.source-label.source-label__steam{background:linear-gradient(30deg, #1387b8, #111d2e);color:white;border:none;font-weight:600;padding-top:2px}.source-label.source-label__bangumi{background:#FCFCFC;color:#F09199;font-style:italic;font-weight:600}.source-label.source-label__goodreads{background:#F4F1EA;color:#372213;font-weight:lighter}.source-label.source-label__tmdb{background:linear-gradient(90deg, #91CCA3, #1FB4E2);color:white;border:none;font-weight:lighter;padding-top:2px}.source-label.source-label__googlebooks{color:white;background-color:#4285F4;border-color:#4285F4}.source-label.source-label__bandcamp{color:#fff;background-color:#28A0C1;display:inline-block}.source-label.source-label__bandcamp span{display:inline-block;margin:0 4px}.main-section-wrapper{padding:32px 48px 32px 36px;background-color:#f7f7f7;overflow:auto}.main-section-wrapper input,.main-section-wrapper select{width:100%}.entity-list .entity-list__title{margin-bottom:20px}.entity-list .entity-list__entity{display:flex;margin-bottom:36px}.entity-list .entity-list__entity::after{content:' ';clear:both;display:table}.entity-list .entity-list__entity-img{object-fit:contain;min-width:130px;max-width:130px}.entity-list .entity-list__entity-text{margin-left:20px;overflow:hidden;width:100%}.entity-list .entity-list__entity-text .tag-collection{margin-left:-3px}.entity-list .entity-list__entity-link{font-size:1.2em}.entity-list .entity-list__entity-title{display:block}.entity-list .entity-list__entity-category{color:#bbb;margin-left:5px;position:relative;top:-1px}.entity-list .entity-list__entity-info{max-width:73%;white-space:nowrap;overflow:hidden;display:inline-block;text-overflow:ellipsis;position:relative;top:0.52em}.entity-list .entity-list__entity-info--full-length{max-width:100%}.entity-list .entity-list__entity-brief{margin-top:8px;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:4;overflow:hidden;margin-bottom:0}.entity-list .entity-list__rating{display:inline-block;margin:0}.entity-list .entity-list__rating--empty{margin-right:5px}.entity-list .entity-list__rating-score{margin-right:5px;position:relative;top:1px}.entity-list .entity-list__rating-star{display:inline;position:relative;top:0.3em;left:-0.3em}.entity-detail .entity-detail__img{height:210px;float:left;object-fit:contain;max-width:150px;object-position:top}.entity-detail .entity-detail__img-origin{cursor:zoom-in}.entity-detail .entity-detail__info{float:left;margin-left:20px;overflow:hidden;text-overflow:ellipsis;width:70%}.entity-detail .entity-detail__title{font-weight:bold}.entity-detail .entity-detail__title--secondary{color:#bbb}.entity-detail .entity-detail__fields{display:inline-block;vertical-align:top;width:46%;margin-left:2%}.entity-detail .entity-detail__fields div,.entity-detail .entity-detail__fields span{margin:1px 0}.entity-detail .entity-detail__fields+.tag-collection{margin-top:5px;margin-left:6px}.entity-detail .entity-detail__rating{position:relative;top:-5px}.entity-detail .entity-detail__rating-star{position:relative;left:-4px;top:3px}.entity-detail .entity-detail__rating-score{font-weight:bold}.entity-detail::after{content:' ';clear:both;display:table}.entity-desc{margin-bottom:28px}.entity-desc .entity-desc__title{display:inline-block;margin-bottom:8px}.entity-desc .entity-desc__content{overflow:hidden}.entity-desc .entity-desc__content--folded{max-height:202px}.entity-desc .entity-desc__unfold-button{display:flex;color:#00a1cc;background-color:transparent;justify-content:center;text-align:center}.entity-desc .entity-desc__unfold-button--hidden{display:none}.entity-marks{margin-bottom:28px}.entity-marks .entity-marks__title{margin-bottom:8px;display:inline-block}.entity-marks .entity-marks__title>a{margin-right:5px}.entity-marks .entity-marks__title--stand-alone{margin-bottom:20px}.entity-marks .entity-marks__more-link{margin-left:5px}.entity-marks .entity-marks__mark{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-marks .entity-marks__mark:last-child{border:none}.entity-marks .entity-marks__mark--wider{padding:6px 0}.entity-marks .entity-marks__mark-content{margin-bottom:0}.entity-marks .entity-marks__mark-time{color:#ccc;margin-left:2px}.entity-marks .entity-marks__rating-star{position:relative;top:4px}.entity-reviews:first-child{margin-bottom:28px}.entity-reviews .entity-reviews__title{display:inline-block;margin-bottom:8px}.entity-reviews .entity-reviews__title>a{margin-right:5px}.entity-reviews .entity-reviews__title--stand-alone{margin-bottom:20px}.entity-reviews .entity-reviews__more-link{margin-left:5px}.entity-reviews .entity-reviews__review{margin:0;padding:3px 0;border-bottom:1px dashed #e5e5e5}.entity-reviews .entity-reviews__review:last-child{border:none}.entity-reviews .entity-reviews__review--wider{padding:6px 0}.entity-reviews .entity-reviews__review-time{color:#ccc;margin-left:2px}.dividing-line{height:0;width:100%;margin:40px 0 24px 0;border-top:solid 1px #ccc}.dividing-line.dividing-line--dashed{margin:0;margin-top:10px;margin-bottom:2px;border-top:1px dashed #e5e5e5}.entity-sort{position:relative;margin-bottom:30px}.entity-sort .entity-sort__label{font-size:large;display:inline-block;margin-bottom:20px}.entity-sort .entity-sort__more-link{margin-left:8px}.entity-sort .entity-sort__count{color:#bbb}.entity-sort .entity-sort__count::before{content:'('}.entity-sort .entity-sort__count::after{content:')'}.entity-sort .entity-sort__entity-list{display:flex;justify-content:flex-start;flex-wrap:wrap}.entity-sort .entity-sort__entity{padding:0 10px;flex-basis:20%;text-align:center;display:inline-block;color:#606c76}.entity-sort .entity-sort__entity:hover{color:#00a1cc}.entity-sort .entity-sort__entity>a{color:inherit}.entity-sort .entity-sort__entity-img{height:110px}.entity-sort .entity-sort__entity-name{text-overflow:ellipsis;overflow:hidden;display:-webkit-box;-webkit-box-orient:vertical;-webkit-line-clamp:2}.entity-sort--placeholder{border:dashed #bbb 4px}.entity-sort--hover{padding:10px;border:dashed #00a1cc 2px !important;border-radius:3px}.entity-sort--sortable{padding:10px;margin:10px 0;border:dashed #bbb 2px;cursor:all-scroll}.entity-sort--hidden{opacity:0.4}.entity-sort-control{display:flex;justify-content:flex-end}.entity-sort-control__button{margin-top:5px;margin-left:12px;padding:0 2px;cursor:pointer;color:#bbb}.entity-sort-control__button:hover{color:#00a1cc}.entity-sort-control__button:hover>.icon-save svg,.entity-sort-control__button:hover>.icon-edit svg{fill:#00a1cc}.entity-sort-control__button--float-right{position:absolute;top:4px;right:10px}.related-user-list .related-user-list__title{margin-bottom:20px}.related-user-list .related-user-list__user{display:flex;justify-content:flex-start;margin-bottom:20px}.related-user-list .related-user-list__user-info{margin-left:15px;overflow:auto}.related-user-list .related-user-list__user-avatar{max-height:72px;min-width:72px}.review-head .review-head__title{display:inline-block;font-weight:bold}.review-head .review-head__body{margin-bottom:10px}.review-head .review-head__body::after{content:' ';clear:both;display:table}.review-head .review-head__info{float:left}.review-head .review-head__owner-link{color:#ccc}.review-head .review-head__owner-link:hover{color:#00a1cc}.review-head .review-head__time{color:#ccc}.review-head .review-head__rating-star{position:relative;top:3px;left:-1px}.review-head .review-head__actions{float:right}.review-head .review-head__action-link:not(:first-child){margin-left:5px}.tag-collection{margin-left:-9px}.tag-collection .tag-collection__tag{position:relative;display:block;float:left;color:white;background:#ccc;padding:5px;border-radius:.3rem;line-height:1.2em;font-size:80%;margin:3px}.tag-collection .tag-collection__tag a{color:white}.tag-collection .tag-collection__tag a:hover{color:#00a1cc}.track-carousel{position:relative;margin-top:5px}.track-carousel__content{overflow:auto;scroll-behavior:smooth;scrollbar-width:none;display:flex;margin:auto;box-sizing:border-box;padding-bottom:10px}.track-carousel__content::-webkit-scrollbar{height:3px;width:1px;background-color:#e5e5e5}.track-carousel__content::-webkit-scrollbar-thumb{background-color:#bbb}.track-carousel__track{text-align:center;overflow:hidden;text-overflow:ellipsis;min-width:18%;max-width:18%;margin-right:2.5%}.track-carousel__track img{object-fit:contain}.track-carousel__track-title{white-space:nowrap}.track-carousel__button{display:flex;justify-content:center;align-content:center;background:white;border:none;padding:8px;border-radius:50%;outline:0;cursor:pointer;position:absolute;top:50%}.track-carousel__button--prev{left:0;transform:translate(50%, -50%)}.track-carousel__button--next{right:0;transform:translate(-50%, -50%)}@media (max-width: 575.98px){.entity-list .entity-list__entity{flex-direction:column;margin-bottom:30px}.entity-list .entity-list__entity-text{margin-left:0}.entity-list .entity-list__entity-img-wrapper{margin-bottom:8px}.entity-list .entity-list__entity-info{max-width:unset}.entity-list .entity-list__rating--empty+.entity-list__entity-info{max-width:70%}.entity-list .entity-list__entity-brief{-webkit-line-clamp:5}.entity-detail{flex-direction:column}.entity-detail .entity-detail__title{margin-bottom:5px}.entity-detail .entity-detail__info{margin-left:0;float:none;display:flex;flex-direction:column;width:100%}.entity-detail .entity-detail__img{margin-bottom:24px;float:none;height:unset;max-width:170px}.entity-detail .entity-detail__fields{width:unset;margin-left:unset}.entity-detail .entity-detail__fields+.tag-collection{margin-left:-3px}.dividing-line{margin-top:24px}.entity-sort .entity-sort__entity{flex-basis:50%}.entity-sort .entity-sort__entity-img{height:130px}.review-head .review-head__info{float:unset}.review-head .review-head__actions{float:unset}.track-carousel__content{padding-bottom:10px}.track-carousel__track{min-width:31%;max-width:31%;margin-right:4.5%}}@media (max-width: 991.98px){.main-section-wrapper{padding:32px 28px 28px 28px}.entity-detail{display:flex}}.aside-section-wrapper{display:flex;flex:1;flex-direction:column;width:100%;padding:28px 25px 12px 25px;background-color:#f7f7f7;margin-bottom:30px;overflow:auto}.aside-section-wrapper--transparent{background-color:unset}.aside-section-wrapper--collapse{padding:unset}.add-entity-entries .add-entity-entries__entry{margin-bottom:10px}.add-entity-entries .add-entity-entries__label{font-size:1.2em;margin-bottom:8px}.add-entity-entries .add-entity-entries__button{line-height:unset;height:unset;padding:4px 15px;margin:5px}.action-panel{margin-bottom:20px}.action-panel .action-panel__label{font-weight:bold;margin-bottom:12px}.action-panel .action-panel__button-group{display:flex;justify-content:space-between}.action-panel .action-panel__button-group--center{justify-content:center}.action-panel .action-panel__button{line-height:unset;height:unset;padding:4px 15px;margin:0 5px}.mark-panel{margin-bottom:20px}.mark-panel .mark-panel__status{font-weight:bold}.mark-panel .mark-panel__rating-star{position:relative;top:2px}.mark-panel .mark-panel__actions{float:right}.mark-panel .mark-panel__actions form{display:inline}.mark-panel .mark-panel__time{color:#ccc;margin-bottom:10px}.mark-panel .mark-panel__clear{content:' ';clear:both;display:table}.review-panel .review-panel__label{font-weight:bold}.review-panel .review-panel__actions{float:right}.review-panel .review-panel__time{color:#ccc;margin-bottom:10px}.review-panel .review-panel__review-title{display:block;margin-bottom:15px;font-weight:bold}.review-panel .review-panel__clear{content:' ';clear:both;display:table}.user-profile .user-profile__header{display:flex;align-items:flex-start;margin-bottom:15px}.user-profile .user-profile__avatar{width:72px}.user-profile .user-profile__username{font-size:large;margin-left:10px;margin-bottom:0}.user-profile .user-profile__report-link{color:#ccc}.user-relation .user-relation__label{display:inline-block;font-size:large;margin-bottom:10px}.user-relation .user-relation__more-link{margin-left:5px}.user-relation .user-relation__related-user-list{display:flex;justify-content:flex-start}.user-relation .user-relation__related-user-list:last-of-type{margin-bottom:0}.user-relation .user-relation__related-user{flex-basis:25%;padding:0px 3px;text-align:center;display:inline-block;overflow:hidden}.user-relation .user-relation__related-user>a:hover{color:#606c76}.user-relation .user-relation__related-user-avatar{background-image:url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");width:48px;height:48px}@media (min-width: 575.98px) and (max-width: 991.98px){.user-relation .user-relation__related-user-avatar{height:unset;width:60%;max-width:96px}}.user-relation .user-relation__related-user-name{color:inherit;overflow:hidden;text-overflow:ellipsis;-webkit-box-orient:vertical;-webkit-line-clamp:2}.report-panel .report-panel__label{display:inline-block;margin-bottom:10px}.report-panel .report-panel__body{padding-left:0}.report-panel .report-panel__report{margin:2px 0}.report-panel .report-panel__user-link{margin:0 2px}.report-panel .report-panel__all-link{margin-left:5px}.import-panel{overflow-x:hidden}.import-panel .import-panel__label{display:inline-block;margin-bottom:10px}.import-panel .import-panel__body{padding-left:0;border:2px dashed #00a1cc;padding:6px 9px}.import-panel .import-panel__body form{margin:0}@media (max-width: 991.98px){.import-panel .import-panel__body{border:unset;padding-left:0}}.import-panel .import-panel__help{background-color:#e5e5e5;border-radius:100000px;display:inline-block;width:16px;height:16px;text-align:center;font-size:12px;cursor:help}.import-panel .import-panel__checkbox{display:inline-block;margin-right:10px}.import-panel .import-panel__checkbox label{display:inline}.import-panel .import-panel__checkbox input[type="checkbox"]{margin:0;position:relative;top:2px}.import-panel .import-panel__checkbox--last{margin-right:0}.import-panel .import-panel__file-input{margin-top:10px}.import-panel .import-panel__button{line-height:unset;height:unset;padding:4px 15px}.import-panel .import-panel__progress{padding-top:10px}.import-panel .import-panel__progress:not(:first-child){border-top:#bbb 1px dashed}.import-panel .import-panel__progress label{display:inline}.import-panel .import-panel__progress progress{background-color:#d5d5d5;border-radius:0;height:10px;width:54%}.import-panel .import-panel__progress progress::-webkit-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__progress progress::-webkit-progress-value{background-color:#00a1cc}.import-panel .import-panel__progress progress::-moz-progress-bar{background-color:#d5d5d5}.import-panel .import-panel__last-task:not(:first-child){padding-top:4px;border-top:#bbb 1px dashed}.import-panel .import-panel__last-task .index:not(:last-of-type){margin-right:8px}.import-panel .import-panel__fail-urls{margin-top:10px}.import-panel .import-panel__fail-urls li{word-break:break-all}.import-panel .import-panel__fail-urls ul{max-height:100px;overflow-y:auto}.relation-dropdown .relation-dropdown__button{display:none}.entity-card{display:flex;margin-bottom:10px;flex-direction:column}.entity-card--horizontal{flex-direction:row}.entity-card .entity-card__img{height:150px}.entity-card .entity-card__rating-star{position:relative;top:4px;left:-3px}.entity-card .entity-card__rating-score{position:relative;top:1px;margin-left:2px}.entity-card .entity-card__title{margin-bottom:10px;margin-top:5px}.entity-card .entity-card__info-wrapper--horizontal{margin-left:20px}.entity-card .entity-card__img-wrapper{flex-basis:100px}@media (max-width: 575.98px){.add-entity-entries{display:block !important}.add-entity-entries .add-entity-entries__button{width:100%;margin:5px 0 5px 0}.aside-section-wrapper:first-child{margin-right:0 !important;margin-bottom:0 !important}.aside-section-wrapper--singular:first-child{margin-bottom:20px !important}.action-panel{flex-direction:column !important}.entity-card--horizontal{flex-direction:column !important}.entity-card .entity-card__info-wrapper{margin-left:10px !important}.entity-card .entity-card__info-wrapper--horizontal{margin-left:0 !important}}@media (max-width: 991.98px){.add-entity-entries{display:flex;justify-content:space-around}.aside-section-wrapper{padding:24px 25px 10px 25px;margin-top:20px}.aside-section-wrapper:not(:last-child){margin-right:20px}.aside-section-wrapper--collapse{padding:24px 25px 10px 25px !important;margin-top:0;margin-bottom:0}.aside-section-wrapper--collapse:first-child{margin-right:0}.aside-section-wrapper--no-margin{margin:0}.action-panel{flex-direction:row}.action-panel .action-panel__button-group{justify-content:space-evenly}.relation-dropdown{margin-bottom:20px}.relation-dropdown .relation-dropdown__button{padding-bottom:10px;background-color:#f7f7f7;width:100%;display:flex;justify-content:center;align-items:center;cursor:pointer;transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:focus{background-color:red}.relation-dropdown .relation-dropdown__button>.icon-arrow{transition:transform 0.3s}.relation-dropdown .relation-dropdown__button:hover>.icon-arrow>svg{fill:#00a1cc}.relation-dropdown .relation-dropdown__button>.icon-arrow--expand{transform:rotate(-180deg)}.relation-dropdown .relation-dropdown__button+.relation-dropdown__body--expand{max-height:2000px;transition:max-height 1s ease-in}.relation-dropdown .relation-dropdown__body{background-color:#f7f7f7;max-height:0;transition:max-height 1s ease-out;overflow:hidden}.entity-card{flex-direction:row}.entity-card .entity-card__info-wrapper{margin-left:30px}}.single-section-wrapper{padding:32px 36px;background-color:#f7f7f7;overflow:auto}.single-section-wrapper .single-section-wrapper__link--secondary{display:inline-block;color:#ccc;margin-bottom:20px}.single-section-wrapper .single-section-wrapper__link--secondary:hover{color:#00a1cc}.entity-form,.review-form{overflow:auto}.entity-form>input[type='email'],.entity-form>input[type='number'],.entity-form>input[type='password'],.entity-form>input[type='search'],.entity-form>input[type='tel'],.entity-form>input[type='text'],.entity-form>input[type='url'],.entity-form textarea,.review-form>input[type='email'],.review-form>input[type='number'],.review-form>input[type='password'],.review-form>input[type='search'],.review-form>input[type='tel'],.review-form>input[type='text'],.review-form>input[type='url'],.review-form textarea{width:100%}.entity-form img,.review-form img{display:block}.review-form .review-form__preview-button{color:#00a1cc;font-weight:bold;cursor:pointer}.review-form .review-form__fyi{color:#ccc}.review-form .review-form__main-content,.review-form textarea{margin-bottom:5px;resize:vertical;height:400px}.review-form .review-form__option{margin-top:24px;margin-bottom:10px}.review-form .review-form__option::after{content:' ';clear:both;display:table}.review-form .review-form__visibility-radio{float:left}.review-form .review-form__visibility-radio ul,.review-form .review-form__visibility-radio li,.review-form .review-form__visibility-radio label{display:inline}.review-form .review-form__visibility-radio label{font-size:normal}.review-form .review-form__visibility-radio input[type="radio"]{position:relative;top:2px}.review-form .review-form__share-checkbox{float:right}.review-form .review-form__share-checkbox input[type="checkbox"]{position:relative;top:2px}.report-form input,.report-form select{width:100%}@media (max-width: 575.98px){.review-form .review-form__visibility-radio{float:unset}.review-form .review-form__share-checkbox{float:unset;position:relative;left:-3px}}.markdownx-preview{min-height:100px}.markdownx-preview ul li{list-style:circle inside}.rating-star .jq-star{cursor:unset !important}.ms-parent>.ms-choice{margin-bottom:1.5rem;appearance:none;background-color:transparent;border:0.1rem solid #ccc;border-radius:.4rem;box-shadow:none;box-sizing:inherit;padding:.6rem 1.0rem;width:100%;height:30.126px}.ms-parent>.ms-choice:focus{border-color:#00a1cc}.ms-parent>.ms-choice>.icon-caret{top:15.5px}.ms-parent>.ms-choice>span{color:black;font-weight:initial;font-size:13.3333px;top:2.5px;left:2px}.ms-parent>.ms-choice>span:hover,.ms-parent>.ms-choice>span:focus{color:black}.ms-parent>.ms-drop>ul>li>label>span{margin-left:10px}.ms-parent>.ms-drop>ul>li>label>input{width:unset}.tippy-box{border:#606c76 1px solid;background-color:#f7f7f7;padding:3px 5px}.tag-input input{flex-grow:1}.tools-section-wrapper input,.tools-section-wrapper select{width:unset} diff --git a/common/static/img/fediverse.svg b/common/static/img/fediverse.svg new file mode 100644 index 00000000..c3ab108c --- /dev/null +++ b/common/static/img/fediverse.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/common/static/img/logo.svg b/common/static/img/logo.svg index 0570333b..cc10cff8 100644 --- a/common/static/img/logo.svg +++ b/common/static/img/logo.svg @@ -1 +1,140 @@ -logo \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/common/static/img/logo_square.jpg b/common/static/img/logo_square.jpg index 37ed7679..073cc571 100644 Binary files a/common/static/img/logo_square.jpg and b/common/static/img/logo_square.jpg differ diff --git a/common/static/img/logo_square.svg b/common/static/img/logo_square.svg index ccc0ff8d..76251fed 100644 --- a/common/static/img/logo_square.svg +++ b/common/static/img/logo_square.svg @@ -1 +1,193 @@ -logo \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/common/static/js/create_update_review.js b/common/static/js/create_update_review.js index 9dad74dc..60ab409d 100644 --- a/common/static/js/create_update_review.js +++ b/common/static/js/create_update_review.js @@ -1,9 +1,9 @@ $(document).ready( function() { $(".markdownx-preview").hide(); - $(".markdownx textarea").attr("placeholder", "拖拽图片至编辑框即可插入哦~"); + $(".markdownx textarea").attr("placeholder", "从剪贴板粘贴或者拖拽文件至编辑框即可插入图片"); - $(".review-form__preview-button").click(function() { + $(".review-form__preview-button").on('click', function() { if ($(".markdownx-preview").is(":visible")) { $(".review-form__preview-button").text("预览"); $(".markdownx-preview").hide(); diff --git a/common/static/js/detail.js b/common/static/js/detail.js index 01feac46..fe61e7c9 100644 --- a/common/static/js/detail.js +++ b/common/static/js/detail.js @@ -7,7 +7,7 @@ $(document).ready( function() { // pop up new rating modal $("#addMarkPanel button").each(function() { - $(this).click(function(e) { + $(this).on('click', function(e) { e.preventDefault(); let title = $(this).text().trim(); $(".mark-modal__title").text(title); @@ -29,7 +29,7 @@ $(document).ready( function() { }) // pop up modify mark modal - $(".mark-panel a.edit").click(function(e) { + $(".mark-panel a.edit").on('click', function(e) { e.preventDefault(); let title = $(".mark-panel__status").text().trim(); $(".mark-modal__title").text(title); @@ -79,7 +79,7 @@ $(document).ready( function() { if ($("#statusSelection input[type='radio']:checked").val() == WISH_CODE) { $(".mark-modal .rating-star-edit").hide(); } - $("#statusSelection input[type='radio']").click(function() { + $("#statusSelection input[type='radio']").on('click', function() { if ($(this).val() == WISH_CODE) { $(".mark-modal .rating-star-edit").hide(); } else { @@ -89,14 +89,14 @@ $(document).ready( function() { }); // show confirm modal - $(".mark-panel a.delete").click(function(e) { + $(".mark-panel a.delete").on('click', function(e) { e.preventDefault(); $(".confirm-modal").show(); $(".bg-mask").show(); }); // confirm modal - $(".confirm-modal input[type='submit']").click(function(e) { + $(".confirm-modal input[type='submit']").on('click', function(e) { e.preventDefault(); $(".mark-panel form").submit(); }); @@ -116,20 +116,20 @@ $(document).ready( function() { }); // expand hidden long text - $(".entity-desc__unfold-button a").click(function() { + $(".entity-desc__unfold-button a").on('click', function() { $(this).parent().siblings(".entity-desc__content").removeClass('entity-desc__content--folded'); $(this).parent(".entity-desc__unfold-button").remove(); }); // disable delete mark button after click const confirmDeleteMarkButton = $('.confirm-modal__confirm-button > input'); - confirmDeleteMarkButton.click(function() { + confirmDeleteMarkButton.on('click', function() { confirmDeleteMarkButton.prop("disabled", true); }); // disable sumbit button after click const confirmSumbitMarkButton = $('.mark-modal__confirm-button > input'); - confirmSumbitMarkButton.click(function() { + confirmSumbitMarkButton.on('click', function() { confirmSumbitMarkButton.prop("disabled", true); confirmSumbitMarkButton.closest('form')[0].submit(); }); diff --git a/common/static/js/home.js b/common/static/js/home.js index 456c637b..ca1cb32b 100644 --- a/common/static/js/home.js +++ b/common/static/js/home.js @@ -1,114 +1,98 @@ - $(document).ready( function() { + $("#userInfoCard .mast-brief").text($("
"+$("#userInfoCard .mast-brief").text().replace(/\
").text()); + $("#userInfoCard .mast-brief").html($("#userInfoCard .mast-brief").html().replace(/\n/g,'
')); let token = $("#oauth2Token").text(); - let mast_uri = $("#mastodonURI").text(); - let mast_domain = new URL(mast_uri); - mast_domain = mast_domain.hostname; + let mast_domain = $("#mastodonURI").text(); + let mast_uri = 'https://' + mast_domain let id = $("#userMastodonID").text(); - let userInfoSpinner = $("#spinner").clone().removeAttr("hidden"); - let followersSpinner = $("#spinner").clone().removeAttr("hidden"); - let followingSpinner = $("#spinner").clone().removeAttr("hidden"); - $("#userInfoCard").append(userInfoSpinner); - $("#followings h5").after(followingSpinner); - $("#followers h5").after(followersSpinner); - $(".mast-following-more").hide(); - $(".mast-followers-more").hide(); + if (id && id != 'None' && mast_domain != 'twitter.com') { + // let userInfoSpinner = $("#spinner").clone().removeAttr("hidden"); + let followersSpinner = $("#spinner").clone().removeAttr("hidden"); + let followingSpinner = $("#spinner").clone().removeAttr("hidden"); + // $("#userInfoCard").append(userInfoSpinner); + $("#followings h5").after(followingSpinner); + $("#followers h5").after(followersSpinner); + $(".mast-following-more").hide(); + $(".mast-followers-more").hide(); - getUserInfo( - id, - mast_uri, - token, - function(userData) { - let userName; - if (userData.display_name) { - userName = translateEmojis(userData.display_name, userData.emojis, true); - } else { - userName = userData.username; - } - $("#userInfoCard .mast-avatar").attr("src", userData.avatar); - $("#userInfoCard .mast-displayname").html(userName); - $("#userInfoCard .mast-brief").text($(userData.note).text()); - $(userInfoSpinner).remove(); - } - ); + getFollowers( + id, + mast_uri, + token, + function(userList, request) { + if (userList.length == 0) { + $(".mast-followers").hide(); + $(".mast-followers").before('
暂无
'); - getFollowers( - id, - mast_uri, - token, - function(userList, request) { - if (userList.length == 0) { - $(".mast-followers").hide(); - $(".mast-followers").before('
暂无
'); - - } else { - if (userList.length > 4){ - userList = userList.slice(0, 4); - $(".mast-followers-more").show(); + } else { + if (userList.length > 4){ + userList = userList.slice(0, 4); + $(".mast-followers-more").show(); + } + let template = $(".mast-followers li").clone(); + $(".mast-followers").html(""); + userList.forEach(data => { + temp = $(template).clone(); + temp.find("img").attr("src", data.avatar); + if (data.display_name) { + temp.find(".mast-displayname").html(translateEmojis(data.display_name, data.emojis)); + } else { + temp.find(".mast-displayname").text(data.username); + } + let url; + if (data.acct.includes('@')) { + url = $("#userPageURL").text().replace('0', data.acct); + } else { + url = $("#userPageURL").text().replace('0', data.acct + '@' + mast_domain); + } + temp.find("a").attr('href', url); + $(".mast-followers").append(temp); + }); } - let template = $(".mast-followers li").clone(); - $(".mast-followers").html(""); - userList.forEach(data => { - temp = $(template).clone(); - temp.find("img").attr("src", data.avatar); - if (data.display_name) { - temp.find(".mast-displayname").html(translateEmojis(data.display_name, data.emojis)); - } else { - temp.find(".mast-displayname").text(data.username); - } - let url; - if (data.acct.includes('@')) { - url = $("#userPageURL").text().replace('0', data.acct); - } else { - url = $("#userPageURL").text().replace('0', data.acct + '@' + mast_domain); - } - temp.find("a").attr('href', url); - $(".mast-followers").append(temp); - }); + $(followersSpinner).remove(); } - $(followersSpinner).remove(); - } - ); + ); - getFollowing( - id, - mast_uri, - token, - function(userList, request) { - if (userList.length == 0) { - $(".mast-following").hide(); - $(".mast-following").before('
暂无
'); - } else { - if (userList.length > 4){ - userList = userList.slice(0, 4); - $(".mast-following-more").show(); + getFollowing( + id, + mast_uri, + token, + function(userList, request) { + if (userList.length == 0) { + $(".mast-following").hide(); + $(".mast-following").before('
暂无
'); + } else { + if (userList.length > 4){ + userList = userList.slice(0, 4); + $(".mast-following-more").show(); + } + let template = $(".mast-following li").clone(); + $(".mast-following").html(""); + userList.forEach(data => { + temp = $(template).clone() + temp.find("img").attr("src", data.avatar); + if (data.display_name) { + temp.find(".mast-displayname").html(translateEmojis(data.display_name, data.emojis)); + } else { + temp.find(".mast-displayname").text(data.username); + } + let url; + if (data.acct.includes('@')) { + url = $("#userPageURL").text().replace('0', data.acct); + } else { + url = $("#userPageURL").text().replace('0', data.acct + '@' + mast_domain); + } + temp.find("a").attr('href', url); + $(".mast-following").append(temp); + }); } - let template = $(".mast-following li").clone(); - $(".mast-following").html(""); - userList.forEach(data => { - temp = $(template).clone() - temp.find("img").attr("src", data.avatar); - if (data.display_name) { - temp.find(".mast-displayname").html(translateEmojis(data.display_name, data.emojis)); - } else { - temp.find(".mast-displayname").text(data.username); - } - let url; - if (data.acct.includes('@')) { - url = $("#userPageURL").text().replace('0', data.acct); - } else { - url = $("#userPageURL").text().replace('0', data.acct + '@' + mast_domain); - } - temp.find("a").attr('href', url); - $(".mast-following").append(temp); - }); - } - $(followingSpinner).remove(); + $(followingSpinner).remove(); - } - ); + } + ); + } // mobile dropdown $(".relation-dropdown__button").data("collapse", true); @@ -118,7 +102,7 @@ $(document).ready( function() { button.children('.icon-arrow').toggleClass("icon-arrow--expand"); button.siblings('.relation-dropdown__body').toggleClass("relation-dropdown__body--expand"); } - $(".relation-dropdown__button").click(onClickDropdownButton) + $(".relation-dropdown__button").on('click', onClickDropdownButton); // close when click outside window.onclick = evt => { @@ -129,7 +113,7 @@ $(document).ready( function() { }; // import panel - $("#uploadBtn").click(e => { + $("#uploadBtn").on('click', e => { const btn = $("#uploadBtn") const form = $(".import-panel__body form") @@ -201,7 +185,8 @@ $(document).ready( function() { if (!data.total_items == 0) { progress.attr("max", data.total_items); progress.attr("value", data.finished_items); - percent.text(Math.floor(100 * data.finished_items / data.total_items) + '%'); + progress.attr("value", data.finished_items); + percent.text("" + data.finished_items + "/" + data.total_items); } setTimeout(() => { poll(); diff --git a/common/static/js/mastodon.js b/common/static/js/mastodon.js index 91defcbe..02a54679 100644 --- a/common/static/js/mastodon.js +++ b/common/static/js/mastodon.js @@ -54,38 +54,50 @@ const NUMBER_PER_REQUEST = 20 // "fields": [] // } // ] -function getFollowers(id, mastodonURI, token, callback) { - let url = mastodonURI + API_FOLLOWERS.replace(":id", id); - $.ajax({ - url: url, - method: 'GET', - headers: { - 'Authorization': 'Bearer ' + token, - }, - data: { - 'limit': NUMBER_PER_REQUEST - }, - success: function(data, status, request){ - callback(data, request); - }, - }); +async function getFollowers(id, mastodonURI, token, callback) { + const url = mastodonURI + API_FOLLOWERS.replace(":id", id); + var response; + try { + response = await fetch(url+'?limit='+NUMBER_PER_REQUEST, {headers: {'Authorization': 'Bearer ' + token}}); + } catch (e) { + console.error('loading followers failed.'); + return; + } + const json = await response.json(); + let nextUrl = null; + let links = response.headers.get('link'); + if (links) { + links.split(',').forEach(link => { + if (link.includes('next')) { + let regex = /<(.*?)>/; + nextUrl = link.match(regex)[1]; + } + }); + } + callback(json, nextUrl); } -function getFollowing(id, mastodonURI, token, callback) { - let url = mastodonURI + API_FOLLOWING.replace(":id", id); - $.ajax({ - url: url, - method: 'GET', - headers: { - 'Authorization': 'Bearer ' + token, - }, - data: { - 'limit': NUMBER_PER_REQUEST - }, - success: function(data, status, request){ - callback(data, request); - }, - }); +async function getFollowing(id, mastodonURI, token, callback) { + const url = mastodonURI + API_FOLLOWING.replace(":id", id); + var response; + try { + response = await fetch(url+'?limit='+NUMBER_PER_REQUEST, {headers: {'Authorization': 'Bearer ' + token}}); + } catch (e) { + console.error('loading following failed.'); + return; + } + const json = await response.json(); + let nextUrl = null; + let links = response.headers.get('link'); + if (links) { + links.split(',').forEach(link => { + if (link.includes('next')) { + let regex = /<(.*?)>/; + nextUrl = link.match(regex)[1]; + } + }); + } + callback(json, nextUrl); } // { diff --git a/common/static/js/rating-star-readonly.js b/common/static/js/rating-star-readonly.js index 76802ea8..a552a6b1 100644 --- a/common/static/js/rating-star-readonly.js +++ b/common/static/js/rating-star-readonly.js @@ -1,5 +1,5 @@ $(document).ready( function() { - +let render = function() { let ratingLabels = $(".rating-star"); $(ratingLabels).each( function(index, value) { let ratingScore = $(this).data("rating-score") / 2; @@ -8,5 +8,9 @@ $(document).ready( function() { readOnly: true }); }); - +}; +document.body.addEventListener('htmx:load', function(evt) { + render(); +}); +render(); }); \ No newline at end of file diff --git a/common/static/js/scrape.js b/common/static/js/scrape.js index 67ce3e3a..bd05bbd8 100644 --- a/common/static/js/scrape.js +++ b/common/static/js/scrape.js @@ -1,6 +1,6 @@ $(document).ready( function() { - $(".submit").click(function(e) { + $(".submit").on('click', function(e) { e.preventDefault(); let form = $("#scrapeForm form"); if (form.data('submitted') === true) { diff --git a/common/static/js/sort_layout.js b/common/static/js/sort_layout.js index 1ac72eba..c20e370d 100644 --- a/common/static/js/sort_layout.js +++ b/common/static/js/sort_layout.js @@ -8,7 +8,7 @@ $(() => { $(e).data("visibility", true); } let btn = $("#toggleDisplayButtonTemplate").clone().removeAttr("id"); - btn.click(e => { + btn.on('click', e => { if ($(e.currentTarget).parent().data('visibility') === true) { // flip text $(e.currentTarget).children("span.showText").show(); @@ -72,7 +72,7 @@ $(() => { }); // activate sorting - $("#sortEditButton").click(evt => { + $("#sortEditButton").on('click', evt => { // test if edit mode is activated isActivated = $("#sortSaveIcon").is(":visible"); @@ -134,7 +134,7 @@ $(() => { }); // exit edit mode - $("#sortExitButton").click(evt => { + $("#sortExitButton").on('click', evt => { initialLayoutData.forEach(elem => { // set visiblity $('#' + elem.id).data('visibility', elem.visibility); diff --git a/common/static/lib/css/milligram.css b/common/static/lib/css/milligram.css deleted file mode 100644 index be254df8..00000000 --- a/common/static/lib/css/milligram.css +++ /dev/null @@ -1,605 +0,0 @@ -/*! - * Milligram v1.3.0 - * https://milligram.github.io - * - * Copyright (c) 2017 CJ Patoilo - * Licensed under the MIT license - */ - -*, -*:after, -*:before { - box-sizing: inherit; -} - -html { - box-sizing: border-box; - font-size: 62.5%; -} - -body { - color: #606c76; - font-family: 'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif; - font-size: 1.6em; - font-weight: 300; - letter-spacing: .01em; - line-height: 1.6; -} - -textarea { - font-family: 'Roboto', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif; -} - -blockquote { - border-left: 0.3rem solid #d1d1d1; - margin-left: 0; - margin-right: 0; - padding: 1rem 1.5rem; -} - -blockquote *:last-child { - margin-bottom: 0; -} - -.button, -button, -input[type='button'], -input[type='reset'], -input[type='submit'] { - background-color: #00a1cc; - border: 0.1rem solid #00a1cc; - border-radius: .4rem; - color: #fff; - cursor: pointer; - display: inline-block; - font-size: 1.1rem; - font-weight: 700; - height: 3.8rem; - letter-spacing: .1rem; - line-height: 3.8rem; - padding: 0 3.0rem; - text-align: center; - text-decoration: none; - text-transform: uppercase; - white-space: nowrap; -} - -.button:focus, .button:hover, -button:focus, -button:hover, -input[type='button']:focus, -input[type='button']:hover, -input[type='reset']:focus, -input[type='reset']:hover, -input[type='submit']:focus, -input[type='submit']:hover { - background-color: #606c76; - border-color: #606c76; - color: #fff; - outline: 0; -} - -.button[disabled], -button[disabled], -input[type='button'][disabled], -input[type='reset'][disabled], -input[type='submit'][disabled] { - cursor: default; - opacity: .5; -} - -.button[disabled]:focus, .button[disabled]:hover, -button[disabled]:focus, -button[disabled]:hover, -input[type='button'][disabled]:focus, -input[type='button'][disabled]:hover, -input[type='reset'][disabled]:focus, -input[type='reset'][disabled]:hover, -input[type='submit'][disabled]:focus, -input[type='submit'][disabled]:hover { - background-color: #00a1cc; - border-color: #00a1cc; -} - -.button.button-outline, -button.button-outline, -input[type='button'].button-outline, -input[type='reset'].button-outline, -input[type='submit'].button-outline { - background-color: transparent; - color: #00a1cc; -} - -.button.button-outline:focus, .button.button-outline:hover, -button.button-outline:focus, -button.button-outline:hover, -input[type='button'].button-outline:focus, -input[type='button'].button-outline:hover, -input[type='reset'].button-outline:focus, -input[type='reset'].button-outline:hover, -input[type='submit'].button-outline:focus, -input[type='submit'].button-outline:hover { - background-color: transparent; - border-color: #606c76; - color: #606c76; -} - -.button.button-outline[disabled]:focus, .button.button-outline[disabled]:hover, -button.button-outline[disabled]:focus, -button.button-outline[disabled]:hover, -input[type='button'].button-outline[disabled]:focus, -input[type='button'].button-outline[disabled]:hover, -input[type='reset'].button-outline[disabled]:focus, -input[type='reset'].button-outline[disabled]:hover, -input[type='submit'].button-outline[disabled]:focus, -input[type='submit'].button-outline[disabled]:hover { - border-color: inherit; - color: #00a1cc; -} - -.button.button-clear, -button.button-clear, -input[type='button'].button-clear, -input[type='reset'].button-clear, -input[type='submit'].button-clear { - background-color: transparent; - border-color: transparent; - color: #00a1cc; -} - -.button.button-clear:focus, .button.button-clear:hover, -button.button-clear:focus, -button.button-clear:hover, -input[type='button'].button-clear:focus, -input[type='button'].button-clear:hover, -input[type='reset'].button-clear:focus, -input[type='reset'].button-clear:hover, -input[type='submit'].button-clear:focus, -input[type='submit'].button-clear:hover { - background-color: transparent; - border-color: transparent; - color: #606c76; -} - -.button.button-clear[disabled]:focus, .button.button-clear[disabled]:hover, -button.button-clear[disabled]:focus, -button.button-clear[disabled]:hover, -input[type='button'].button-clear[disabled]:focus, -input[type='button'].button-clear[disabled]:hover, -input[type='reset'].button-clear[disabled]:focus, -input[type='reset'].button-clear[disabled]:hover, -input[type='submit'].button-clear[disabled]:focus, -input[type='submit'].button-clear[disabled]:hover { - color: #00a1cc; -} - -code { - background: #f4f5f6; - border-radius: .4rem; - font-size: 86%; - margin: 0 .2rem; - padding: .2rem .5rem; - white-space: nowrap; -} - -pre { - background: #f4f5f6; - border-left: 0.3rem solid #00a1cc; - overflow-y: hidden; -} - -pre > code { - border-radius: 0; - display: block; - padding: 1rem 1.5rem; - white-space: pre; -} - -hr { - border: 0; - border-top: 0.1rem solid #f4f5f6; - margin: 3.0rem 0; -} - -input[type='email'], -input[type='number'], -input[type='password'], -input[type='search'], -input[type='tel'], -input[type='text'], -input[type='url'], -textarea, -select { - -webkit-appearance: none; - -moz-appearance: none; - appearance: none; - background-color: transparent; - border: 0.1rem solid #d1d1d1; - border-radius: .4rem; - box-shadow: none; - box-sizing: inherit; - height: 3.8rem; - padding: .6rem 1.0rem; - width: 100%; -} - -input[type='email']:focus, -input[type='number']:focus, -input[type='password']:focus, -input[type='search']:focus, -input[type='tel']:focus, -input[type='text']:focus, -input[type='url']:focus, -textarea:focus, -select:focus { - border-color: #00a1cc; - outline: 0; -} - -select { - background: url('data:image/svg+xml;utf8,') center right no-repeat; - padding-right: 3.0rem; -} - -select:focus { - background-image: url('data:image/svg+xml;utf8,'); -} - -textarea { - min-height: 6.5rem; -} - -label, -legend { - display: block; - font-size: 1.6rem; - font-weight: 700; - margin-bottom: .5rem; -} - -fieldset { - border-width: 0; - padding: 0; -} - -input[type='checkbox'], -input[type='radio'] { - display: inline; -} - -.label-inline { - display: inline-block; - font-weight: normal; - margin-left: .5rem; -} - -.container { - margin: 0 auto; - max-width: 112.0rem; - padding: 0 2.0rem; - position: relative; - width: 100%; -} - -.row { - display: flex; - flex-direction: column; - padding: 0; - width: 100%; -} - -.row.row-no-padding { - padding: 0; -} - -.row.row-no-padding > .column { - padding: 0; -} - -.row.row-wrap { - flex-wrap: wrap; -} - -.row.row-top { - align-items: flex-start; -} - -.row.row-bottom { - align-items: flex-end; -} - -.row.row-center { - align-items: center; -} - -.row.row-stretch { - align-items: stretch; -} - -.row.row-baseline { - align-items: baseline; -} - -.row .column { - display: block; - flex: 1 1 auto; - margin-left: 0; - max-width: 100%; - width: 100%; -} - -.row .column.column-offset-10 { - margin-left: 10%; -} - -.row .column.column-offset-20 { - margin-left: 20%; -} - -.row .column.column-offset-25 { - margin-left: 25%; -} - -.row .column.column-offset-33, .row .column.column-offset-34 { - margin-left: 33.3333%; -} - -.row .column.column-offset-50 { - margin-left: 50%; -} - -.row .column.column-offset-66, .row .column.column-offset-67 { - margin-left: 66.6666%; -} - -.row .column.column-offset-75 { - margin-left: 75%; -} - -.row .column.column-offset-80 { - margin-left: 80%; -} - -.row .column.column-offset-90 { - margin-left: 90%; -} - -.row .column.column-10 { - flex: 0 0 10%; - max-width: 10%; -} - -.row .column.column-20 { - flex: 0 0 20%; - max-width: 20%; -} - -.row .column.column-25 { - flex: 0 0 25%; - max-width: 25%; -} - -.row .column.column-33, .row .column.column-34 { - flex: 0 0 33.3333%; - max-width: 33.3333%; -} - -.row .column.column-40 { - flex: 0 0 40%; - max-width: 40%; -} - -.row .column.column-50 { - flex: 0 0 50%; - max-width: 50%; -} - -.row .column.column-60 { - flex: 0 0 60%; - max-width: 60%; -} - -.row .column.column-66, .row .column.column-67 { - flex: 0 0 66.6666%; - max-width: 66.6666%; -} - -.row .column.column-75 { - flex: 0 0 75%; - max-width: 75%; -} - -.row .column.column-80 { - flex: 0 0 80%; - max-width: 80%; -} - -.row .column.column-90 { - flex: 0 0 90%; - max-width: 90%; -} - -.row .column .column-top { - align-self: flex-start; -} - -.row .column .column-bottom { - align-self: flex-end; -} - -.row .column .column-center { - -ms-grid-row-align: center; - align-self: center; -} - -@media (min-width: 40rem) { - .row { - flex-direction: row; - margin-left: -1.0rem; - width: calc(100% + 2.0rem); - } - .row .column { - margin-bottom: inherit; - padding: 0 1.0rem; - } -} - -a { - color: #00a1cc; - text-decoration: none; -} - -a:focus, a:hover { - color: #606c76; -} - -dl, -ol, -ul { - list-style: none; - margin-top: 0; - padding-left: 0; -} - -dl dl, -dl ol, -dl ul, -ol dl, -ol ol, -ol ul, -ul dl, -ul ol, -ul ul { - font-size: 90%; - margin: 1.5rem 0 1.5rem 3.0rem; -} - -ol { - list-style: decimal inside; -} - -ul { - list-style: circle inside; -} - -.button, -button, -dd, -dt, -li { - margin-bottom: 1.0rem; -} - -fieldset, -input, -select, -textarea { - margin-bottom: 1.5rem; -} - -blockquote, -dl, -figure, -form, -ol, -p, -pre, -table, -ul { - margin-bottom: 2.5rem; -} - -table { - border-spacing: 0; - width: 100%; -} - -td, -th { - border-bottom: 0.1rem solid #e1e1e1; - padding: 1.2rem 1.5rem; - text-align: left; -} - -td:first-child, -th:first-child { - padding-left: 0; -} - -td:last-child, -th:last-child { - padding-right: 0; -} - -b, -strong { - font-weight: bold; -} - -p { - margin-top: 0; -} - -h1, -h2, -h3, -h4, -h5, -h6 { - font-weight: 300; - letter-spacing: -.1rem; - margin-bottom: 2.0rem; - margin-top: 0; -} - -h1 { - font-size: 4.6rem; - line-height: 1.2; -} - -h2 { - font-size: 3.6rem; - line-height: 1.25; -} - -h3 { - font-size: 2.8rem; - line-height: 1.3; -} - -h4 { - font-size: 2.2rem; - letter-spacing: -.08rem; - line-height: 1.35; -} - -h5 { - font-size: 1.8rem; - letter-spacing: -.05rem; - line-height: 1.5; -} - -h6 { - font-size: 1.6rem; - letter-spacing: 0; - line-height: 1.4; -} - -img { - max-width: 100%; -} - -.clearfix:after { - clear: both; - content: ' '; - display: table; -} - -.float-left { - float: left; -} - -.float-right { - float: right; -} - diff --git a/common/static/lib/css/multiple-select.min.css b/common/static/lib/css/multiple-select.min.css deleted file mode 100644 index 5a6f1b19..00000000 --- a/common/static/lib/css/multiple-select.min.css +++ /dev/null @@ -1,10 +0,0 @@ -/** - * multiple-select - Multiple select is a jQuery plugin to select multiple elements with checkboxes :). - * - * @version v1.5.2 - * @homepage http://multiple-select.wenzhixin.net.cn - * @author wenzhixin (http://wenzhixin.net.cn/) - * @license MIT - */ - -@charset "UTF-8";.ms-offscreen{clip:rect(0 0 0 0)!important;width:1px!important;height:1px!important;border:0!important;margin:0!important;padding:0!important;overflow:hidden!important;position:absolute!important;outline:0!important;left:auto!important;top:auto!important}.ms-parent{display:inline-block;position:relative;vertical-align:middle}.ms-choice{display:block;width:100%;height:26px;padding:0;overflow:hidden;cursor:pointer;border:1px solid #aaa;text-align:left;white-space:nowrap;line-height:26px;color:#444;text-decoration:none;border-radius:4px;background-color:#fff}.ms-choice.disabled{background-color:#f4f4f4;background-image:none;border:1px solid #ddd;cursor:default}.ms-choice>span{position:absolute;top:0;left:0;right:20px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;display:block;padding-left:8px}.ms-choice>span.placeholder{color:#999}.ms-choice>div.icon-close{position:absolute;top:0;right:16px;height:100%;width:16px}.ms-choice>div.icon-close:before{content:'×';color:#888;font-weight:bold;position:absolute;top:50%;margin-top:-14px}.ms-choice>div.icon-close:hover:before{color:#333}.ms-choice>div.icon-caret{position:absolute;width:0;height:0;top:50%;right:8px;margin-top:-2px;border-color:#888 transparent transparent transparent;border-style:solid;border-width:5px 4px 0 4px}.ms-choice>div.icon-caret.open{border-color:transparent transparent #888 transparent;border-width:0 4px 5px 4px}.ms-drop{width:auto;min-width:100%;overflow:hidden;display:none;margin-top:-1px;padding:0;position:absolute;z-index:1000;background:#fff;color:#000;border:1px solid #aaa;border-radius:4px}.ms-drop.bottom{top:100%;box-shadow:0 4px 5px rgba(0,0,0,0.15)}.ms-drop.top{bottom:100%;box-shadow:0 -4px 5px rgba(0,0,0,0.15)}.ms-search{display:inline-block;margin:0;min-height:26px;padding:2px;position:relative;white-space:nowrap;width:100%;z-index:10000;box-sizing:border-box}.ms-search input{width:100%;height:auto!important;min-height:24px;padding:0 5px;margin:0;outline:0;font-family:sans-serif;border:1px solid #aaa;border-radius:5px;box-shadow:none}.ms-drop ul{overflow:auto;margin:0;padding:0}.ms-drop ul>li{list-style:none;display:list-item;background-image:none;position:static;padding:.25rem 8px}.ms-drop ul>li .disabled{font-weight:normal!important;opacity:.35;filter:Alpha(Opacity=35);cursor:default}.ms-drop ul>li.multiple{display:block;float:left}.ms-drop ul>li.group{clear:both}.ms-drop ul>li.multiple label{width:100%;display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.ms-drop ul>li label{position:relative;padding-left:1.25rem;margin-bottom:0;font-weight:normal;display:block;white-space:nowrap;cursor:pointer}.ms-drop ul>li label.optgroup{font-weight:bold}.ms-drop ul>li.hide-radio{padding:0}.ms-drop ul>li.hide-radio:focus,.ms-drop ul>li.hide-radio:hover{background-color:#f8f9fa}.ms-drop ul>li.hide-radio.selected{color:#fff;background-color:#007bff}.ms-drop ul>li.hide-radio label{margin-bottom:0;padding:5px 8px}.ms-drop ul>li.hide-radio input{display:none}.ms-drop ul>li.option-level-1 label{padding-left:28px}.ms-drop input[type="radio"],.ms-drop input[type="checkbox"]{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.ms-drop .ms-no-results{display:none} \ No newline at end of file diff --git a/common/static/lib/css/neo.css b/common/static/lib/css/neo.css new file mode 100644 index 00000000..2f2fcbce --- /dev/null +++ b/common/static/lib/css/neo.css @@ -0,0 +1,166 @@ +.markdownx-preview h1 { + font-size: 2.5em; +} + +.markdownx-preview h2 { + font-size: 2.0em; +} + +.markdownx-preview h3 { + font-size: 1.6em; +} + +.markdownx-preview blockquote { + border-left: lightgray solid 0.4em; + padding-left: 0.4em; +} + +.collection-item-position-edit { + float: right; +} + +.collection-item-position-edit a { + cursor: pointer; + color: #ccc; +} + +.action-icon svg { + cursor: pointer; + fill: #ccc; + height: 12px; + vertical-align: text-bottom; +} + +.entity-list__entity-img-wrapper { + position: relative; +} + +.entity-list__entity-action-icon { + position: absolute; + top:0; + right:0; + mix-blend-mode: hard-light; + text-stroke: 1px black; + background-color: lightgray; + border-radius: 0 0 0 8px; + padding: 0 4px; + cursor: pointer; +} + + +/***** MODAL DIALOG ****/ +#modal { + /* Underlay covers entire screen. */ + position: fixed; + top:0px; + bottom: 0px; + left:0px; + right:0px; + background-color:rgba(0,0,0,0.5); + z-index:1000; + + /* Flexbox centers the .modal-content vertically and horizontally */ + display:flex; + flex-direction:column; + align-items:center; + + /* Animate when opening */ + animation-name: fadeIn; + animation-duration:150ms; + animation-timing-function: ease; +} + +#modal > .modal-underlay { + /* underlay takes up the entire viewport. This is only + required if you want to click to dismiss the popup */ + position: absolute; + z-index: -1; + .collection_list_position_edittop:0px; + + + bottom:0px; + left: 0px; + right: 0px; +} + +#modal > .modal-content { + /* Position visible dialog near the top of the window */ + margin-top:10vh; + + /* Sizing for visible dialog */ + width:80%; + max-width:600px; + + /* Display properties for visible dialog*/ + background-color: #f7f7f7; + padding: 20px 20px 10px 20px; + color: #606c76; + + /* Animate when opening */ + animation-name:zoomIn; + animation-duration:150ms; + animation-timing-function: ease; +} + +#modal.closing { + /* Animate when closing */ + animation-name: fadeOut; + animation-duration:150ms; + animation-timing-function: ease; +} + +#modal.closing > .modal-content { + /* Aniate when closing */ + animation-name: zoomOut; + animation-duration:150ms; + animation-timing-function: ease; +} + +@keyframes fadeIn { + 0% {opacity: 0;} + 100% {opacity: 1;} +} + +@keyframes fadeOut { + 0% {opacity: 1;} + 100% {opacity: 0;} +} + +@keyframes zoomIn { + 0% {transform: scale(0.9);} + 100% {transform: scale(1);} +} + +@keyframes zoomOut { + 0% {transform: scale(1);} + 100% {transform: scale(0.9);} +} + +#modal .add-to-list-modal__head { + margin-bottom: 20px; +} + +#modal .add-to-list-modal__head::after { + content: ' '; + clear: both; + display: table; +} + +#modal .add-to-list-modal__title { + font-weight: bold; + font-size: 1.2em; + float: left; +} + +#modal .add-to-list-modal__close-button { + float: right; + cursor: pointer; +} + +#modal .add-to-list-modal__confirm-button { + float: right; +} + +#modal li, #modal ul, #modal label { + display: inline; +} diff --git a/common/static/lib/js/hyperscript-0.9.5.min.js b/common/static/lib/js/hyperscript-0.9.5.min.js new file mode 100644 index 00000000..a17cb5e1 --- /dev/null +++ b/common/static/lib/js/hyperscript-0.9.5.min.js @@ -0,0 +1,2 @@ +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):(e||self)._hyperscript=t()}(this,function(){function e(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);n=e.length?{done:!0}:{done:!1,value:e[r++]}}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}function c(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n]);return e}function f(e,t){var n=e[t];if(n)return n;var r={};return e[t]=r,r}function m(e,t){return new(e.bind.apply(e,[e].concat(t)))}var p,d=globalThis,v=function(e){function n(e,t,n){this._css=e,this.relativeToElement=t,this.escape=n}var r=n.prototype;return r.contains=function(e){for(var t,n=s(this);!(t=n()).done;)if(t.value.contains(e))return!0;return!1},r[e]=function(){return this.selectMatches()[Symbol.iterator]()},r.selectMatches=function(){return T.getRootNode(this.relativeToElement).querySelectorAll(this.css)},t(n,[{key:"css",get:function(){return this.escape?T.escapeSelector(this._css):this._css}},{key:"className",get:function(){return this._css.substr(1)}},{key:"id",get:function(){return this.className()}},{key:"length",get:function(){return this.selectMatches().length}}]),n}(Symbol.iterator),h=function(){var e={"+":"PLUS","-":"MINUS","*":"MULTIPLY","/":"DIVIDE",".":"PERIOD","..":"ELLIPSIS","\\":"BACKSLASH",":":"COLON","%":"PERCENT","|":"PIPE","!":"EXCLAMATION","?":"QUESTION","#":"POUND","&":"AMPERSAND",$:"DOLLAR",";":"SEMI",",":"COMMA","(":"L_PAREN",")":"R_PAREN","<":"L_ANG",">":"R_ANG","<=":"LTE_ANG",">=":"GTE_ANG","==":"EQ","===":"EQQ","!=":"NEQ","!==":"NEQQ","{":"L_BRACE","}":"R_BRACE","[":"L_BRACKET","]":"R_BRACKET","=":"EQUALS"};function t(e){return i(e)||a(e)||"-"===e||"_"===e||":"===e}function n(e){return i(e)||a(e)||"-"===e||"_"===e||":"===e}function r(e){return" "===e||"\t"===e||o(e)}function o(e){return"\r"===e||"\n"===e}function a(e){return e>="0"&&e<="9"}function i(e){return e>="a"&&e<="z"||e>="A"&&e<="Z"}function u(e,t){return"_"===e||"$"===e}function l(e,t,n){o();var r=null;function o(){for(;"WHITESPACE"===f(0,!0).type;)t.push(e.shift())}function a(e,t){E.raiseParseError(e,t)}function i(e){if(m()&&m().op&&m().value===e)return s()}function u(e,t,n,r){if(m()&&m().type&&[e,t,n,r].indexOf(m().type)>=0)return s()}function l(e,t){if(-1===p.indexOf(e))return t=t||"IDENTIFIER",m()&&m().value===e&&m().type===t?s():void 0}function s(){var n=e.shift();return t.push(n),r=n,o(),n}function c(n,r){for(var a=[],i=f(0,!0);!(null!=r&&i.type===r||null!=n&&i.value===n||"EOF"===i.type);){var u=e.shift();t.push(u),a.push(i),i=f(0,!0)}return o(),a}function f(t,n){var r,o=0;do{if(!n)for(;e[o]&&"WHITESPACE"===e[o].type;)o++;r=e[o],t--,o++}while(t>-1);return r||{type:"EOF",value:"<<>>"}}function m(){return f(0)}var p=[];return{pushFollow:function(e){p.push(e)},popFollow:function(){p.pop()},clearFollow:function(){var e=p;return p=[],e},restoreFollow:function(e){p=e},matchAnyToken:function(e,t,n){for(var r=0;r0},currentToken:m,lastMatch:function(){return r},token:f,consumeUntil:c,consumeUntilWhitespace:function(){return c(null,"WHITESPACE")},lastWhitespace:function(){return t[t.length-1]&&"WHITESPACE"===t[t.length-1].type?t[t.length-1].value:""},sourceFor:function(){return n.substring(this.startToken.start,this.endToken.end)},lineFor:function(){return n.split("\n")[this.startToken.line-1]}}}function s(e){if(e.length>0){var t=e[e.length-1];if("IDENTIFIER"===t.type||"CLASS_REF"===t.type||"ID_REF"===t.type)return!1;if(t.op&&(">"===t.value||")"===t.value))return!1}return!0}return{tokenize:function(c,f){var m,p=[],d=c,v=0,h=0,E=1,y="",T=0;function k(){return f&&0===T}for(;v0;){var a=t.shift(),i=o[a];null==i&&(o[a]=i={}),o=i}o[n]=r},registerHyperTrace:function(e,t){for(var n=[],r=null;null!=e;)n.push(e),r=e,e=e.meta.caller;null==r.meta.traceMap&&(r.meta.traceMap=new Map),r.meta.traceMap.get(t)||r.meta.traceMap.set(t,{trace:n,print:function(e){(e=e||console.error)("hypertrace /// ");for(var t=0,r=0;r",o.meta.feature.displayName.padEnd(t+2),"-",o.meta.owner)}}})},getHyperTrace:function(e,t){for(var n=e;n.meta.caller;)n=n.meta.caller;if(n.meta.traceMap)return n.meta.traceMap.get(t,[])},getInternalData:O,getHyperscriptFeatures:q,escapeSelector:function(e){return e.replace(/:/g,function(e){return"\\"+e})},nullCheck:function(e,t){if(null==e)throw new Error("'"+t.sourceFor()+"' is null")},isEmpty:function(e){return null==e||0===e.length},doesExist:function(e){if(null==e)return!1;if(o(e))for(var t=s(e);!t().done;)return!0;return!1},getRootNode:function(e){if(e&&e instanceof Node){var t=e.getRootNode();if(t instanceof Document||t instanceof ShadowRoot)return t}return document},getEventQueueFor:function(e,t){var n=O(e),r=n.eventQueues;null==r&&(r=new Map,n.eventQueues=r);var o=r.get(t);return null==o&&r.set(t,o={queue:[],executing:!1}),o},hyperscriptUrl:"document"in d?"undefined"==typeof document&&"undefined"==typeof location?new(require("url").URL)("file:"+__filename).href:"undefined"==typeof document?location.href:document.currentScript&&document.currentScript.src||new URL("_hyperscript_web.min.js",document.baseURI).href:null,HALT:l}}(),k=function(e,t,n){if(t.contains)return t.contains(n);if(t.includes)return t.includes(n);throw Error("The value of "+e.sourceFor()+" does not have a contains or includes method on it")},g=function(e,t,n){if(t.match)return!!t.match(n);if(t.matches)return t.matches(n);throw Error("The value of "+e.sourceFor()+" does not have a match or matches method on it")},x=function(e,t,n,r){var o=t.requireElement("eventName",r),a=t.parseElement("namedArgumentList",r);if("send"===e&&r.matchToken("to")||"trigger"===e&&r.matchToken("on"))var i=t.requireElement("expression",r);else i=t.requireElement("implicitMeTarget",r);var u={eventName:o,details:a,to:i,args:[i,o,a],op:function(e,t,r,o){return n.nullCheck(t,i),n.forEach(t,function(t){n.triggerEvent(t,r,o,e.me)}),n.findNext(u,e)}};return u},b=function(e,t){var n,r="text";return e.matchToken("a")||e.matchToken("an"),e.matchToken("json")||e.matchToken("Object")?r="json":e.matchToken("response")?r="response":e.matchToken("html")?r="html":e.matchToken("text")||(n=t.requireElement("dotOrColonPath",e).evaluate()),{type:r,conversion:n}};E.addLeafExpression("parenthesized",function(e,t,n){if(n.matchOpToken("(")){var r=n.clearFollow();try{var o=e.requireElement("expression",n)}finally{n.restoreFollow(r)}return n.requireOpToken(")"),o}}),E.addLeafExpression("string",function(e,t,n){var r=n.matchTokenType("STRING");if(r){var o,a=r.value;if(r.template){var i=h.tokenize(a,!0);o=e.parseStringTemplate(i)}else o=[];return{type:"string",token:r,args:o,op:function(e){for(var t="",n=1;n");var o=r.map(function(e){return"STRING"===e.type?'"'+e.value+'"':e.value}).join("");if(o.indexOf("$")>=0)var a=!0,i=h.tokenize(o,!0),u=e.parseStringTemplate(i);return{type:"queryRef",css:o,args:u,op:function(e){return a?new q(o,e.me,[].slice.call(arguments,1)):new v(o,e.me)},evaluate:function(e){return t.unifiedEval(this,e)}}}}),E.addLeafExpression("attributeRef",function(e,t,n){var r=n.matchTokenType("ATTRIBUTE_REF");if(r){var o=r.value;if(0===o.indexOf("["))var a=o.substring(2,o.length-1);else a=o.substring(1);var i="["+a+"]",u=a.split("="),l=u[0],s=u[1];return s&&0===s.indexOf('"')&&(s=s.substring(1,s.length-1)),{type:"attributeRef",name:l,css:i,value:s,op:function(e){var t=e.beingTold||e.me;if(t)return t.getAttribute(l)},evaluate:function(e){return t.unifiedEval(this,e)}}}}),E.addLeafExpression("styleRef",function(e,t,n){var r=n.matchTokenType("STYLE_REF");if(r){var o=r.value.substr(1);return o.startsWith("computed-")?{type:"computedStyleRef",name:o=o.substr("computed-".length),op:function(e){var n=e.beingTold||e.me;if(n)return t.resolveComputedStyle(n,o)},evaluate:function(e){return t.unifiedEval(this,e)}}:{type:"styleRef",name:o,op:function(e){var n=e.beingTold||e.me;if(n)return t.resolveStyle(n,o)},evaluate:function(e){return t.unifiedEval(this,e)}}}}),E.addGrammarElement("objectKey",function(e,t,n){var r;if(r=n.matchTokenType("STRING"))return{type:"objectKey",key:r.value,evaluate:function(){return r.value}};if(n.matchOpToken("[")){var o=e.parseElement("expression",n);return n.requireOpToken("]"),{type:"objectKey",expr:o,args:[o],op:function(e,t){return t},evaluate:function(e){return t.unifiedEval(this,e)}}}var a="";do{(r=n.matchTokenType("IDENTIFIER")||n.matchOpToken("-"))&&(a+=r.value)}while(r);return{type:"objectKey",key:a,evaluate:function(){return a}}}),E.addLeafExpression("objectLiteral",function(e,t,n){if(n.matchOpToken("{")){var r=[],o=[];if(!n.matchOpToken("}")){do{var a=e.requireElement("objectKey",n);n.requireOpToken(":");var i=e.requireElement("expression",n);o.push(i),r.push(a)}while(n.matchOpToken(","));n.requireOpToken("}")}return{type:"objectLiteral",args:[r,o],op:function(e,t,n){for(var r={},o=0;o");var a=e.requireElement("expression",n);return{type:"blockLiteral",args:r,expr:a,evaluate:function(e){return function(){for(var t=0;t=0;a--){var i=o[a];if(i.compareDocumentPosition(e)===Node.DOCUMENT_POSITION_FOLLOWING)return i}if(r)return o[o.length-1]}(n,a,s,c)},evaluate:function(e){return t.unifiedEval(this,e)}}}}),E.addGrammarElement("positionalExpression",function(e,t,n){var r=n.matchAnyToken("first","last","random");if(r){n.matchAnyToken("in","from","of");var o=e.requireElement("unaryExpression",n),a=r.value;return{type:"positionalExpression",rhs:o,operator:r.value,args:[o],op:function(e,t){if(t&&!Array.isArray(t)&&(t=t.children?t.children:Array.from(t)),t){if("first"===a)return t[0];if("last"===a)return t[t.length-1];if("random"===a)return t[Math.floor(Math.random()*t.length)]}},evaluate:function(e){return t.unifiedEval(this,e)}}}}),E.addGrammarElement("mathOperator",function(e,t,n){var r,o=e.parseElement("unaryExpression",n),a=null;for(r=n.matchAnyOpToken("+","-","*","/","%");r;){var i=r.value;(a=a||r).value!==i&&e.raiseParseError(n,"You must parenthesize math operations with different operators");var u=e.parseElement("unaryExpression",n);o={type:"mathOperator",lhs:o,rhs:u,operator:i,args:[o,u],op:function(e,t,n){return"+"===i?t+n:"-"===i?t-n:"*"===i?t*n:"/"===i?t/n:"%"===i?t%n:void 0},evaluate:function(e){return t.unifiedEval(this,e)}},r=n.matchAnyOpToken("+","-","*","/","%")}return o}),E.addGrammarElement("mathExpression",function(e,t,n){return e.parseAnyOf(["mathOperator","unaryExpression"],n)}),E.addGrammarElement("comparisonOperator",function(e,t,n){var r=e.parseElement("mathExpression",n),o=n.matchAnyOpToken("<",">","<=",">=","==","===","!=","!=="),a=o?o.value:null,i=!0,u=!1;if(null==a&&(n.matchToken("is")||n.matchToken("am")?n.matchToken("not")?n.matchToken("in")?a="not in":n.matchToken("a")?(a="not a",u=!0):n.matchToken("empty")?(a="not empty",i=!1):a="!=":n.matchToken("in")?a="in":n.matchToken("a")?(a="a",u=!0):n.matchToken("empty")?(a="empty",i=!1):n.matchToken("less")?(n.requireToken("than"),n.matchToken("or")?(n.requireToken("equal"),n.requireToken("to"),a="<="):a="<"):n.matchToken("greater")?(n.requireToken("than"),n.matchToken("or")?(n.requireToken("equal"),n.requireToken("to"),a=">="):a=">"):a="==":n.matchToken("exist")||n.matchToken("exists")?(a="exist",i=!1):n.matchToken("matches")||n.matchToken("match")?a="match":n.matchToken("contains")||n.matchToken("contain")?a="contain":n.matchToken("includes")||n.matchToken("include")?a="include":(n.matchToken("do")||n.matchToken("does"))&&(n.requireToken("not"),n.matchToken("matches")||n.matchToken("match")?a="not match":n.matchToken("contains")||n.matchToken("contain")?a="not contain":n.matchToken("exist")||n.matchToken("exist")?(a="not exist",i=!1):n.matchToken("include")?a="not include":e.raiseParseError(n,"Expected matches or contains"))),a){if(u)var l=n.requireTokenType("IDENTIFIER"),s=!n.matchOpToken("!");else if(i){var c=e.requireElement("mathExpression",n);"match"!==a&&"not match"!==a||(c=c.css?c.css:c)}var f=r;r={type:"comparisonOperator",operator:a,typeName:l,nullOk:s,lhs:r,rhs:c,args:[r,c],op:function(e,n,r){if("=="===a)return n==r;if("!="===a)return n!=r;if("match"===a)return null!=n&&g(f,n,r);if("not match"===a)return null==n||!g(f,n,r);if("in"===a)return null!=r&&k(c,r,n);if("not in"===a)return null==r||!k(c,r,n);if("contain"===a)return null!=n&&k(f,n,r);if("not contain"===a)return null==n||!k(f,n,r);if("include"===a)return null!=n&&k(f,n,r);if("not include"===a)return null==n||!k(f,n,r);if("==="===a)return n===r;if("!=="===a)return n!==r;if("<"===a)return n"===a)return n>r;if("<="===a)return n<=r;if(">="===a)return n>=r;if("empty"===a)return t.isEmpty(n);if("not empty"===a)return!t.isEmpty(n);if("exist"===a)return t.doesExist(n);if("not exist"===a)return!t.doesExist(n);if("a"===a)return t.typeCheck(n,l.value,s);if("not a"===a)return!t.typeCheck(n,l.value,s);throw"Unknown comparison : "+a},evaluate:function(e){return t.unifiedEval(this,e)}}}return r}),E.addGrammarElement("comparisonExpression",function(e,t,n){return e.parseAnyOf(["comparisonOperator","mathExpression"],n)}),E.addGrammarElement("logicalOperator",function(e,t,n){var r,o=e.parseElement("comparisonExpression",n),a=null;r=n.matchToken("and")||n.matchToken("or");for(var i=function(){(a=a||r).value!==r.value&&e.raiseParseError(n,"You must parenthesize logical operations with different operators"),u=e.requireElement("comparisonExpression",n);var i=r.value;o={type:"logicalOperator",operator:i,lhs:o,rhs:u,args:[o,u],op:function(e,t,n){return"and"===i?t&&n:t||n},evaluate:function(e){return t.unifiedEval(this,e)}},r=n.matchToken("and")||n.matchToken("or")};r;){var u;i()}return o}),E.addGrammarElement("logicalExpression",function(e,t,n){return e.parseAnyOf(["logicalOperator","mathExpression"],n)}),E.addGrammarElement("asyncExpression",function(e,t,n){return n.matchToken("async")?{type:"asyncExpression",value:e.requireElement("logicalExpression",n),evaluate:function(e){return{asyncWrapper:!0,value:this.value.evaluate(e)}}}:e.parseElement("logicalExpression",n)}),E.addGrammarElement("expression",function(e,t,n){return n.matchToken("the"),e.parseElement("asyncExpression",n)}),E.addGrammarElement("assignableExpression",function(e,t,n){n.matchToken("the");var r=e.parseElement("primaryExpression",n);return!r||"symbol"!==r.type&&"ofExpression"!==r.type&&"propertyAccess"!==r.type&&"attributeRefAccess"!==r.type&&"attributeRef"!==r.type&&"styleRef"!==r.type&&"arrayIndex"!==r.type&&"possessive"!==r.type?(E.raiseParseError(n,"A target expression must be writable. The expression type '"+(r&&r.type)+"' is not."),r):r}),E.addGrammarElement("hyperscript",function(e,t,n){var r=[];if(n.hasMore())for(;e.featureStart(n.currentToken())||"("===n.currentToken().value;){var o=e.requireElement("feature",n);r.push(o),n.matchToken("end")}return{type:"hyperscript",features:r,apply:function(e,t,n){for(var o,a=s(r);!(o=a()).done;)o.value.install(e,t,n)}}});var N=function(e){var t=[];if("("===e.token(0).value&&(")"===e.token(1).value||","===e.token(2).value||")"===e.token(2).value)){e.matchOpToken("(");do{t.push(e.requireTokenType("IDENTIFIER"))}while(e.matchOpToken(","));e.requireOpToken(")")}return t};E.addFeature("on",function(e,t,n){if(n.matchToken("on")){var r=!1;n.matchToken("every")&&(r=!0);var o=[],a=null;do{var i=e.requireElement("eventName",n,"Expected event name").evaluate();a=a?a+" or "+i:"on "+i;var u=N(n),l=null;if(n.matchOpToken("[")&&(l=e.requireElement("expression",n),n.requireOpToken("]")),"NUMBER"===n.currentToken().type){var f=n.consumeToken(),m=parseInt(f.value);if(n.matchToken("to"))var p=n.consumeToken(),d=parseInt(p.value);else if(n.matchToken("and")){var v=!0;n.requireToken("on")}}if("intersection"===i){var h={};if(n.matchToken("with")&&(h.with=e.requireElement("expression",n).evaluate()),n.matchToken("having"))do{n.matchToken("margin")?h.rootMargin=e.requireElement("stringLike",n).evaluate():n.matchToken("threshold")?h.threshold=e.requireElement("expression",n).evaluate():e.raiseParseError(n,"Unknown intersection config specification")}while(n.matchToken("and"))}else if("mutation"===i){var E={};if(n.matchToken("of"))do{if(n.matchToken("anything"))E.attributes=!0,E.subtree=!0,E.characterData=!0,E.childList=!0;else if(n.matchToken("childList"))E.childList=!0;else if(n.matchToken("attributes"))E.attributes=!0,E.attributeOldValue=!0;else if(n.matchToken("subtree"))E.subtree=!0;else if(n.matchToken("characterData"))E.characterData=!0,E.characterDataOldValue=!0;else if("ATTRIBUTE_REF"===n.currentToken().type){var y=n.consumeToken();null==E.attributeFilter&&(E.attributeFilter=[]),0==y.value.indexOf("@")?E.attributeFilter.push(y.value.substring(1)):e.raiseParseError(n,"Only shorthand attribute references are allowed here")}else e.raiseParseError(n,"Unknown mutation config specification")}while(n.matchToken("or"));else E.attributes=!0,E.characterData=!0,E.childList=!0}var k=null,g=!1;if(n.matchToken("from")&&(n.matchToken("elsewhere")?g=!0:(k=e.parseElement("expression",n))||e.raiseParseError(n,'Expected either target value or "elsewhere".')),null===k&&!1===g&&n.matchToken("elsewhere")&&(g=!0),n.matchToken("in"))var x=e.parseElement("unaryExpression",n);if(n.matchToken("debounced")){n.requireToken("at");var b=e.requireElement("expression",n).evaluate({})}else if(n.matchToken("throttled")){n.requireToken("at");var q=e.requireElement("expression",n).evaluate({})}o.push({execCount:0,every:r,on:i,args:u,filter:l,from:k,inExpr:x,elsewhere:g,startCount:m,endCount:d,unbounded:v,debounceTime:b,throttleTime:q,mutationSpec:E,intersectionSpec:h,debounced:void 0,lastExec:void 0})}while(n.matchToken("or"));var w=!0;if(!r&&n.matchToken("queue"))if(n.matchToken("all"))w=!1;else if(n.matchToken("first"))var S=!0;else if(n.matchToken("none"))var I=!0;else n.requireToken("last");var R=e.requireElement("commandList",n);if(e.ensureTerminated(R),n.matchToken("catch")){var O=n.requireTokenType("IDENTIFIER").value,L=e.requireElement("commandList",n);e.ensureTerminated(L)}if(n.matchToken("finally")){var C=e.requireElement("commandList",n);e.ensureTerminated(C)}var A={displayName:a,events:o,start:R,every:r,execCount:0,errorHandler:L,errorSymbol:O,execute:function(e){var n=t.getEventQueueFor(e.me,A);if(n.executing&&!1===r){if(I||S&&n.queue.length>0)return;return w&&(n.queue.length=0),void n.queue.push(e)}A.execCount++,n.executing=!0,e.meta.onHalt=function(){n.executing=!1;var e=n.queue.shift();e&&setTimeout(function(){A.execute(e)},1)},e.meta.reject=function(n){console.error(n.message?n.message:n);var r=t.getHyperTrace(e,n);r&&r.print(),t.triggerEvent(e.me,"exception",{error:n})},R.execute(e)},install:function(e,n){for(var r,o=function(){var n=r.value;i=n.elsewhere?[document]:n.from?n.from.evaluate(t.makeContext(e,A,e,null)):[e],t.implicitLoop(i,function(r){var o=n.on;if(n.mutationSpec&&(o="hyperscript:mutation",new MutationObserver(function(e,t){A.executing||T.triggerEvent(r,o,{mutationList:e,observer:t})}).observe(r,n.mutationSpec)),n.intersectionSpec){o="hyperscript:insersection";var a=new IntersectionObserver(function(e){for(var t,n=s(e);!(t=n()).done;){var i=t.value,u={observer:a};(u=c(u,i)).intersecting=i.isIntersecting,T.triggerEvent(r,o,u)}},n.intersectionSpec);a.observe(r)}(r.addEventListener||r.on).call(r,o,function a(i){if("undefined"!=typeof Node&&e instanceof Node&&r!==e&&!e.isConnected)r.removeEventListener(o,a);else{var u=t.makeContext(e,A,e,i);if(!n.elsewhere||!e.contains(i.target)){n.from&&(u.result=r);for(var l,c=s(n.args);!(l=c()).done;){var f=l.value,m=u.event[f.value];void 0!==m?u[f.value]=m:"detail"in u.event&&(u[f.value]=u.event.detail[f.value])}if(u.meta.errorHandler=L,u.meta.errorSymbol=O,u.meta.finallyHandler=C,n.filter){var p=u.meta.context;u.meta.context=u.event;try{if(!n.filter.evaluate(u))return}finally{u.meta.context=p}}if(n.inExpr)for(var d=i.target;;){if(d.matches&&d.matches(n.inExpr.css)){u.result=d;break}if(null==(d=d.parentElement))return}if(n.execCount++,n.startCount)if(n.endCount){if(n.execCountn.endCount)return}else if(n.unbounded){if(n.execCount (http://wenzhixin.net.cn/) - * @license MIT - */ - -!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(require("jquery")):"function"==typeof define&&define.amd?define(["jquery"],e):e((t=t||self).jQuery)}(this,(function(t){"use strict";function e(t){return(e="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function i(t,e){for(var n=0;n0?ct:at)(t)},ft=Math.min,pt=function(t){return t>0?ft(ht(t),9007199254740991):0},dt=Math.max,vt=Math.min,gt=function(t,e){var n=ht(t);return n<0?dt(n+e,0):vt(n,e)},yt=function(t){return function(e,n,i){var r,u=D(e),o=pt(u.length),s=gt(i,o);if(t&&n!=n){for(;o>s;)if((r=u[s++])!=r)return!0}else for(;o>s;s++)if((t||s in u)&&u[s]===n)return t||s||0;return!t&&-1}},Et={includes:yt(!0),indexOf:yt(!1)},bt=Et.indexOf,mt=function(t,e){var n,i=D(t),r=0,u=[];for(n in i)!$(Q,n)&&$(i,n)&&u.push(n);for(;e.length>r;)$(i,n=e[r++])&&(~bt(u,n)||u.push(n));return u},At=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"],Ft=At.concat("length","prototype"),St={f:Object.getOwnPropertyNames||function(t){return mt(t,Ft)}},Ct={f:Object.getOwnPropertySymbols},kt=lt("Reflect","ownKeys")||function(t){var e=St.f(R(t)),n=Ct.f;return n?e.concat(n(t)):e},Dt=function(t,e){for(var n=kt(e),i=P.f,r=I.f,u=0;uu;)P.f(t,n=i[u++],e[n]);return t},Gt=lt("document","documentElement"),Ut=J("IE_PROTO"),Wt=function(){},Vt=function(){var t,e=T("iframe"),n=At.length;for(e.style.display="none",Gt.appendChild(e),e.src=String("javascript:"),(t=e.contentWindow.document).open(),t.write(" + {{ site_name }} - {% trans '搜索结果' %} + + + @@ -43,399 +45,39 @@
@@ -500,7 +142,7 @@ {% endif %}
-
+
@@ -573,15 +215,11 @@
- {% comment %} - - - - - {% endcomment %} diff --git a/common/templates/partial/_announcement.html b/common/templates/partial/_announcement.html new file mode 100644 index 00000000..166d2856 --- /dev/null +++ b/common/templates/partial/_announcement.html @@ -0,0 +1,61 @@ +{% load static %} +{% load i18n %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} +
+ +
+
+

{% trans '公告' %}

+ + + + + + + + + +
+
+
    + {% for ann in unread_announcements %} +
  • + +
    {{ ann.title }}
    +     + {{ ann.created_time }} +

    {{ ann.get_plain_content | truncate:200 }}

    +
    • + {% if not forloop.last %} +
    + {% endif %} + {% endfor %} +
+
+ {% trans '查看全部公告' %} +
+
+
+
+
+ diff --git a/common/templates/partial/_common_libs.html b/common/templates/partial/_common_libs.html new file mode 100644 index 00000000..9460e2da --- /dev/null +++ b/common/templates/partial/_common_libs.html @@ -0,0 +1,23 @@ +{% load static %} +{% if sentry_dsn %} + + +{% endif %} +{% if jquery %} + +{% else %} + +{% endif %} + + + + + + diff --git a/common/templates/partial/_footer.html b/common/templates/partial/_footer.html index e5710473..412f00ad 100644 --- a/common/templates/partial/_footer.html +++ b/common/templates/partial/_footer.html @@ -1,13 +1,12 @@ \ No newline at end of file diff --git a/common/templates/partial/_navbar.html b/common/templates/partial/_navbar.html index 171eda8d..90d7a459 100644 --- a/common/templates/partial/_navbar.html +++ b/common/templates/partial/_navbar.html @@ -1,24 +1,24 @@ {% load static %} {% load i18n %} {% load admin_url %} +
\ No newline at end of file + +
diff --git a/common/templates/partial/_sidebar.html b/common/templates/partial/_sidebar.html new file mode 100644 index 00000000..90bc2c0d --- /dev/null +++ b/common/templates/partial/_sidebar.html @@ -0,0 +1,186 @@ +{% load static %} +{% load i18n %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} +{% load neo %} +
+
+ +
+ +
+
+ + + + + +
+ {% if user == request.user %} +
+
+ +
+
+ {% trans '关注的人' %} +
+ {% trans '更多' %} + +
+ +
+
+ {% trans '被他们关注' %} +
+ {% trans '更多' %} + +
+ +
+
+ {% trans '常用标签' %} +
+ {% trans '更多' %} +
+ {% if book_tags %} +
{% trans '书籍' %}
+ {% for v in book_tags %} + + {{ v.content }} + + {% endfor %} +
+ {% endif %} + + {% if movie_tags %} +
{% trans '电影和剧集' %}
+ {% for v in movie_tags %} + + {{ v.content }} + + {% endfor %} +
+ {% endif %} + + {% if music_tags %} +
{% trans '音乐' %}
+ {% for v in music_tags %} + + {{ v.content }} + + {% endfor %} +
+ {% endif %} + + {% if game_tags %} +
{% trans '游戏' %}
+ {% for v in game_tags %} + + {{ v.content }} + + {% endfor %} +
+ {% endif %} +
+
+ +
+ +
+ {% if request.user.is_staff and request.user == user%} +
+
{% trans '投诉信息' %}
+ 全部投诉 +
+ +
+
+ {% endif %} +
+
+ {% endif %} +
+
+ +{% if user == request.user %} + + + + + + +{% endif %} diff --git a/common/templates/partial/list_item.html b/common/templates/partial/list_item.html new file mode 100644 index 00000000..9b4eb9ad --- /dev/null +++ b/common/templates/partial/list_item.html @@ -0,0 +1,9 @@ +{% if item.category_name|lower == 'book' %} +{% include "partial/list_item_book.html" with book=item %} +{% elif item.category_name|lower == 'movie' %} +{% include "partial/list_item_movie.html" with movie=item %} +{% elif item.category_name|lower == 'game' %} +{% include "partial/list_item_game.html" with game=item %} +{% elif item.category_name|lower == 'album' or item.category_name|lower == 'song' %} +{% include "partial/list_item_music.html" with music=item %} +{% endif %} \ No newline at end of file diff --git a/common/templates/partial/list_item_book.html b/common/templates/partial/list_item_book.html new file mode 100644 index 00000000..3c5dc9eb --- /dev/null +++ b/common/templates/partial/list_item_book.html @@ -0,0 +1,159 @@ +{% load thumb %} +{% load highlight %} +{% load i18n %} +{% load l10n %} +{% load neo %} +{% current_user_marked_item book as marked %} +
  • +
    + + + + {% if not marked %} + ➕ + {% endif %} +
    + +
    + {% if editable %} +
    + {% if not forloop.first %} + ▲ + {% endif %} + {% if not forloop.last %} + ▼ + {% endif %} + ✖ +
    + {% endif %} + +
    + + + {% if request.GET.q %} + {{ book.title | highlight:request.GET.q }} + {% else %} + {{ book.title }} + {% endif %} + + + {% if not request.GET.c and not hide_category %} + [{{book.verbose_category_name}}] + {% endif %} + + {{ book.get_source_site_display }} + +
    + + {% if book.rating %} +
    + {{ book.rating }} + {% else %} +
    {% trans '暂无评分' %}
    + {% endif %} + + + {% if book.pub_year %} / + {{ book.pub_year }}{% trans '年' %}{% if book.pub_month %}{{book.pub_month }}{% trans '月' %}{% endif %} + {% endif %} + + {% if book.author %} / + {% for author in book.author %} + {% if request.GET.q %} + {{ author | highlight:request.GET.q }} + {% else %} + {{ author }} + {% endif %} + {% if not forloop.last %},{% endif %} + {% endfor %} + {% endif %} + + {% if book.translator %} / + {% trans '翻译' %}: + {% for translator in book.translator %} + {% if request.GET.q %} + {{ translator | highlight:request.GET.q }} + {% else %} + {{ translator }} + {% endif %} + {% if not forloop.last %},{% endif %} + {% endfor %} + {% endif %} + + {% if book.subtitle %} / + {% trans '副标题' %}: + {% if request.GET.q %} + {{ book.subtitle | highlight:request.GET.q }} + {% else %} + {{ book.subtitle }} + {% endif %} + {% endif %} + + {% if book.orig_title %} / + {% trans '原名' %}: + {% if request.GET.q %} + {{ book.orig_title | highlight:request.GET.q }} + {% else %} + {{ book.orig_title }} + {% endif %} + {% endif %} + +

    + {{ book.brief }} +

    + +
    + {% for tag_dict in book.top_tags %} + + {{ tag_dict.content }} + + {% endfor %} +
    + + {% if mark %} +
    +
    +
    +
      +
    • + {% if mark.rating %} + + {% endif %} + {% if mark.visibility > 0 %} + + + + {% endif %} + + {% trans '于' %} {{ mark.created_time }} + {% if status == 'reviewed' %} + {% trans '评论' %}: {{ mark.title }} + {% else %} + {% trans '标记' %} + {% endif %} + + {% if mark.text %} +

      {{ mark.text }}

      + {% endif %} +
      • +
    +
    + {% endif %} + + {% if collectionitem %} +
    +
    +
    +
      +
    • +

      + {% include "show_item_comment.html" %} +

      +
      • +
    +
    + {% endif %} +
    +
    • \ No newline at end of file diff --git a/common/templates/partial/list_item_game.html b/common/templates/partial/list_item_game.html new file mode 100644 index 00000000..42346910 --- /dev/null +++ b/common/templates/partial/list_item_game.html @@ -0,0 +1,139 @@ +{% load thumb %} +{% load highlight %} +{% load i18n %} +{% load l10n %} +{% load neo %} +{% current_user_marked_item game as marked %} +
  • +
    + + + + {% if not marked %} + ➕ + {% endif %} +
    +
    + {% if editable %} +
    + {% if not forloop.first %} + ▲ + {% endif %} + {% if not forloop.last %} + ▼ + {% endif %} + ✖ +
    + {% endif %} + +
    + + {% if request.GET.q %} + {{ game.title | highlight:request.GET.q }} + {% else %} + {{ game.title }} + {% endif %} + + + {% if not request.GET.c and not hide_category %} + [{{item.verbose_category_name}}] + {% endif %} + + {{ game.get_source_site_display }} + +
    + + {% if game.rating %} +
    + {{ game.rating }} + {% else %} +
    {% trans '暂无评分' %}
    + {% endif %} + + + + {% if game.other_title %}{% trans '别名' %}: + {% for other_title in game.other_title %} + {{ other_title }}{% if not forloop.last %} {% endif %} + {% endfor %}/ + {% endif %} + + {% if game.developer %}{% trans '开发商' %}: + {% for developer in game.developer %} + {{ developer }}{% if not forloop.last %} {% endif %} + {% endfor %}/ + {% endif %} + + {% if game.genre %}{% trans '类型' %}: + {% for genre in game.genre %} + {{ genre }}{% if not forloop.last %} {% endif %} + {% endfor %}/ + {% endif %} + + {% if game.platform %}{% trans '平台' %}: + {% for platform in game.platform %} + {{ platform }}{% if not forloop.last %} {% endif %} + {% endfor %} + {% endif %} + + +

    + {{ game.brief }} +

    + +
    + {% for tag_dict in game.top_tags %} + + {{ tag_dict.content }} + + {% endfor %} +
    + + {% if mark %} +
    +
    +
    +
      +
    • + {% if mark.rating %} + + {% endif %} + {% if mark.visibility > 0 %} + + + + {% endif %} + + {% trans '于' %} {{ mark.created_time }} + {% if status == 'reviewed' %} + {% trans '评论' %}: {{ mark.title }} + {% else %} + {% trans '标记' %} + {% endif %} + + {% if mark.text %} +

      {{ mark.text }}

      + {% endif %} +
      • +
    +
    + {% endif %} + + {% if collectionitem %} +
    +
    +
    +
      +
    • +

      + {% include "show_item_comment.html" %} +

      +
      • +
    +
    + {% endif %} +
    + +
    • \ No newline at end of file diff --git a/common/templates/partial/list_item_movie.html b/common/templates/partial/list_item_movie.html new file mode 100644 index 00000000..f20f40f5 --- /dev/null +++ b/common/templates/partial/list_item_movie.html @@ -0,0 +1,164 @@ +{% load thumb %} +{% load highlight %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load neo %} +{% current_user_marked_item movie as marked %} +
  • +
    + + + + {% if not marked %} + ➕ + {% endif %} +
    +
    + {% if editable %} +
    + {% if not forloop.first %} + ▲ + {% endif %} + {% if not forloop.last %} + ▼ + {% endif %} + ✖ +
    + {% endif %} + +
    + + {% if movie.season %} + + {% if request.GET.q %} + {{ movie.title | highlight:request.GET.q }} {% trans '第' %}{{ movie.season|apnumber }}{% trans '季' %} + {{ movie.orig_title | highlight:request.GET.q }} Season {{ movie.season }} + {% if movie.year %}({{ movie.year }}){% endif %} + {% else %} + {{ movie.title }} {% trans '第' %}{{ movie.season|apnumber }}{% trans '季' %} + {{ movie.orig_title }} Season {{ movie.season }} + {% if movie.year %}({{ movie.year }}){% endif %} + {% endif %} + + {% else %} + {% if request.GET.q %} + {{ movie.title | highlight:request.GET.q }} {{ movie.orig_title | highlight:request.GET.q }} + {% if movie.year %}({{ movie.year }}){% endif %} + {% else %} + {{ movie.title }} {{ movie.orig_title }} + {% if movie.year %}({{ movie.year }}){% endif %} + {% endif %} + {% endif %} + + + {% if not request.GET.c and not hide_category %} + [{{movie.verbose_category_name}}] + {% endif %} + + {{ movie.get_source_site_display }} + +
    + + {% if movie.rating %} +
    + {{ movie.rating }} + {% else %} +
    {% trans '暂无评分' %}
    + {% endif %} + + + + {% if movie.director %}{% trans '导演' %}: + {% for director in movie.director %} + {% if request.GET.q %} + {{ director | highlight:request.GET.q }} + {% else %} + {{ director }} + {% endif %} + {% if not forloop.last %},{% endif %} + {% endfor %}/ + {% endif %} + + {% if movie.genre %}{% trans '类型' %}: + {% for genre in movie.get_genre_display %} + {{ genre }}{% if not forloop.last %} {% endif %} + {% endfor %}/ + {% endif %} + + + + {% if movie.actor %}{% trans '主演' %}: + {% for actor in movie.actor %} + 5 %}style="display: none;" {% endif %}> + {% if request.GET.q %} + {{ actor | highlight:request.GET.q }} + {% else %} + {{ actor }} + {% endif %} + + {% if forloop.counter <= 5 %} + {% if not forloop.counter == 5 and not forloop.last %} {% endif %} + {% endif %} + {% endfor %} + {% endif %} + +

    + {{ movie.brief }} +

    +
    + {% for tag_dict in movie.top_tags %} + + {{ tag_dict.content }} + + {% endfor %} +
    + + {% if mark %} +
    +
    +
    +
      +
    • + {% if mark.rating %} + + {% endif %} + {% if mark.visibility > 0 %} + + + + {% endif %} + + {% trans '于' %} {{ mark.created_time }} + {% if status == 'reviewed' %} + {% trans '评论' %}: {{ mark.title }} + {% else %} + {% trans '标记' %} + {% endif %} + + {% if mark.text %} +

      {{ mark.text }}

      + {% endif %} +
      • +
    +
    + {% endif %} + + {% if collectionitem %} +
    +
    +
    +
      +
    • +

      + {% include "show_item_comment.html" %} +

      +
      • +
    +
    + {% endif %} +
    + +
    • \ No newline at end of file diff --git a/common/templates/partial/list_item_music.html b/common/templates/partial/list_item_music.html new file mode 100644 index 00000000..bcf0d29a --- /dev/null +++ b/common/templates/partial/list_item_music.html @@ -0,0 +1,171 @@ +{% load thumb %} +{% load highlight %} +{% load i18n %} +{% load l10n %} +{% load neo %} +{% current_user_marked_item music as marked %} +
  • +
    + + {% if music.category_name|lower == 'album' %} + + + + {% if not marked %} + ➕ + {% endif %} + {% elif music.category_name|lower == 'song' %} + + + + {% if not marked %} + ➕ + {% endif %} + {% endif %} +
    +
    + {% if editable %} +
    + {% if not forloop.first %} + ▲ + {% endif %} + {% if not forloop.last %} + ▼ + {% endif %} + ✖ +
    + {% endif %} + +
    + + {% if music.category_name|lower == 'album' %} + + {% if request.GET.q %} + {{ music.title | highlight:request.GET.q }} + {% else %} + {{ music.title }} + {% endif %} + + {% elif music.category_name|lower == 'song' %} + + {% if request.GET.q %} + {{ music.title | highlight:request.GET.q }} + {% else %} + {{ music.title }} + {% endif %} + + {% endif %} + + + {% if not request.GET.c and not hide_category %} + [{{music.verbose_category_name}}] + {% endif %} + + {{ music.get_source_site_display }} + +
    + + {% if music.rating %} +
    + {{ music.rating }} + {% else %} +
    {% trans '暂无评分' %}
    + {% endif %} + + + {% if music.artist %}{% trans '艺术家' %}: + {% for artist in music.artist %} + {{ artist }} + {% if not forloop.last %} {% endif %} + {% endfor %} + {% endif %} + + {% if music.genre %}/ {% trans '流派' %}: + {{ music.genre }} + {% endif %} + + {% if music.release_date %}/ {% trans '发行日期' %}: + {{ music.release_date }} + {% endif %} + + + + + + {% if music.brief %} +

    + {{ music.brief }} +

    + {% elif music.category_name|lower == 'album' %} +

    + {% trans '曲目：' %}{{ music.track_list }} +

    + {% else %} + +

    + {% trans '所属专辑：' %}{{ music.album }} +

    + {% endif %} + +
    + {% for tag_dict in music.top_tags %} + + {{ tag_dict.content }} + + {% endfor %} +
    + + {% if mark %} +
    +
    +
    +
      +
    • + {% if mark.rating %} + + {% endif %} + {% if mark.visibility > 0 %} + + + + {% endif %} + + {% trans '于' %} {{ mark.created_time }} + {% if status == 'reviewed' %} + {% trans '评论' %}: + {% if music.category_name|lower == 'album' %} + {{ mark.title }} + {% else %} + {{ mark.title }} + {% endif %} + {% else %} + {% trans '标记' %} + {% endif %} + + {% if mark.text %} +

      {{ mark.text }}

      + {% endif %} +
      • +
    +
    + {% endif %} + + {% if collectionitem %} +
    +
    +
    +
      +
    • +

      + {% include "show_item_comment.html" %} +

      +
      • +
    +
    + {% endif %} + +
    + +
    • \ No newline at end of file diff --git a/common/templates/partial/mark_list.html b/common/templates/partial/mark_list.html new file mode 100644 index 00000000..71c4102a --- /dev/null +++ b/common/templates/partial/mark_list.html @@ -0,0 +1,37 @@ +{% load i18n %} + + \ No newline at end of file diff --git a/common/templatetags/highlight.py b/common/templatetags/highlight.py index a2276800..02d79bbb 100644 --- a/common/templatetags/highlight.py +++ b/common/templatetags/highlight.py @@ -1,17 +1,19 @@ from django import template from django.utils.safestring import mark_safe from django.template.defaultfilters import stringfilter -from django.utils.html import format_html +from opencc import OpenCC -import re +cc = OpenCC('t2s') register = template.Library() + @register.filter @stringfilter def highlight(text, search): - to_be_replaced_words = set(re.findall(search, text, flags=re.IGNORECASE)) - - for word in to_be_replaced_words: - text = text.replace(word, f'{word}') + for s in cc.convert(search.strip().lower()).split(' '): + if s: + p = cc.convert(text.lower()).find(s) + if p != -1: + text = f'{text[0:p]}{text[p:p+len(s)]}{text[p+len(s):]}' return mark_safe(text) diff --git a/common/templatetags/neo.py b/common/templatetags/neo.py new file mode 100644 index 00000000..b915b7b9 --- /dev/null +++ b/common/templatetags/neo.py @@ -0,0 +1,48 @@ +from django import template +import datetime +from django.utils import timezone +from collection.models import Collection + + +register = template.Library() + + +@register.simple_tag(takes_context=True) +def current_user_marked_item(context, item): + user = context['request'].user + if user and user.is_authenticated: + if isinstance(item, Collection) and item.owner == user: + return item + else: + return context['request'].user.get_mark_for_item(item) + return None + + +@register.simple_tag(takes_context=True) +def current_user_relationship(context, user): + current_user = context['request'].user + if current_user and current_user.is_authenticated: + if current_user.is_following(user): + if current_user.is_followed_by(user): + return '互相关注' + else: + return '已关注' + elif current_user.is_followed_by(user): + return '被ta关注' + return None + + +@register.filter +def prettydate(d): + diff = timezone.now() - d + s = diff.seconds + if diff.days > 14 or diff.days < 0: + return d.strftime('%Y年%m月%d日') + elif diff.days >= 1: + return '{} 天前'.format(diff.days) + elif s < 120: + return '刚刚' + elif s < 3600: + return '{} 分钟前'.format(s // 60) + else: + return '{} 小时前'.format(s // 3600) diff --git a/common/templatetags/oauth_token.py b/common/templatetags/oauth_token.py index 7aac83a1..b2f24677 100644 --- a/common/templatetags/oauth_token.py +++ b/common/templatetags/oauth_token.py @@ -7,7 +7,7 @@ register = template.Library() class OAuthTokenNode(template.Node): def render(self, context): request = context.get('request') - oauth_token = request.session.get('oauth_token', default='') + oauth_token = request.user.mastodon_token return format_html(oauth_token) diff --git a/common/templatetags/thumb.py b/common/templatetags/thumb.py index 2e21c6ca..aa698abb 100644 --- a/common/templatetags/thumb.py +++ b/common/templatetags/thumb.py @@ -12,4 +12,7 @@ def thumb(source, alias): if source.url.endswith('.svg'): return source.url else: - return thumbnail_url(source, alias) \ No newline at end of file + try: + return thumbnail_url(source, alias) + except Exception as e: + return '' diff --git a/common/urls.py b/common/urls.py index 6de4f66a..b803a47d 100644 --- a/common/urls.py +++ b/common/urls.py @@ -6,4 +6,6 @@ urlpatterns = [ path('', home), path('home/', home, name='home'), path('search/', search, name='search'), + path('search.json/', search, name='search.json'), + path('external_search/', external_search, name='external_search'), ] diff --git a/common/utils.py b/common/utils.py index 5d26a3d9..4d51ca16 100644 --- a/common/utils.py +++ b/common/utils.py @@ -10,6 +10,8 @@ class PageLinksGenerator: def __init__(self, length, current_page, total_pages): current_page = int(current_page) self.current_page = current_page + self.previous_page = current_page - 1 if current_page > 1 else None + self.next_page = current_page + 1 if current_page < total_pages else None self.start_page = None self.end_page = None self.page_range = None diff --git a/common/views.py b/common/views.py index c028184a..e527e09a 100644 --- a/common/views.py +++ b/common/views.py @@ -17,21 +17,93 @@ from music.models import Album, Song, AlbumMark, SongMark from users.models import Report, User, Preference from mastodon.decorators import mastodon_request_included from users.views import home as user_home +from timeline.views import timeline as user_timeline from common.models import MarkStatusEnum from common.utils import PageLinksGenerator -from common.scraper import scraper_registry +from common.scraper import get_scraper_by_url, get_normalized_url from common.config import * +from common.searcher import ExternalSources from management.models import Announcement +from django.conf import settings +from common.index import Indexer +from django.http import JsonResponse +from django.db.utils import IntegrityError + logger = logging.getLogger(__name__) + @login_required def home(request): - return user_home(request, request.user.id) + if request.user.get_preference().classic_homepage: + return redirect(reverse("users:home", args=[request.user.mastodon_username])) + else: + return redirect(reverse("timeline:timeline")) @login_required +def external_search(request): + category = request.GET.get("c", default='all').strip().lower() + if category == 'all': + category = None + keywords = request.GET.get("q", default='').strip() + page_number = int(request.GET.get('page', default=1)) + return render( + request, + "common/external_search_result.html", + { + "external_items": ExternalSources.search(category, keywords, page_number) if keywords else [], + } + ) + + def search(request): + if settings.SEARCH_BACKEND is None: + return search2(request) + category = request.GET.get("c", default='all').strip().lower() + if category == 'all': + category = None + keywords = request.GET.get("q", default='').strip() + tag = request.GET.get("tag", default='').strip() + p = request.GET.get('page', default='1') + page_number = int(p) if p.isdigit() else 1 + if not (keywords or tag): + return render( + request, + "common/search_result.html", + { + "items": None, + } + ) + if request.user.is_authenticated: + url_validator = URLValidator() + try: + url_validator(keywords) + # validation success + return jump_or_scrape(request, keywords) + except ValidationError as e: + pass + + result = Indexer.search(keywords, page=page_number, category=category, tag=tag) + for item in result.items: + item.tag_list = item.all_tag_list[:TAG_NUMBER_ON_LIST] + if request.path.endswith('.json/'): + return JsonResponse({ + 'num_pages': result.num_pages, + 'items':list(map(lambda i:i.get_json(), result.items)) + }) + return render( + request, + "common/search_result.html", + { + "items": result.items, + "pagination": PageLinksGenerator(PAGE_LINK_NUMBER, page_number, result.num_pages), + "categories": ['book', 'movie', 'music', 'game'], + } + ) + + +def search2(request): if request.method == 'GET': # test if input serach string is empty or not excluding param ?c= @@ -109,7 +181,7 @@ def search(request): else: ordered_queryset = list(queryset) return ordered_queryset - + def movie_param_handler(**kwargs): # keywords keywords = kwargs.get('keywords') @@ -240,7 +312,7 @@ def search(request): elif music.__class__ == Song: similarity += 1/2 * SequenceMatcher(None, keyword, music.title).quick_ratio() \ + 1/6 * SequenceMatcher(None, keyword, artist_dump).quick_ratio() \ - + 1/6 * SequenceMatcher(None, keyword, music.album.title).quick_ratio() + + 1/6 * (SequenceMatcher(None, keyword, music.album.title).quick_ratio() if music.album is not None else 0) n += 1 music.similarity = similarity / n elif tag: @@ -322,32 +394,50 @@ def jump_or_scrape(request, url): if this_site in url: return redirect(url) - # match url to registerd sites - matched_host = None - for host in scraper_registry: - if host in url: - matched_host = host - break - - if matched_host is None: + url = get_normalized_url(url) + scraper = get_scraper_by_url(url) + if scraper is None: # invalid url - return render(request, 'common/error.html', {'msg': _("链接非法，查询失败")}) + return render(request, 'common/error.html', {'msg': _("链接无效，查询失败")}) else: - scraper = scraper_registry[matched_host] + try: + effective_url = scraper.get_effective_url(url) + except ValueError: + return render(request, 'common/error.html', {'msg': _("链接无效，查询失败")}) try: # raise ObjectDoesNotExist - effective_url = scraper.get_effective_url(url) entity = scraper.data_class.objects.get(source_url=effective_url) # if exists then jump to detail page + if request.path.endswith('.json/'): + return JsonResponse({ + 'num_pages': 1, + 'items': [entity.get_json()] + }) return redirect(entity) except ObjectDoesNotExist: # scrape if not exists try: scraper.scrape(url) form = scraper.save(request_user=request.user) + except IntegrityError as ie: # duplicate key on source_url may be caused by user's double submission + try: + entity = scraper.data_class.objects.get(source_url=effective_url) + return redirect(entity) + except Exception as e: + logger.error(f"Scrape Failed URL: {url}\n{e}") + if settings.DEBUG: + logger.error("Expections during saving scraped data:", exc_info=e) + return render(request, 'common/error.html', {'msg': _("爬取数据失败😫")}) except Exception as e: - logger.error(f"Scrape Failed URL: {url}") - logger.error("Expections during saving scraped data:", exc_info=e) + logger.error(f"Scrape Failed URL: {url}\n{e}") + if settings.DEBUG: + logger.error("Expections during saving scraped data:", exc_info=e) return render(request, 'common/error.html', {'msg': _("爬取数据失败😫")}) return redirect(form.instance) + +def go_relogin(request): + return render(request, 'common/error.html', { + 'url': reverse("users:connect") + '?domain=' + request.user.mastodon_site, + 'msg': _("信息已保存，但是未能分享到联邦网络"), + 'secondary_msg': _("可能是你在联邦网络(Mastodon/Pleroma/...)的登录状态过期了，正在跳转到联邦网络重新登录😼")}) diff --git a/doc/GUIDE.md b/doc/GUIDE.md new file mode 100644 index 00000000..055ca2bb --- /dev/null +++ b/doc/GUIDE.md @@ -0,0 +1,106 @@ +NiceDB / NeoDB - Getting Start +============================== +This is a very basic guide with limited detail, contributions welcomed + +Install +------- +Install PostgreSQL, Redis and Python if not yet + +Setup database +``` +CREATE DATABASE neodb ENCODING 'UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8' TEMPLATE template0; +\c neodb; +CREATE EXTENSION hstore WITH SCHEMA public; +CREATE ROLE neodb with LOGIN ENCRYPTED PASSWORD 'abadface'; +GRANT ALL ON DATABASE neodb TO neodb; +``` + +Create and edit your own configuration file (optional but very much recommended) +``` +mkdir mysite && cp boofilsic/settings.py mysite/ +export DJANGO_SETTINGS_MODULE=mysite.settings +``` + +Create and use `venv` as you normally would, then install packages +``` +python3 -m pip install -r requirements.txt +``` + +Quick check +``` +python3 manage.py check +``` + +Initialize database +``` +python3 manage.py makemigrations users books movies games music sync mastodon management collection +python3 manage.py migrate users +python3 manage.py migrate +``` + +Build static assets +``` +python3 manage.py collectstatic +``` + + +Start services +-------------- +Make sure PostgreSQL and Redis are running + +Start job queue server +``` +export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES # required and only for macOS, otherwise it may crash +python3 manage.py rqworker --with-scheduler doufen export mastodon +``` + +Run web server in dev mode +``` +python3 manage.py runserver 0.0.0.0:80 +``` + +It should be ready to serve from here, to run web server for production, consider `gunicorn -w 8 boofilsic.wsgi` in systemd or sth similar + + +Migrate from an earlier version +------------------------------- +Update database +``` +python3 manage.py makemigrations +python3 manage.py migrate +``` + +Rebuild static assets +``` +python3 manage.py sass common/static/sass/boofilsic.sass common/static/css/boofilsic.min.css -t compressed +python3 manage.py sass common/static/sass/boofilsic.sass common/static/css/boofilsic.css +python3 manage.py collectstatic +``` + +Add Cron Jobs +------------- +add `python manage.py refresh_mastodon` to crontab to run hourly, it will refresh cached users' follow/mute/block from mastodon + +Index and Search +---------------- +Install TypeSense or Meilisearch, change `SEARCH_BACKEND` and coniguration for search server in `settings.py` + +Build initial index, it may take a few minutes or hours +``` +python3 manage.py init_index +python3 manage.py reindex +``` + +Other maintenance tasks +----------------------- +Requeue failed jobs +``` +rq requeue --all --queue doufen +``` + +Run in Docker +``` +docker-compose build +docker-compose up db && docker exec -it app_db_1 psql -U postgres postgres -c 'CREATE EXTENSION hstore WITH SCHEMA public;' # first time only +docker-compose up +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..86f8c09f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3' + +services: + redis: + image: redis:alpine + + db: + image: postgres:14-alpine + volumes: + - /tmp/data/db:/var/lib/postgresql/data + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + environment: + - DB_HOST=db + - DB_NAME=postgres + - DB_USER=postgres + - DB_PASSWORD=postgres + - REDIS_HOST=redis + - DJANGO_SETTINGS_MODULE=neodb.dev + depends_on: + - db + - redis diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..5feb0354 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +python manage.py collectstatic --noinput +python manage.py makemigrations users books movies games music sync mastodon management collection +python manage.py makemigrations +python manage.py migrate users +python manage.py migrate + +exec "$@" diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 00000000..a54a76c7 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +cd /app + +if [ $# -eq 0 ]; then + echo "Usage: start.sh " + exit 1 +fi + +PROCESS_TYPE=$1 + +if [ "$PROCESS_TYPE" = "server" ]; then + if [ "$DJANGO_DEBUG" = "true" ]; then + gunicorn \ + --reload \ + --bind 0.0.0.0:8000 \ + --workers 2 \ + --worker-class eventlet \ + --log-level DEBUG \ + --access-logfile "-" \ + --error-logfile "-" \ + boofilsic.wsgi + else + gunicorn \ + --bind 0.0.0.0:8000 \ + --workers 2 \ + --worker-class eventlet \ + --log-level DEBUG \ + --access-logfile "-" \ + --error-logfile "-" \ + boofilsic.wsgi + fi +elif [ "$PROCESS_TYPE" = "rq" ]; then + rqworker --with-scheduler doufen export mastodon +fi + diff --git a/games/admin.py b/games/admin.py index fe72bd9a..36cc44fa 100644 --- a/games/admin.py +++ b/games/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin from .models import * +from simple_history.admin import SimpleHistoryAdmin -admin.site.register(Game) +admin.site.register(Game, SimpleHistoryAdmin) admin.site.register(GameMark) admin.site.register(GameReview) admin.site.register(GameTag) diff --git a/games/apps.py b/games/apps.py index b74f62c9..a204c094 100644 --- a/games/apps.py +++ b/games/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class GamesConfig(AppConfig): name = 'games' + + def ready(self): + from common.index import Indexer + from .models import Game + Indexer.update_model_indexable(Game) diff --git a/games/forms.py b/games/forms.py index 5aa53330..e758f9fd 100644 --- a/games/forms.py +++ b/games/forms.py @@ -1,28 +1,24 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField from django.utils.translation import gettext_lazy as _ -from .models import Game, GameMark, GameReview +from .models import Game, GameMark, GameReview, GameMarkStatusTranslation from common.models import MarkStatusEnum from common.forms import * def GameMarkStatusTranslator(status): - trans_dict = { - MarkStatusEnum.DO.value: _("在玩"), - MarkStatusEnum.WISH.value: _("想玩"), - MarkStatusEnum.COLLECT.value: _("玩过") - } - return trans_dict[status] + return GameMarkStatusTranslation[status] class GameForm(forms.ModelForm): - # id = forms.IntegerField(required=False, widget=forms.HiddenInput()) + id = forms.IntegerField(required=False, widget=forms.HiddenInput()) other_info = JSONField(required=False, label=_("其他信息")) class Meta: model = Game fields = [ + 'id', 'title', 'source_site', 'source_url', @@ -66,11 +62,8 @@ class GameMarkForm(MarkForm): 'status', 'rating', 'text', - 'is_private', + 'visibility', ] - labels = { - 'rating': _("评分"), - } widgets = { 'game': forms.TextInput(attrs={"hidden": ""}), } @@ -85,14 +78,8 @@ class GameReviewForm(ReviewForm): 'game', 'title', 'content', - 'is_private' + 'visibility' ] - labels = { - 'book': "", - 'title': _("标题"), - 'content': _("正文"), - 'share_to_mastodon': _("分享到长毛象") - } widgets = { 'game': forms.TextInput(attrs={"hidden": ""}), } diff --git a/games/models.py b/games/models.py index a2614f0a..fba9e639 100644 --- a/games/models.py +++ b/games/models.py @@ -1,17 +1,25 @@ import uuid import django.contrib.postgres.fields as postgres -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.db import models from django.core.serializers.json import DjangoJSONEncoder from django.shortcuts import reverse -from common.models import Entity, Mark, Review, Tag +from common.models import Entity, Mark, Review, Tag, MarkStatusEnum from common.utils import ChoicesDictGenerator, GenerateDateUUIDMediaFilePath -from boofilsic.settings import GAME_MEDIA_PATH_ROOT, DEFAULT_GAME_IMAGE from django.utils import timezone +from django.conf import settings +from simple_history.models import HistoricalRecords + + +GameMarkStatusTranslation = { + MarkStatusEnum.DO.value: _("在玩"), + MarkStatusEnum.WISH.value: _("想玩"), + MarkStatusEnum.COLLECT.value: _("玩过") +} def game_cover_path(instance, filename): - return GenerateDateUUIDMediaFilePath(instance, filename, GAME_MEDIA_PATH_ROOT) + return GenerateDateUUIDMediaFilePath(instance, filename, settings.GAME_MEDIA_PATH_ROOT) class Game(Entity): @@ -53,7 +61,7 @@ class Game(Entity): ) genre = postgres.ArrayField( - models.CharField(blank=True, default='', max_length=50), + models.CharField(blank=True, default='', max_length=200), null=True, blank=True, default=list, @@ -61,23 +69,39 @@ class Game(Entity): ) platform = postgres.ArrayField( - models.CharField(blank=True, default='', max_length=50), + models.CharField(blank=True, default='', max_length=200), null=True, blank=True, default=list, verbose_name=_("平台") ) - cover = models.ImageField(_("封面"), upload_to=game_cover_path, default=DEFAULT_GAME_IMAGE, blank=True) - + cover = models.ImageField(_("封面"), upload_to=game_cover_path, default=settings.DEFAULT_GAME_IMAGE, blank=True) + history = HistoricalRecords() def __str__(self): return self.title + def get_json(self): + r = { + 'developer': self.developer, + 'other_title': self.other_title, + 'publisher': self.publisher, + 'release_date': self.release_date, + 'platform': self.platform, + 'genre': self.genre, + } + r.update(super().get_json()) + return r + def get_absolute_url(self): return reverse("games:retrieve", args=[self.id]) + @property + def wish_url(self): + return reverse("games:wish", args=[self.id]) + def get_tags_manager(self): return self.game_tags @@ -85,6 +109,14 @@ class Game(Entity): def verbose_category_name(self): return _("游戏") + @property + def mark_class(self): + return GameMark + + @property + def tag_class(self): + return GameTag + class GameMark(Mark): game = models.ForeignKey( @@ -96,6 +128,10 @@ class GameMark(Mark): fields=['owner', 'game'], name='unique_game_mark') ] + @property + def translated_status(self): + return GameMarkStatusTranslation[self.status] + class GameReview(Review): game = models.ForeignKey( @@ -107,6 +143,14 @@ class GameReview(Review): fields=['owner', 'game'], name='unique_game_review') ] + @property + def url(self): + return settings.APP_WEBSITE + reverse("games:retrieve_review", args=[self.id]) + + @property + def item(self): + return self.game + class GameTag(Tag): game = models.ForeignKey( @@ -119,3 +163,7 @@ class GameTag(Tag): models.UniqueConstraint( fields=['content', 'mark'], name="unique_gamemark_tag") ] + + @property + def item(self): + return self.game diff --git a/games/templates/games/create_update.html b/games/templates/games/create_update.html index 40a768e2..178089c4 100644 --- a/games/templates/games/create_update.html +++ b/games/templates/games/create_update.html @@ -10,8 +10,8 @@ - {% trans 'NiceDB - ' %}{{ title }} - + {{ site_name }} - {{ title }} + @@ -22,9 +22,24 @@
    + {% if is_update and form.source_site.value != 'in-site' %} +
    +
    +
    +
    {% trans '源网站' %}: {{ form.source_site.value }}
    +
    +
    + {% csrf_token %} + +
    +
    +
    +
    +
    + {% endif %} +
    - {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
    {% csrf_token %} {{ form.media }} @@ -55,12 +70,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -94,7 +94,7 @@
    - {{ form.is_private.label }}{{ form.is_private }} + {{ form.visibility.label }}{{ form.visibility }}
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除电影/剧集' %} + @@ -55,7 +55,7 @@ {% if game.last_editor %}
    {% trans '最近编辑者：' %} - + {{ game.last_editor | default:"" }}
    @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除评论' %} + @@ -35,7 +35,7 @@
    {{ review.title }}
    - {% if review.is_private %} + {% if review.visibility > 0 %}
    - {{ review.owner.username }} {% if mark %} @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {% include "partial/_common_libs.html" with jquery=1 %} + - - @@ -54,11 +53,12 @@
    - {% if game.rating %} + {% if game.rating and game.rating_number >= 5 %} {{ game.rating }} + ({{ game.rating_number }}人评分) {% else %} - {% trans '评分：暂无评分' %} + {% trans '评分：评分人数不足' %} {% endif %}
    @@ -72,7 +72,7 @@ {% if game.other_title|length > 5 %} {% trans '更多' %} + {{ site_name }} - {{ game.title }}{% trans '的标记' %} + @@ -35,37 +35,7 @@
    {{ game.title }}{% trans ' 的标记' %}
    -
      - - {% for mark in marks %} - -
    • - {{ mark.owner.username }} - {{ mark.get_status_display }} - {% if mark.rating %} - - {% endif %} - {% if mark.is_private %} - - - - {% endif %} - {{ mark.edited_time }} - {% if mark.text %} -

      {{ mark.text }}

      - {% endif %} -
      • - - {% empty %} -
      - {% trans '无结果' %} -
      - {% endfor %} - -
    + {% include "partial/mark_list.html" with mark_list=marks current_item=game %}
    - {% comment %} - - - - - {% endcomment %} + + {{ site_name }}游戏评论 - {{ review.title }} + + @@ -39,7 +40,7 @@
    {{ review.title }}
    - {% if review.is_private %} + {% if review.visibility > 0 %} @@ -48,7 +49,7 @@
    - {{ review.owner.username }} + {{ review.owner.username }} {% if mark %} @@ -73,6 +74,7 @@ {{ form.content }}
    {{ form.media }} + {% csrf_token %}
    @@ -134,16 +136,8 @@
    - {% comment %} - - - - - {% endcomment %} diff --git a/games/templates/games/review_list.html b/games/templates/games/review_list.html index 10b01d43..8b05318d 100644 --- a/games/templates/games/review_list.html +++ b/games/templates/games/review_list.html @@ -14,8 +14,8 @@ - {% trans 'NiceDB - ' %}{{ game.title }}{% trans '的评论' %} - + {{ site_name }} - {{ game.title }}{% trans '的评论' %} + @@ -41,8 +41,8 @@
  • - {{ review.owner.username }} - {% if review.is_private %} + {{ review.owner.username }} + {% if review.visibility > 0 %} {% endif %} {{ review.edited_time }} @@ -137,12 +137,6 @@ - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '从豆瓣获取数据' %} + diff --git a/games/urls.py b/games/urls.py index ff984632..88ce1629 100644 --- a/games/urls.py +++ b/games/urls.py @@ -1,4 +1,4 @@ -from django.urls import path +from django.urls import path, re_path from .views import * @@ -8,9 +8,10 @@ urlpatterns = [ path('/', retrieve, name='retrieve'), path('update//', update, name='update'), path('delete//', delete, name='delete'), + path('rescrape//', rescrape, name='rescrape'), path('mark/', create_update_mark, name='create_update_mark'), - path('/mark/list/', - retrieve_mark_list, name='retrieve_mark_list'), + path('wish//', wish, name='wish'), + re_path('(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_mark_list, name='retrieve_mark_list'), path('mark/delete//', delete_mark, name='delete_mark'), path('/review/create/', create_review, name='create_review'), path('review/update//', update_review, name='update_review'), diff --git a/games/views.py b/games/views.py index f69e4918..0f0fc166 100644 --- a/games/views.py +++ b/games/views.py @@ -2,21 +2,23 @@ import logging from django.shortcuts import render, get_object_or_404, redirect, reverse from django.contrib.auth.decorators import login_required, permission_required from django.utils.translation import gettext_lazy as _ -from django.http import HttpResponseBadRequest, HttpResponseServerError +from django.http import HttpResponseBadRequest, HttpResponseServerError, HttpResponse from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import IntegrityError, transaction from django.db.models import Count from django.utils import timezone from django.core.paginator import Paginator from mastodon import mastodon_request_included -from mastodon.api import check_visibility, post_toot, TootVisibilityEnum -from mastodon.utils import rating_to_emoji +from mastodon.models import MastodonApplication +from mastodon.api import share_mark, share_review from common.utils import PageLinksGenerator -from common.views import PAGE_LINK_NUMBER, jump_or_scrape +from common.views import PAGE_LINK_NUMBER, jump_or_scrape, go_relogin from common.models import SourceSiteEnum from .models import * from .forms import * -from boofilsic.settings import MASTODON_TAGS +from django.conf import settings +from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -87,6 +89,18 @@ def create(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Game, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("games:retrieve", args=[form.instance.id])) + + @login_required def update(request, id): if request.method == 'GET': @@ -98,6 +112,7 @@ def update(request, id): 'games/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("games:update", args=[game.id]), # provided for frontend js @@ -127,6 +142,7 @@ def update(request, id): 'games/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("games:update", args=[game.id]), # provided for frontend js @@ -167,6 +183,7 @@ def retrieve(request, id): else: mark_form = GameMarkForm(initial={ 'game': game, + 'visibility': request.user.get_preference().default_visibility if request.user.is_authenticated else 0, 'tags': mark_tags }) @@ -186,10 +203,8 @@ def retrieve(request, id): mark_list_more = None review_list_more = None else: - mark_list = GameMark.get_available( - game, request.user, request.session['oauth_token']) - review_list = GameReview.get_available( - game, request.user, request.session['oauth_token']) + mark_list = GameMark.get_available(game, request.user) + review_list = GameReview.get_available(game, request.user) mark_list_more = True if len(mark_list) > MARK_NUMBER else False mark_list = mark_list[:MARK_NUMBER] for m in mark_list: @@ -197,6 +212,7 @@ def retrieve(request, id): review_list_more = True if len( review_list) > REVIEW_NUMBER else False review_list = review_list[:REVIEW_NUMBER] + collection_list = filter(lambda c: c.is_visible_to(request.user), map(lambda i: i.collection, CollectionItem.objects.filter(game=game))) # def strip_html_tags(text): # import re @@ -221,6 +237,7 @@ def retrieve(request, id): 'review_list_more': review_list_more, 'game_tag_list': game_tag_list, 'mark_tags': mark_tags, + 'collection_list': collection_list, } ) else: @@ -265,12 +282,19 @@ def create_update_mark(request): pk = request.POST.get('id') old_rating = None old_tags = None + if not pk: + game_id = request.POST.get('game') + mark = GameMark.objects.filter(game_id=game_id, owner=request.user).first() + if mark: + pk = mark.id if pk: mark = get_object_or_404(GameMark, pk=pk) if request.user != mark.owner: return HttpResponseBadRequest() old_rating = mark.rating old_tags = mark.gamemark_tags.all() + if mark.status != request.POST.get('status'): + mark.created_time = timezone.now() # update form = GameMarkForm(request.POST, instance=mark) else: @@ -278,7 +302,7 @@ def create_update_mark(request): form = GameMarkForm(request.POST) if form.is_valid(): - if form.instance.status == MarkStatusEnum.WISH.value: + if form.instance.status == MarkStatusEnum.WISH.value or form.instance.rating == 0: form.instance.rating = None form.cleaned_data['rating'] = None form.instance.owner = request.user @@ -306,28 +330,10 @@ def create_update_mark(request): return HttpResponseServerError("integrity error") if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("games:retrieve", - args=[game.id]) - words = GameMarkStatusTranslator(form.cleaned_data['status']) +\ - f"《{game.title}》" + \ - rating_to_emoji(form.cleaned_data['rating']) - - # tags = MASTODON_TAGS % {'category': '书', 'type': '标记'} - tags = '' - content = words + '\n' + url + '\n' + \ - form.cleaned_data['text'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_mark(form.instance): + return go_relogin(request) else: - return HttpResponseBadRequest("invalid form data") + return HttpResponseBadRequest(f"invalid form data {form.errors}") return redirect(reverse("games:retrieve", args=[form.instance.game.id])) else: @@ -336,11 +342,30 @@ def create_update_mark(request): @mastodon_request_included @login_required -def retrieve_mark_list(request, game_id): +def wish(request, id): + if request.method == 'POST': + game = get_object_or_404(Game, pk=id) + params = { + 'owner': request.user, + 'status': MarkStatusEnum.WISH, + 'visibility': 0, + 'game': game, + } + try: + GameMark.objects.create(**params) + except Exception: + pass + return HttpResponse("✔️") + else: + return HttpResponseBadRequest("invalid method") + + +@mastodon_request_included +@login_required +def retrieve_mark_list(request, game_id, following_only=False): if request.method == 'GET': game = get_object_or_404(Game, pk=game_id) - queryset = GameMark.get_available( - game, request.user, request.session['oauth_token']) + queryset = GameMark.get_available(game, request.user, following_only=following_only) paginator = Paginator(queryset, MARK_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) @@ -401,23 +426,8 @@ def create_review(request, game_id): form.instance.owner = request.user form.save() if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("games:retrieve_review", - args=[form.instance.id]) - words = "发布了关于" + f"《{form.instance.game.title}》" + "的评论" - # tags = MASTODON_TAGS % {'category': '书', 'type': '评论'} - tags = '' - content = words + '\n' + url + \ - '\n' + form.cleaned_data['title'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_review(form.instance): + return go_relogin(request) return redirect(reverse("games:retrieve_review", args=[form.instance.id])) else: return HttpResponseBadRequest() @@ -453,23 +463,8 @@ def update_review(request, id): form.instance.edited_time = timezone.now() form.save() if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("games:retrieve_review", - args=[form.instance.id]) - words = "发布了关于" + f"《{form.instance.game.title}》" + "的评论" - # tags = MASTODON_TAGS % {'category': '书', 'type': '评论'} - tags = '' - content = words + '\n' + url + \ - '\n' + form.cleaned_data['title'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_review(form.instance): + return go_relogin(request) return redirect(reverse("games:retrieve_review", args=[form.instance.id])) else: return HttpResponseBadRequest() @@ -504,11 +499,10 @@ def delete_review(request, id): @mastodon_request_included -@login_required def retrieve_review(request, id): if request.method == 'GET': review = get_object_or_404(GameReview, pk=id) - if not check_visibility(review, request.session['oauth_token'], request.user): + if not review.is_visible_to(request.user): msg = _("你没有访问这个页面的权限😥") return render( request, @@ -543,8 +537,7 @@ def retrieve_review(request, id): def retrieve_review_list(request, game_id): if request.method == 'GET': game = get_object_or_404(Game, pk=game_id) - queryset = GameReview.get_available( - game, request.user, request.session['oauth_token']) + queryset = GameReview.get_available(game, request.user) paginator = Paginator(queryset, REVIEW_PER_PAGE) page_number = request.GET.get('page', default=1) reviews = paginator.get_page(page_number) diff --git a/management/models.py b/management/models.py index 9816781e..c5bf2f64 100644 --- a/management/models.py +++ b/management/models.py @@ -1,7 +1,7 @@ import re from django.db import models from django.shortcuts import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from markdownx.models import MarkdownxField from markdown import markdown diff --git a/management/templates/management/create_update.html b/management/templates/management/create_update.html index 9ffd568c..692dc126 100644 --- a/management/templates/management/create_update.html +++ b/management/templates/management/create_update.html @@ -4,8 +4,8 @@ - - + + Create/Update Announcement diff --git a/management/templates/management/delete.html b/management/templates/management/delete.html index 11f08209..d48b740d 100644 --- a/management/templates/management/delete.html +++ b/management/templates/management/delete.html @@ -4,8 +4,8 @@ - - + + Delete Announcement diff --git a/management/templates/management/detail.html b/management/templates/management/detail.html index a7cf64e5..76150b1d 100644 --- a/management/templates/management/detail.html +++ b/management/templates/management/detail.html @@ -6,9 +6,9 @@ - + - NiceDB - {{ object.title }} + {{ site_name }} - {{ object.title }} diff --git a/management/templates/management/list.html b/management/templates/management/list.html index c8199e1f..ab5a2b24 100644 --- a/management/templates/management/list.html +++ b/management/templates/management/list.html @@ -7,10 +7,10 @@ - + - {% trans 'NiceDB - 公告栏' %} + {{ site_name }} - {% trans '公告栏' %} diff --git a/mastodon/admin.py b/mastodon/admin.py index b29c2a3c..e81bc74b 100644 --- a/mastodon/admin.py +++ b/mastodon/admin.py @@ -38,7 +38,7 @@ class MastodonApplicationModelAdmin(admin.ModelAdmin): try: response = create_app(request.POST.get('domain_name')) except (Timeout, ConnectionError): - request.POST['domain_name'] = _("长毛象请求超时。") + request.POST['domain_name'] = _("联邦网络请求超时。") except Exception as e: request.POST['domain_name'] = str(e) else: diff --git a/mastodon/api.py b/mastodon/api.py index 5e48c944..1919f099 100644 --- a/mastodon/api.py +++ b/mastodon/api.py @@ -2,10 +2,16 @@ import requests import string import random import functools +import logging from django.core.exceptions import ObjectDoesNotExist -from boofilsic.settings import MASTODON_TIMEOUT -from boofilsic.settings import CLIENT_NAME, APP_WEBSITE, REDIRECT_URIS -from .models import CrossSiteUserInfo +from django.conf import settings +from django.shortcuts import reverse +from urllib.parse import quote +from .models import CrossSiteUserInfo, MastodonApplication +from mastodon.utils import rating_to_emoji + + +logger = logging.getLogger(__name__) # See https://docs.joinmastodon.org/methods/accounts/ @@ -46,39 +52,76 @@ API_CREATE_APP = '/api/v1/apps' # GET API_SEARCH = '/api/v2/search' +TWITTER_DOMAIN = 'twitter.com' -get = functools.partial(requests.get, timeout=MASTODON_TIMEOUT) -post = functools.partial(requests.post, timeout=MASTODON_TIMEOUT) +TWITTER_API_ME = 'https://api.twitter.com/2/users/me' + +TWITTER_API_POST = 'https://api.twitter.com/2/tweets' + +TWITTER_API_TOKEN = 'https://api.twitter.com/2/oauth2/token' + +USER_AGENT = f"{settings.CLIENT_NAME}/1.0" + +get = functools.partial(requests.get, timeout=settings.MASTODON_TIMEOUT) +post = functools.partial(requests.post, timeout=settings.MASTODON_TIMEOUT) # low level api below -def get_relationships(site, id_list, token): +def get_relationships(site, id_list, token): # no longer in use url = 'https://' + site + API_GET_RELATIONSHIPS payload = {'id[]': id_list} headers = { + 'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}' } - response = get(url, headers=headers, data=payload) + response = get(url, headers=headers, params=payload) return response.json() def post_toot(site, content, visibility, token, local_only=False): - url = 'https://' + site + API_PUBLISH_TOOT headers = { + 'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}', 'Idempotency-Key': random_string_generator(16) } - payload = { - 'status': content, - 'visibility': visibility, - 'local_only': True, - } - if not local_only: - del payload['local_only'] - response = post(url, headers=headers, data=payload) + if site == TWITTER_DOMAIN: + url = TWITTER_API_POST + payload = { + 'text': content if len(content) <= 150 else content[0:150] + '...' + } + response = post(url, headers=headers, json=payload) + else: + url = 'https://' + site + API_PUBLISH_TOOT + payload = { + 'status': content, + 'visibility': visibility, + } + if local_only: + payload['local_only'] = True + try: + response = post(url, headers=headers, data=payload) + if response.status_code == 201: + response.status_code = 200 + if response.status_code != 200: + logger.error(f"Error {url} {response.status_code}") + except Exception: + response = None return response +def get_instance_info(domain_name): + if domain_name.lower().strip() == TWITTER_DOMAIN: + return TWITTER_DOMAIN, '' + try: + url = f'https://{domain_name}/api/v1/instance' + response = get(url, headers={'User-Agent': USER_AGENT}) + j = response.json() + return j['uri'].lower().split('//')[-1].split('/')[0], j['version'] + except Exception: + logger.error(f"Error {url}") + return domain_name, '' + + def create_app(domain_name): # naive protocal strip is_http = False @@ -96,18 +139,13 @@ def create_app(domain_name): url = 'http://' + domain_name + API_CREATE_APP payload = { - 'client_name': CLIENT_NAME, - 'scopes': 'read write follow', - 'redirect_uris': REDIRECT_URIS, - 'website': APP_WEBSITE + 'client_name': settings.CLIENT_NAME, + 'scopes': settings.MASTODON_CLIENT_SCOPE, + 'redirect_uris': settings.REDIRECT_URIS, + 'website': settings.APP_WEBSITE } - from boofilsic.settings import DEBUG - if DEBUG: - payload['redirect_uris'] = 'http://localhost/users/OAuth2_login/\nurn:ietf:wg:oauth:2.0:oob' - payload['client_name'] = 'test_do_not_authorise' - - response = post(url, data=payload) + response = post(url, data=payload, headers={'User-Agent': USER_AGENT}) return response @@ -116,26 +154,35 @@ def get_site_id(username, user_site, target_site, token): payload = { 'limit': 1, 'type': 'accounts', + 'resolve': True, 'q': f"{username}@{user_site}" } headers = { + 'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}' } - response = get(url, data=payload, headers=headers) - data = response.json() - if not data['accounts']: + response = get(url, params=payload, headers=headers) + try: + data = response.json() + except Exception: + logger.error(f"Error parsing JSON from {url}") + return None + if 'accounts' not in data: + return None + elif len(data['accounts']) == 0: # target site may return empty if no cache of this user + return None + elif data['accounts'][0]['acct'] != f"{username}@{user_site}": # or return another user with a similar id which needs to be skipped return None else: return data['accounts'][0]['id'] # high level api below -def get_relationship(request_user, target_user, token): - if request_user.mastodon_site == target_user.mastodon_site: - return get_relationships(request_user.mastodon_site, target_user.mastodon_id, token) - else: - cross_site_id = get_cross_site_id(target_user, request_user.mastodon_site, token) - return get_relationships(request_user.mastodon_site, [cross_site_id,], token) +def get_relationship(request_user, target_user, useless_token=None): + return [{ + 'blocked_by': target_user.is_blocking(request_user), + 'following': request_user.is_following(target_user), + }] def get_cross_site_id(target_user, target_site, token): @@ -147,6 +194,8 @@ def get_cross_site_id(target_user, target_site, token): """ if target_site == target_user.mastodon_site: return target_user.mastodon_id + if target_site == TWITTER_DOMAIN: + return None try: cross_site_info = CrossSiteUserInfo.objects.get( @@ -157,6 +206,7 @@ def get_cross_site_id(target_user, target_site, token): cross_site_id = get_site_id( target_user.username, target_user.mastodon_site, target_site, token) if not cross_site_id: + logger.error(f'unable to find cross_site_id for {target_user} on {target_site}') return None cross_site_info = CrossSiteUserInfo.objects.create( uid=f"{target_user.username}@{target_user.mastodon_site}", @@ -167,30 +217,251 @@ def get_cross_site_id(target_user, target_site, token): return cross_site_info.site_id -def check_visibility(user_owned_entity, token, visitor): - """ - check if given user can see the user owned entity - """ - if not visitor == user_owned_entity.owner: - # mastodon request - relationship = get_relationship(visitor, user_owned_entity.owner, token)[0] - if relationship['blocked_by']: - return False - if not relationship['following'] and user_owned_entity.is_private: - return False - return True - else: - return True - - # utils below def random_string_generator(n): s = string.ascii_letters + string.punctuation + string.digits return ''.join(random.choice(s) for i in range(n)) +def verify_account(site, token): + if site == TWITTER_DOMAIN: + url = TWITTER_API_ME + '?user.fields=id,username,name,description,profile_image_url,created_at,protected' + try: + response = get(url, headers={'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}'}) + if response.status_code != 200: + logger.error(f"Error {url} {response.status_code}") + return response.status_code, None + r = response.json()['data'] + r['display_name'] = r['name'] + r['note'] = r['description'] + r['avatar'] = r['profile_image_url'] + r['avatar_static'] = r['profile_image_url'] + r['locked'] = r['protected'] + r['url'] = f'https://{TWITTER_DOMAIN}/{r["username"]}' + return 200, r + except Exception: + return -1, None + url = 'https://' + site + API_VERIFY_ACCOUNT + try: + response = get(url, headers={'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}'}) + return response.status_code, (response.json() if response.status_code == 200 else None) + except Exception: + return -1, None + + +def get_related_acct_list(site, token, api): + if site == TWITTER_DOMAIN: + return [] + url = 'https://' + site + api + results = [] + while url: + response = get(url, headers={'User-Agent': USER_AGENT, 'Authorization': f'Bearer {token}'}) + url = None + if response.status_code == 200: + results.extend(map(lambda u: (u['acct'] if u['acct'].find('@') != -1 else u['acct'] + '@' + site) if 'acct' in u else u, response.json())) + if 'Link' in response.headers: + for ls in response.headers['Link'].split(','): + li = ls.strip().split(';') + if li[1].strip() == 'rel="next"': + url = li[0].strip().replace('>', '').replace('<', '') + return results + + class TootVisibilityEnum: PUBLIC = 'public' PRIVATE = 'private' DIRECT = 'direct' UNLISTED = 'unlisted' + + +def get_mastodon_application(domain): + app = MastodonApplication.objects.filter(domain_name=domain).first() + if app is not None: + return app, '' + if domain == TWITTER_DOMAIN: + return None, 'Twitter未配置' + error_msg = None + try: + response = create_app(domain) + except (requests.exceptions.Timeout, ConnectionError): + error_msg = "联邦网络请求超时。" + logger.error(f'Error creating app for {domain}: Timeout') + except Exception as e: + error_msg = "联邦网络请求失败 " + str(e) + logger.error(f'Error creating app for {domain}: {e}') + else: + # fill the form with returned data + if response.status_code != 200: + error_msg = "实例连接错误，代码: " + str(response.status_code) + logger.error(f'Error creating app for {domain}: {response.status_code}') + else: + try: + data = response.json() + except Exception: + error_msg = "实例返回内容无法识别" + logger.error(f'Error creating app for {domain}: unable to parse response') + else: + if settings.MASTODON_ALLOW_ANY_SITE: + app = MastodonApplication.objects.create(domain_name=domain, app_id=data['id'], client_id=data['client_id'], + client_secret=data['client_secret'], vapid_key=data['vapid_key'] if 'vapid_key' in data else '') + else: + error_msg = "不支持其它实例登录" + logger.error(f'Disallowed to create app for {domain}') + return app, error_msg + + +def get_mastodon_login_url(app, login_domain, version, request): + url = request.scheme + "://" + request.get_host() + reverse('users:OAuth2_login') + if login_domain == TWITTER_DOMAIN: + return f"https://twitter.com/i/oauth2/authorize?response_type=code&client_id={app.client_id}&redirect_uri={quote(url)}&scope={quote(settings.TWITTER_CLIENT_SCOPE)}&state=state&code_challenge=challenge&code_challenge_method=plain" + scope = settings.MASTODON_LEGACY_CLIENT_SCOPE if 'Pixelfed' in version else settings.MASTODON_CLIENT_SCOPE + return "https://" + login_domain + "/oauth/authorize?client_id=" + app.client_id + "&scope=" + quote(scope) + "&redirect_uri=" + url + "&response_type=code" + + +def obtain_token(site, request, code): + """ Returns token if success else None. """ + mast_app = MastodonApplication.objects.get(domain_name=site) + redirect_uri = request.scheme + "://" + request.get_host() + reverse('users:OAuth2_login') + payload = { + 'client_id': mast_app.client_id, + 'client_secret': mast_app.client_secret, + 'redirect_uri': redirect_uri, + 'grant_type': 'authorization_code', + 'code': code + } + headers = {'User-Agent': USER_AGENT} + auth = None + if mast_app.is_proxy: + url = 'https://' + mast_app.proxy_to + API_OBTAIN_TOKEN + elif site == TWITTER_DOMAIN: + url = TWITTER_API_TOKEN + auth = (mast_app.client_id, mast_app.client_secret) + del payload['client_secret'] + payload['code_verifier'] = 'challenge' + else: + url = 'https://' + mast_app.domain_name + API_OBTAIN_TOKEN + try: + response = post(url, data=payload, headers=headers, auth=auth) + # {"token_type":"bearer","expires_in":7200,"access_token":"VGpkOEZGR3FQRDJ5NkZ0dmYyYWIwS0dqeHpvTnk4eXp0NV9nWDJ2TEpmM1ZTOjE2NDg3ODMxNTU4Mzc6MToxOmF0OjE","scope":"block.read follows.read offline.access tweet.write users.read mute.read","refresh_token":"b1pXbGEzeUF1WE5yZHJOWmxTeWpvMTBrQmZPd0czLU0tQndZQTUyU3FwRDVIOjE2NDg3ODMxNTU4Mzg6MToxOnJ0OjE"} + if response.status_code != 200: + logger.error(f"Error {url} {response.status_code}") + return None, None + except Exception as e: + logger.error(f"Error {url} {e}") + return None, None + data = response.json() + return data.get('access_token'), data.get('refresh_token', '') + + +def refresh_access_token(site, refresh_token): + if site != TWITTER_DOMAIN: + return None + mast_app = MastodonApplication.objects.get(domain_name=site) + url = TWITTER_API_TOKEN + payload = { + 'client_id': mast_app.client_id, + 'refresh_token': refresh_token, + 'grant_type': 'refresh_token', + } + headers = {'User-Agent': USER_AGENT} + auth = (mast_app.client_id, mast_app.client_secret) + response = post(url, data=payload, headers=headers, auth=auth) + if response.status_code != 200: + logger.error(f"Error {url} {response.status_code}") + return None + data = response.json() + return data.get('access_token') + + +def revoke_token(site, token): + mast_app = MastodonApplication.objects.get(domain_name=site) + + payload = { + 'client_id': mast_app.client_id, + 'client_secret': mast_app.client_secret, + 'token': token + } + + if mast_app.is_proxy: + url = 'https://' + mast_app.proxy_to + API_REVOKE_TOKEN + else: + url = 'https://' + site + API_REVOKE_TOKEN + post(url, data=payload, headers={'User-Agent': USER_AGENT}) + + +def share_mark(mark): + user = mark.owner + if mark.visibility == 2: + visibility = TootVisibilityEnum.DIRECT + elif mark.visibility == 1: + visibility = TootVisibilityEnum.PRIVATE + elif user.get_preference().mastodon_publish_public: + visibility = TootVisibilityEnum.PUBLIC + else: + visibility = TootVisibilityEnum.UNLISTED + tags = '\n' + user.get_preference().mastodon_append_tag.replace('[category]', str(mark.item.verbose_category_name)) if user.get_preference().mastodon_append_tag else '' + stars = rating_to_emoji(mark.rating, MastodonApplication.objects.get(domain_name=user.mastodon_site).star_mode) + content = f"{mark.translated_status}《{mark.item.title}》{stars}\n{mark.item.url}\n{mark.text}{tags}" + response = post_toot(user.mastodon_site, content, visibility, user.mastodon_token) + if response and response.status_code in [200, 201]: + j = response.json() + if 'url' in j: + mark.shared_link = j['url'] + elif 'data' in j: + mark.shared_link = f"https://twitter.com/{user.username}/status/{j['data']['id']}" + if mark.shared_link: + mark.save(update_fields=['shared_link']) + return True + else: + return False + + +def share_review(review): + user = review.owner + if review.visibility == 2: + visibility = TootVisibilityEnum.DIRECT + elif review.visibility == 1: + visibility = TootVisibilityEnum.PRIVATE + elif user.get_preference().mastodon_publish_public: + visibility = TootVisibilityEnum.PUBLIC + else: + visibility = TootVisibilityEnum.UNLISTED + tags = '\n' + user.get_preference().mastodon_append_tag.replace('[category]', str(review.item.verbose_category_name)) if user.get_preference().mastodon_append_tag else '' + content = f"发布了关于《{review.item.title}》的评论\n{review.url}\n{review.title}{tags}" + response = post_toot(user.mastodon_site, content, visibility, user.mastodon_token) + if response and response.status_code in [200, 201]: + j = response.json() + if 'url' in j: + review.shared_link = j['url'] + elif 'data' in j: + review.shared_link = f"https://twitter.com/{user.username}/status/{j['data']['id']}" + if review.shared_link: + review.save(update_fields=['shared_link']) + return True + else: + return False + + +def share_collection(collection, comment, user, visibility_no): + if visibility_no == 2: + visibility = TootVisibilityEnum.DIRECT + elif visibility_no == 1: + visibility = TootVisibilityEnum.PRIVATE + elif user.get_preference().mastodon_publish_public: + visibility = TootVisibilityEnum.PUBLIC + else: + visibility = TootVisibilityEnum.UNLISTED + tags = '\n' + user.get_preference().mastodon_append_tag.replace('[category]', '收藏单') if user.get_preference().mastodon_append_tag else '' + content = f"分享收藏单《{collection.title}》\n{collection.url}\n{comment}{tags}" + response = post_toot(user.mastodon_site, content, visibility, user.mastodon_token) + if response and response.status_code in [200, 201]: + j = response.json() + if 'url' in j: + shared_link = j['url'] + elif 'data' in j: + shared_link = f"https://twitter.com/{user.username}/status/{j['data']['id']}" + if shared_link: + pass + return True + else: + return False diff --git a/mastodon/auth.py b/mastodon/auth.py index 29a691e8..02d7703f 100644 --- a/mastodon/auth.py +++ b/mastodon/auth.py @@ -1,74 +1,5 @@ from django.contrib.auth.backends import ModelBackend, UserModel -from django.shortcuts import reverse -from .api import * -from .models import MastodonApplication - - -def obtain_token(site, request, code): - """ Returns token if success else None. """ - mast_app = MastodonApplication.objects.get(domain_name=site) - payload = { - 'client_id': mast_app.client_id, - 'client_secret': mast_app.client_secret, - 'redirect_uri': f"https://{request.get_host()}{reverse('users:OAuth2_login')}", - 'grant_type': 'authorization_code', - 'code': code, - 'scope': 'read write' - } - from boofilsic.settings import DEBUG - if DEBUG: - payload['redirect_uri']= f"http://{request.get_host()}{reverse('users:OAuth2_login')}", - if mast_app.is_proxy: - url = 'https://' + mast_app.proxy_to + API_OBTAIN_TOKEN - else: - url = 'https://' + mast_app.domain_name + API_OBTAIN_TOKEN - response = post(url, data=payload) - if response.status_code != 200: - return - data = response.json() - return data.get('access_token') - - -def get_user_data(site, token): - url = 'https://' + site + API_VERIFY_ACCOUNT - headers = { - 'Authorization': f'Bearer {token}' - } - response = get(url, headers=headers) - if response.status_code != 200: - return None - return response.json() - - -def revoke_token(site, token): - mast_app = MastodonApplication.objects.get(domain_name=site) - - payload = { - 'client_id': mast_app.client_id, - 'client_secret': mast_app.client_secret, - 'scope': token - } - - if mast_app.is_proxy: - url = 'https://' + mast_app.proxy_to + API_REVOKE_TOKEN - else: - url = 'https://' + site + API_REVOKE_TOKEN - response = post(url, data=payload) - - -def verify_token(site, token): - """ Check if the token is valid and is of local instance. """ - url = 'https://' + site + API_VERIFY_ACCOUNT - headers = { - 'Authorization': f'Bearer {token}' - } - response = get(url, headers=headers) - if response.status_code == 200: - res_data = response.json() - # check if is local instance user - if res_data['acct'] == res_data['username']: - return True - return False +from .api import verify_account class OAuth2Backend(ModelBackend): @@ -76,22 +7,23 @@ class OAuth2Backend(ModelBackend): # "authenticate() should check the credentials it gets and returns # a user object that matches those credentials." # arg request is an interface specification, not used in this implementation - def authenticate(self, request, token=None, username=None, site=None, **kwargs): + + def authenticate(self, request, token=None, username=None, site=None, **kwargs): """ when username is provided, assume that token is newly obtained and valid """ if token is None or site is None: return if username is None: - user_data = get_user_data(site, token) - if user_data: - username = user_data['username'] + code, user_data = verify_account(site, token) + if code == 200: + userid = user_data['id'] else: # aquiring user data fail means token is invalid thus auth fail return None # when username is provided, assume that token is newly obtained and valid try: - user = UserModel._default_manager.get_by_natural_key(user_data['username']) + user = UserModel._default_manager.get(mastodon_id=userid, mastodon_site=site) except UserModel.DoesNotExist: return None else: diff --git a/mastodon/decorators.py b/mastodon/decorators.py index 3f2ec5cd..ef7a31d1 100644 --- a/mastodon/decorators.py +++ b/mastodon/decorators.py @@ -16,7 +16,7 @@ def mastodon_request_included(func): args[0], 'common/error.html', { - 'msg': _("长毛象请求超时叻_(´ཀ`」 ∠)__ ") + 'msg': _("联邦网络请求超时叻_(´ཀ`」 ∠)__ ") } ) return wrapper diff --git a/mastodon/management/commands/wrong_sites.py b/mastodon/management/commands/wrong_sites.py new file mode 100644 index 00000000..f985a1e7 --- /dev/null +++ b/mastodon/management/commands/wrong_sites.py @@ -0,0 +1,21 @@ +from django.core.management.base import BaseCommand +from mastodon.models import MastodonApplication +from django.conf import settings +from mastodon.api import get_instance_info +from users.models import User + + +class Command(BaseCommand): + help = 'Find wrong sites' + + def handle(self, *args, **options): + for site in MastodonApplication.objects.all(): + d = site.domain_name + login_domain = d.strip().lower().split('//')[-1].split('/')[0].split('@')[-1] + domain, version = get_instance_info(login_domain) + if d != domain: + print(f'{d} should be {domain}') + for u in User.objects.filter(mastodon_site=d, is_active=True): + u.mastodon_site = domain + print(f'fixing {u}') + u.save() diff --git a/mastodon/models.py b/mastodon/models.py index b6ac1207..49346810 100644 --- a/mastodon/models.py +++ b/mastodon/models.py @@ -1,14 +1,16 @@ from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class MastodonApplication(models.Model): domain_name = models.CharField(_('site domain name'), max_length=100, unique=True) - app_id = models.PositiveIntegerField(_('in-site app id')) + app_id = models.PositiveIntegerField(_('in-site app id')) # TODO Remove? bc 1) it seems useless 2) GoToSocial returns a hash text id client_id = models.CharField(_('client id'), max_length=100) client_secret = models.CharField(_('client secret'), max_length=100) - vapid_key = models.CharField(_('vapid key'), max_length=200) + vapid_key = models.CharField(_('vapid key'), max_length=200, null=True, blank=True) + star_mode = models.PositiveIntegerField(_('0: custom emoji; 1: unicode moon; 2: text'), blank=False, default=0) + max_status_len = models.PositiveIntegerField(_('max toot len'), blank=False, default=500) is_proxy = models.BooleanField(default=False, blank=True) proxy_to = models.CharField(max_length=100, blank=True, default='') @@ -27,7 +29,7 @@ class CrossSiteUserInfo(models.Model): # target site domain name target_site = models.CharField(_("target site domain name"), max_length=100) # target site id - site_id = models.PositiveIntegerField() + site_id = models.CharField(max_length=100, blank=False) class Meta: constraints = [ diff --git a/mastodon/utils.py b/mastodon/utils.py index 7da7f94e..8bada43a 100644 --- a/mastodon/utils.py +++ b/mastodon/utils.py @@ -1,14 +1,17 @@ -from boofilsic.settings import STAR_EMPTY, STAR_HALF, STAR_SOLID +from django.conf import settings -def rating_to_emoji(score): +def rating_to_emoji(score, star_mode = 0): """ convert score to mastodon star emoji code """ if score is None or score == '' or score == 0: return '' solid_stars = score // 2 half_star = int(bool(score % 2)) empty_stars = 5 - solid_stars if not half_star else 5 - solid_stars - 1 - emoji_code = STAR_SOLID * solid_stars + STAR_HALF * half_star + STAR_EMPTY * empty_stars + if star_mode == 1: + emoji_code = "🌕" * solid_stars + "🌗" * half_star + "🌑" * empty_stars + else: + emoji_code = settings.STAR_SOLID * solid_stars + settings.STAR_HALF * half_star + settings.STAR_EMPTY * empty_stars emoji_code = emoji_code.replace("::", ": :") emoji_code = ' ' + emoji_code + ' ' return emoji_code \ No newline at end of file diff --git a/movies/admin.py b/movies/admin.py index ca4d1b0e..bb2b1a54 100644 --- a/movies/admin.py +++ b/movies/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin from .models import * +from simple_history.admin import SimpleHistoryAdmin -admin.site.register(Movie) +admin.site.register(Movie, SimpleHistoryAdmin) admin.site.register(MovieMark) admin.site.register(MovieReview) admin.site.register(MovieTag) diff --git a/movies/apps.py b/movies/apps.py index bda16f08..3b025da4 100644 --- a/movies/apps.py +++ b/movies/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class MoviesConfig(AppConfig): name = 'movies' + + def ready(self): + from common.index import Indexer + from .models import Movie + Indexer.update_model_indexable(Movie) diff --git a/movies/forms.py b/movies/forms.py index aa320bd5..587d92e9 100644 --- a/movies/forms.py +++ b/movies/forms.py @@ -1,18 +1,13 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField from django.utils.translation import gettext_lazy as _ -from .models import Movie, MovieMark, MovieReview, MovieGenreEnum +from .models import Movie, MovieMark, MovieReview, MovieGenreEnum, MovieMarkStatusTranslation from common.models import MarkStatusEnum from common.forms import * def MovieMarkStatusTranslator(status): - trans_dict = { - MarkStatusEnum.DO.value: _("在看"), - MarkStatusEnum.WISH.value: _("想看"), - MarkStatusEnum.COLLECT.value: _("看过") - } - return trans_dict[status] + return MovieMarkStatusTranslation[status] class MovieForm(forms.ModelForm): @@ -119,11 +114,8 @@ class MovieMarkForm(MarkForm): 'status', 'rating', 'text', - 'is_private', + 'visibility', ] - labels = { - 'rating': _("评分"), - } widgets = { 'movie': forms.TextInput(attrs={"hidden": ""}), } @@ -138,14 +130,8 @@ class MovieReviewForm(ReviewForm): 'movie', 'title', 'content', - 'is_private' + 'visibility' ] - labels = { - 'book': "", - 'title': _("标题"), - 'content': _("正文"), - 'share_to_mastodon': _("分享到长毛象") - } widgets = { 'movie': forms.TextInput(attrs={"hidden": ""}), } diff --git a/movies/management/commands/fix-movie-poster.py b/movies/management/commands/fix-movie-poster.py new file mode 100644 index 00000000..b8a35f85 --- /dev/null +++ b/movies/management/commands/fix-movie-poster.py @@ -0,0 +1,203 @@ +from django.core.management.base import BaseCommand +from django.core.files.uploadedfile import SimpleUploadedFile +from common.scraper import * +from django.conf import settings +from movies.models import Movie +from movies.forms import MovieForm +import requests +import re +import filetype +from lxml import html +from PIL import Image +from io import BytesIO + + +class DoubanPatcherMixin: + @classmethod + def download_page(cls, url, headers): + url = cls.get_effective_url(url) + r = None + error = 'DoubanScrapper: error occured when downloading ' + url + content = None + + def get(url, timeout): + nonlocal r + # print('Douban GET ' + url) + try: + r = requests.get(url, timeout=timeout) + except Exception as e: + r = requests.Response() + r.status_code = f"Exception when GET {url} {e}" + url + # print('Douban CODE ' + str(r.status_code)) + return r + + def check_content(): + nonlocal r, error, content + content = None + if r.status_code == 200: + content = r.content.decode('utf-8') + if content.find('关于豆瓣') == -1: + if content.find('你的 IP 发出') == -1: + error = error + 'Content not authentic' # response is garbage + else: + error = error + 'IP banned' + content = None + elif re.search('不存在[^<]+', content, re.MULTILINE): + content = None + error = error + 'Not found or hidden by Douban' + else: + error = error + str(r.status_code) + + def fix_wayback_links(): + nonlocal content + # fix links + content = re.sub(r'href="http[^"]+http', r'href="http', content) + # https://img9.doubanio.com/view/subject/{l|m|s}/public/s1234.jpg + content = re.sub(r'src="[^"]+/(s\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/subject/m/public/\1"', content) + # https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2681329386.jpg + # https://img9.doubanio.com/view/photo/{l|m|s}/public/p1234.webp + content = re.sub(r'src="[^"]+/(p\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/photo/m/public/\1"', content) + + # Wayback Machine: get latest available + def wayback(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://archive.org/wayback/available?url=' + url, 10) + if r.status_code == 200: + w = r.json() + if w['archived_snapshots'] and w['archived_snapshots']['closest']: + get(w['archived_snapshots']['closest']['url'], 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + # Wayback Machine: guess via CDX API + def wayback_cdx(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://web.archive.org/cdx/search/cdx?url=' + url, 10) + if r.status_code == 200: + dates = re.findall(r'[^\s]+\s+(\d+)\s+[^\s]+\s+[^\s]+\s+\d+\s+[^\s]+\s+\d{5,}', + r.content.decode('utf-8')) + # assume snapshots whose size >9999 contain real content, use the latest one of them + if len(dates) > 0: + get('http://web.archive.org/web/' + dates[-1] + '/' + url, 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + def latest(): + nonlocal r, error, content + if settings.SCRAPESTACK_KEY is None: + error = error + '\nDirect: ' + get(url, 60) + else: + error = error + '\nScraperAPI: ' + get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}', 60) + check_content() + + # wayback_cdx() + # if content is None: + latest() + + if content is None: + logger.error(error) + content = '' + # with open('/tmp/temp.html', 'w', encoding='utf-8') as fp: + # fp.write(content) + return html.fromstring(content) + + @classmethod + def download_image(cls, url, item_url=None): + if url is None: + return None, None + raw_img = None + ext = None + + dl_url = url + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + # raise RuntimeError(f"Douban: download image failed {img_response.status_code} {dl_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + if raw_img is None and settings.SCRAPESTACK_KEY is not None: + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + return raw_img, ext + + +class DoubanMoviePatcher(DoubanPatcherMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'movie.douban.com' + data_class = Movie + form_class = MovieForm + + regex = re.compile(r"https://movie\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + img_url_elem = content.xpath("//img[@rel='v:image']/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + return raw_img, ext + + +class Command(BaseCommand): + help = 'fix cover image' + + def add_arguments(self, parser): + parser.add_argument('threadId', type=int, help='% 8') + + def handle(self, *args, **options): + t = int(options['threadId']) + for m in Movie.objects.filter(cover='movie/default.svg', source_site='douban'): + if m.id % 8 == t: + print(f'Re-fetching {m.source_url}') + try: + raw_img, img_ext = DoubanMoviePatcher.scrape(m.source_url) + if img_ext is not None: + m.cover = SimpleUploadedFile('temp.' + img_ext, raw_img) + m.save() + print(f'Saved {m.source_url}') + else: + print(f'Skipped {m.source_url}') + except Exception as e: + print(e) + # return diff --git a/movies/models.py b/movies/models.py index be6d4ba3..ae3a9990 100644 --- a/movies/models.py +++ b/movies/models.py @@ -1,17 +1,27 @@ import uuid import django.contrib.postgres.fields as postgres -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.db import models from django.core.serializers.json import DjangoJSONEncoder from django.shortcuts import reverse -from common.models import Entity, Mark, Review, Tag +from common.models import Entity, Mark, Review, Tag, MarkStatusEnum from common.utils import ChoicesDictGenerator, GenerateDateUUIDMediaFilePath -from boofilsic.settings import MOVIE_MEDIA_PATH_ROOT, DEFAULT_MOVIE_IMAGE from django.utils import timezone +from django.conf import settings +from django.db.models import Q +import re +from simple_history.models import HistoricalRecords + + +MovieMarkStatusTranslation = { + MarkStatusEnum.DO.value: _("在看"), + MarkStatusEnum.WISH.value: _("想看"), + MarkStatusEnum.COLLECT.value: _("看过") +} def movie_cover_path(instance, filename): - return GenerateDateUUIDMediaFilePath(instance, filename, MOVIE_MEDIA_PATH_ROOT) + return GenerateDateUUIDMediaFilePath(instance, filename, settings.MOVIE_MEDIA_PATH_ROOT) class MovieGenreEnum(models.TextChoices): @@ -47,6 +57,10 @@ class MovieGenreEnum(models.TextChoices): REALITY_TV = 'Reality-TV', _('真人秀') FAMILY = 'Family', _('家庭') TALK_SHOW = 'Talk-Show', _('脱口秀') + NEWS = 'News', _('新闻') + SOAP = 'Soap', _('肥皂剧') + TV_MOVIE = 'TV Movie', _('电视电影') + THEATRE = 'Theatre', _('舞台艺术') OTHER = 'Other', _('其他') @@ -58,13 +72,13 @@ class Movie(Entity): Can either be movie or series. ''' # widely recognized name, usually in Chinese - title = models.CharField(_("title"), max_length=200) + title = models.CharField(_("title"), max_length=500) # original name, for books in foreign language orig_title = models.CharField( - _("original title"), blank=True, default='', max_length=200) + _("original title"), blank=True, default='', max_length=500) other_title = postgres.ArrayField( models.CharField(_("other title"), blank=True, - default='', max_length=300), + default='', max_length=500), null=True, blank=True, default=list, @@ -73,21 +87,21 @@ class Movie(Entity): blank=True, max_length=10, null=False, db_index=True, default='') director = postgres.ArrayField( models.CharField(_("director"), blank=True, - default='', max_length=100), + default='', max_length=200), null=True, blank=True, default=list, ) playwright = postgres.ArrayField( models.CharField(_("playwright"), blank=True, - default='', max_length=100), + default='', max_length=200), null=True, blank=True, default=list, ) actor = postgres.ArrayField( models.CharField(_("actor"), blank=True, - default='', max_length=100), + default='', max_length=200), null=True, blank=True, default=list, @@ -112,7 +126,7 @@ class Movie(Entity): default=list, ) site = models.URLField(_('site url'), blank=True, default='', max_length=200) - + # country or region area = postgres.ArrayField( models.CharField( @@ -140,7 +154,7 @@ class Movie(Entity): year = models.PositiveIntegerField(null=True, blank=True) duration = models.CharField(blank=True, default='', max_length=200) - cover = models.ImageField(_("poster"), upload_to=movie_cover_path, default=DEFAULT_MOVIE_IMAGE, blank=True) + cover = models.ImageField(_("poster"), upload_to=movie_cover_path, default=settings.DEFAULT_MOVIE_IMAGE, blank=True) ############################################ # exclusive fields to series @@ -157,27 +171,67 @@ class Movie(Entity): ############################################ is_series = models.BooleanField(default=False) + history = HistoricalRecords() def __str__(self): if self.year: - return self.title + f"({self.year})" + return self.title + f"({self.year})" else: return self.title + def get_json(self): + r = { + 'other_title': self.other_title, + 'original_title': self.orig_title, + 'director': self.director, + 'playwright': self.playwright, + 'actor': self.actor, + 'release_year': self.year, + 'genre': self.genre, + 'language': self.language, + 'season': self.season, + 'duration': self.duration, + 'imdb_code': self.imdb_code, + } + r.update(super().get_json()) + return r def get_absolute_url(self): return reverse("movies:retrieve", args=[self.id]) + @property + def wish_url(self): + return reverse("movies:wish", args=[self.id]) + def get_tags_manager(self): return self.movie_tags - def get_genre_display(self): translated_genre = [] for g in self.genre: translated_genre.append(MovieGenreTranslator[g]) return translated_genre + def get_related_movies(self): + imdb = 'no match' if self.imdb_code is None or self.imdb_code == '' else self.imdb_code + qs = Q(imdb_code=imdb) + if self.is_series: + prefix = re.sub(r'\d+', '', re.sub(r'\s+第.+季', '', self.title)) + if not prefix: + prefix = self.title + qs = qs | Q(title__startswith=prefix) + qs = qs & ~Q(id=self.id) + return Movie.objects.filter(qs).order_by('season') + + def get_identicals(self): + qs = Q(orig_title=self.title) + if self.imdb_code: + qs = Q(imdb_code=self.imdb_code) + # qs = qs & ~Q(id=self.id) + return Movie.objects.filter(qs) + else: + return [self] # Book.objects.filter(id=self.id) + @property def verbose_category_name(self): if self.is_series: @@ -185,23 +239,45 @@ class Movie(Entity): else: return _("电影") + @property + def mark_class(self): + return MovieMark + + @property + def tag_class(self): + return MovieTag + class MovieMark(Mark): movie = models.ForeignKey(Movie, on_delete=models.CASCADE, related_name='movie_marks', null=True) + class Meta: constraints = [ models.UniqueConstraint(fields=['owner', 'movie'], name='unique_movie_mark') ] + @property + def translated_status(self): + return MovieMarkStatusTranslation[self.status] + class MovieReview(Review): movie = models.ForeignKey(Movie, on_delete=models.CASCADE, related_name='movie_reviews', null=True) + class Meta: constraints = [ models.UniqueConstraint( fields=['owner', 'movie'], name='unique_movie_review') ] + @property + def url(self): + return settings.APP_WEBSITE + reverse("movies:retrieve_review", args=[self.id]) + + @property + def item(self): + return self.movie + class MovieTag(Tag): movie = models.ForeignKey(Movie, on_delete=models.CASCADE, related_name='movie_tags', null=True) @@ -211,3 +287,7 @@ class MovieTag(Tag): models.UniqueConstraint( fields=['content', 'mark'], name="unique_moviemark_tag") ] + + @property + def item(self): + return self.movie diff --git a/movies/templates/movies/create_update.html b/movies/templates/movies/create_update.html index f55c026b..497c9bce 100644 --- a/movies/templates/movies/create_update.html +++ b/movies/templates/movies/create_update.html @@ -10,8 +10,8 @@ - {% trans 'NiceDB - ' %}{{ title }} - + {{ site_name }} - {{ title }} + @@ -21,10 +21,25 @@ {% include "partial/_navbar.html" %}
    -
    +
    + {% if is_update and form.source_site.value != 'in-site' %} +
    +
    +
    +
    {% trans '源网站' %}: {{ form.source_site.value }}
    +
    + + {% csrf_token %} + + +
    +
    +
    +
    + {% endif %} +
    - {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
    {% csrf_token %} {{ form.media }} @@ -53,12 +68,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -61,7 +61,7 @@ {% if movie.director|length > 5 %} {% trans '更多' %} + {{ site_name }} - {% trans '删除电影/剧集' %} + @@ -62,7 +62,7 @@ {% if movie.last_editor %}
    {% trans '最近编辑者：' %} - + {{ movie.last_editor | default:"" }}
    @@ -96,12 +96,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除评论' %} + @@ -35,7 +35,7 @@
    {{ review.title }}
    - {% if review.is_private %} + {% if review.visibility > 0 %}
    - {{ review.owner.username }} {% if mark %} @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {% include "partial/_common_libs.html" with jquery=1 %} + - - - @@ -78,11 +76,12 @@
    - {% if movie.rating %} + {% if movie.rating and movie.rating_number >= 5 %} {{ movie.rating }} + ({{ movie.rating_number }}人评分) {% else %} - {% trans '评分：暂无评分' %} + {% trans '评分：评分人数不足' %} {% endif %}
    {% if movie.imdb_code %} @@ -99,7 +98,7 @@ {% if movie.director|length > 5 %} {% trans '更多' %} + {{ site_name }} - {{ movie.title }}{% trans '的标记' %} + @@ -35,37 +35,7 @@
    {{ movie.title }}{% trans ' 的标记' %}
    -
      - - {% for mark in marks %} - -
    • - {{ mark.owner.username }} - {{ mark.get_status_display }} - {% if mark.rating %} - - {% endif %} - {% if mark.is_private %} - - - - {% endif %} - {{ mark.edited_time }} - {% if mark.text %} -

      {{ mark.text }}

      - {% endif %} -
      • - - {% empty %} -
      - {% trans '无结果' %} -
      - {% endfor %} - -
    + {% include "partial/mark_list.html" with mark_list=marks current_item=movie %}
    @@ -108,7 +110,7 @@ {% if movie.director|length > 5 %} {% trans '更多' %} diff --git a/movies/templates/movies/review_list.html b/movies/templates/movies/review_list.html index ed32cfe0..daedf296 100644 --- a/movies/templates/movies/review_list.html +++ b/movies/templates/movies/review_list.html @@ -14,8 +14,8 @@ - {% trans 'NiceDB - ' %}{{ movie.title }}{% trans '的评论' %} - + {{ site_name }} - {{ movie.title }}{% trans '的评论' %} + @@ -41,11 +41,14 @@
  • - {{ review.owner.username }} - {% if review.is_private %} + {{ review.owner.username }} + {% if review.visibility > 0 %} {% endif %} {{ review.edited_time }} + {% if review.movie != movie %} + {{ review.movie.get_source_site_display }} + {% endif %} {{ review.title }} @@ -116,7 +119,7 @@ {% if movie.director|length > 5 %} {% trans '更多' %} + {{ site_name }} - {% trans '从豆瓣获取数据' %} + diff --git a/movies/urls.py b/movies/urls.py index 9f359e9e..0495a9f0 100644 --- a/movies/urls.py +++ b/movies/urls.py @@ -1,4 +1,4 @@ -from django.urls import path +from django.urls import path, re_path from .views import * @@ -8,8 +8,10 @@ urlpatterns = [ path('/', retrieve, name='retrieve'), path('update//', update, name='update'), path('delete//', delete, name='delete'), + path('rescrape//', rescrape, name='rescrape'), path('mark/', create_update_mark, name='create_update_mark'), - path('/mark/list/', retrieve_mark_list, name='retrieve_mark_list'), + path('wish//', wish, name='wish'), + re_path('(?P[0-9]+)/mark/list/(?:(?P\\d+))?', retrieve_mark_list, name='retrieve_mark_list'), path('mark/delete//', delete_mark, name='delete_mark'), path('/review/create/', create_review, name='create_review'), path('review/update//', update_review, name='update_review'), diff --git a/movies/views.py b/movies/views.py index f63cdeef..d4fc21f5 100644 --- a/movies/views.py +++ b/movies/views.py @@ -2,21 +2,23 @@ import logging from django.shortcuts import render, get_object_or_404, redirect, reverse from django.contrib.auth.decorators import login_required, permission_required from django.utils.translation import gettext_lazy as _ -from django.http import HttpResponseBadRequest, HttpResponseServerError +from django.http import HttpResponseBadRequest, HttpResponseServerError, HttpResponse from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import IntegrityError, transaction from django.db.models import Count from django.utils import timezone from django.core.paginator import Paginator from mastodon import mastodon_request_included -from mastodon.api import check_visibility, post_toot, TootVisibilityEnum -from mastodon.utils import rating_to_emoji +from mastodon.models import MastodonApplication +from mastodon.api import share_mark, share_review from common.utils import PageLinksGenerator -from common.views import PAGE_LINK_NUMBER, jump_or_scrape +from common.views import PAGE_LINK_NUMBER, jump_or_scrape, go_relogin from common.models import SourceSiteEnum from .models import * from .forms import * -from boofilsic.settings import MASTODON_TAGS +from django.conf import settings +from collection.models import CollectionItem +from common.scraper import get_scraper_by_url, get_normalized_url logger = logging.getLogger(__name__) @@ -87,6 +89,18 @@ def create(request): return HttpResponseBadRequest() +@login_required +def rescrape(request, id): + if request.method != 'POST': + return HttpResponseBadRequest() + item = get_object_or_404(Movie, pk=id) + url = get_normalized_url(item.source_url) + scraper = get_scraper_by_url(url) + scraper.scrape(url) + form = scraper.save(request_user=request.user, instance=item) + return redirect(reverse("movies:retrieve", args=[form.instance.id])) + + @login_required def update(request, id): if request.method == 'GET': @@ -98,6 +112,7 @@ def update(request, id): 'movies/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("movies:update", args=[movie.id]), # provided for frontend js @@ -127,6 +142,7 @@ def update(request, id): 'movies/create_update.html', { 'form': form, + 'is_update': True, 'title': page_title, 'submit_url': reverse("movies:update", args=[movie.id]), # provided for frontend js @@ -167,6 +183,7 @@ def retrieve(request, id): else: mark_form = MovieMarkForm(initial={ 'movie': movie, + 'visibility': request.user.get_preference().default_visibility if request.user.is_authenticated else 0, 'tags': mark_tags }) @@ -185,10 +202,8 @@ def retrieve(request, id): mark_list_more = None review_list_more = None else: - mark_list = MovieMark.get_available( - movie, request.user, request.session['oauth_token']) - review_list = MovieReview.get_available( - movie, request.user, request.session['oauth_token']) + mark_list = MovieMark.get_available_for_identicals(movie, request.user) + review_list = MovieReview.get_available_for_identicals(movie, request.user) mark_list_more = True if len(mark_list) > MARK_NUMBER else False mark_list = mark_list[:MARK_NUMBER] for m in mark_list: @@ -196,6 +211,7 @@ def retrieve(request, id): review_list_more = True if len( review_list) > REVIEW_NUMBER else False review_list = review_list[:REVIEW_NUMBER] + collection_list = filter(lambda c: c.is_visible_to(request.user), map(lambda i: i.collection, CollectionItem.objects.filter(movie=movie))) # def strip_html_tags(text): # import re @@ -220,6 +236,7 @@ def retrieve(request, id): 'review_list_more': review_list_more, 'movie_tag_list': movie_tag_list, 'mark_tags': mark_tags, + 'collection_list': collection_list, } ) else: @@ -264,12 +281,19 @@ def create_update_mark(request): pk = request.POST.get('id') old_rating = None old_tags = None + if not pk: + movie_id = request.POST.get('movie') + mark = MovieMark.objects.filter(movie_id=movie_id, owner=request.user).first() + if mark: + pk = mark.id if pk: mark = get_object_or_404(MovieMark, pk=pk) if request.user != mark.owner: return HttpResponseBadRequest() old_rating = mark.rating old_tags = mark.moviemark_tags.all() + if mark.status != request.POST.get('status'): + mark.created_time = timezone.now() # update form = MovieMarkForm(request.POST, instance=mark) else: @@ -277,7 +301,7 @@ def create_update_mark(request): form = MovieMarkForm(request.POST) if form.is_valid(): - if form.instance.status == MarkStatusEnum.WISH.value: + if form.instance.status == MarkStatusEnum.WISH.value or form.instance.rating == 0: form.instance.rating = None form.cleaned_data['rating'] = None form.instance.owner = request.user @@ -305,28 +329,10 @@ def create_update_mark(request): return HttpResponseServerError("integrity error") if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("movies:retrieve", - args=[movie.id]) - words = MovieMarkStatusTranslator(form.cleaned_data['status']) +\ - f"《{movie.title}》" + \ - rating_to_emoji(form.cleaned_data['rating']) - - # tags = MASTODON_TAGS % {'category': '书', 'type': '标记'} - tags = '' - content = words + '\n' + url + '\n' + \ - form.cleaned_data['text'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_mark(form.instance): + return go_relogin(request) else: - return HttpResponseBadRequest("invalid form data") + return HttpResponseBadRequest(f"invalid form data {form.errors}") return redirect(reverse("movies:retrieve", args=[form.instance.movie.id])) else: @@ -335,11 +341,30 @@ def create_update_mark(request): @mastodon_request_included @login_required -def retrieve_mark_list(request, movie_id): +def wish(request, id): + if request.method == 'POST': + movie = get_object_or_404(Movie, pk=id) + params = { + 'owner': request.user, + 'status': MarkStatusEnum.WISH, + 'visibility': 0, + 'movie': movie, + } + try: + MovieMark.objects.create(**params) + except Exception: + pass + return HttpResponse("✔️") + else: + return HttpResponseBadRequest("invalid method") + + +@mastodon_request_included +@login_required +def retrieve_mark_list(request, movie_id, following_only=False): if request.method == 'GET': movie = get_object_or_404(Movie, pk=movie_id) - queryset = MovieMark.get_available( - movie, request.user, request.session['oauth_token']) + queryset = MovieMark.get_available_for_identicals(movie, request.user, following_only=following_only) paginator = Paginator(queryset, MARK_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) @@ -400,23 +425,8 @@ def create_review(request, movie_id): form.instance.owner = request.user form.save() if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("movies:retrieve_review", - args=[form.instance.id]) - words = "发布了关于" + f"《{form.instance.movie.title}》" + "的评论" - # tags = MASTODON_TAGS % {'category': '书', 'type': '评论'} - tags = '' - content = words + '\n' + url + \ - '\n' + form.cleaned_data['title'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_review(form.instance): + return go_relogin(request) return redirect(reverse("movies:retrieve_review", args=[form.instance.id])) else: return HttpResponseBadRequest() @@ -452,23 +462,8 @@ def update_review(request, id): form.instance.edited_time = timezone.now() form.save() if form.cleaned_data['share_to_mastodon']: - if form.cleaned_data['is_private']: - visibility = TootVisibilityEnum.PRIVATE - else: - visibility = TootVisibilityEnum.UNLISTED - url = "https://" + request.get_host() + reverse("movies:retrieve_review", - args=[form.instance.id]) - words = "发布了关于" + f"《{form.instance.movie.title}》" + "的评论" - # tags = MASTODON_TAGS % {'category': '书', 'type': '评论'} - tags = '' - content = words + '\n' + url + \ - '\n' + form.cleaned_data['title'] + '\n' + tags - response = post_toot(request.user.mastodon_site, content, visibility, - request.session['oauth_token']) - if response.status_code != 200: - mastodon_logger.error( - f"CODE:{response.status_code} {response.text}") - return HttpResponseServerError("publishing mastodon status failed") + if not share_review(form.instance): + return go_relogin(request) return redirect(reverse("movies:retrieve_review", args=[form.instance.id])) else: return HttpResponseBadRequest() @@ -503,11 +498,10 @@ def delete_review(request, id): @mastodon_request_included -@login_required def retrieve_review(request, id): if request.method == 'GET': review = get_object_or_404(MovieReview, pk=id) - if not check_visibility(review, request.session['oauth_token'], request.user): + if not review.is_visible_to(request.user): msg = _("你没有访问这个页面的权限😥") return render( request, @@ -542,8 +536,7 @@ def retrieve_review(request, id): def retrieve_review_list(request, movie_id): if request.method == 'GET': movie = get_object_or_404(Movie, pk=movie_id) - queryset = MovieReview.get_available( - movie, request.user, request.session['oauth_token']) + queryset = MovieReview.get_available_for_identicals(movie, request.user) paginator = Paginator(queryset, REVIEW_PER_PAGE) page_number = request.GET.get('page', default=1) reviews = paginator.get_page(page_number) diff --git a/music/admin.py b/music/admin.py index eb770458..33eb056d 100644 --- a/music/admin.py +++ b/music/admin.py @@ -1,11 +1,12 @@ from django.contrib import admin from .models import * +from simple_history.admin import SimpleHistoryAdmin -admin.site.register(Song) +admin.site.register(Song, SimpleHistoryAdmin) admin.site.register(SongMark) admin.site.register(SongReview) admin.site.register(SongTag) -admin.site.register(Album) +admin.site.register(Album, SimpleHistoryAdmin) admin.site.register(AlbumMark) admin.site.register(AlbumReview) admin.site.register(AlbumTag) diff --git a/music/apps.py b/music/apps.py index d909c7fb..6fb97b37 100644 --- a/music/apps.py +++ b/music/apps.py @@ -3,3 +3,9 @@ from django.apps import AppConfig class MusicConfig(AppConfig): name = 'music' + + def ready(self): + from common.index import Indexer + from .models import Album, Song + Indexer.update_model_indexable(Album) + Indexer.update_model_indexable(Song) diff --git a/music/forms.py b/music/forms.py index 3eec5646..9e487592 100644 --- a/music/forms.py +++ b/music/forms.py @@ -7,12 +7,7 @@ from common.forms import * def MusicMarkStatusTranslator(status): - trans_dict = { - MarkStatusEnum.DO.value: _("在听"), - MarkStatusEnum.WISH.value: _("想听"), - MarkStatusEnum.COLLECT.value: _("听过") - } - return trans_dict[status] + return MusicMarkStatusTranslation[status] class SongForm(forms.ModelForm): @@ -65,11 +60,8 @@ class SongMarkForm(MarkForm): 'status', 'rating', 'text', - 'is_private', + 'visibility', ] - labels = { - 'rating': _("评分"), - } widgets = { 'song': forms.TextInput(attrs={"hidden": ""}), } @@ -84,14 +76,8 @@ class SongReviewForm(ReviewForm): 'song', 'title', 'content', - 'is_private' + 'visibility' ] - labels = { - 'song': "", - 'title': _("标题"), - 'content': _("正文"), - 'share_to_mastodon': _("分享到长毛象") - } widgets = { 'song': forms.TextInput(attrs={"hidden": ""}), } @@ -148,11 +134,8 @@ class AlbumMarkForm(MarkForm): 'status', 'rating', 'text', - 'is_private', + 'visibility', ] - labels = { - 'rating': _("评分"), - } widgets = { 'album': forms.TextInput(attrs={"hidden": ""}), } @@ -167,14 +150,8 @@ class AlbumReviewForm(ReviewForm): 'album', 'title', 'content', - 'is_private' + 'visibility' ] - labels = { - 'album': "", - 'title': _("标题"), - 'content': _("正文"), - 'share_to_mastodon': _("分享到长毛象") - } widgets = { 'album': forms.TextInput(attrs={"hidden": ""}), } diff --git a/music/management/commands/fix-album-cover.py b/music/management/commands/fix-album-cover.py new file mode 100644 index 00000000..9b9f4d3e --- /dev/null +++ b/music/management/commands/fix-album-cover.py @@ -0,0 +1,199 @@ +from django.core.management.base import BaseCommand +from django.core.files.uploadedfile import SimpleUploadedFile +from common.scraper import * +from django.conf import settings +from music.models import Album +from music.forms import AlbumForm +import requests +import re +import filetype +from lxml import html +from PIL import Image +from io import BytesIO + + +class DoubanPatcherMixin: + @classmethod + def download_page(cls, url, headers): + url = cls.get_effective_url(url) + r = None + error = 'DoubanScrapper: error occured when downloading ' + url + content = None + + def get(url, timeout): + nonlocal r + # print('Douban GET ' + url) + try: + r = requests.get(url, timeout=timeout) + except Exception as e: + r = requests.Response() + r.status_code = f"Exception when GET {url} {e}" + url + # print('Douban CODE ' + str(r.status_code)) + return r + + def check_content(): + nonlocal r, error, content + content = None + if r.status_code == 200: + content = r.content.decode('utf-8') + if content.find('关于豆瓣') == -1: + content = None + error = error + 'Content not authentic' # response is garbage + elif re.search('不存在[^<]+', content, re.MULTILINE): + content = None + error = error + 'Not found or hidden by Douban' + else: + error = error + str(r.status_code) + + def fix_wayback_links(): + nonlocal content + # fix links + content = re.sub(r'href="http[^"]+http', r'href="http', content) + # https://img9.doubanio.com/view/subject/{l|m|s}/public/s1234.jpg + content = re.sub(r'src="[^"]+/(s\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/subject/m/public/\1"', content) + # https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2681329386.jpg + # https://img9.doubanio.com/view/photo/{l|m|s}/public/p1234.webp + content = re.sub(r'src="[^"]+/(p\d+\.\w+)"', + r'src="https://img9.doubanio.com/view/photo/m/public/\1"', content) + + # Wayback Machine: get latest available + def wayback(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://archive.org/wayback/available?url=' + url, 10) + if r.status_code == 200: + w = r.json() + if w['archived_snapshots'] and w['archived_snapshots']['closest']: + get(w['archived_snapshots']['closest']['url'], 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + # Wayback Machine: guess via CDX API + def wayback_cdx(): + nonlocal r, error, content + error = error + '\nWayback: ' + get('http://web.archive.org/cdx/search/cdx?url=' + url, 10) + if r.status_code == 200: + dates = re.findall(r'[^\s]+\s+(\d+)\s+[^\s]+\s+[^\s]+\s+\d+\s+[^\s]+\s+\d{5,}', + r.content.decode('utf-8')) + # assume snapshots whose size >9999 contain real content, use the latest one of them + if len(dates) > 0: + get('http://web.archive.org/web/' + dates[-1] + '/' + url, 10) + check_content() + if content is not None: + fix_wayback_links() + else: + error = error + 'No snapshot available' + else: + error = error + str(r.status_code) + + def latest(): + nonlocal r, error, content + if settings.SCRAPESTACK_KEY is None: + error = error + '\nDirect: ' + get(url, 60) + else: + error = error + '\nScraperAPI: ' + # get(f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}', 60) + get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}', 60) + check_content() + + wayback_cdx() + if content is None: + latest() + + if content is None: + logger.error(error) + content = '' + return html.fromstring(content) + + @classmethod + def download_image(cls, url, item_url=None): + if url is None: + return None, None + raw_img = None + ext = None + + dl_url = url + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + # f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}' + + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + # raise RuntimeError(f"Douban: download image failed {img_response.status_code} {dl_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + if raw_img is None and settings.SCRAPESTACK_KEY is not None: + try: + img_response = requests.get(dl_url, timeout=90) + if img_response.status_code == 200: + raw_img = img_response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception + content_type = img_response.headers.get('Content-Type') + ext = filetype.get_type(mime=content_type.partition(';')[0].strip()).extension + else: + logger.error(f"Douban: download image failed {img_response.status_code} {dl_url} {item_url}") + except Exception as e: + raw_img = None + ext = None + logger.error(f"Douban: download image failed {e} {dl_url} {item_url}") + return raw_img, ext + + +class DoubanAlbumPatcher(DoubanPatcherMixin, AbstractScraper): + site_name = SourceSiteEnum.DOUBAN.value + host = 'music.douban.com' + data_class = Album + form_class = AlbumForm + + regex = re.compile(r"https://music\.douban\.com/subject/\d+/{0,1}") + + def scrape(self, url): + headers = DEFAULT_REQUEST_HEADERS.copy() + headers['Host'] = self.host + content = self.download_page(url, headers) + img_url_elem = content.xpath("//div[@id='mainpic']//img/@src") + img_url = img_url_elem[0].strip() if img_url_elem else None + raw_img, ext = self.download_image(img_url, url) + return raw_img, ext + + +class Command(BaseCommand): + help = 'fix cover image' + + def add_arguments(self, parser): + parser.add_argument('threadId', type=int, help='% 8') + + def handle(self, *args, **options): + t = int(options['threadId']) + for m in Album.objects.filter(cover='album/default.svg', source_site='douban'): + if m.id % 8 == t: + self.stdout.write(f'Re-fetching {m.source_url}') + try: + raw_img, img_ext = DoubanAlbumPatcher.scrape(m.source_url) + if img_ext is not None: + m.cover = SimpleUploadedFile('temp.' + img_ext, raw_img) + m.save() + self.stdout.write(self.style.SUCCESS(f'Saved {m.source_url}')) + else: + self.stdout.write(self.style.ERROR(f'Skipped {m.source_url}')) + except Exception as e: + print(e) diff --git a/music/models.py b/music/models.py index c738db5b..131f9a14 100644 --- a/music/models.py +++ b/music/models.py @@ -1,21 +1,29 @@ import uuid import django.contrib.postgres.fields as postgres -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.db import models from django.core.serializers.json import DjangoJSONEncoder from django.shortcuts import reverse -from common.models import Entity, Mark, Review, Tag +from common.models import Entity, Mark, Review, Tag, SourceSiteEnum, MarkStatusEnum from common.utils import ChoicesDictGenerator, GenerateDateUUIDMediaFilePath -from boofilsic.settings import SONG_MEDIA_PATH_ROOT, DEFAULT_SONG_IMAGE, ALBUM_MEDIA_PATH_ROOT, DEFAULT_ALBUM_IMAGE from django.utils import timezone +from django.conf import settings +from simple_history.models import HistoricalRecords + + +MusicMarkStatusTranslation = { + MarkStatusEnum.DO.value: _("在听"), + MarkStatusEnum.WISH.value: _("想听"), + MarkStatusEnum.COLLECT.value: _("听过") +} def song_cover_path(instance, filename): - return GenerateDateUUIDMediaFilePath(instance, filename, SONG_MEDIA_PATH_ROOT) + return GenerateDateUUIDMediaFilePath(instance, filename, settings.SONG_MEDIA_PATH_ROOT) def album_cover_path(instance, filename): - return GenerateDateUUIDMediaFilePath(instance, filename, ALBUM_MEDIA_PATH_ROOT) + return GenerateDateUUIDMediaFilePath(instance, filename, settings.ALBUM_MEDIA_PATH_ROOT) class Album(Entity): @@ -23,11 +31,11 @@ class Album(Entity): release_date = models.DateField( _('发行日期'), auto_now=False, auto_now_add=False, null=True, blank=True) cover = models.ImageField( - _("封面"), upload_to=album_cover_path, default=DEFAULT_ALBUM_IMAGE, blank=True) + _("封面"), upload_to=album_cover_path, default=settings.DEFAULT_ALBUM_IMAGE, blank=True) duration = models.PositiveIntegerField(_("时长"), null=True, blank=True) artist = postgres.ArrayField( models.CharField(_("artist"), blank=True, - default='', max_length=100), + default='', max_length=200), null=True, blank=True, default=list, @@ -45,12 +53,36 @@ class Album(Entity): ) track_list = models.TextField(_("曲目"), blank=True, default="") + history = HistoricalRecords() + def __str__(self): return self.title + def get_json(self): + r = { + 'artist': self.artist, + 'release_date': self.release_date, + 'genre': self.genre, + 'publisher': self.company, + } + r.update(super().get_json()) + return r + + def get_embed_link(self): + if self.source_site == SourceSiteEnum.SPOTIFY.value: + return self.source_url.replace("open.spotify.com/", "open.spotify.com/embed/") + elif self.source_site == SourceSiteEnum.BANDCAMP.value and self.other_info and 'bandcamp_album_id' in self.other_info: + return f"https://bandcamp.com/EmbeddedPlayer/album={self.other_info['bandcamp_album_id']}/size=large/bgcol=ffffff/linkcol=19A2CA/artwork=small/transparent=true/" + else: + return None + def get_absolute_url(self): return reverse("music:retrieve_album", args=[self.id]) + @property + def wish_url(self): + return reverse("music:wish_album", args=[self.id]) + def get_tags_manager(self): return self.album_tags @@ -58,6 +90,14 @@ class Album(Entity): def verbose_category_name(self): return _("专辑") + @property + def mark_class(self): + return AlbumMark + + @property + def tag_class(self): + return AlbumTag + class Song(Entity): ''' @@ -70,7 +110,7 @@ class Song(Entity): # duration in ms duration = models.PositiveIntegerField(_("时长"), null=True, blank=True) cover = models.ImageField( - _("封面"), upload_to=song_cover_path, default=DEFAULT_SONG_IMAGE, blank=True) + _("封面"), upload_to=song_cover_path, default=settings.DEFAULT_SONG_IMAGE, blank=True) artist = postgres.ArrayField( models.CharField(blank=True, default='', max_length=100), @@ -84,19 +124,46 @@ class Song(Entity): album = models.ForeignKey( Album, models.SET_NULL, "album_songs", null=True, blank=True, verbose_name=_("所属专辑")) + history = HistoricalRecords() + def __str__(self): return self.title + def get_json(self): + r = { + 'artist': self.artist, + 'release_date': self.release_date, + 'genre': self.genre, + } + r.update(super().get_json()) + return r + + def get_embed_link(self): + return self.source_url.replace("open.spotify.com/", "open.spotify.com/embed/") if self.source_site == SourceSiteEnum.SPOTIFY.value else None + def get_absolute_url(self): return reverse("music:retrieve_song", args=[self.id]) + @property + def wish_url(self): + return reverse("music:wish_song", args=[self.id]) + def get_tags_manager(self): return self.song_tags - + @property def verbose_category_name(self): return _("单曲") + @property + def mark_class(self): + return SongMark + + @property + def tag_class(self): + return SongTag + + class SongMark(Mark): song = models.ForeignKey( Song, on_delete=models.CASCADE, related_name='song_marks', null=True) @@ -107,6 +174,10 @@ class SongMark(Mark): fields=['owner', 'song'], name='unique_song_mark') ] + @property + def translated_status(self): + return MusicMarkStatusTranslation[self.status] + class SongReview(Review): song = models.ForeignKey( @@ -118,6 +189,14 @@ class SongReview(Review): fields=['owner', 'song'], name='unique_song_review') ] + @property + def url(self): + return settings.APP_WEBSITE + reverse("music:retrieve_song_review", args=[self.id]) + + @property + def item(self): + return self.song + class SongTag(Tag): song = models.ForeignKey( @@ -131,6 +210,10 @@ class SongTag(Tag): fields=['content', 'mark'], name="unique_songmark_tag") ] + @property + def item(self): + return self.song + class AlbumMark(Mark): album = models.ForeignKey( @@ -142,6 +225,10 @@ class AlbumMark(Mark): fields=['owner', 'album'], name='unique_album_mark') ] + @property + def translated_status(self): + return MusicMarkStatusTranslation[self.status] + class AlbumReview(Review): album = models.ForeignKey( @@ -153,6 +240,14 @@ class AlbumReview(Review): fields=['owner', 'album'], name='unique_album_review') ] + @property + def url(self): + return settings.APP_WEBSITE + reverse("music:retrieve_album_review", args=[self.id]) + + @property + def item(self): + return self.album + class AlbumTag(Tag): album = models.ForeignKey( @@ -165,3 +260,7 @@ class AlbumTag(Tag): models.UniqueConstraint( fields=['content', 'mark'], name="unique_albummark_tag") ] + + @property + def item(self): + return self.album diff --git a/music/templates/music/album_detail.html b/music/templates/music/album_detail.html index 68fe1143..7de3d0ae 100644 --- a/music/templates/music/album_detail.html +++ b/music/templates/music/album_detail.html @@ -6,6 +6,7 @@ {% load mastodon %} {% load oauth_token %} {% load truncate %} +{% load strip_scheme %} {% load thumb %} @@ -13,21 +14,19 @@ - + - - + + - {% trans 'NiceDB - 音乐详情' %} | {{ album.title }} + {{ site_name }} - {% trans '音乐详情' %} | {{ album.title }} - + {% include "partial/_common_libs.html" with jquery=1 %} + - - - @@ -55,11 +54,12 @@
    - {% if album.rating %} + {% if album.rating and album.rating_number >= 5 %} {{ album.rating }} + ({{ album.rating_number }}人评分) {% else %} - {% trans '评分：暂无评分' %} + {% trans '评分：评分人数不足' %} {% endif %}
    {% if album.artist %}{% trans '艺术家：' %} @@ -72,7 +72,7 @@ {% if album.artist|length > 5 %} {% trans '更多' %} + {{ site_name }} - {{ album.title }}{% trans '的标记' %} + @@ -35,38 +35,7 @@
    {{ album.title }}{% trans '的标记' %}
    -
      - - {% for mark in marks %} - -
    • - {{ mark.owner.username }} - {{ mark.get_status_display }} - {% if mark.rating %} - - {% endif %} - {% if mark.is_private %} - - - - {% endif %} - {{ mark.edited_time }} - {% if mark.text %} -

      {{ mark.text }}

      - {% endif %} -
      • - - {% empty %} -
      - {% trans '无结果' %} -
      - {% endfor %} - -
    + {% include "partial/mark_list.html" with mark_list=marks current_item=album %}
    @@ -109,7 +111,7 @@ {% if album.artist|length > 5 %} {% trans '更多' %} diff --git a/music/templates/music/album_review_list.html b/music/templates/music/album_review_list.html index a4e4f04a..40ce5b6d 100644 --- a/music/templates/music/album_review_list.html +++ b/music/templates/music/album_review_list.html @@ -13,8 +13,8 @@ - {% trans 'NiceDB - ' %}{{ album.title }}{% trans '的评论' %} - + {{ site_name }} - {{ album.title }}{% trans '的评论' %} + @@ -40,9 +40,9 @@
  • - {{ review.owner.username }} - {% if review.is_private %} + {% if review.visibility > 0 %} 5 %} {% trans '更多' %}
    • - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -22,9 +22,24 @@
    + {% if is_update and form.source_site.value != 'in-site' %} +
    +
    +
    +
    {% trans '源网站' %}: {{ form.source_site.value }}
    +
    + + {% csrf_token %} + + +
    +
    +
    +
    + {% endif %} +
    - {% trans '>>> 试试一键剽取~ <<<' %} + {% comment %} {% trans '>>> 试试一键剽取~ <<<' %} {% endcomment %}
    {% csrf_token %} {{ form.media }} @@ -54,12 +69,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -52,7 +52,7 @@ {% if album.artist|length > 5 %} {% trans '更多' %} + {{ site_name }} - {{ title }} + @@ -57,12 +57,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {{ title }} + @@ -52,7 +52,7 @@ {% if song.artist|length > 5 %} {% trans '更多' %} + {{ site_name }} - {% trans '删除音乐' %} + @@ -55,7 +55,7 @@ {% if album.last_editor %}
    {% trans '最近编辑者：' %} - + {{ album.last_editor | default:"" }}
    @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除评论' %} + @@ -35,7 +35,7 @@
    {{ review.title }}
    - {% if review.is_private %} + {% if review.visibility > 0 %}
    - {{ review.owner.username }} {% if mark %} @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除音乐' %} + @@ -55,7 +55,7 @@ {% if song.last_editor %}
    {% trans '最近编辑者：' %} - + {{ song.last_editor | default:"" }}
    @@ -89,12 +89,6 @@ - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '删除评论' %} + @@ -35,7 +35,7 @@
    {{ review.title }}
    - {% if review.is_private %} + {% if review.visibility > 0 %}
    - {{ review.owner.username }} {% if mark %} @@ -89,12 +89,6 @@
    - {% comment %} - - - - - {% endcomment %} + {{ site_name }} - {% trans '从豆瓣获取数据' %} + diff --git a/music/templates/music/scrape_song.html b/music/templates/music/scrape_song.html index e2cd4730..2ffa1b3b 100644 --- a/music/templates/music/scrape_song.html +++ b/music/templates/music/scrape_song.html @@ -10,8 +10,8 @@ - {% trans 'NiceDB - 从豆瓣获取数据' %} - + {{ site_name }} - {% trans '从豆瓣获取数据' %} + diff --git a/music/templates/music/song_detail.html b/music/templates/music/song_detail.html index 6aa9b87b..48a8c7be 100644 --- a/music/templates/music/song_detail.html +++ b/music/templates/music/song_detail.html @@ -6,6 +6,7 @@ {% load mastodon %} {% load oauth_token %} {% load truncate %} +{% load strip_scheme %} {% load thumb %} @@ -13,21 +14,19 @@ - + - - + + - {% trans 'NiceDB - 音乐详情' %} | {{ song.title }} + {{ site_name }} - {% trans '音乐详情' %} | {{ song.title }} - + {% include "partial/_common_libs.html" with jquery=1 %} + - - - @@ -54,11 +53,12 @@
    - {% if song.rating %} + {% if song.rating and song.rating_number >= 5 %} {{ song.rating }} + ({{ song.rating_number }}人评分) {% else %} - {% trans '评分：暂无评分' %} + {% trans '评分：评分人数不足' %} {% endif %}
    {% if song.artist %}{% trans '艺术家：' %} @@ -71,7 +71,7 @@ {% if song.artist|length > 5 %} {% trans '更多' %} + {{ site_name }} - {{ song.title }}{% trans '的标记' %} + @@ -32,41 +33,9 @@
    - {{ song.title }}{% trans ' - 的标记' %} + {{ song.title }}{% trans '的标记' %}
    -
      - - {% for mark in marks %} - -
    • - {{ mark.owner.username }} - {{ mark.get_status_display }} - {% if mark.rating %} - - {% endif %} - {% if mark.is_private %} - - - - {% endif %} - {{ mark.edited_time }} - {% if mark.text %} -

      {{ mark.text }}

      - {% endif %} -
      • - - {% empty %} -
      - {% trans '无结果' %} -
      - {% endfor %} - -
    + {% include "partial/mark_list.html" with mark_list=marks current_item=song %}
    + + {% if user == request.user %} + +
    +
    + + {% trans '编辑布局' %} + + + + + + + + +
    -
    -
    - -
    -
    - {% trans '关注的人' %} -
    - {% trans '更多' %} - -
    - -
    -
    - {% trans '被他们关注' %} -
    - {% trans '更多' %} - -
    - -
    - - - {% if user == request.user %} -
    -
    -
    {% trans '导入豆瓣标记数据' %}
    - ? -
    -
    - - {% csrf_token %} - - {% trans '导入：' %} -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - {% trans '覆盖：' %} -
    - - -
    - ? -
    - {% trans '可见性：' %} -
    - - -
    - ? -
    -
    - -
    - -
    - - -
    -
    -
    - {% endif %} - -
    - {% if request.user.is_staff and request.user == user%} -
    -
    {% trans '举报信息' %}
    - 全部举报 -
    - -
    -
    - {% endif %} -
    +
    + +
    + {% csrf_token %} + +
    + + + {% endif %} +
    + + {% include "partial/_sidebar.html" %}
    - - {% include "partial/_footer.html" %} - - -
    -
    - - - {% if user == request.user %} - - {% else %} - - {% endif %} - - - - - {% if unread_announcements %} -
    - -
    -
    -

    {% trans '公告' %}

    - - - - - - - - - -
    -
    -
      - {% for ann in unread_announcements %} -
    • - -
      {{ ann.title }}
      -       - {{ ann.created_time }} -

      {{ ann.get_plain_content | truncate:200 }}

      -
      • - {% if not forloop.last %} -
      - {% endif %} - {% endfor %} -
    -
    - {% trans '查看全部公告' %} -
    -
    -
    -
    -
    + {% include "partial/_announcement.html" %} {% endif %} - - - - - - - - \ No newline at end of file diff --git a/users/templates/users/home_anonymous.html b/users/templates/users/home_anonymous.html new file mode 100644 index 00000000..946ac96e --- /dev/null +++ b/users/templates/users/home_anonymous.html @@ -0,0 +1,17 @@ +{% load static %} +{% load i18n %} + + + + + + {{ site_name }} - {{ username }}@{{ site }} + + + + + + + Mastodon homepage + + \ No newline at end of file diff --git a/users/templates/users/item_list.html b/users/templates/users/item_list.html new file mode 100644 index 00000000..321ad0fe --- /dev/null +++ b/users/templates/users/item_list.html @@ -0,0 +1,94 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} + + + + + + + {{ site_name }} - {{ user.mastodon_username }} {{ list_title }} + + + + + + + + + + + + +
    +
    + {% include "partial/_navbar.html" %} + +
    +
    +
    +
    +
    + +
    +
    + {{ user.mastodon_username }} {{ list_title }} +
    +
    +
      + {% for mark in marks %} + {% include "partial/list_item.html" with item=mark.item hide_category=True %} + {% empty %} +
      {% trans '无结果' %}
      + {% endfor %} +
    +
    + +
    +
    + + {% include "partial/_sidebar.html" %} +
    +
    +
    + {% include "partial/_footer.html" %} +
    + + + + + + diff --git a/users/templates/users/login.html b/users/templates/users/login.html index 27a0a0fa..d843543c 100644 --- a/users/templates/users/login.html +++ b/users/templates/users/login.html @@ -1,4 +1,3 @@ - {% load i18n %} {% load static %} @@ -6,74 +5,81 @@ - + + - - + {{ site_name }} - {% trans '登录' %} + {% include "partial/_common_libs.html" %} + - - - {% trans 'NiceDB - 登录' %} - - - + +
    - - - -
    - - + +
    {% if user.is_authenticated %} - {% trans '前往我的主页' %} + {% trans '前往首页' %} + {% else %} +
    + {% if allow_any_site %} + + +
    {% trans '了解更多' %} + {% else %} - {% for site in sites %} {% endfor %} - + {% endif %} - +
    + {% endif %} +
    网页加载超时，请检查网络（翻墙）设置。
    - -
    - {% if not user.is_authenticated %} - - - - {% endif %} - \ No newline at end of file + diff --git a/users/templates/users/manage_report.html b/users/templates/users/manage_report.html index 35d3604b..f077a434 100644 --- a/users/templates/users/manage_report.html +++ b/users/templates/users/manage_report.html @@ -10,8 +10,8 @@ - {% trans 'NiceDB - 管理举报' %} - + {{ site_name }} - {% trans '管理举报' %} + @@ -27,9 +27,9 @@ {% for report in reports %}
    - {{ report.submit_user.username }} + {{ report.submit_user.username }} {% trans '举报了' %} - {{ report.reported_user.username }} + {{ report.reported_user.username }} @{{ report.submitted_time }} {% if report.image %} @@ -49,12 +49,6 @@
    - {% comment %} - - - - - {% endcomment %} - - - - - - - - - -
    -
    - {% include "partial/_navbar.html" %} - -
    -
    -
    -
    -
    - -
    -
    - {{ user.username }}{{ list_title }} -
    -
    - -
    - -
    -
    - -
    -
    - -
    - -
    -
    - - - - - -
    -
    -
    - -
    -
    - {% trans '关注的人' %} -
    - {% trans '更多' %} - -
    - -
    -
    - {% trans '被他们关注' %} -
    - {% trans '更多' %} - -
    - -
    -
    -
    - -
    -
    -
    -
    - {% include "partial/_footer.html" %} -
    - - - - - - {% if user == request.user %} - - {% else %} - - {% endif %} - - - - - - - diff --git a/users/templates/users/music_list.html b/users/templates/users/music_list.html deleted file mode 100644 index 18c0fe74..00000000 --- a/users/templates/users/music_list.html +++ /dev/null @@ -1,290 +0,0 @@ -{% load static %} -{% load i18n %} -{% load l10n %} -{% load humanize %} -{% load admin_url %} -{% load mastodon %} -{% load oauth_token %} -{% load truncate %} -{% load thumb %} - - - - - - - - {% trans 'NiceDB - ' %}{{ user.username }}{{ list_title }} - - - - - - - - - - -
    -
    - {% include "partial/_navbar.html" %} - -
    -
    -
    -
    -
    - -
    -
    - {{ user.username }}{{ list_title }} -
    -
    -
      - - {% for mark in marks %} - - {% with mark.music as music %} - -
    • -
      - {% if music.category_name|lower == 'album' %} - - - - {% elif music.category_name|lower == 'song' %} - - - - {% endif %} -
      -
      -
      - {% if music.category_name|lower == 'album' %} - - {{ music.title }} - - {% elif music.category_name|lower == 'song' %} - - {{ music.title }} - - {% endif %} - - {{ music.get_source_site_display }} - -
      - - {% if music.artist %}{% trans '艺术家' %} - {% for artist in music.artist %} - {{ artist }} - {% if not forloop.last %} {% endif %} - {% endfor %} - {% endif %} - - {% if music.genre %}/ {% trans '流派' %} - {{ music.genre }} - {% endif %} - - {% if music.release_date %}/ {% trans '发行日期' %} - {{ music.release_date }} - {% endif %} - - {% if music.brief %} -

      - {{ music.brief }} -

      - {% elif music.category_name|lower == 'album' %} -

      - {% trans '曲目：' %}{{ music.track_list }} -

      - {% else %} - -

      - {% trans '所属专辑：' %}{{ music.album }} -

      - {% endif %} -
      - {% for tag_dict in music.tag_list %} - {% for k, v in tag_dict.items %} - {% if k == 'content' %} - - {{ v }} - - {% endif %} - {% endfor %} - {% endfor %} -
      -
      -
      -
      -
        -
      • - - {% if mark.rating %} - - {% endif %} - {% if mark.is_private %} - - - - {% endif %} - {% trans '于' %} {{ mark.edited_time }} {% trans '标记' %} - {% if mark.text %} -

        {{ mark.text }}

        - {% endif %} -
        • -
      -
      -
      - -
      • - - {% endwith %} - - {% empty %} -
      {% trans '无结果' %}
      - {% endfor %} - - - -
    -
    - -
    -
    - -
    -
    - -
    - -
    -
    - - - - - -
    -
    -
    - -
    -
    - {% trans '关注的人' %} -
    - {% trans '更多' %} - -
    - -
    -
    - {% trans '被他们关注' %} -
    - {% trans '更多' %} - -
    - -
    -
    -
    - -
    -
    -
    -
    - {% include "partial/_footer.html" %} -
    - - - - - - {% if user == request.user %} - - {% else %} - - {% endif %} - - - - - - - diff --git a/users/templates/users/preferences.html b/users/templates/users/preferences.html new file mode 100644 index 00000000..cca66196 --- /dev/null +++ b/users/templates/users/preferences.html @@ -0,0 +1,93 @@ +{% load static %} +{% load i18n %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load thumb %} + + + + + + + {{ site_name }} - 设置 + {% include "partial/_common_libs.html" %} + + + + + +
    +
    + {% include "partial/_navbar.html" %} + +
    +
    +
    +
    +
    +
    +
    +
    {% trans '使用偏好设置' %}
    +
    + {% csrf_token %} + {% trans '新标记默认可见性：' %} +
    + + + +
    +
    + {% trans '登录后显示个人主页：' %} +
    + + +
    +
    +
    +
    + +
    +
    +
    {% trans '社交网络分享相关设置' %}
    +
    + {% csrf_token %} + {% trans '在联邦网络上以公开方式分享的帖文是否发布到公共时间轴上：' %} +
    + + +
    +

    + {% trans '在联邦网络上分享帖文时附加标签：' %} +
    + + +
    +
    +
    +
    + +
    + +
    +
    +
    +
    + + {% include "partial/_sidebar.html" %} +
    +
    + +
    + + {% include "partial/_footer.html" %} +
    + + + + + \ No newline at end of file diff --git a/users/templates/users/register.html b/users/templates/users/register.html index 9b750a66..cb63b5c6 100644 --- a/users/templates/users/register.html +++ b/users/templates/users/register.html @@ -6,11 +6,10 @@ - - + - {% trans 'NiceDB - 注册' %} + {{ site_name }} - {% trans '注册' %} @@ -19,18 +18,18 @@
    -

    欢迎来到NiceDB书影音！

    +

    欢迎来到{{ site_name }}！

    - NiceDB书影音继承了长毛象的用户关系，比如您在里瓣屏蔽了某人，那您将不会在书影音的公共区域看到TA的痕迹。 - 这里仍是一片处女地，丰富的内容需要大家共同创造。 - 请注意虽然您可以随意发表任何言论，但试图添加垃圾数据到公共数据领域（如添加不存在的乱码的书籍）将会受到制裁！ - BTW欧盟惯例本站使用了Cookie，请理解！ + {{ site_name }}还在不断完善中，丰富的内容需要大家共同创造。 + 试图添加垃圾数据（如添加信息混乱或缺失的书籍、以推广为主要目的的评论）将会受到严肃处理。 + {{ site_name }}继承了联邦宇宙的用户关系，比如您在联邦宇宙屏蔽了某人，那您将不会在书影音的公共区域看到TA的痕迹。 + 本站为非盈利站点，cookie和其他数据保管使用原则请参阅站内公告。

    - 此外NiceDB书影音现处于“公开阿尔法测试”阶段，您的数据存在丢失的可能。使用过程中遇到的问题或者Bug欢迎向作者提出。 + 此外，{{ site_name }}现处于测试阶段，疏漏在所难免，请妥善备份您的数据。 + 使用过程中遇到的问题或者错误欢迎向维护者提出。感谢理解和支持！

    -
    - {% csrf_token %} +
    diff --git a/users/templates/users/relation_list.html b/users/templates/users/relation_list.html index 6984bbd3..ae14e2ea 100644 --- a/users/templates/users/relation_list.html +++ b/users/templates/users/relation_list.html @@ -11,11 +11,13 @@ {% if is_followers_page %} - {% trans 'NiceDB - 被他们关注' %} + {{ site_name }} - {% trans '被他们关注' %} {% else %} - {% trans 'NiceDB - 关注的人' %} + {{ site_name }} - {% trans '关注的人' %} {% endif %} - + + {% include "partial/_common_libs.html" with jquery=1 %} + {% if is_followers_page %} @@ -23,8 +25,6 @@ {% else %} {% endif %} - - @@ -64,7 +64,7 @@ @@ -94,7 +94,7 @@
    {% trans '关注的人' %}
    - {% trans '更多' %}
    • @@ -110,7 +110,7 @@
      {% trans '被他们关注' %}
      - {% trans '更多' %}
      • @@ -140,7 +140,7 @@ {% if user == request.user %} {% else %} - + {% endif %} - {% comment %} - - - - - {% endcomment %} + + + + + + + + + +
        +
        + {% include "partial/_navbar.html" %} + +
        +
        +
        +
        +
        +
        +
        {% trans '书籍' %}
        + {% for v in book_tags %} + + + {{ v.content }} + + ({{ v.total }}) + + {% empty %} + {% trans '暂无标签' %} + {% endfor %} +
        + +
        {% trans '电影和剧集' %}
        + {% for v in movie_tags %} + + + {{ v.content }} + + ({{ v.total }}) + + {% empty %} + {% trans '暂无标签' %} + {% endfor %} +
        + +
        {% trans '音乐' %}
        + {% for v in music_tags %} + + + {{ v.content }} + + ({{ v.total }}) + + {% empty %} + {% trans '暂无标签' %} + {% endfor %} +
        + +
        {% trans '游戏' %}
        + {% for v in game_tags %} + + + {{ v.content }} + + ({{ v.total }}) + + {% empty %} + {% trans '暂无标签' %} + {% endfor %} +
        +
        +
        +
        +
        + + {% include "partial/_sidebar.html" %} + +
        +
        +
        + {% include "partial/_footer.html" %} +
        + + + + + + + + diff --git a/users/urls.py b/users/urls.py index 121c1afe..3b0d2b34 100644 --- a/users/urls.py +++ b/users/urls.py @@ -5,22 +5,38 @@ app_name = 'users' urlpatterns = [ path('login/', login, name='login'), path('register/', register, name='register'), + path('connect/', connect, name='connect'), + path('reconnect/', reconnect, name='reconnect'), + path('data/', data, name='data'), + path('data/import_goodreads', import_goodreads, name='import_goodreads'), + path('data/import_douban', import_douban, name='import_douban'), + path('data/export_reviews', export_reviews, name='export_reviews'), + path('data/export_marks', export_marks, name='export_marks'), + path('data/sync_mastodon', sync_mastodon, name='sync_mastodon'), + path('data/reset_visibility', reset_visibility, name='reset_visibility'), + path('data/clear_data', clear_data, name='clear_data'), + path('preferences/', preferences, name='preferences'), path('logout/', logout, name='logout'), - path('delete/', delete, name='delete'), path('layout/', set_layout, name='set_layout'), path('OAuth2_login/', OAuth2_login, name='OAuth2_login'), - path('/', home, name='home'), - path('/followers/', followers, name='followers'), - path('/following/', following, name='following'), - path('/book//', book_list, name='book_list'), - path('/movie//', movie_list, name='movie_list'), - path('/music//', music_list, name='music_list'), - path('/game//', game_list, name='game_list'), + path('/', home_redirect, name='home_redirect'), + # path('/followers/', followers, name='followers'), + # path('/following/', following, name='following'), + # path('/collections/', collection_list, name='collection_list'), + # path('/book//', book_list, name='book_list'), + # path('/movie//', movie_list, name='movie_list'), + # path('/music//', music_list, name='music_list'), + # path('/game//', game_list, name='game_list'), path('/', home, name='home'), path('/followers/', followers, name='followers'), path('/following/', following, name='following'), + path('/tags/', tag_list, name='tag_list'), + path('/collections/', collection_list, name='collection_list'), + path('/collections/marked/', marked_collection_list, name='marked_collection_list'), path('/book//', book_list, name='book_list'), path('/movie//', movie_list, name='movie_list'), + path('/music//', music_list, name='music_list'), + path('/game//', game_list, name='game_list'), path('report/', report, name='report'), path('manage_report/', manage_report, name='manage_report'), ] diff --git a/users/views.py b/users/views.py index ff77d066..e216f806 100644 --- a/users/views.py +++ b/users/views.py @@ -5,11 +5,10 @@ from django.contrib import auth from django.contrib.auth import authenticate from django.core.paginator import Paginator from django.utils.translation import gettext_lazy as _ -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db.models import Count from .models import User, Report, Preference from .forms import ReportForm -from mastodon.auth import * from mastodon.api import * from mastodon import mastodon_request_included from common.config import * @@ -25,156 +24,69 @@ from movies.forms import MovieMarkStatusTranslator from music.forms import MusicMarkStatusTranslator from games.forms import GameMarkStatusTranslator from mastodon.models import MastodonApplication +from mastodon.api import verify_account +from django.conf import settings +from urllib.parse import quote +import django_rq +from .account import * +from .data import * +from datetime import timedelta +from django.utils import timezone +import json +from django.contrib import messages +from books.models import BookMark, BookReview +from movies.models import MovieMark, MovieReview +from games.models import GameMark, GameReview +from music.models import AlbumMark, SongMark, AlbumReview, SongReview +from collection.models import Collection +from common.importers.goodreads import GoodreadsImporter +from common.importers.douban import DoubanImporter -# Views -######################################## - -# no page rendered -@mastodon_request_included -def OAuth2_login(request): - """ oauth authentication and logging user into django system """ - if request.method == 'GET': - code = request.GET.get('code') - site = request.COOKIES.get('mastodon_domain') - - # Network IO - try: - token = obtain_token(site, request, code) - except ObjectDoesNotExist: - return HttpResponseBadRequest("Mastodon site not registered") - if token: - # oauth is completed when token aquired - user = authenticate(request, token=token, site=site) - if user: - auth_login(request, user, token) - if request.session.get('next_url') is not None: - response = redirect(request.session.get('next_url')) - del request.session['next_url'] - else: - response = redirect(reverse('common:home')) - - response.delete_cookie('mastodon_domain') - return response - else: - # will be passed to register page - request.session['new_user_token'] = token - return redirect(reverse('users:register')) - else: - return render( - request, - 'common/error.html', - { - 'msg': _("认证失败😫") - } - ) - else: - return HttpResponseBadRequest() +def render_user_not_found(request): + msg = _("😖哎呀，这位用户还没有加入本站，快去联邦宇宙呼唤TA来注册吧！") + sec_msg = _("") + return render( + request, + 'common/error.html', + { + 'msg': msg, + 'secondary_msg': sec_msg, + } + ) -# the 'login' page that user can see -def login(request): - if request.method == 'GET': - selected_site = request.GET.get('site', default='') +def home_redirect(request, id): + try: + query_kwargs = {'pk': id} + user = User.objects.get(**query_kwargs) + return redirect(reverse("users:home", args=[user.mastodon_username])) + except Exception: + return redirect(settings.LOGIN_URL) - sites = MastodonApplication.objects.all().order_by("domain_name") - # store redirect url in the cookie - if request.GET.get('next'): - request.session['next_url'] = request.GET.get('next') - - return render( - request, - 'users/login.html', - { - 'sites': sites, - 'selected_site': selected_site, - } - ) - else: - return HttpResponseBadRequest() +def home_anonymous(request, id): + login_url = settings.LOGIN_URL + "?next=" + request.get_full_path() + try: + username = id.split('@')[0] + site = id.split('@')[1] + return render(request, 'users/home_anonymous.html', { + 'login_url': login_url, + 'username': username, + 'site': site, + }) + except Exception: + return redirect(login_url) @mastodon_request_included -@login_required -def logout(request): - if request.method == 'GET': - revoke_token(request.user.mastodon_site, request.session['oauth_token']) - auth_logout(request) - return redirect(reverse("users:login")) - else: - return HttpResponseBadRequest() - - -@mastodon_request_included -def register(request): - """ register confirm page """ - if request.method == 'GET': - if request.session.get('oauth_token'): - return redirect(reverse('common:home')) - elif request.session.get('new_user_token'): - return render( - request, - 'users/register.html' - ) - else: - return HttpResponseBadRequest() - elif request.method == 'POST': - token = request.session['new_user_token'] - user_data = get_user_data(request.COOKIES['mastodon_domain'], token) - if user_data is None: - return render( - request, - 'common/error.html', - { - 'msg': _("长毛象访问失败😫") - } - ) - new_user = User( - username=user_data['username'], - mastodon_id=user_data['id'], - mastodon_site=request.COOKIES['mastodon_domain'], - ) - new_user.save() - del request.session['new_user_token'] - auth_login(request, new_user, token) - response = redirect(reverse('common:home')) - response.delete_cookie('mastodon_domain') - return response - else: - return HttpResponseBadRequest() - - -def delete(request): - raise NotImplementedError - - -@mastodon_request_included -@login_required def home(request, id): + if not request.user.is_authenticated: + return home_anonymous(request, id) if request.method == 'GET': - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) + user = User.get(id) + if user is None: + return render_user_not_found(request) # access one's own home page if user == request.user: @@ -194,23 +106,19 @@ def home(request, id): album_marks = request.user.user_albummarks.all() song_marks = request.user.user_songmarks.all() game_marks = request.user.user_gamemarks.all() - - latest_task = user.user_synctasks.order_by("-id").first() + book_reviews = request.user.user_bookreviews.all() + movie_reviews = request.user.user_moviereviews.all() + album_reviews = request.user.user_albumreviews.all() + song_reviews = request.user.user_songreviews.all() + game_reviews = request.user.user_gamereviews.all() # visit other's home page else: - latest_task = None # no these value on other's home page reports = None unread_announcements = None - # cross site info for visiting other's home page - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) - - # make queries - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: + if request.user.is_blocked_by(user) or request.user.is_blocking(user): msg = _("你没有访问TA主页的权限😥") return render( request, @@ -219,20 +127,28 @@ def home(request, id): 'msg': msg, } ) - book_marks = BookMark.get_available_by_user(user, relation['following']) - movie_marks = MovieMark.get_available_by_user(user, relation['following']) - song_marks = SongMark.get_available_by_user(user, relation['following']) - album_marks = AlbumMark.get_available_by_user(user, relation['following']) - game_marks = GameMark.get_available_by_user(user, relation['following']) + is_following = request.user.is_following(user) + book_marks = BookMark.get_available_by_user(user, is_following) + movie_marks = MovieMark.get_available_by_user(user, is_following) + song_marks = SongMark.get_available_by_user(user, is_following) + album_marks = AlbumMark.get_available_by_user(user, is_following) + game_marks = GameMark.get_available_by_user(user, is_following) + book_reviews = BookReview.get_available_by_user(user, is_following) + movie_reviews = MovieReview.get_available_by_user(user, is_following) + song_reviews = SongReview.get_available_by_user(user, is_following) + album_reviews = AlbumReview.get_available_by_user(user, is_following) + game_reviews = GameReview.get_available_by_user(user, is_following) + collections = Collection.objects.filter(owner=user) + marked_collections = Collection.objects.filter(pk__in=CollectionMark.objects.filter(owner=user).values_list('collection', flat=True)) # book marks - filtered_book_marks = filter_marks(book_marks, BOOKS_PER_SET, 'book') + filtered_book_marks = filter_marks(book_marks, BOOKS_PER_SET, 'book') book_marks_count = count_marks(book_marks, "book") # movie marks filtered_movie_marks = filter_marks(movie_marks, MOVIES_PER_SET, 'movie') - movie_marks_count= count_marks(movie_marks, "movie") + movie_marks_count = count_marks(movie_marks, "movie") # game marks filtered_game_marks = filter_marks(game_marks, GAMES_PER_SET, 'game') @@ -241,7 +157,6 @@ def home(request, id): # music marks filtered_music_marks = filter_marks([song_marks, album_marks], MUSIC_PER_SET, 'music') music_marks_count = count_marks([song_marks, album_marks], "music") - for mark in filtered_music_marks["do_music_marks"] +\ filtered_music_marks["wish_music_marks"] +\ @@ -251,12 +166,12 @@ def home(request, id): mark.type = "album" else: mark.type = "song" - - try: - layout = user.preference.get_serialized_home_layout() - except ObjectDoesNotExist: - Preference.objects.create(user=user) - layout = user.preference.get_serialized_home_layout() + + music_reviews = list(album_reviews.order_by("-edited_time")) + list(song_reviews.order_by("-edited_time")) + for review in music_reviews: + review.type = 'album' if review.__class__ == AlbumReview else 'song' + + layout = user.get_preference().get_serialized_home_layout() return render( request, @@ -271,10 +186,36 @@ def home(request, id): **movie_marks_count, **music_marks_count, **game_marks_count, + + 'book_tags': BookTag.all_by_user(user)[:10] if user == request.user else [], + 'movie_tags': MovieTag.all_by_user(user)[:10] if user == request.user else [], + 'music_tags': AlbumTag.all_by_user(user)[:10] if user == request.user else [], + 'game_tags': GameTag.all_by_user(user)[:10] if user == request.user else [], + + 'book_reviews': book_reviews.order_by("-edited_time")[:BOOKS_PER_SET], + 'movie_reviews': movie_reviews.order_by("-edited_time")[:MOVIES_PER_SET], + 'music_reviews': music_reviews[:MUSIC_PER_SET], + 'game_reviews': game_reviews[:GAMES_PER_SET], + 'book_reviews_more': book_reviews.count() > BOOKS_PER_SET, + 'movie_reviews_more': movie_reviews.count() > MOVIES_PER_SET, + 'music_reviews_more': len(music_reviews) > MUSIC_PER_SET, + 'game_reviews_more': game_reviews.count() > GAMES_PER_SET, + 'book_reviews_count': book_reviews.count(), + 'movie_reviews_count': movie_reviews.count(), + 'music_reviews_count': len(music_reviews), + 'game_reviews_count': game_reviews.count(), + + 'collections': collections.order_by("-edited_time")[:BOOKS_PER_SET], + 'collections_count': collections.count(), + 'collections_more': collections.count() > BOOKS_PER_SET, + + 'marked_collections': marked_collections.order_by("-edited_time")[:BOOKS_PER_SET], + 'marked_collections_count': marked_collections.count(), + 'marked_collections_more': marked_collections.count() > BOOKS_PER_SET, + 'layout': layout, 'reports': reports, 'unread_announcements': unread_announcements, - 'latest_task': latest_task, } ) else: @@ -283,7 +224,7 @@ def home(request, id): def filter_marks(querysets, maximum, type_name): """ - Filter marks by amount limits and order them edited time, store results in a dict, + Filter marks by amount limits and order them edited time, store results in a dict, which could be directly used in template. @param querysets: one queryset or multiple querysets as a list """ @@ -295,9 +236,9 @@ def filter_marks(querysets, maximum, type_name): marks = [] count = 0 for queryset in querysets: - marks += list(queryset.filter(status=MarkStatusEnum[status.upper()]).order_by("-edited_time")[:maximum]) + marks += list(queryset.filter(status=MarkStatusEnum[status.upper()]).order_by("-created_time")[:maximum]) count += queryset.filter(status=MarkStatusEnum[status.upper()]).count() - + # marks marks = sorted(marks, key=lambda e: e.edited_time, reverse=True)[:maximum] result[f"{status}_{type_name}_marks"] = marks @@ -309,6 +250,7 @@ def filter_marks(querysets, maximum, type_name): return result + def count_marks(querysets, type_name): """ Count all available marks, then assembly a dict to be used in template @@ -327,44 +269,11 @@ def count_marks(querysets, type_name): @mastodon_request_included @login_required -def followers(request, id): +def followers(request, id): if request.method == 'GET': - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) - # mastodon request - if not user == request.user: - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: - msg = _("你没有访问TA主页的权限😥") - return render( - request, - 'common/error.html', - { - 'msg': msg, - } - ) - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) + user = User.get(id) + if user is None or user != request.user: + return render_user_not_found(request) return render( request, 'users/relation_list.html', @@ -381,42 +290,9 @@ def followers(request, id): @login_required def following(request, id): if request.method == 'GET': - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) - # mastodon request - if not user == request.user: - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: - msg = _("你没有访问TA主页的权限😥") - return render( - request, - 'common/error.html', - { - 'msg': msg, - } - ) - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) + user = User.get(id) + if user is None or user != request.user: + return render_user_not_found(request) return render( request, 'users/relation_list.html', @@ -433,35 +309,15 @@ def following(request, id): @login_required def book_list(request, id, status): if request.method == 'GET': - if not status.upper() in MarkStatusEnum.names: + if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']: return HttpResponseBadRequest() - - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) - if not user == request.user: - # mastodon request - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: + + user = User.get(id) + if user is None: + return render_user_not_found(request) + tag = request.GET.get('t', default='') + if user != request.user: + if request.user.is_blocked_by(user) or request.user.is_blocking(user): msg = _("你没有访问TA主页的权限😥") return render( request, @@ -470,13 +326,22 @@ def book_list(request, id, status): 'msg': msg, } ) - queryset = BookMark.get_available_by_user(user, relation['following']).filter( - status=MarkStatusEnum[status.upper()]).order_by("-edited_time") - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) + is_following = request.user.is_following(user) + if status == 'reviewed': + queryset = BookReview.get_available_by_user(user, is_following).order_by("-edited_time") + elif status == 'tagged': + queryset = BookTag.find_by_user(tag, user, request.user).order_by("-mark__created_time") + else: + queryset = BookMark.get_available_by_user(user, is_following).filter( + status=MarkStatusEnum[status.upper()]).order_by("-created_time") else: - queryset = BookMark.objects.filter( - owner=user, status=MarkStatusEnum[status.upper()]).order_by("-edited_time") + if status == 'reviewed': + queryset = BookReview.objects.filter(owner=user).order_by("-edited_time") + elif status == 'tagged': + queryset = BookTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time") + else: + queryset = BookMark.objects.filter( + owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time") paginator = Paginator(queryset, ITEMS_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) @@ -484,14 +349,20 @@ def book_list(request, id, status): mark.book.tag_list = mark.book.get_tags_manager().values('content').annotate( tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages) - list_title = str(BookMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的书")) + if status == 'reviewed': + list_title = str(_("评论过的书")) + elif status == 'tagged': + list_title = str(_(f"标记为「{tag}」的书")) + else: + list_title = str(BookMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的书")) return render( request, - 'users/book_list.html', + 'users/item_list.html', { 'marks': marks, 'user': user, - 'list_title' : list_title, + 'status': status, + 'list_title': list_title, } ) else: @@ -502,35 +373,15 @@ def book_list(request, id, status): @login_required def movie_list(request, id, status): if request.method == 'GET': - if not status.upper() in MarkStatusEnum.names: + if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']: return HttpResponseBadRequest() - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) - if not user == request.user: - # mastodon request - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: + user = User.get(id) + if user is None: + return render_user_not_found(request) + tag = request.GET.get('t', default='') + if user != request.user: + if request.user.is_blocked_by(user) or request.user.is_blocking(user): msg = _("你没有访问TA主页的权限😥") return render( request, @@ -539,14 +390,21 @@ def movie_list(request, id, status): 'msg': msg, } ) - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) - - queryset = MovieMark.get_available_by_user(user, relation['following']).filter( - status=MarkStatusEnum[status.upper()]).order_by("-edited_time") + is_following = request.user.is_following(user) + if status == 'reviewed': + queryset = MovieReview.get_available_by_user(user, is_following).order_by("-edited_time") + elif status == 'tagged': + queryset = MovieTag.find_by_user(tag, user, request.user).order_by("-mark__created_time") + else: + queryset = MovieMark.get_available_by_user(user, is_following).filter( + status=MarkStatusEnum[status.upper()]).order_by("-created_time") else: - queryset = MovieMark.objects.filter( - owner=user, status=MarkStatusEnum[status.upper()]).order_by("-edited_time") + if status == 'reviewed': + queryset = MovieReview.objects.filter(owner=user).order_by("-edited_time") + elif status == 'tagged': + queryset = MovieTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time") + else: + queryset = MovieMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time") paginator = Paginator(queryset, ITEMS_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) @@ -554,53 +412,40 @@ def movie_list(request, id, status): mark.movie.tag_list = mark.movie.get_tags_manager().values('content').annotate( tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages) - list_title = str(MovieMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的电影和剧集")) + if status == 'reviewed': + list_title = str(_("评论过的电影和剧集")) + elif status == 'tagged': + list_title = str(_(f"标记为「{tag}」的电影和剧集")) + else: + list_title = str(MovieMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的电影和剧集")) + return render( request, - 'users/movie_list.html', + 'users/item_list.html', { 'marks': marks, 'user': user, - 'list_title' : list_title, + 'status': status, + 'list_title': list_title, } ) else: return HttpResponseBadRequest() - + @mastodon_request_included @login_required def game_list(request, id, status): if request.method == 'GET': - if not status.upper() in MarkStatusEnum.names: + if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']: return HttpResponseBadRequest() - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) - if not user == request.user: - # mastodon request - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: + user = User.get(id) + if user is None: + return render_user_not_found(request) + tag = request.GET.get('t', default='') + if user != request.user: + if request.user.is_blocked_by(user) or request.user.is_blocking(user): msg = _("你没有访问TA主页的权限😥") return render( request, @@ -609,14 +454,22 @@ def game_list(request, id, status): 'msg': msg, } ) - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) - - queryset = GameMark.get_available_by_user(user, relation['following']).filter( - status=MarkStatusEnum[status.upper()]).order_by("-edited_time") + is_following = request.user.is_following(user) + if status == 'reviewed': + queryset = GameReview.get_available_by_user(user, is_following).order_by("-edited_time") + elif status == 'tagged': + queryset = GameTag.find_by_user(tag, user, request.user).order_by("-mark__created_time") + else: + queryset = GameMark.get_available_by_user(user, is_following).filter( + status=MarkStatusEnum[status.upper()]).order_by("-created_time") else: - queryset = GameMark.objects.filter( - owner=user, status=MarkStatusEnum[status.upper()]).order_by("-edited_time") + if status == 'reviewed': + queryset = GameReview.objects.filter(owner=user).order_by("-edited_time") + elif status == 'tagged': + queryset = GameTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time") + else: + queryset = GameMark.objects.filter( + owner=user, status=MarkStatusEnum[status.upper()]).order_by("-created_time") paginator = Paginator(queryset, ITEMS_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) @@ -624,53 +477,39 @@ def game_list(request, id, status): mark.game.tag_list = mark.game.get_tags_manager().values('content').annotate( tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages) - list_title = str(GameMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的游戏")) + if status == 'reviewed': + list_title = str(_("评论过的游戏")) + elif status == 'tagged': + list_title = str(_(f"标记为「{tag}」的游戏")) + else: + list_title = str(GameMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的游戏")) return render( request, - 'users/game_list.html', + 'users/item_list.html', { 'marks': marks, 'user': user, - 'list_title' : list_title, + 'status': status, + 'list_title': list_title, } ) else: return HttpResponseBadRequest() - + @mastodon_request_included @login_required def music_list(request, id, status): if request.method == 'GET': - if not status.upper() in MarkStatusEnum.names: + if status.upper() not in MarkStatusEnum.names and status not in ['reviewed', 'tagged']: return HttpResponseBadRequest() - if isinstance(id, str): - try: - username = id.split('@')[0] - site = id.split('@')[1] - except IndexError as e: - return HttpResponseBadRequest("Invalid user id") - query_kwargs = {'username': username, 'mastodon_site': site} - elif isinstance(id, int): - query_kwargs = {'pk': id} - try: - user = User.objects.get(**query_kwargs) - except ObjectDoesNotExist: - msg = _("😖哎呀这位老师还没有注册书影音呢，快去长毛象喊TA来吧！") - sec_msg = _("目前只开放本站用户注册") - return render( - request, - 'common/error.html', - { - 'msg': msg, - 'secondary_msg': sec_msg, - } - ) + user = User.get(id) + if user is None: + return render_user_not_found(request) + tag = request.GET.get('t', default='') if not user == request.user: - # mastodon request - relation = get_relationship(request.user, user, request.session['oauth_token'])[0] - if relation['blocked_by']: + if request.user.is_blocked_by(user) or request.user.is_blocking(user): msg = _("你没有访问TA主页的权限😥") return render( request, @@ -679,39 +518,55 @@ def music_list(request, id, status): 'msg': msg, } ) - queryset = list(AlbumMark.get_available_by_user(user, relation['following']).filter( - status=MarkStatusEnum[status.upper()])) \ - + list(SongMark.get_available_by_user(user, relation['following']).filter( - status=MarkStatusEnum[status.upper()])) - - user.target_site_id = get_cross_site_id( - user, request.user.mastodon_site, request.session['oauth_token']) + is_following = request.user.is_following(user) + if status == 'reviewed': + queryset = list(AlbumReview.get_available_by_user(user, is_following).order_by("-edited_time")) + \ + list(SongReview.get_available_by_user(user, is_following).order_by("-edited_time")) + elif status == 'tagged': + queryset = list(AlbumTag.find_by_user(tag, user, request.user).order_by("-mark__created_time")) + else: + queryset = list(AlbumMark.get_available_by_user(user, is_following).filter( + status=MarkStatusEnum[status.upper()])) \ + + list(SongMark.get_available_by_user(user, is_following).filter( + status=MarkStatusEnum[status.upper()])) else: - queryset = list(AlbumMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()])) \ - + list(SongMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()])) + if status == 'reviewed': + queryset = list(AlbumReview.objects.filter(owner=user).order_by("-edited_time")) + \ + list(SongReview.objects.filter(owner=user).order_by("-edited_time")) + elif status == 'tagged': + queryset = list(AlbumTag.objects.filter(content=tag, mark__owner=user).order_by("-mark__created_time")) + else: + queryset = list(AlbumMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()])) \ + + list(SongMark.objects.filter(owner=user, status=MarkStatusEnum[status.upper()])) queryset = sorted(queryset, key=lambda e: e.edited_time, reverse=True) paginator = Paginator(queryset, ITEMS_PER_PAGE) page_number = request.GET.get('page', default=1) marks = paginator.get_page(page_number) for mark in marks: - if mark.__class__ == AlbumMark: + if mark.__class__ in [AlbumMark, AlbumReview, AlbumTag]: mark.music = mark.album mark.music.tag_list = mark.album.get_tags_manager().values('content').annotate( tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] - elif mark.__class__ == SongMark: + elif mark.__class__ == SongMark or mark.__class__ == SongReview: mark.music = mark.song mark.music.tag_list = mark.song.get_tags_manager().values('content').annotate( tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] marks.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages) - list_title = str(MusicMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的音乐")) + if status == 'reviewed': + list_title = str(_("评论过的音乐")) + elif status == 'tagged': + list_title = str(_(f"标记为「{tag}」的音乐")) + else: + list_title = str(MusicMarkStatusTranslator(MarkStatusEnum[status.upper()])) + str(_("的音乐")) return render( request, - 'users/music_list.html', + 'users/item_list.html', { 'marks': marks, 'user': user, - 'list_title' : list_title, + 'status': status, + 'list_title': list_title, } ) else: @@ -724,7 +579,7 @@ def set_layout(request): layout = json.loads(request.POST.get('layout')) request.user.preference.home_layout = layout request.user.preference.save() - return redirect(reverse("common:home")) + return redirect(reverse("users:home", args=[request.user.mastodon_username])) else: return HttpResponseBadRequest() @@ -751,14 +606,14 @@ def report(request): form.instance.is_read = False form.instance.submit_user = request.user form.save() - return redirect(reverse("users:home", args=[form.instance.reported_user.id])) + return redirect(reverse("users:home", args=[form.instance.reported_user.mastodon_username])) else: return render( request, 'users/report.html', { 'form': form, - } + } ) else: return HttpResponseBadRequest() @@ -782,15 +637,37 @@ def manage_report(request): return HttpResponseBadRequest() -# Utils -######################################## -def auth_login(request, user, token): - """ Decorates django ``login()``. Attach token to session.""" - request.session['oauth_token'] = token - auth.login(request, user) +@login_required +def collection_list(request, id): + from collection.views import list + user = User.get(id) + if user is None: + return render_user_not_found(request) + return list(request, user.id) -def auth_logout(request): - """ Decorates django ``logout()``. Release token in session.""" - del request.session['oauth_token'] - auth.logout(request) +@login_required +def marked_collection_list(request, id): + from collection.views import list + user = User.get(id) + if user is None: + return render_user_not_found(request) + return list(request, user.id, True) + + +@login_required +def tag_list(request, id): + user = User.get(id) + if user is None: + return render_user_not_found(request) + if user != request.user: + raise PermissionDenied() # tag list is for user's own view only, for now + return render( + request, + 'users/tags.html', { + 'book_tags': BookTag.all_by_user(user), + 'movie_tags': MovieTag.all_by_user(user), + 'music_tags': AlbumTag.all_by_user(user), + 'game_tags': GameTag.all_by_user(user), + } + )