From 376357ec9024b658b2ccc6921615a327c42cba63 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 Jan 2024 22:20:57 -0500 Subject: [PATCH] letterboxd import ui --- ...alter_externalresource_id_type_and_more.py | 2 + catalog/sites/douban_music.py | 2 +- common/utils.py | 2 +- journal/exporters/doufen.py | 2 +- journal/importers/douban.py | 8 +-- journal/importers/letterboxd.py | 49 ++++++++++++++----- journal/migrations/0022_letterboxdimporter.py | 24 +++++++++ users/data.py | 27 +++++++++- users/models/report.py | 4 +- users/models/task.py | 36 ++++++-------- users/templates/users/data.html | 48 ++++++++++++++++++ users/urls.py | 1 + 12 files changed, 159 insertions(+), 46 deletions(-) create mode 100644 journal/migrations/0022_letterboxdimporter.py diff --git a/catalog/migrations/0011_alter_externalresource_id_type_and_more.py b/catalog/migrations/0011_alter_externalresource_id_type_and_more.py index 3659a6d5..25fd5de3 100644 --- a/catalog/migrations/0011_alter_externalresource_id_type_and_more.py +++ b/catalog/migrations/0011_alter_externalresource_id_type_and_more.py @@ -51,6 +51,7 @@ class Migration(migrations.Migration): ("spotify_artist", "Spotify艺术家"), ("tmdb_person", "TMDB影人"), ("igdb", "IGDB游戏"), + ("bgg", "BGG桌游"), ("steam", "Steam游戏"), ("bangumi", "Bangumi"), ("apple_podcast", "苹果播客"), @@ -104,6 +105,7 @@ class Migration(migrations.Migration): ("spotify_artist", "Spotify艺术家"), ("tmdb_person", "TMDB影人"), ("igdb", "IGDB游戏"), + ("bgg", "BGG桌游"), ("steam", "Steam游戏"), ("bangumi", "Bangumi"), ("apple_podcast", "苹果播客"), diff --git a/catalog/sites/douban_music.py b/catalog/sites/douban_music.py index fc92cc21..5523ae7c 100644 --- a/catalog/sites/douban_music.py +++ b/catalog/sites/douban_music.py @@ -84,7 +84,7 @@ class DoubanMusic(AbstractSite): "genre": genre, "release_date": release_date, "duration": None, - "company": [company], + "company": [company] if company else [], "track_list": track_list, "brief": brief, "cover_image_url": img_url, diff --git a/common/utils.py b/common/utils.py index 8c6d6191..804ad100 100644 --- a/common/utils.py +++ b/common/utils.py @@ -151,7 +151,7 @@ class PageLinksGenerator: # assert self.has_prev is not None and self.has_next is not None -def GenerateDateUUIDMediaFilePath(instance, filename, path_root): +def GenerateDateUUIDMediaFilePath(filename, path_root): ext = filename.split(".")[-1] filename = "%s.%s" % (uuid.uuid4(), ext) root = "" diff --git a/journal/exporters/doufen.py b/journal/exporters/doufen.py index c4017adb..5cb4b7a6 100644 --- a/journal/exporters/doufen.py +++ b/journal/exporters/doufen.py @@ -33,7 +33,7 @@ def export_marks_task(user): user.preference.export_status["marks_pending"] = True user.preference.save(update_fields=["export_status"]) filename = GenerateDateUUIDMediaFilePath( - None, "f.xlsx", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT + "f.xlsx", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT ) if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) diff --git a/journal/importers/douban.py b/journal/importers/douban.py index f79e8554..a78c83e1 100644 --- a/journal/importers/douban.py +++ b/journal/importers/douban.py @@ -28,9 +28,7 @@ def _fetch_remote_image(url): imgdl = ProxiedImageDownloader(url) raw_img = imgdl.download().content ext = imgdl.extention - f = GenerateDateUUIDMediaFilePath( - None, f"x.{ext}", settings.MARKDOWNX_MEDIA_PATH - ) + f = GenerateDateUUIDMediaFilePath(f"x.{ext}", settings.MARKDOWNX_MEDIA_PATH) file = settings.MEDIA_ROOT + "/" + f local_url = settings.MEDIA_URL + f os.makedirs(os.path.dirname(file), exist_ok=True) @@ -96,9 +94,7 @@ class DoubanImporter: file = ( settings.MEDIA_ROOT + "/" - + GenerateDateUUIDMediaFilePath( - None, "x.xlsx", settings.SYNC_FILE_PATH_ROOT - ) + + GenerateDateUUIDMediaFilePath("x.xlsx", settings.SYNC_FILE_PATH_ROOT) ) os.makedirs(os.path.dirname(file), exist_ok=True) with open(file, "wb") as destination: diff --git a/journal/importers/letterboxd.py b/journal/importers/letterboxd.py index 669e1c7c..35adfd67 100644 --- a/journal/importers/letterboxd.py +++ b/journal/importers/letterboxd.py @@ -1,14 +1,10 @@ import csv -import re import tempfile import zipfile -from datetime import datetime -from django.utils import timezone +import pytz from django.utils.dateparse import parse_datetime -from django.utils.timezone import make_aware from loguru import logger -from user_messages import api as msg from catalog.common import * from catalog.common.downloaders import * @@ -16,6 +12,8 @@ from catalog.models import * from journal.models import * from users.models import * +_tz_sh = pytz.timezone("Asia/Shanghai") + class LetterboxdImporter(Task): TaskQueue = "import" @@ -30,13 +28,20 @@ class LetterboxdImporter(Task): "file": None, } + class Meta: + proxy = True + def get_item_by_url(self, url): try: - h = BasicDownloader(url).html() # type:ignore - tt = h.xpath("//body/@data-tmdb-type")[0].strip() - ti = h.xpath("//body/@data-tmdb-type")[0].strip() - if tt != "movie": - logger.error(f"Unknown TMDB type {tt} / {ti}") + h = BasicDownloader(url).download().html() + if not h.xpath("//body/@data-tmdb-type"): + i = h.xpath('//span[@class="film-title-wrapper"]/a/@href') + u2 = "https://letterboxd.com" + i[0] # type:ignore + h = BasicDownloader(u2).download().html() + tt = h.xpath("//body/@data-tmdb-type")[0].strip() # type:ignore + ti = str(h.xpath("//body/@data-tmdb-id")[0].strip()) # type:ignore + if tt != "movie" or not ti: + logger.error(f"Unknown TMDB ({tt}/{ti}) for {url}") return None site = SiteManager.get_site_by_id(IdType.TMDB_Movie, ti) if not site: @@ -50,6 +55,7 @@ class LetterboxdImporter(Task): item = self.get_item_by_url(url) if not item: logger.error(f"Unable to get item for {url}") + self.progress(-1) return owner = self.user.identity mark = Mark(owner, item) @@ -61,7 +67,8 @@ class LetterboxdImporter(Task): and shelf_type == ShelfType.WISHLIST ) ): - return + self.progress(0) + return 0 visibility = self.metadata["visibility"] shelf_time_offset = { ShelfType.WISHLIST: " 20:00:00", @@ -69,16 +76,32 @@ class LetterboxdImporter(Task): ShelfType.COMPLETE: " 22:00:00", } dt = parse_datetime(date + shelf_time_offset[shelf_type]) + if dt: + dt = dt.replace(tzinfo=_tz_sh) mark.update( shelf_type, comment_text=review or None, - rating_grade=round(rating * 2) if rating else None, + rating_grade=round(float(rating) * 2) if rating else None, visibility=visibility, created_time=dt, ) if tags: tag_titles = [s.strip() for s in tags.split(",")] TagManager.tag_item(item, owner, tag_titles, visibility) + self.progress(1) + + def progress(self, mark_state: int): + self.metadata["total"] += 1 + self.metadata["processed"] += 1 + match mark_state: + case 1: + self.metadata["imported"] += 1 + case 0: + self.metadata["skipped"] += 1 + case _: + self.metadata["failed"] += 1 + self.message = f"{self.metadata['imported']} imported, {self.metadata['skipped']} skipped, {self.metadata['failed']} failed" + self.save(update_fields=["metadata", "message"]) def run(self): uris = set() @@ -95,8 +118,8 @@ class LetterboxdImporter(Task): row["Letterboxd URI"], ShelfType.COMPLETE, row["Watched Date"], - row["Review"], row["Rating"], + row["Review"], row["Tags"], ) with open(tmpdirname + "/ratings.csv") as f: diff --git a/journal/migrations/0022_letterboxdimporter.py b/journal/migrations/0022_letterboxdimporter.py new file mode 100644 index 00000000..9027671c --- /dev/null +++ b/journal/migrations/0022_letterboxdimporter.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.9 on 2024-01-11 01:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0019_task"), + ("journal", "0021_pieceinteraction_pieceinteraction_unique_interaction"), + ] + + operations = [ + migrations.CreateModel( + name="LetterboxdImporter", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("users.task",), + ), + ] diff --git a/users/data.py b/users/data.py index 1a332c9f..87a075d5 100644 --- a/users/data.py +++ b/users/data.py @@ -1,3 +1,5 @@ +import os + import django_rq from django.conf import settings from django.contrib import messages @@ -8,14 +10,15 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ from common.config import * +from common.utils import GenerateDateUUIDMediaFilePath from journal.exporters.doufen import export_marks_task from journal.importers.douban import DoubanImporter from journal.importers.goodreads import GoodreadsImporter +from journal.importers.letterboxd import LetterboxdImporter from journal.importers.opml import OPMLImporter from journal.models import reset_journal_visibility_for_user from mastodon.api import * from social.models import reset_social_visibility_for_user -from takahe.models import Identity from .account import * from .tasks import * @@ -71,6 +74,7 @@ def data(request): "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE, "import_status": request.user.preference.import_status, "export_status": request.user.preference.export_status, + "letterboxd_task": LetterboxdImporter.latest_task(request.user), }, ) @@ -175,6 +179,27 @@ def import_douban(request): return redirect(reverse("users:data")) +@login_required +def import_letterboxd(request): + if request.method == "POST": + f = ( + settings.MEDIA_ROOT + + "/" + + GenerateDateUUIDMediaFilePath("x.zip", settings.SYNC_FILE_PATH_ROOT) + ) + os.makedirs(os.path.dirname(f), exist_ok=True) + with open(f, "wb+") as destination: + for chunk in request.FILES["file"].chunks(): + destination.write(chunk) + LetterboxdImporter.enqueue( + request.user, + visibility=int(request.POST.get("visibility", 0)), + file=f, + ) + messages.add_message(request, messages.INFO, _("文件上传成功，等待后台导入。")) + return redirect(reverse("users:data")) + + @login_required def import_opml(request): if request.method == "POST": diff --git a/users/models/report.py b/users/models/report.py index 4a65d2a2..d89103a3 100644 --- a/users/models/report.py +++ b/users/models/report.py @@ -10,9 +10,7 @@ from .user import User def report_image_path(instance, filename): - return GenerateDateUUIDMediaFilePath( - instance, filename, settings.REPORT_MEDIA_PATH_ROOT - ) + return GenerateDateUUIDMediaFilePath(filename, settings.REPORT_MEDIA_PATH_ROOT) class Report(models.Model): diff --git a/users/models/task.py b/users/models/task.py index c69e4b2c..691bf73f 100644 --- a/users/models/task.py +++ b/users/models/task.py @@ -1,27 +1,9 @@ -import hashlib -import re -from functools import cached_property -from operator import index - +import django_rq from auditlog.context import set_actor -from django.conf import settings -from django.contrib.auth.models import AbstractUser -from django.core import validators -from django.core.exceptions import ValidationError -from django.core.serializers.json import DjangoJSONEncoder from django.db import models -from django.db.models import F, Q, Value -from django.db.models.functions import Concat, Lower -from django.templatetags.static import static -from django.urls import reverse -from django.utils import timezone -from django.utils.deconstruct import deconstructible from django.utils.translation import gettext_lazy as _ from loguru import logger - -from management.models import Announcement -from mastodon.api import * -from takahe.utils import Takahe +from user_messages import api as msg from .user import User @@ -57,6 +39,14 @@ class Task(models.Model): def __str__(self): return self.job_id + @classmethod + def latest_task(cls, user: User): + return ( + cls.objects.filter(user=user, type=cls.TaskType) + .order_by("-created_time") + .first() + ) + @classmethod def enqueue(cls, user: User, **kwargs) -> "Task": d = cls.DefaultMetadata.copy() @@ -80,6 +70,12 @@ class Task(models.Model): task.message = "Error occured." task.state = cls.States.failed task.save(update_fields=["state", "message"]) + task = cls.objects.get(pk=task_id) + if task.message: + if task.state == cls.States.complete: + msg.success(task.user, f"[{task.type}] {task.message}") + else: + msg.error(task.user, f"[{task.type}] {task.message}") def run(self) -> None: raise NotImplemented diff --git a/users/templates/users/data.html b/users/templates/users/data.html index 15874846..f0b02a5f 100644 --- a/users/templates/users/data.html +++ b/users/templates/users/data.html @@ -104,6 +104,54 @@ +
+
+ {% trans '导入Letterboxd标记' %} +
+ {% csrf_token %} + 在Letterboxd网站的Settings页面中选择DATA，或在其app的Settings菜单中选择Advanced Settings，点击 EXPORT YOUR DATA，即可下载导出名称类似letterboxd-username-2018-03-11-07-52-utc.zip的文件，勿需解压。 +
+ +

+ 可见性: +
+ + + +

+ + {% if letterboxd_task %} + 最近导入于{{ letterboxd_task.created_time }}，状态：{{ letterboxd_task.get_state_display }}。 {{ letterboxd_task.message }} + {% endif %} +
+
+
{% trans '导入播客订阅列表' %} diff --git a/users/urls.py b/users/urls.py index 0f9c8959..7495604c 100644 --- a/users/urls.py +++ b/users/urls.py @@ -19,6 +19,7 @@ urlpatterns = [ path("data/import/status", data_import_status, name="import_status"), path("data/import/goodreads", import_goodreads, name="import_goodreads"), path("data/import/douban", import_douban, name="import_douban"), + path("data/import/letterboxd", import_letterboxd, name="import_letterboxd"), path("data/import/opml", import_opml, name="import_opml"), path("data/export/reviews", export_reviews, name="export_reviews"), path("data/export/marks", export_marks, name="export_marks"),