letterboxd import ui

catalog/migrations/0011_alter_externalresource_id_type_and_more.py

@@ -51,6 +51,7 @@ class Migration(migrations.Migration):
                     ("spotify_artist", "Spotify艺术家"),
                     ("tmdb_person", "TMDB影人"),
                     ("igdb", "IGDB游戏"),
+                    ("bgg", "BGG桌游"),
                     ("steam", "Steam游戏"),
                     ("bangumi", "Bangumi"),
                     ("apple_podcast", "苹果播客"),
@@ -104,6 +105,7 @@ class Migration(migrations.Migration):
                     ("spotify_artist", "Spotify艺术家"),
                     ("tmdb_person", "TMDB影人"),
                     ("igdb", "IGDB游戏"),
+                    ("bgg", "BGG桌游"),
                     ("steam", "Steam游戏"),
                     ("bangumi", "Bangumi"),
                     ("apple_podcast", "苹果播客"),

catalog/sites/douban_music.py

@@ -84,7 +84,7 @@ class DoubanMusic(AbstractSite):
             "genre": genre,
             "release_date": release_date,
             "duration": None,
-            "company": [company],
+            "company": [company] if company else [],
             "track_list": track_list,
             "brief": brief,
             "cover_image_url": img_url,

common/utils.py

@@ -151,7 +151,7 @@ class PageLinksGenerator:
         # assert self.has_prev is not None and self.has_next is not None
 
 
-def GenerateDateUUIDMediaFilePath(instance, filename, path_root):
+def GenerateDateUUIDMediaFilePath(filename, path_root):
     ext = filename.split(".")[-1]
     filename = "%s.%s" % (uuid.uuid4(), ext)
     root = ""

journal/exporters/doufen.py

@@ -33,7 +33,7 @@ def export_marks_task(user):
     user.preference.export_status["marks_pending"] = True
     user.preference.save(update_fields=["export_status"])
     filename = GenerateDateUUIDMediaFilePath(
-        None, "f.xlsx", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT
+        "f.xlsx", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT
     )
     if not os.path.exists(os.path.dirname(filename)):
         os.makedirs(os.path.dirname(filename))

journal/importers/douban.py

@@ -28,9 +28,7 @@ def _fetch_remote_image(url):
         imgdl = ProxiedImageDownloader(url)
         raw_img = imgdl.download().content
         ext = imgdl.extention
-        f = GenerateDateUUIDMediaFilePath(
+        f = GenerateDateUUIDMediaFilePath(f"x.{ext}", settings.MARKDOWNX_MEDIA_PATH)
-            None, f"x.{ext}", settings.MARKDOWNX_MEDIA_PATH
-        )
         file = settings.MEDIA_ROOT + "/" + f
         local_url = settings.MEDIA_URL + f
         os.makedirs(os.path.dirname(file), exist_ok=True)
@@ -96,9 +94,7 @@ class DoubanImporter:
             file = (
                 settings.MEDIA_ROOT
                 + "/"
-                + GenerateDateUUIDMediaFilePath(
+                + GenerateDateUUIDMediaFilePath("x.xlsx", settings.SYNC_FILE_PATH_ROOT)
-                    None, "x.xlsx", settings.SYNC_FILE_PATH_ROOT
-                )
             )
             os.makedirs(os.path.dirname(file), exist_ok=True)
             with open(file, "wb") as destination:

journal/importers/letterboxd.py

@@ -1,14 +1,10 @@
 import csv
-import re
 import tempfile
 import zipfile
-from datetime import datetime
 
-from django.utils import timezone
+import pytz
 from django.utils.dateparse import parse_datetime
-from django.utils.timezone import make_aware
 from loguru import logger
-from user_messages import api as msg
 
 from catalog.common import *
 from catalog.common.downloaders import *
@@ -16,6 +12,8 @@ from catalog.models import *
 from journal.models import *
 from users.models import *
 
+_tz_sh = pytz.timezone("Asia/Shanghai")
+
 
 class LetterboxdImporter(Task):
     TaskQueue = "import"
@@ -30,13 +28,20 @@ class LetterboxdImporter(Task):
         "file": None,
     }
 
+    class Meta:
+        proxy = True
+
     def get_item_by_url(self, url):
         try:
-            h = BasicDownloader(url).html()  # type:ignore
+            h = BasicDownloader(url).download().html()
-            tt = h.xpath("//body/@data-tmdb-type")[0].strip()
+            if not h.xpath("//body/@data-tmdb-type"):
-            ti = h.xpath("//body/@data-tmdb-type")[0].strip()
+                i = h.xpath('//span[@class="film-title-wrapper"]/a/@href')
-            if tt != "movie":
+                u2 = "https://letterboxd.com" + i[0]  # type:ignore
-                logger.error(f"Unknown TMDB type {tt} / {ti}")
+                h = BasicDownloader(u2).download().html()
+            tt = h.xpath("//body/@data-tmdb-type")[0].strip()  # type:ignore
+            ti = str(h.xpath("//body/@data-tmdb-id")[0].strip())  # type:ignore
+            if tt != "movie" or not ti:
+                logger.error(f"Unknown TMDB ({tt}/{ti}) for {url}")
                 return None
             site = SiteManager.get_site_by_id(IdType.TMDB_Movie, ti)
             if not site:
@@ -50,6 +55,7 @@ class LetterboxdImporter(Task):
         item = self.get_item_by_url(url)
         if not item:
             logger.error(f"Unable to get item for {url}")
+            self.progress(-1)
             return
         owner = self.user.identity
         mark = Mark(owner, item)
@@ -61,7 +67,8 @@ class LetterboxdImporter(Task):
                 and shelf_type == ShelfType.WISHLIST
             )
         ):
-            return
+            self.progress(0)
+            return 0
         visibility = self.metadata["visibility"]
         shelf_time_offset = {
             ShelfType.WISHLIST: " 20:00:00",
@@ -69,16 +76,32 @@ class LetterboxdImporter(Task):
             ShelfType.COMPLETE: " 22:00:00",
         }
         dt = parse_datetime(date + shelf_time_offset[shelf_type])
+        if dt:
+            dt = dt.replace(tzinfo=_tz_sh)
         mark.update(
             shelf_type,
             comment_text=review or None,
-            rating_grade=round(rating * 2) if rating else None,
+            rating_grade=round(float(rating) * 2) if rating else None,
             visibility=visibility,
             created_time=dt,
         )
         if tags:
             tag_titles = [s.strip() for s in tags.split(",")]
             TagManager.tag_item(item, owner, tag_titles, visibility)
+        self.progress(1)
+
+    def progress(self, mark_state: int):
+        self.metadata["total"] += 1
+        self.metadata["processed"] += 1
+        match mark_state:
+            case 1:
+                self.metadata["imported"] += 1
+            case 0:
+                self.metadata["skipped"] += 1
+            case _:
+                self.metadata["failed"] += 1
+        self.message = f"{self.metadata['imported']} imported, {self.metadata['skipped']} skipped, {self.metadata['failed']} failed"
+        self.save(update_fields=["metadata", "message"])
 
     def run(self):
         uris = set()
@@ -95,8 +118,8 @@ class LetterboxdImporter(Task):
                             row["Letterboxd URI"],
                             ShelfType.COMPLETE,
                             row["Watched Date"],
-                            row["Review"],
                             row["Rating"],
+                            row["Review"],
                             row["Tags"],
                         )
                 with open(tmpdirname + "/ratings.csv") as f:

journal/migrations/0022_letterboxdimporter.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.9 on 2024-01-11 01:47
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("users", "0019_task"),
+        ("journal", "0021_pieceinteraction_pieceinteraction_unique_interaction"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="LetterboxdImporter",
+            fields=[],
+            options={
+                "proxy": True,
+                "indexes": [],
+                "constraints": [],
+            },
+            bases=("users.task",),
+        ),
+    ]

users/data.py

@@ -1,3 +1,5 @@
+import os
+
 import django_rq
 from django.conf import settings
 from django.contrib import messages
@@ -8,14 +10,15 @@ from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
 from common.config import *
+from common.utils import GenerateDateUUIDMediaFilePath
 from journal.exporters.doufen import export_marks_task
 from journal.importers.douban import DoubanImporter
 from journal.importers.goodreads import GoodreadsImporter
+from journal.importers.letterboxd import LetterboxdImporter
 from journal.importers.opml import OPMLImporter
 from journal.models import reset_journal_visibility_for_user
 from mastodon.api import *
 from social.models import reset_social_visibility_for_user
-from takahe.models import Identity
 
 from .account import *
 from .tasks import *
@@ -71,6 +74,7 @@ def data(request):
             "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE,
             "import_status": request.user.preference.import_status,
             "export_status": request.user.preference.export_status,
+            "letterboxd_task": LetterboxdImporter.latest_task(request.user),
         },
     )
 
@@ -175,6 +179,27 @@ def import_douban(request):
     return redirect(reverse("users:data"))
 
 
+@login_required
+def import_letterboxd(request):
+    if request.method == "POST":
+        f = (
+            settings.MEDIA_ROOT
+            + "/"
+            + GenerateDateUUIDMediaFilePath("x.zip", settings.SYNC_FILE_PATH_ROOT)
+        )
+        os.makedirs(os.path.dirname(f), exist_ok=True)
+        with open(f, "wb+") as destination:
+            for chunk in request.FILES["file"].chunks():
+                destination.write(chunk)
+        LetterboxdImporter.enqueue(
+            request.user,
+            visibility=int(request.POST.get("visibility", 0)),
+            file=f,
+        )
+        messages.add_message(request, messages.INFO, _("文件上传成功，等待后台导入。"))
+    return redirect(reverse("users:data"))
+
+
 @login_required
 def import_opml(request):
     if request.method == "POST":

users/models/report.py

@@ -10,9 +10,7 @@ from .user import User
 
 
 def report_image_path(instance, filename):
-    return GenerateDateUUIDMediaFilePath(
+    return GenerateDateUUIDMediaFilePath(filename, settings.REPORT_MEDIA_PATH_ROOT)
-        instance, filename, settings.REPORT_MEDIA_PATH_ROOT
-    )
 
 
 class Report(models.Model):

users/models/task.py

@@ -1,27 +1,9 @@
-import hashlib
+import django_rq
-import re
-from functools import cached_property
-from operator import index
-
 from auditlog.context import set_actor
-from django.conf import settings
-from django.contrib.auth.models import AbstractUser
-from django.core import validators
-from django.core.exceptions import ValidationError
-from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
-from django.db.models import F, Q, Value
-from django.db.models.functions import Concat, Lower
-from django.templatetags.static import static
-from django.urls import reverse
-from django.utils import timezone
-from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 from loguru import logger
-
+from user_messages import api as msg
-from management.models import Announcement
-from mastodon.api import *
-from takahe.utils import Takahe
 
 from .user import User
 
@@ -57,6 +39,14 @@ class Task(models.Model):
     def __str__(self):
         return self.job_id
 
+    @classmethod
+    def latest_task(cls, user: User):
+        return (
+            cls.objects.filter(user=user, type=cls.TaskType)
+            .order_by("-created_time")
+            .first()
+        )
+
     @classmethod
     def enqueue(cls, user: User, **kwargs) -> "Task":
         d = cls.DefaultMetadata.copy()
@@ -80,6 +70,12 @@ class Task(models.Model):
             task.message = "Error occured."
             task.state = cls.States.failed
             task.save(update_fields=["state", "message"])
+        task = cls.objects.get(pk=task_id)
+        if task.message:
+            if task.state == cls.States.complete:
+                msg.success(task.user, f"[{task.type}] {task.message}")
+            else:
+                msg.error(task.user, f"[{task.type}] {task.message}")
 
     def run(self) -> None:
         raise NotImplemented

users/templates/users/data.html

@@ -104,6 +104,54 @@
             </form>
           </details>
         </article>
+        <article>
+          <details>
+            <summary>{% trans '导入Letterboxd标记' %}</summary>
+            <form action="{% url 'users:import_letterboxd' %}"
+                  method="post"
+                  enctype="multipart/form-data">
+              {% csrf_token %}
+              在Letterboxd网站的<a href="https://letterboxd.com/settings/data/"
+   target="_blank"
+   rel="noopener">Settings页面中选择DATA</a>，或在其app的Settings菜单中选择Advanced Settings，点击 EXPORT YOUR DATA，即可下载导出名称类似<code>letterboxd-username-2018-03-11-07-52-utc.zip</code>的文件，勿需解压。
+              <br>
+              <input type="file" name="file" required accept=".zip">
+              <p>
+                可见性:
+                <br>
+                <label for="l_visibility_0">
+                  <input type="radio"
+                         name="visibility"
+                         value="0"
+                         required=""
+                         id="l_visibility_0"
+                         checked>
+                  公开
+                </label>
+                <label for="l_visibility_1">
+                  <input type="radio"
+                         name="visibility"
+                         value="1"
+                         required=""
+                         id="l_visibility_1">
+                  仅关注者
+                </label>
+                <label for="l_visibility_2">
+                  <input type="radio"
+                         name="visibility"
+                         value="2"
+                         required=""
+                         id="l_visibility_2">
+                  仅自己
+                </label>
+              </p>
+              <input type="submit" value="{% trans '导入' %}" />
+              {% if letterboxd_task %}
+                最近导入于{{ letterboxd_task.created_time }}，状态：{{ letterboxd_task.get_state_display }}。 {{ letterboxd_task.message }}
+              {% endif %}
+            </form>
+          </details>
+        </article>
         <article>
           <details>
             <summary>{% trans '导入播客订阅列表' %}</summary>

users/urls.py

@@ -19,6 +19,7 @@ urlpatterns = [
     path("data/import/status", data_import_status, name="import_status"),
     path("data/import/goodreads", import_goodreads, name="import_goodreads"),
     path("data/import/douban", import_douban, name="import_douban"),
+    path("data/import/letterboxd", import_letterboxd, name="import_letterboxd"),
     path("data/import/opml", import_opml, name="import_opml"),
     path("data/export/reviews", export_reviews, name="export_reviews"),
     path("data/export/marks", export_marks, name="export_marks"),