new data model: rework douban import

catalog/common/sites.py

@@ -6,11 +6,12 @@ a Site should map to a unique set of url patterns.
 a Site may scrape a url and store result in ResourceContent
 ResourceContent persists as an ExternalResource which may link to an Item
 """
-from typing import *
+from typing import Callable
 import re
 from .models import ExternalResource
 from dataclasses import dataclass, field
 import logging
+import json
 
 
 _logger = logging.getLogger(__name__)
@@ -20,8 +21,8 @@ _logger = logging.getLogger(__name__)
 class ResourceContent:
     lookup_ids: dict = field(default_factory=dict)
     metadata: dict = field(default_factory=dict)
-    cover_image: bytes = None
-    cover_image_extention: str = None
+    cover_image: bytes | None = None
+    cover_image_extention: str | None = None
 
     def dict(self):
         return {"metadata": self.metadata, "lookup_ids": self.lookup_ids}
@@ -42,25 +43,25 @@ class AbstractSite:
     URL_PATTERNS = [r"\w+://undefined/(\d+)"]
 
     @classmethod
-    def validate_url(self, url: str):
+    def validate_url(cls, url: str):
         u = next(
-            iter([re.match(p, url) for p in self.URL_PATTERNS if re.match(p, url)]),
+            iter([re.match(p, url) for p in cls.URL_PATTERNS if re.match(p, url)]),
             None,
         )
         return u is not None
 
     @classmethod
-    def validate_url_fallback(self, url: str):
+    def validate_url_fallback(cls, url: str):
         return False
 
     @classmethod
-    def id_to_url(self, id_value):
+    def id_to_url(cls, id_value):
         return "https://undefined/" + id_value
 
     @classmethod
-    def url_to_id(self, url: str):
+    def url_to_id(cls, url: str):
         u = next(
-            iter([re.match(p, url) for p in self.URL_PATTERNS if re.match(p, url)]),
+            iter([re.match(p, url) for p in cls.URL_PATTERNS if re.match(p, url)]),
             None,
         )
         return u[1] if u else None
@@ -121,7 +122,7 @@ class AbstractSite:
         auto_link=True,
         preloaded_content=None,
         ignore_existing_content=False,
-    ):
+    ) -> ExternalResource | None:
         """
         Returns an ExternalResource in scraped state if possible
 
@@ -195,7 +196,9 @@ class SiteManager:
         return SiteManager.registry[typ]() if typ in SiteManager.registry else None
 
     @staticmethod
-    def get_site_by_url(url: str):
+    def get_site_by_url(url: str) -> AbstractSite | None:
+        if not url:
+            return None
         cls = next(
             filter(lambda p: p.validate_url(url), SiteManager.registry.values()), None
         )

catalog/templates/item_base.html

@@ -61,8 +61,8 @@
                   <div class="entity-detail__fields">
                     <div class="entity-detail__rating">
                       {% if item.rating %}
-                      <span class="entity-detail__rating-star rating-star" data-rating-score="{{ item.rating | floatformat:'0' }}"></span>
-                      <span class="entity-detail__rating-score"> {{ item.rating }} </span>
+                      <span class="entity-detail__rating-star rating-star" data-rating-score="{{ item.rating | floatformat:0 }}"></span>
+                      <span class="entity-detail__rating-score"> {{ item.rating | floatformat:1 }} </span>
                       <small>({{ item.rating_count }}人评分)</small>
                       {% else %}
                       <span> {% trans '评分：评分人数不足' %}</span>

catalog/templates/sidebar_item.html

@@ -23,7 +23,7 @@
       <div>{% if item.pub_house %}{% trans '出版社：' %}{{ item.pub_house }}{% endif %}</div>
       {% if item.rating %}
       {% trans '评分: ' %}<span class="entity-card__rating-star rating-star" data-rating-score="{{ item.rating }}"></span>
-      <span class="entity-card__rating-score rating-score">{{ item.rating }}</span>
+      <span class="entity-card__rating-score rating-score">{{ item.rating | floatformat:1 }}</span>
       {% endif %}
     </div>
 

catalog/templates/tvseason.html

@@ -38,7 +38,7 @@
 	<div class="entity-detail__rating">
 		{% if item.rating and item.rating_count >= 5 %}
 		<span class="entity-detail__rating-star rating-star" data-rating-score="{{ item.rating | floatformat:"0" }}"></span>
-		<span class="entity-detail__rating-score"> {{ item.rating }} </span>
+		<span class="entity-detail__rating-score"> {{ item.rating | floatformat:1 }} </span>
 		<small>({{ item.rating_count }}人评分)</small>
 		{% else %}
 		<span> {% trans '评分：评分人数不足' %}</span>

journal/importers/douban.py

@@ -0,0 +1,343 @@
+import openpyxl
+import re
+from markdownify import markdownify as md
+from datetime import datetime
+import logging
+import pytz
+from django.conf import settings
+from user_messages import api as msg
+import django_rq
+from common.utils import GenerateDateUUIDMediaFilePath
+import os
+from catalog.common import *
+from catalog.common.downloaders import *
+from catalog.sites.douban import DoubanDownloader
+from journal.models import *
+
+_logger = logging.getLogger(__name__)
+_tz_sh = pytz.timezone("Asia/Shanghai")
+
+
+def _fetch_remote_image(url):
+    try:
+        print(f"fetching remote image {url}")
+        imgdl = ProxiedImageDownloader(url)
+        raw_img = imgdl.download().content
+        ext = imgdl.extention
+        f = GenerateDateUUIDMediaFilePath(
+            None, "x." + ext, settings.MARKDOWNX_MEDIA_PATH
+        )
+        file = settings.MEDIA_ROOT + f
+        local_url = settings.MEDIA_URL + f
+        os.makedirs(os.path.dirname(file), exist_ok=True)
+        with open(file, "wb") as binary_file:
+            binary_file.write(raw_img)
+        # print(f'remote image saved as {local_url}')
+        return local_url
+    except Exception:
+        print(f"unable to fetch remote image {url}")
+        return url
+
+
+class DoubanImporter:
+    total = 0
+    processed = 0
+    skipped = 0
+    imported = 0
+    failed = []
+    user = None
+    visibility = 0
+    file = None
+
+    def __init__(self, user, visibility):
+        self.user = user
+        self.visibility = visibility
+
+    def update_user_import_status(self, status):
+        self.user.preference.import_status["douban_pending"] = status
+        self.user.preference.import_status["douban_file"] = self.file
+        self.user.preference.import_status["douban_visibility"] = self.visibility
+        self.user.preference.import_status["douban_total"] = self.total
+        self.user.preference.import_status["douban_processed"] = self.processed
+        self.user.preference.import_status["douban_skipped"] = self.skipped
+        self.user.preference.import_status["douban_imported"] = self.imported
+        self.user.preference.import_status["douban_failed"] = self.failed
+        self.user.preference.save(update_fields=["import_status"])
+
+    def import_from_file(self, uploaded_file):
+        try:
+            wb = openpyxl.open(
+                uploaded_file, read_only=True, data_only=True, keep_links=False
+            )
+            wb.close()
+            file = settings.MEDIA_ROOT + GenerateDateUUIDMediaFilePath(
+                None, "x.xlsx", settings.SYNC_FILE_PATH_ROOT
+            )
+            os.makedirs(os.path.dirname(file), exist_ok=True)
+            with open(file, "wb") as destination:
+                for chunk in uploaded_file.chunks():
+                    destination.write(chunk)
+            self.file = file
+            self.update_user_import_status(2)
+            jid = f"Douban_{self.user.id}_{os.path.basename(self.file)}"
+            django_rq.get_queue("import").enqueue(
+                self.import_from_file_task, job_id=jid
+            )
+        except Exception:
+            return False
+        # self.import_from_file_task(file, user, visibility)
+        return True
+
+    mark_sheet_config = {
+        "想读": [ShelfType.WISHLIST],
+        "在读": [ShelfType.PROGRESS],
+        "读过": [ShelfType.COMPLETE],
+        "想看": [ShelfType.WISHLIST],
+        "在看": [ShelfType.PROGRESS],
+        "想看": [ShelfType.COMPLETE],
+        "想听": [ShelfType.WISHLIST],
+        "在听": [ShelfType.PROGRESS],
+        "听过": [ShelfType.COMPLETE],
+        "想玩": [ShelfType.WISHLIST],
+        "在玩": [ShelfType.PROGRESS],
+        "玩过": [ShelfType.COMPLETE],
+    }
+    review_sheet_config = {
+        "书评": [Edition],
+        "影评": [Movie],
+        "乐评": [Album],
+        "游戏评论&攻略": [Game],
+    }
+    mark_data = {}
+    review_data = {}
+    entity_lookup = {}
+
+    def load_sheets(self):
+        """Load data into mark_data / review_data / entity_lookup"""
+        f = open(self.file, "rb")
+        wb = openpyxl.load_workbook(f, read_only=True, data_only=True, keep_links=False)
+        for data, config in [
+            (self.mark_data, self.mark_sheet_config),
+            (self.review_data, self.review_sheet_config),
+        ]:
+            for name in config:
+                data[name] = []
+                if name in wb:
+                    print(f"{self.user} parsing {name}")
+                    for row in wb[name].iter_rows(min_row=2, values_only=True):
+                        cells = [cell for cell in row]
+                        if len(cells) > 6 and cells[0]:
+                            data[name].append(cells)
+        for sheet in self.mark_data.values():
+            for cells in sheet:
+                # entity_lookup["title|rating"] = [(url, time), ...]
+                k = f"{cells[0]}|{cells[5]}"
+                v = (cells[3], cells[4])
+                if k in self.entity_lookup:
+                    self.entity_lookup[k].append(v)
+                else:
+                    self.entity_lookup[k] = [v]
+        self.total = sum(map(lambda a: len(a), self.mark_data.values()))
+        self.total += sum(map(lambda a: len(a), self.review_data.values()))
+
+    def guess_entity_url(self, title, rating, timestamp):
+        k = f"{title}|{rating}"
+        if k not in self.entity_lookup:
+            return None
+        v = self.entity_lookup[k]
+        if len(v) > 1:
+            v.sort(
+                key=lambda c: abs(
+                    timestamp
+                    - (
+                        datetime.strptime(c[1], "%Y-%m-%d %H:%M:%S")
+                        if type(c[1]) == str
+                        else c[1]
+                    ).replace(tzinfo=_tz_sh)
+                )
+            )
+        return v[0][0]
+        # for sheet in self.mark_data.values():
+        #     for cells in sheet:
+        #         if cells[0] == title and cells[5] == rating:
+        #             return cells[3]
+
+    def import_from_file_task(self):
+        print(f"{self.user} import start")
+        msg.info(self.user, f"开始导入豆瓣标记和评论")
+        self.update_user_import_status(1)
+        self.load_sheets()
+        print(f"{self.user} sheet loaded, {self.total} lines total")
+        self.update_user_import_status(1)
+        for name, param in self.mark_sheet_config.items():
+            self.import_mark_sheet(self.mark_data[name], param[0], name)
+        for name, param in self.review_sheet_config.items():
+            self.import_review_sheet(self.review_data[name], name)
+        self.update_user_import_status(0)
+        msg.success(
+            self.user,
+            f"豆瓣标记和评论导入完成，共处理{self.total}篇，已存在{self.skipped}篇，新增{self.imported}篇。",
+        )
+        if len(self.failed):
+            msg.error(self.user, f'豆瓣评论导入时未能处理以下网址：\n{" , ".join(self.failed)}')
+
+    def import_mark_sheet(self, worksheet, shelf_type, sheet_name):
+        prefix = f"{self.user} {sheet_name}|"
+        if worksheet is None:  # or worksheet.max_row < 2:
+            print(f"{prefix} empty sheet")
+            return
+        for cells in worksheet:
+            if len(cells) < 6:
+                continue
+            # title = cells[0] or ""
+            url = cells[3]
+            time = cells[4]
+            rating = cells[5]
+            rating_grade = int(rating) * 2 if rating else None
+            tags = cells[6] if len(cells) >= 7 else ""
+            tags = tags.split(",") if tags else []
+            comment = cells[7] if len(cells) >= 8 else None
+            self.processed += 1
+            if time:
+                if type(time) == str:
+                    time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
+                time = time.replace(tzinfo=_tz_sh)
+            else:
+                time = None
+            r = self.import_mark(url, shelf_type, comment, rating_grade, tags, time)
+            if r == 1:
+                self.imported += 1
+            elif r == 2:
+                self.skipped += 1
+            self.update_user_import_status(1)
+
+    def import_mark(self, url, shelf_type, comment, rating_grade, tags, time):
+        """
+        Import one mark: return 1: done / 2: skipped / None: failed
+        """
+        item = self.get_item_by_url(url)
+        if not item:
+            print(f"{self.user} | match/fetch {url} failed")
+            return
+        mark = Mark(self.user, item)
+        if (
+            mark.shelf_type == shelf_type
+            or mark.shelf_type == ShelfType.COMPLETE
+            or (
+                mark.shelf_type == ShelfType.PROGRESS
+                and shelf_type == ShelfType.WISHLIST
+            )
+        ):
+            return 2
+        mark.update(
+            shelf_type, comment, rating_grade, self.visibility, created_time=time
+        )
+        if tags:
+            TagManager.tag_item_by_user(item, self.user, tags)
+        return 1
+
+    def import_review_sheet(self, worksheet, sheet_name):
+        prefix = f"{self.user} {sheet_name}|"
+        if worksheet is None:  # or worksheet.max_row < 2:
+            print(f"{prefix} empty sheet")
+            return
+        for cells in worksheet:
+            if len(cells) < 6:
+                continue
+            title = cells[0]
+            entity_title = (
+                re.sub("^《", "", re.sub("》$", "", cells[1])) if cells[1] else ""
+            )
+            review_url = cells[2]
+            time = cells[3]
+            rating = cells[4]
+            content = cells[6]
+            self.processed += 1
+            if time:
+                if type(time) == str:
+                    time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
+                time = time.replace(tzinfo=_tz_sh)
+            else:
+                time = None
+            if not content:
+                content = ""
+            if not title:
+                title = ""
+            r = self.import_review(
+                entity_title, rating, title, review_url, content, time
+            )
+            if r == 1:
+                self.imported += 1
+            elif r == 2:
+                self.skipped += 1
+            else:
+                self.failed.append(review_url)
+            self.update_user_import_status(1)
+
+    def get_item_by_url(self, url):
+        item = None
+        try:
+            site = SiteManager.get_site_by_url(url)
+            item = site.get_item()
+            if not item:
+                print(f"fetching {url}")
+                site.get_resource_ready()
+                item = site.get_item()
+            else:
+                print(f"matched {url}")
+        except Exception as e:
+            print(f"fetching exception: {url} {e}")
+            _logger.error(f"scrape failed: {url}", exc_info=e)
+        if item is None:
+            self.failed.append(url)
+        return item
+
+    def import_review(self, entity_title, rating, title, review_url, content, time):
+        """
+        Import one review: return 1: done / 2: skipped / None: failed
+        """
+        prefix = f"{self.user} |"
+        url = self.guess_entity_url(entity_title, rating, time)
+        if url is None:
+            print(f"{prefix} fetching review {review_url}")
+            try:
+                h = DoubanDownloader(review_url).download().html()
+                for u in h.xpath("//header[@class='main-hd']/a/@href"):
+                    if ".douban.com/subject/" in u:
+                        url = u
+                if not url:
+                    print(
+                        f"{prefix} fetching error {review_url} unable to locate entity url"
+                    )
+                    return
+            except Exception:
+                print(f"{prefix} fetching review exception {review_url}")
+                return
+        item = self.get_item_by_url(url)
+        if not item:
+            print(f"{prefix} match/fetch {url} failed")
+            return
+        if Review.objects.filter(owner=self.user, item=item).exists():
+            return 2
+        content = re.sub(
+            r'<span style="font-weight: bold;">([^<]+)</span>', r"<b>\1</b>", content
+        )
+        content = re.sub(r"(<img [^>]+>)", r"\1<br>", content)
+        content = re.sub(
+            r'<div class="image-caption">([^<]+)</div>', r"<br><i>\1</i><br>", content
+        )
+        content = md(content)
+        content = re.sub(
+            r"(?<=!\[\]\()([^)]+)(?=\))", lambda x: _fetch_remote_image(x[1]), content
+        )
+        params = {
+            "owner": self.user,
+            "created_time": time,
+            "edited_time": time,
+            "title": title,
+            "body": content,
+            "visibility": self.visibility,
+            "item": item,
+        }
+        Review.objects.create(**params)
+        return 1

journal/models.py

@@ -68,6 +68,33 @@ def query_item_category(item_category):
     return Q(item__polymorphic_ctype__in=contenttype_ids)
 
 
+# class ImportStatus(Enum):
+#     QUEUED = 0
+#     PROCESSING = 1
+#     FINISHED = 2
+
+
+# class ImportSession(models.Model):
+#     owner = models.ForeignKey(User, on_delete=models.CASCADE)
+#     status = models.PositiveSmallIntegerField(default=ImportStatus.QUEUED)
+#     importer = models.CharField(max_length=50)
+#     file = models.CharField()
+#     default_visibility = models.PositiveSmallIntegerField()
+#     total = models.PositiveIntegerField()
+#     processed = models.PositiveIntegerField()
+#     skipped = models.PositiveIntegerField()
+#     imported = models.PositiveIntegerField()
+#     failed = models.PositiveIntegerField()
+#     logs = models.JSONField(default=list)
+#     created_time = models.DateTimeField(auto_now_add=True)
+#     edited_time = models.DateTimeField(auto_now=True)
+
+#     class Meta:
+#         indexes = [
+#             models.Index(fields=["owner", "importer", "created_time"]),
+#         ]
+
+
 class Piece(PolymorphicModel, UserOwnedObjectMixin):
     url_path = "piece"  # subclass must specify this
     uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
@@ -274,7 +301,6 @@ class Reply(Piece):
     )
     title = models.CharField(max_length=500, null=True)
     body = MarkdownxField()
-    pass
 
 
 """

journal/templates/profile.html

@@ -46,13 +46,12 @@
                   {{ shelf.count }}
                 </span>
                 {% if shelf.count > 5 %}
-                <a href="{% url 'journal:user_mark_list' user.mastodon_username shelf_type category %}"
-                class="entity-sort__more-link">{% trans '更多' %}</a>
+                <a href="{% if shelf_type == 'reviewed' %}{% url 'journal:user_review_list' user.mastodon_username  category %}{% else %}{% url 'journal:user_mark_list' user.mastodon_username shelf_type category %}{% endif %}" class="entity-sort__more-link">{% trans '更多' %}</a>
                 {% endif %}
                 <ul class="entity-sort__entity-list">
                   {% for member in shelf.members %}
                   <li class="entity-sort__entity">
-                    <a href="{{ member.item.url }}">
+                    <a href="{% if shelf_type == 'reviewed' %}{{ member.url }}{% else %}{{ member.item.url }}{% endif %}">
                       <img src="{{ member.item.cover.url }}" alt="{{ member.item.title }}" class="entity-sort__entity-img"> 
                       <div class="entity-sort__entity-name" title="{{ member.item.title }}"> {{ member.item.title }}</div>
                     </a>

journal/views.py

@@ -24,6 +24,7 @@ from .forms import *
 from mastodon.api import share_review
 from users.views import render_user_blocked, render_user_not_found
 from users.models import User, Report, Preference
+from common.utils import PageLinksGenerator
 
 _logger = logging.getLogger(__name__)
 PAGE_SIZE = 10
@@ -397,6 +398,7 @@ def _render_list(
     paginator = Paginator(queryset, PAGE_SIZE)
     page_number = request.GET.get("page", default=1)
     members = paginator.get_page(page_number)
+    members.pagination = PageLinksGenerator(PAGE_SIZE, page_number, paginator.num_pages)
     return render(
         request,
         f"user_{type}_list.html",
@@ -560,7 +562,11 @@ def home(request, user_name):
                 "count": members.count(),
                 "members": members[:5].prefetch_related("item"),
             }
-        reviews = Review.objects.filter(owner=user).filter(qv)
+        reviews = (
+            Review.objects.filter(owner=user)
+            .filter(qv)
+            .filter(query_item_category(category))
+        )
         shelf_list[category]["reviewed"] = {
             "title": "评论过的" + category.label,
             "count": reviews.count(),

social/models.py

@@ -115,6 +115,7 @@ class DefaultActivityProcessor:
             "visibility": self.action_object.visibility,
             "template": self.template,
             "action_object": self.action_object,
+            "created_time": self.action_object.created_time,
         }
         LocalActivity.objects.create(**params)
 

users/data.py

@@ -41,75 +41,110 @@ from music.models import AlbumMark, SongMark, AlbumReview, SongReview
 from timeline.models import Activity
 from collection.models import Collection
 from common.importers.goodreads import GoodreadsImporter
-from common.importers.douban import DoubanImporter
+
+if settings.ENABLE_NEW_MODEL:
+    from journal.importers.douban import DoubanImporter
+else:
+    from common.importers.douban import DoubanImporter
 
 
 @mastodon_request_included
 @login_required
 def preferences(request):
     preference = request.user.get_preference()
-    if request.method == 'POST':
-        preference.default_visibility = int(request.POST.get('default_visibility'))
-        preference.classic_homepage = bool(request.POST.get('classic_homepage'))
-        preference.mastodon_publish_public = bool(request.POST.get('mastodon_publish_public'))
-        preference.show_last_edit = bool(request.POST.get('show_last_edit'))
-        preference.mastodon_append_tag = request.POST.get('mastodon_append_tag', '').strip()
-        preference.save(update_fields=['default_visibility', 'classic_homepage', 'mastodon_publish_public', 'mastodon_append_tag', 'show_last_edit'])
-    return render(request, 'users/preferences.html')
+    if request.method == "POST":
+        preference.default_visibility = int(request.POST.get("default_visibility"))
+        preference.classic_homepage = bool(request.POST.get("classic_homepage"))
+        preference.mastodon_publish_public = bool(
+            request.POST.get("mastodon_publish_public")
+        )
+        preference.show_last_edit = bool(request.POST.get("show_last_edit"))
+        preference.mastodon_append_tag = request.POST.get(
+            "mastodon_append_tag", ""
+        ).strip()
+        preference.save(
+            update_fields=[
+                "default_visibility",
+                "classic_homepage",
+                "mastodon_publish_public",
+                "mastodon_append_tag",
+                "show_last_edit",
+            ]
+        )
+    return render(request, "users/preferences.html")
 
 
 @mastodon_request_included
 @login_required
 def data(request):
-    return render(request, 'users/data.html', {
-        'allow_any_site': settings.MASTODON_ALLOW_ANY_SITE,
-        'latest_task': request.user.user_synctasks.order_by("-id").first(),
-        'import_status': request.user.get_preference().import_status,
-        'export_status': request.user.get_preference().export_status
-    })
+    return render(
+        request,
+        "users/data.html",
+        {
+            "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE,
+            "latest_task": request.user.user_synctasks.order_by("-id").first(),
+            "import_status": request.user.get_preference().import_status,
+            "export_status": request.user.get_preference().export_status,
+        },
+    )
+
+
+@login_required
+def data_import_status(request):
+    return render(
+        request,
+        "users/data_import_status.html",
+        {
+            "import_status": request.user.get_preference().import_status,
+        },
+    )
 
 
 @mastodon_request_included
 @login_required
 def export_reviews(request):
-    if request.method != 'POST':
+    if request.method != "POST":
         return redirect(reverse("users:data"))
-    return render(request, 'users/data.html')
+    return render(request, "users/data.html")
 
 
 @mastodon_request_included
 @login_required
 def export_marks(request):
-    if request.method == 'POST':
-        if not request.user.preference.export_status.get('marks_pending'):
-            django_rq.get_queue('export').enqueue(export_marks_task, request.user)
-            request.user.preference.export_status['marks_pending'] = True
+    if request.method == "POST":
+        if not request.user.preference.export_status.get("marks_pending"):
+            django_rq.get_queue("export").enqueue(export_marks_task, request.user)
+            request.user.preference.export_status["marks_pending"] = True
             request.user.preference.save()
-        messages.add_message(request, messages.INFO, _('导出已开始。'))
+        messages.add_message(request, messages.INFO, _("导出已开始。"))
         return redirect(reverse("users:data"))
     else:
         try:
-            with open(request.user.preference.export_status['marks_file'], 'rb') as fh:
-                response = HttpResponse(fh.read(), content_type="application/vnd.ms-excel")
-                response['Content-Disposition'] = 'attachment;filename="marks.xlsx"'
+            with open(request.user.preference.export_status["marks_file"], "rb") as fh:
+                response = HttpResponse(
+                    fh.read(), content_type="application/vnd.ms-excel"
+                )
+                response["Content-Disposition"] = 'attachment;filename="marks.xlsx"'
                 return response
         except Exception:
-            messages.add_message(request, messages.ERROR, _('导出文件已过期，请重新导出'))
+            messages.add_message(request, messages.ERROR, _("导出文件已过期，请重新导出"))
             return redirect(reverse("users:data"))
 
 
 @login_required
 def sync_mastodon(request):
-    if request.method == 'POST':
-        django_rq.get_queue('mastodon').enqueue(refresh_mastodon_data_task, request.user)
-        messages.add_message(request, messages.INFO, _('同步已开始。'))
+    if request.method == "POST":
+        django_rq.get_queue("mastodon").enqueue(
+            refresh_mastodon_data_task, request.user
+        )
+        messages.add_message(request, messages.INFO, _("同步已开始。"))
     return redirect(reverse("users:data"))
 
 
 @login_required
 def reset_visibility(request):
-    if request.method == 'POST':
-        visibility = int(request.POST.get('visibility'))
+    if request.method == "POST":
+        visibility = int(request.POST.get("visibility"))
         visibility = visibility if visibility >= 0 and visibility <= 2 else 0
         BookMark.objects.filter(owner=request.user).update(visibility=visibility)
         MovieMark.objects.filter(owner=request.user).update(visibility=visibility)
@@ -117,27 +152,27 @@ def reset_visibility(request):
         AlbumMark.objects.filter(owner=request.user).update(visibility=visibility)
         SongMark.objects.filter(owner=request.user).update(visibility=visibility)
         Activity.objects.filter(owner=request.user).update(visibility=visibility)
-        messages.add_message(request, messages.INFO, _('已重置。'))
+        messages.add_message(request, messages.INFO, _("已重置。"))
     return redirect(reverse("users:data"))
 
 
 @login_required
 def import_goodreads(request):
-    if request.method == 'POST':
-        raw_url = request.POST.get('url')
+    if request.method == "POST":
+        raw_url = request.POST.get("url")
         if GoodreadsImporter.import_from_url(raw_url, request.user):
-            messages.add_message(request, messages.INFO, _('链接已保存，等待后台导入。'))
+            messages.add_message(request, messages.INFO, _("链接已保存，等待后台导入。"))
         else:
-            messages.add_message(request, messages.ERROR, _('无法识别链接。'))
+            messages.add_message(request, messages.ERROR, _("无法识别链接。"))
     return redirect(reverse("users:data"))
 
 
 @login_required
 def import_douban(request):
-    if request.method == 'POST':
-        importer = DoubanImporter(request.user, request.POST.get('visibility'))
-        if importer.import_from_file(request.FILES['file']):
-            messages.add_message(request, messages.INFO, _('文件上传成功，等待后台导入。'))
+    if request.method == "POST":
+        importer = DoubanImporter(request.user, request.POST.get("visibility"))
+        if importer.import_from_file(request.FILES["file"]):
+            messages.add_message(request, messages.INFO, _("文件上传成功，等待后台导入。"))
         else:
-            messages.add_message(request, messages.ERROR, _('无法识别文件。'))
+            messages.add_message(request, messages.ERROR, _("无法识别文件。"))
     return redirect(reverse("users:data"))

users/templates/users/data.html

@@ -148,6 +148,9 @@
                                     <div class="import-panel__body">
                                         <form action="{% url 'users:import_douban' %}" method="POST" enctype="multipart/form-data" >
                                             {% csrf_token %}
+                                            <div>
+                                                请在豆伴(豆坟)导出时勾选「书影音游剧」和「评论」；已经存在的评论不会被覆盖。
+                                            </div>
                                             <div class="import-panel__checkbox">
                                                 <p>从<a href="https://doufen.org" target="_blank">豆伴(豆坟)</a>备份导出的.xlsx文件:
                                                     <input type="file" name="file" id="excel" required accept=".xlsx">
@@ -166,20 +169,11 @@
                                                 <input type="submit" class="import-panel__button" value="{% trans '导入' %}"/>
                                                 {% endif %}
 
-                                                {% if import_status.douban_pending == 2 %}
-                                                正在等待
-                                                {% elif import_status.douban_pending == 1 %}
-                                                正在导入
-                                                    {% if import_status.douban_total %}
-                                                    共{{ import_status.douban_total }}篇，目前已处理{{ import_status.douban_processed }}篇，其中已存在{{ import_status.douban_skipped }}篇，新增{{ import_status.douban_imported }}篇
-                                                    {% endif %}
-                                                {% elif import_status.douban_file %}
-                                                    上次结果
-                                                    共计{{ import_status.douban_total }}篇，处理{{ import_status.douban_processed }}篇，其中已存在{{ import_status.douban_skipped }}篇，新增{{ import_status.douban_imported }}篇
-                                                {% endif %}
-                                            </div>
-                                            <div>
-                                                请务必在豆伴(豆坟)导出时勾选「书影音游剧」和「评论」；已经存在的评论不会被覆盖。
+                                                <div hx-get="{% url 'users:import_status' %}"
+                                                    hx-trigger="load delay:1s"
+                                                    hx-swap="outerHTML"
+                                                >
+                                                </div>
                                             </div>
                                         </form>
                                     </div>

users/templates/users/data_import_status.html

@@ -0,0 +1,15 @@
+{% if import_status.douban_pending == 2 %}
+正在等待
+{% elif import_status.douban_pending == 1 %}
+<div hx-get="{% url 'users:import_status' %} "hx-trigger="every 15s" hx-swap="outerHTML">
+正在导入
+    {% if import_status.douban_total %}
+    <br>
+    <progress value="{{ import_status.douban_processed }}" max="{{ import_status.douban_total }}"></progress> 
+    共{{ import_status.douban_total }}篇，目前已处理{{ import_status.douban_processed }}篇，其中已存在{{ import_status.douban_skipped }}篇，新增{{ import_status.douban_imported }}篇
+    {% endif %}
+</div>
+{% elif import_status.douban_file %}
+    上次结果
+    共计{{ import_status.douban_total }}篇，处理{{ import_status.douban_processed }}篇，其中已存在{{ import_status.douban_skipped }}篇，新增{{ import_status.douban_imported }}篇
+{% endif %}

users/urls.py

@@ -2,25 +2,26 @@ from django.urls import path
 from .views import *
 from .feeds import ReviewFeed
 
-app_name = 'users'
+app_name = "users"
 urlpatterns = [
-    path('login/', login, name='login'),
-    path('register/', register, name='register'),
-    path('connect/', connect, name='connect'),
-    path('reconnect/', reconnect, name='reconnect'),
-    path('data/', data, name='data'),
-    path('data/import_goodreads', import_goodreads, name='import_goodreads'),
-    path('data/import_douban', import_douban, name='import_douban'),
-    path('data/export_reviews', export_reviews, name='export_reviews'),
-    path('data/export_marks', export_marks, name='export_marks'),
-    path('data/sync_mastodon', sync_mastodon, name='sync_mastodon'),
-    path('data/reset_visibility', reset_visibility, name='reset_visibility'),
-    path('data/clear_data', clear_data, name='clear_data'),
-    path('preferences/', preferences, name='preferences'),
-    path('logout/', logout, name='logout'),
-    path('layout/', set_layout, name='set_layout'),
-    path('OAuth2_login/', OAuth2_login, name='OAuth2_login'),
-    path('<int:id>/', home_redirect, name='home_redirect'),
+    path("login/", login, name="login"),
+    path("register/", register, name="register"),
+    path("connect/", connect, name="connect"),
+    path("reconnect/", reconnect, name="reconnect"),
+    path("data/", data, name="data"),
+    path("data/import_status/", data_import_status, name="import_status"),
+    path("data/import_goodreads", import_goodreads, name="import_goodreads"),
+    path("data/import_douban", import_douban, name="import_douban"),
+    path("data/export_reviews", export_reviews, name="export_reviews"),
+    path("data/export_marks", export_marks, name="export_marks"),
+    path("data/sync_mastodon", sync_mastodon, name="sync_mastodon"),
+    path("data/reset_visibility", reset_visibility, name="reset_visibility"),
+    path("data/clear_data", clear_data, name="clear_data"),
+    path("preferences/", preferences, name="preferences"),
+    path("logout/", logout, name="logout"),
+    path("layout/", set_layout, name="set_layout"),
+    path("OAuth2_login/", OAuth2_login, name="OAuth2_login"),
+    path("<int:id>/", home_redirect, name="home_redirect"),
     # path('<int:id>/followers/', followers, name='followers'),
     # path('<int:id>/following/', following, name='following'),
     # path('<int:id>/collections/', collection_list, name='collection_list'),
@@ -28,17 +29,21 @@ urlpatterns = [
     # path('<int:id>/movie/<str:status>/', movie_list, name='movie_list'),
     # path('<int:id>/music/<str:status>/', music_list, name='music_list'),
     # path('<int:id>/game/<str:status>/', game_list, name='game_list'),
-    path('<str:id>/', home, name='home'),
-    path('<str:id>/followers/', followers, name='followers'),
-    path('<str:id>/following/', following, name='following'),
-    path('<str:id>/tags/', tag_list, name='tag_list'),
-    path('<str:id>/collections/', collection_list, name='collection_list'),
-    path('<str:id>/collections/marked/', marked_collection_list, name='marked_collection_list'),
-    path('<str:id>/book/<str:status>/', book_list, name='book_list'),
-    path('<str:id>/movie/<str:status>/', movie_list, name='movie_list'),
-    path('<str:id>/music/<str:status>/', music_list, name='music_list'),
-    path('<str:id>/game/<str:status>/', game_list, name='game_list'),
-    path('<str:id>/feed/reviews/', ReviewFeed(), name='review_feed'),
-    path('report/', report, name='report'),
-    path('manage_report/', manage_report, name='manage_report'),
+    path("<str:id>/", home, name="home"),
+    path("<str:id>/followers/", followers, name="followers"),
+    path("<str:id>/following/", following, name="following"),
+    path("<str:id>/tags/", tag_list, name="tag_list"),
+    path("<str:id>/collections/", collection_list, name="collection_list"),
+    path(
+        "<str:id>/collections/marked/",
+        marked_collection_list,
+        name="marked_collection_list",
+    ),
+    path("<str:id>/book/<str:status>/", book_list, name="book_list"),
+    path("<str:id>/movie/<str:status>/", movie_list, name="movie_list"),
+    path("<str:id>/music/<str:status>/", music_list, name="music_list"),
+    path("<str:id>/game/<str:status>/", game_list, name="game_list"),
+    path("<str:id>/feed/reviews/", ReviewFeed(), name="review_feed"),
+    path("report/", report, name="report"),
+    path("manage_report/", manage_report, name="manage_report"),
 ]