csv import: auto refresh progress display

journal/importers/csv.py

@@ -57,16 +57,18 @@ class CsvImporter(Task):
             Item if found, None otherwise
         """
         site_url = settings.SITE_INFO["site_url"] + "/"
-
         links = links_str.strip().split()
         # look for local items first
         for link in links:
             if link.startswith("/") or link.startswith(site_url):
-                item = Item.get_by_url(link)
+                item = Item.get_by_url(link, resolve_merge=True)
-                if item:
+                if item and not item.is_deleted:
                     return item
 
-        sites = [SiteManager.get_site_by_url(link) for link in links]
+        sites = [
+            SiteManager.get_site_by_url(link, detect_redirection=False)
+            for link in links
+        ]
         sites = [site for site in sites if site]
         sites.sort(
             key=lambda x: _PREFERRED_SITES.index(x.SITE_NAME)
@@ -74,14 +76,24 @@ class CsvImporter(Task):
             else 99
         )
 
-        # look for external items that already matched
+        # match items without extra requests
         for site in sites:
-            logger.debug(f"matching {site.url}")
             item = site.get_item()
             if item:
                 return item
 
-        # fetch external item if possible
+        # match items after HEAD
+        sites = [
+            SiteManager.get_site_by_url(site.url) if site.url else site
+            for site in sites
+        ]
+        sites = [site for site in sites if site]
+        for site in sites:
+            item = site.get_item()
+            if item:
+                return item
+
+        # fetch from remote
         for site in sites:
             try:
                 logger.debug(f"fetching {site.url}")
@@ -385,7 +397,6 @@ class CsvImporter(Task):
 
         with zipfile.ZipFile(filename, "r") as zipref:
             with tempfile.TemporaryDirectory() as tmpdirname:
-                logger.debug(f"Extracting {filename} to {tmpdirname}")
                 zipref.extractall(tmpdirname)
 
                 # Count total rows in all CSV files first

users/templates/users/data.html

@@ -121,11 +121,16 @@
               <input type="submit" value="{% trans 'Import' %}" />
               <small>
                 {% if csv_import_task %}
-                  <br>
                   {% trans 'Last import started' %}: {{ csv_import_task.created_time }}
-                  {% trans 'Status' %}: {{ csv_import_task.get_state_display }}。
+                  {% if csv_import_task.state == 0 or csv_import_task.state == 1 %}
-                  <br>
+                    <div hx-get="{% url 'users:user_task_status' 'csv_import' %}"
-                  {{ csv_import_task.message }}
+                         hx-target="this"
+                         hx-trigger="load delay:2s, every 10s"
+                         hx-swap="outerHTML"></div>
+                  {% else %}
+                    {% trans 'Status' %}: {{ csv_import_task.get_state_display }}。
+                    {{ csv_import_task.message }}
+                  {% endif %}
                   {% if csv_import_task.metadata.failed_items %}
                     {% trans 'Failed items' %}:
                     <br>

users/templates/users/user_task_status.html

@@ -0,0 +1,19 @@
+{% load i18n %}
+<div hx-get="{% url 'users:user_task_status' 'csv_import' %}"
+     {% if task.state == 0 or task.state == 1 %}hx-target="this" hx-trigger="every 30s"{% endif %}
+     hx-swap="outerHTML">
+  {% trans 'Status' %}: {{ task.get_state_display }}。
+  {{ task.message }}
+  <br>
+  {% if task.metadata.total and task.metadata.processed %}
+    <div class="progress-container">
+      <progress value="{{ task.metadata.processed }}" max="{{ task.metadata.total }}"></progress>
+      <div class="progress-text">
+        {{ task.metadata.processed }} / {{ task.metadata.total }}
+        ({{ task.metadata.imported }} imported,
+        {{ task.metadata.skipped }} skipped,
+        {{ task.metadata.failed }} failed)
+      </div>
+    </div>
+  {% endif %}
+</div>

users/urls.py

@@ -10,6 +10,7 @@ urlpatterns = [
     path("data", data, name="data"),
     path("info", account_info, name="info"),
     path("profile", account_profile, name="profile"),
+    path("task/<str:task_name>/status", user_task_status, name="user_task_status"),
     path("data/import/status", data_import_status, name="import_status"),
     path("data/import/goodreads", import_goodreads, name="import_goodreads"),
     path("data/import/douban", import_douban, name="import_douban"),

users/views/data.py

@@ -120,6 +120,27 @@ def data_import_status(request):
     )
 
 
+@login_required
+def user_task_status(request, task_name: str):
+    match task_name:
+        case "csv_import":
+            task_cls = CsvImporter
+        case "csv_export":
+            task_cls = CsvExporter
+        case "ndjson_export":
+            task_cls = NdjsonExporter
+        case "letterboxd":
+            task_cls = LetterboxdImporter
+        case "goodreads":
+            task_cls = GoodreadsImporter
+        case "douban":
+            task_cls = DoubanImporter
+        case _:
+            return redirect(reverse("users:data"))
+    task = task_cls.latest_task(request.user)
+    return render(request, "users/user_task_status.html", {"task": task})
+
+
 @login_required
 def export_reviews(request):
     if request.method != "POST":