export csv

journal/exporters/__init__.py

@@ -1,3 +1,4 @@
+from .csv import CsvExporter
 from .doufen import DoufenExporter
 
-__all__ = ["DoufenExporter"]
+__all__ = ["DoufenExporter", "CsvExporter"]

journal/exporters/csv.py

@@ -0,0 +1,170 @@
+import csv
+import os
+import shutil
+import tempfile
+
+from django.conf import settings
+
+from catalog.common.models import Item
+from catalog.models import ItemCategory
+from common.utils import GenerateDateUUIDMediaFilePath
+from journal.models import Note, Review, ShelfMember, q_item_in_category
+from users.models import Task
+
+#
+
+_mark_heading = [
+    "title",
+    "info",
+    "links",
+    "timestamp",
+    "status",
+    "rating",
+    "comment",
+    "tags",
+]
+
+_review_heading = [
+    "title",
+    "info",
+    "links",
+    "timestamp",
+    "title",
+    "content",
+]
+
+_note_heading = [
+    "title",
+    "info",
+    "links",
+    "timestamp",
+    "progress",
+    "title",
+    "content",
+]
+
+
+class CsvExporter(Task):
+    class Meta:
+        app_label = "journal"  # workaround bug in TypedModel
+
+    TaskQueue = "export"
+    DefaultMetadata = {
+        "file": None,
+        "total": 0,
+    }
+
+    @property
+    def filename(self) -> str:
+        d = self.created_time.strftime("%Y%m%d%H%M%S")
+        return f"neodb_{self.user.username}_{d}_csv"
+
+    def get_item_info(self, item: Item) -> str:
+        s = []
+        for a in [
+            "imdb",
+            "isbn",
+            "year",
+            "pub_year",
+            "season_number",
+            "episode_number",
+        ]:
+            if hasattr(item, a) and getattr(item, a):
+                s.append(f"{a}:{getattr(item, a)}")
+        for a in ["author", "artist", "director", "host"]:
+            if hasattr(item, a) and getattr(item, a):
+                s.append(f"{a}:{'/'.join(getattr(item, a))}")
+        return " ".join(s)
+
+    def get_item_links(self, item: Item) -> str:
+        links = [item.absolute_url]
+        for ext in item.external_resources.all():
+            links.append(ext.url)
+        return " ".join(links)
+
+    def run(self):
+        user = self.user
+        temp_dir = tempfile.mkdtemp()
+        temp_folder_path = os.path.join(temp_dir, self.filename)
+        os.makedirs(temp_folder_path)
+        for category in [
+            ItemCategory.Movie,
+            ItemCategory.TV,
+            ItemCategory.Music,
+            ItemCategory.Book,
+            ItemCategory.Game,
+            ItemCategory.Podcast,
+            ItemCategory.Performance,
+        ]:
+            q = q_item_in_category(category)
+            csv_file_path = os.path.join(temp_folder_path, f"{category}")
+            marks = (
+                ShelfMember.objects.filter(owner=user.identity)
+                .filter(q)
+                .order_by("created_time")
+            )
+            with open(csv_file_path + "_mark.csv", "w") as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow(_mark_heading)
+                for mark in marks:
+                    item = mark.item
+                    line = [
+                        item.display_title,
+                        self.get_item_info(item),
+                        self.get_item_links(item),
+                        mark.created_time.isoformat(),
+                        mark.shelf_type,
+                        mark.rating_grade,
+                        mark.comment_text,
+                        " ".join(mark.tags),
+                    ]
+                    writer.writerow(line)
+            reviews = (
+                Review.objects.filter(owner=user.identity)
+                .filter(q)
+                .order_by("created_time")
+            )
+            with open(csv_file_path + "_review.csv", "w") as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow(_review_heading)
+                for review in reviews:
+                    item = review.item
+                    line = [
+                        item.display_title,
+                        self.get_item_info(item),
+                        self.get_item_links(item),
+                        review.created_time.isoformat(),
+                        review.title,
+                        review.body,
+                    ]
+                    writer.writerow(line)
+            with open(csv_file_path + "_note.csv", "w") as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow(_note_heading)
+                notes = (
+                    Note.objects.filter(owner=user.identity)
+                    .filter(q)
+                    .order_by("created_time")
+                )
+                for note in notes:
+                    item = note.item
+                    line = [
+                        item.display_title,
+                        self.get_item_info(item),
+                        self.get_item_links(item),
+                        note.created_time.isoformat(),
+                        note.progress_display,
+                        note.title,
+                        note.content,
+                    ]
+                    writer.writerow(line)
+
+        filename = GenerateDateUUIDMediaFilePath(
+            "f.zip", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT
+        )
+        if not os.path.exists(os.path.dirname(filename)):
+            os.makedirs(os.path.dirname(filename))
+        shutil.make_archive(filename[:-4], "zip", temp_folder_path)
+        self.metadata["file"] = filename
+        self.message = "Export complete."
+        self.save()

journal/migrations/0005_csvexporter.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.17 on 2025-01-27 07:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("users", "0007_alter_task_type"),
+        ("journal", "0004_tasks"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CsvExporter",
+            fields=[],
+            options={
+                "proxy": True,
+                "indexes": [],
+                "constraints": [],
+            },
+            bases=("users.task",),
+        ),
+    ]

users/migrations/0007_alter_task_type.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.17 on 2025-01-27 07:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("users", "0006_alter_task_type"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="task",
+            name="type",
+            field=models.CharField(
+                choices=[
+                    ("journal.csvexporter", "csv exporter"),
+                    ("journal.doubanimporter", "douban importer"),
+                    ("journal.doufenexporter", "doufen exporter"),
+                    ("journal.goodreadsimporter", "goodreads importer"),
+                    ("journal.letterboxdimporter", "letterboxd importer"),
+                ],
+                db_index=True,
+                max_length=255,
+            ),
+        ),
+    ]

users/templates/users/data.html

@@ -219,7 +219,8 @@
                   method="post"
                   enctype="multipart/form-data">
               {% csrf_token %}
-              <input type="submit" value="{% trans 'Export marks and reviews' %}" />
+              <input type="submit"
+                     value="{% trans 'Export marks and reviews in XLSX (Doufen format)' %}" />
               {% if export_task %}
                 <br>
                 {% trans 'Last export' %}: {{ export_task.created_time }}
@@ -231,6 +232,24 @@
                 {% endif %}
               {% endif %}
             </form>
+            <hr>
+            <form action="{% url 'users:export_csv' %}"
+                  method="post"
+                  enctype="multipart/form-data">
+              {% csrf_token %}
+              <input type="submit"
+                     value="{% trans 'Export marks, reviews and notes in CSV' %}" />
+              {% if csv_export_task %}
+                <br>
+                {% trans 'Last export' %}: {{ csv_export_task.created_time }}
+                {% trans 'Status' %}: {{ csv_export_task.get_state_display }}
+                <br>
+                {{ csv_export_task.message }}
+                {% if csv_export_task.metadata.file %}
+                  <a href="{% url 'users:export_csv' %}" download>{% trans 'Download' %}</a>
+                {% endif %}
+              {% endif %}
+            </form>
           </details>
         </article>
         <article>

users/urls.py

@@ -17,6 +17,7 @@ urlpatterns = [
     path("data/import/opml", import_opml, name="import_opml"),
     path("data/export/reviews", export_reviews, name="export_reviews"),
     path("data/export/marks", export_marks, name="export_marks"),
+    path("data/export/csv", export_csv, name="export_csv"),
     path("data/sync_mastodon", sync_mastodon, name="sync_mastodon"),
     path(
         "data/sync_mastodon_preference",

users/views/data.py

@@ -8,11 +8,12 @@ from django.db.models import Min
 from django.http import HttpResponse
 from django.shortcuts import redirect, render
 from django.urls import reverse
-from django.utils import translation
+from django.utils import timezone, translation
-from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import GenerateDateUUIDMediaFilePath
 from journal.exporters import DoufenExporter
+from journal.exporters.csv import CsvExporter
 from journal.importers import (
     DoubanImporter,
     GoodreadsImporter,
@@ -97,6 +98,7 @@ def data(request):
             "allow_any_site": settings.MASTODON_ALLOW_ANY_SITE,
             "import_task": DoubanImporter.latest_task(request.user),
             "export_task": DoufenExporter.latest_task(request.user),
+            "csv_export_task": CsvExporter.latest_task(request.user),
             "letterboxd_task": LetterboxdImporter.latest_task(request.user),
             "goodreads_task": GoodreadsImporter.latest_task(request.user),
             "years": years,
@@ -149,6 +151,38 @@ def export_marks(request):
             return redirect(reverse("users:data"))
 
 
+@login_required
+def export_csv(request):
+    if request.method == "POST":
+        task = CsvExporter.latest_task(request.user)
+        if (
+            task
+            and task.state not in [Task.States.complete, Task.States.failed]
+            and task.created_time > (timezone.now() - datetime.timedelta(hours=1))
+        ):
+            messages.add_message(
+                request, messages.INFO, _("Recent export still in progress.")
+            )
+            return redirect(reverse("users:data"))
+        CsvExporter.create(request.user).enqueue()
+        messages.add_message(request, messages.INFO, _("Generating exports."))
+        return redirect(reverse("users:data"))
+    else:
+        task = CsvExporter.latest_task(request.user)
+        if not task or task.state != Task.States.complete:
+            messages.add_message(
+                request, messages.ERROR, _("Export file not available.")
+            )
+            return redirect(reverse("users:data"))
+        response = HttpResponse()
+        response["X-Accel-Redirect"] = (
+            settings.MEDIA_URL + task.metadata["file"][len(settings.MEDIA_ROOT) :]
+        )
+        response["Content-Type"] = "application/zip"
+        response["Content-Disposition"] = f'attachment; filename="{task.filename}.zip"'
+        return response
+
+
 @login_required
 def sync_mastodon(request):
     if request.method == "POST":