diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py index c4da7244..4a1c1b15 100644 --- a/catalog/common/downloaders.py +++ b/catalog/common/downloaders.py @@ -205,6 +205,7 @@ class BasicDownloader: ) return resp, response_type except RequestException as e: + # logger.debug(f"RequestException: {e}") self.logs.append( {"response_type": RESPONSE_NETWORK_ERROR, "url": url, "exception": e} ) @@ -340,16 +341,19 @@ class ImageDownloaderMixin: def validate_response(self, response): if response and response.status_code == 200: try: - raw_img = response.content - img = Image.open(BytesIO(raw_img)) - img.load() # corrupted image will trigger exception - content_type = response.headers.get("Content-Type") + content_type = response.headers["content-type"] + if content_type.startswith("image/svg+xml"): + self.extention = "svg" + return RESPONSE_OK file_type = filetype.get_type( mime=content_type.partition(";")[0].strip() ) if file_type is None: return RESPONSE_NETWORK_ERROR self.extention = file_type.extension + raw_img = response.content + img = Image.open(BytesIO(raw_img)) + img.load() # corrupted image will trigger exception return RESPONSE_OK except Exception: return RESPONSE_NETWORK_ERROR diff --git a/journal/exporters/ndjson.py b/journal/exporters/ndjson.py index cb0457d4..a91ba09d 100644 --- a/journal/exporters/ndjson.py +++ b/journal/exporters/ndjson.py @@ -3,6 +3,7 @@ import os import re import shutil import tempfile +import uuid from django.conf import settings from django.utils import timezone @@ -65,13 +66,15 @@ class NdjsonExporter(Task): def _save_image(url): if url.startswith("http"): - imgdl = ProxiedImageDownloader(url) - raw_img = imgdl.download().content - ext = imgdl.extention - file = GenerateDateUUIDMediaFilePath(f"x.{ext}", attachment_path) - with open(file, "wb") as binary_file: - binary_file.write(raw_img) - return file + try: + raw_img, ext = ProxiedImageDownloader.download_image(url, "") + if raw_img: + file = "%s/%s.%s" % (attachment_path, uuid.uuid4(), ext) + with open(file, "wb") as binary_file: + binary_file.write(raw_img) + return file + except Exception: + logger.debug(f"error downloading {url}") elif url.startswith("/"): p = os.path.abspath( os.path.join(settings.MEDIA_ROOT, url[len(settings.MEDIA_URL) :]) @@ -79,11 +82,8 @@ class NdjsonExporter(Task): if p.startswith(settings.MEDIA_ROOT): try: shutil.copy2(p, attachment_path) - except Exception as e: - logger.error( - f"error copying {p} to {attachment_path}", - extra={"exception": e}, - ) + except Exception: + logger.error(f"error copying {p} to {attachment_path}") return p return url @@ -206,6 +206,25 @@ class NdjsonExporter(Task): for item in self.ref_items: f.write(json.dumps(item.ap_object, default=str) + "\n") + # Export actor.ndjson with Takahe identity data + filename = os.path.join(temp_folder_path, "actor.ndjson") + with open(filename, "w") as f: + f.write(json.dumps(self.get_header()) + "\n") + takahe_identity = self.user.identity.takahe_identity + identity_data = { + "type": "Identity", + "username": takahe_identity.username, + "domain": takahe_identity.domain_id, + "actor_uri": takahe_identity.actor_uri, + "name": takahe_identity.name, + "summary": takahe_identity.summary, + "metadata": takahe_identity.metadata, + "private_key": takahe_identity.private_key, + "public_key": takahe_identity.public_key, + "public_key_id": takahe_identity.public_key_id, + } + f.write(json.dumps(identity_data, default=str) + "\n") + filename = GenerateDateUUIDMediaFilePath( "f.zip", settings.MEDIA_ROOT + "/" + settings.EXPORT_FILE_PATH_ROOT ) diff --git a/journal/importers/ndjson.py b/journal/importers/ndjson.py index b9f7a291..0f0aa3dd 100644 --- a/journal/importers/ndjson.py +++ b/journal/importers/ndjson.py @@ -18,6 +18,7 @@ from journal.models import ( Tag, TagMember, ) +from takahe.utils import Takahe from .base import BaseImporter @@ -401,6 +402,47 @@ class NdjsonImporter(BaseImporter): logger.exception("Error parsing header") return {} + def process_actor(self, file_path: str) -> None: + """Process the actor.ndjson file to update user identity information.""" + logger.debug(f"Processing actor data from {file_path}") + try: + with open(file_path, "r") as jsonfile: + next(jsonfile, None) + for line in jsonfile: + try: + data = json.loads(line) + except json.JSONDecodeError: + logger.error("Error parsing actor data line") + continue + + if data.get("type") == "Identity": + logger.debug("Found identity data in actor.ndjson") + takahe_identity = self.user.identity.takahe_identity + updated = False + if ( + data.get("name") + and data.get("name") != takahe_identity.name + ): + logger.debug( + f"Updating identity name from {takahe_identity.name} to {data.get('name')}" + ) + takahe_identity.name = data.get("name") + updated = True + if ( + data.get("summary") + and data.get("summary") != takahe_identity.summary + ): + logger.debug("Updating identity summary") + takahe_identity.summary = data.get("summary") + updated = True + if updated: + takahe_identity.save() + Takahe.update_state(takahe_identity, "edited") + logger.info("Updated identity") + return + except Exception as e: + logger.exception(f"Error processing actor file: {e}") + def run(self) -> None: """Run the NDJSON import.""" filename = self.metadata["file"] @@ -410,6 +452,15 @@ class NdjsonImporter(BaseImporter): with tempfile.TemporaryDirectory() as tmpdirname: zipref.extractall(tmpdirname) + # Process actor data first if available + actor_path = os.path.join(tmpdirname, "actor.ndjson") + if os.path.exists(actor_path): + actor_header = self.parse_header(actor_path) + logger.debug(f"Found actor.ndjson with {actor_header}") + self.process_actor(actor_path) + else: + logger.debug("No actor.ndjson file found in the archive") + catalog_path = os.path.join(tmpdirname, "catalog.ndjson") if os.path.exists(catalog_path): catalog_header = self.parse_header(catalog_path) diff --git a/journal/tests/ndjson.py b/journal/tests/ndjson.py index cb236a68..431faf5e 100644 --- a/journal/tests/ndjson.py +++ b/journal/tests/ndjson.py @@ -105,7 +105,12 @@ class NdjsonExportImportTest(TestCase): ) def test_ndjson_export_import(self): - # Create marks, reviews and notes for user1 + # set name and summary for user1 + identity1 = self.user1.identity + takahe_identity1 = identity1.takahe_identity + takahe_identity1.name = "Test User" + takahe_identity1.summary = "Test summary" + takahe_identity1.save() # Book marks with ratings and tags mark_book1 = Mark(self.user1.identity, self.book1) @@ -289,6 +294,7 @@ class NdjsonExportImportTest(TestCase): export_path = exporter.metadata["file"] logger.debug(f"exported to {export_path}") self.assertTrue(os.path.exists(export_path)) + self.assertEqual(exporter.metadata["total"], 61) # Validate the NDJSON export file structure with TemporaryDirectory() as extract_dir: @@ -370,7 +376,12 @@ class NdjsonExportImportTest(TestCase): self.assertIn("61 items imported, 0 skipped, 0 failed.", importer.message) # Verify imported data + identity2 = self.user2.identity + takahe_identity2 = identity2.takahe_identity + # Check that name and summary were updated + self.assertEqual(takahe_identity2.name, "Test User") + self.assertEqual(takahe_identity2.summary, "Test summary") # Check marks mark_book1_imported = Mark(self.user2.identity, self.book1) self.assertEqual(mark_book1_imported.shelf_type, ShelfType.COMPLETE)