Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

csv importer: do not count skipped as imported

Browse source
This commit is contained in:
mein Name 2025-03-04 10:06:41 -05:00 • committed by Henri Dickson
parent 584cc9516a
commit d2f6ce4171
1 changed files with 69 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
journal/importers/csv.py
View file

@ -164,8 +164,12 @@ class CsvImporter(Task):
return status_map.get(status_str.lower(), ShelfType.WISHLIST) return status_map.get(status_str.lower(), ShelfType.WISHLIST)
def import_mark(self, row: Dict[str, str]) -> bool: def import_mark(self, row: Dict[str, str]) -> str:
"""Import a mark from a CSV row.""" """Import a mark from a CSV row.
Returns:
str: 'imported', 'skipped', or 'failed' indicating the import result
"""
try: try:
item = self.get_item_by_info_and_links( item = self.get_item_by_info_and_links(
row.get("title", ""), row.get("info", ""), row.get("links", "") row.get("title", ""), row.get("info", ""), row.get("links", "")
@ -176,7 +180,7 @@ class CsvImporter(Task):
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Could not find item: {row.get('links', '')}" f"Could not find item: {row.get('links', '')}"
) )
return False return "failed"
owner = self.user.identity owner = self.user.identity
mark = Mark(owner, item) mark = Mark(owner, item)
@ -203,8 +207,7 @@ class CsvImporter(Task):
and mark.created_time >= created_time and mark.created_time >= created_time
): ):
# skip if existing mark is newer # skip if existing mark is newer
self.metadata["skipped"] = self.metadata.get("skipped", 0) + 1 return "skipped"
return True
# Update the mark # Update the mark
mark.update( mark.update(
@ -215,16 +218,20 @@ class CsvImporter(Task):
created_time=created_time, created_time=created_time,
visibility=self.metadata.get("visibility", 0), visibility=self.metadata.get("visibility", 0),
) )
return True return "imported"
except Exception as e: except Exception as e:
logger.error(f"Error importing mark: {e}") logger.error(f"Error importing mark: {e}")
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Error importing mark for {row.get('title', '')}" f"Error importing mark for {row.get('title', '')}"
) )
return False return "failed"
def import_review(self, row: Dict[str, str]) -> bool: def import_review(self, row: Dict[str, str]) -> str:
"""Import a review from a CSV row.""" """Import a review from a CSV row.
Returns:
str: 'imported', 'skipped', or 'failed' indicating the import result
"""
try: try:
item = self.get_item_by_info_and_links( item = self.get_item_by_info_and_links(
row.get("title", ""), row.get("info", ""), row.get("links", "") row.get("title", ""), row.get("info", ""), row.get("links", "")
@ -235,7 +242,7 @@ class CsvImporter(Task):
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Could not find item for review: {row.get('links', '')}" f"Could not find item for review: {row.get('links', '')}"
) )
return False return "failed"
owner = self.user.identity owner = self.user.identity
review_title = row.get("title", "") # Second "title" field is review title review_title = row.get("title", "") # Second "title" field is review title
@ -258,8 +265,7 @@ class CsvImporter(Task):
logger.debug( logger.debug(
f"Skipping review import for {item.display_title}: existing review is newer or same age" f"Skipping review import for {item.display_title}: existing review is newer or same age"
) )
self.metadata["skipped"] = self.metadata.get("skipped", 0) + 1 return "skipped"
return True
# Create/update the review # Create/update the review
Review.update_item_review( Review.update_item_review(
@ -270,16 +276,20 @@ class CsvImporter(Task):
created_time=created_time, created_time=created_time,
visibility=self.metadata.get("visibility", 0), visibility=self.metadata.get("visibility", 0),
) )
return True return "imported"
except Exception as e: except Exception as e:
logger.error(f"Error importing review: {e}") logger.error(f"Error importing review: {e}")
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Error importing review for {row.get('title', '')}: {str(e)}" f"Error importing review for {row.get('title', '')}: {str(e)}"
) )
return False return "failed"
def import_note(self, row: Dict[str, str]) -> bool: def import_note(self, row: Dict[str, str]) -> str:
"""Import a note from a CSV row.""" """Import a note from a CSV row.
Returns:
str: 'imported', 'skipped', or 'failed' indicating the import result
"""
try: try:
item = self.get_item_by_info_and_links( item = self.get_item_by_info_and_links(
row.get("title", ""), row.get("info", ""), row.get("links", "") row.get("title", ""), row.get("info", ""), row.get("links", "")
@ -290,7 +300,7 @@ class CsvImporter(Task):
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Could not find item for note: {row.get('links', '')}" f"Could not find item for note: {row.get('links', '')}"
) )
return False return "failed"
owner = self.user.identity owner = self.user.identity
title = row.get("title", "") # Second "title" field is note title title = row.get("title", "") # Second "title" field is note title
@ -318,8 +328,7 @@ class CsvImporter(Task):
logger.debug( logger.debug(
f"Skipping note import for {item.display_title}: duplicate note found" f"Skipping note import for {item.display_title}: duplicate note found"
) )
self.metadata["skipped"] = self.metadata.get("skipped", 0) + 1 return "skipped"
return True
# Create the note if no duplicate is found # Create the note if no duplicate is found
Note.objects.create( Note.objects.create(
@ -332,25 +341,32 @@ class CsvImporter(Task):
created_time=created_time, created_time=created_time,
visibility=self.metadata.get("visibility", 0), visibility=self.metadata.get("visibility", 0),
) )
return True return "imported"
except Exception as e: except Exception as e:
logger.error(f"Error importing note: {e}") logger.error(f"Error importing note: {e}")
self.metadata["failed_items"].append( self.metadata["failed_items"].append(
f"Error importing note for {row.get('title', '')}: {str(e)}" f"Error importing note for {row.get('title', '')}: {str(e)}"
) )
return False return "failed"
def progress(self, success: bool) -> None: def progress(self, result: str) -> None:
"""Update import progress.""" """Update import progress.
self.metadata["total"] += 1
Args:
result: The import result ('imported', 'skipped', or 'failed')
"""
self.metadata["processed"] += 1 self.metadata["processed"] += 1
self.metadata[result] = self.metadata.get(result, 0) + 1
if success: progress_percentage = round(
self.metadata["imported"] += 1 self.metadata["processed"] / self.metadata["total"] * 100
else: )
self.metadata["failed"] += 1 self.message = (
f"Progress: {progress_percentage}% - "
self.message = f"{self.metadata['imported']} imported, {self.metadata['skipped']} skipped, {self.metadata['failed']} failed" f"{self.metadata['imported']} imported, "
f"{self.metadata['skipped']} skipped, "
f"{self.metadata['failed']} failed"
)
self.save(update_fields=["metadata", "message"]) self.save(update_fields=["metadata", "message"])
def process_csv_file(self, file_path: str, import_function) -> None: def process_csv_file(self, file_path: str, import_function) -> None:
@ -359,8 +375,8 @@ class CsvImporter(Task):
with open(file_path, "r") as csvfile: with open(file_path, "r") as csvfile:
reader = csv.DictReader(csvfile) reader = csv.DictReader(csvfile)
for row in reader: for row in reader:
success = import_function(row) result = import_function(row)
self.progress(success) self.progress(result)
def run(self) -> None: def run(self) -> None:
"""Run the CSV import.""" """Run the CSV import."""
@ -372,7 +388,10 @@ class CsvImporter(Task):
logger.debug(f"Extracting {filename} to {tmpdirname}") logger.debug(f"Extracting {filename} to {tmpdirname}")
zipref.extractall(tmpdirname) zipref.extractall(tmpdirname)
# Look for mark, review, and note CSV files # Count total rows in all CSV files first
total_rows = 0
csv_files = []
for category in [ for category in [
ItemCategory.Movie, ItemCategory.Movie,
ItemCategory.TV, ItemCategory.TV,
@ -382,20 +401,25 @@ class CsvImporter(Task):
ItemCategory.Podcast, ItemCategory.Podcast,
ItemCategory.Performance, ItemCategory.Performance,
]: ]:
# Import marks for file_type in ["mark", "review", "note"]:
mark_file = os.path.join(tmpdirname, f"{category}_mark.csv") file_path = os.path.join(
if os.path.exists(mark_file): tmpdirname, f"{category}_{file_type}.csv"
self.process_csv_file(mark_file, self.import_mark) )
if os.path.exists(file_path):
with open(file_path, "r") as csvfile:
row_count = sum(1 for _ in csv.DictReader(csvfile))
total_rows += row_count
csv_files.append((file_path, file_type))
# Import reviews # Set the total count in metadata
review_file = os.path.join(tmpdirname, f"{category}_review.csv") self.metadata["total"] = total_rows
if os.path.exists(review_file): self.message = f"Found {total_rows} items to import"
self.process_csv_file(review_file, self.import_review) self.save(update_fields=["metadata", "message"])
# Import notes # Now process all files
note_file = os.path.join(tmpdirname, f"{category}_note.csv") for file_path, file_type in csv_files:
if os.path.exists(note_file): import_function = getattr(self, f"import_{file_type}")
self.process_csv_file(note_file, self.import_note) self.process_csv_file(file_path, import_function)
self.message = _("Import complete") self.message = _("Import complete")
if self.metadata.get("failed_items", []): if self.metadata.get("failed_items", []):