From 4447de4943cfac9f43c601a1127cabecad2c5eb6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 7 Jan 2023 00:35:30 -0500 Subject: [PATCH] use tmdb_tvseason as main key for tvseason clas --- catalog/common/models.py | 35 +++++--- catalog/common/sites.py | 31 +++++-- catalog/models.py | 5 ++ catalog/search/typesense.py | 7 +- catalog/search/views.py | 7 +- catalog/sites/douban_movie.py | 80 ++++++++++++------- catalog/templates/fetch_pending.html | 6 +- catalog/templates/search_results.html | 40 +--------- catalog/templates/search_sidebar.html | 43 ++++++++++ catalog/templates/tvseason.html | 24 ++++-- catalog/templates/tvshow.html | 16 +++- catalog/tv/models.py | 21 ++++- catalog/urls.py | 7 ++ catalog/views.py | 25 +++++- common/templates/partial/_navbar.html | 1 + journal/models.py | 24 +++++- legacy/management/commands/migrate_catalog.py | 15 ++-- legacy/management/commands/refetch_tmdb.py | 31 +++++++ social/models.py | 10 ++- 19 files changed, 315 insertions(+), 113 deletions(-) create mode 100644 catalog/templates/search_sidebar.html create mode 100644 legacy/management/commands/refetch_tmdb.py diff --git a/catalog/common/models.py b/catalog/common/models.py index eae1d19b..c4e344bc 100644 --- a/catalog/common/models.py +++ b/catalog/common/models.py @@ -1,5 +1,6 @@ from polymorphic.models import PolymorphicModel from django.db import models +import logging from catalog.common import jsondata from django.utils.translation import gettext_lazy as _ from django.utils import timezone @@ -13,6 +14,8 @@ from .mixins import SoftDeleteMixin from django.conf import settings from users.models import User +_logger = logging.getLogger(__name__) + class SiteName(models.TextChoices): Douban = "douban", _("豆瓣") @@ -241,23 +244,31 @@ class Item(SoftDeleteMixin, PolymorphicModel): IdType.ISRC, IdType.MusicBrainz, IdType.Feed, - IdType.IMDB, IdType.TMDB_TVSeason, + IdType.IMDB, ] for t in best_id_types: if lookup_ids.get(t): return t, lookup_ids[t] return list(lookup_ids.items())[0] - def merge(self, to_item): + def merge_to(self, to_item): if to_item is None: - raise (ValueError("cannot merge to an empty item")) + raise ValueError("cannot merge to an empty item") elif to_item.merged_to_item is not None: - raise (ValueError("cannot merge with an item aleady merged")) - elif to_item.__class__ != self.__class__: - raise (ValueError("cannot merge with an item in different class")) - else: - self.merged_to_item = to_item + raise ValueError("cannot merge with an item aleady merged") + if to_item.__class__ != self.__class__: + _logger.warn(f"merging item across class from {self} to {to_item}") + self.merged_to_item = to_item + self.save() + for res in self.external_resources.all(): + res.item = to_item + res.save() + + def switch_class_to(self, cls): + _logger.warn(f"switch item across class from {self} to {cls}") + # TODO + pass @property def uuid(self): @@ -379,7 +390,11 @@ class ExternalResource(models.Model): unique_together = [["id_type", "id_value"]] def __str__(self): - return f"{self.id}:{self.id_type}:{self.id_value if self.id_value else ''} ({self.url})" + return f"{self.pk}:{self.id_type}:{self.id_value if self.id_value else ''} ({self.url})" + + def get_site(self): + """place holder only, this will be injected from SiteManager""" + pass @property def site_name(self): @@ -408,7 +423,7 @@ class ExternalResource(models.Model): d = {k: v for k, v in d.items() if bool(v)} return d - def get_preferred_model(self): + def get_preferred_model(self) -> type[Item] | None: model = self.metadata.get("preferred_model") if model: m = ContentType.objects.filter( diff --git a/catalog/common/sites.py b/catalog/common/sites.py index ac67487a..4e7574e4 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -12,6 +12,7 @@ from .models import ExternalResource from dataclasses import dataclass, field import logging import json +import django_rq _logger = logging.getLogger(__name__) @@ -21,8 +22,8 @@ _logger = logging.getLogger(__name__) class ResourceContent: lookup_ids: dict = field(default_factory=dict) metadata: dict = field(default_factory=dict) - cover_image: bytes = None - cover_image_extention: str = None + cover_image: bytes | None = None + cover_image_extention: str | None = None def dict(self): return {"metadata": self.metadata, "lookup_ids": self.lookup_ids} @@ -122,7 +123,7 @@ class AbstractSite: auto_link=True, preloaded_content=None, ignore_existing_content=False, - ) -> ExternalResource: + ) -> ExternalResource | None: """ Returns an ExternalResource in scraped state if possible @@ -158,7 +159,7 @@ class AbstractSite: if not p.ready: _logger.error(f"unable to get resource {self.url} ready") return None - if auto_create and p.item is None: + if auto_create: # and p.item is None: self.get_item() if auto_save: p.save() @@ -175,6 +176,7 @@ class AbstractSite: ) else: _logger.error(f'unable to get site for {linked_resource["url"]}') + django_rq.get_queue("crawl").enqueue(crawl_related_resources_task, p.pk) p.item.update_linked_items_from_external_resource(p) p.item.save() return p @@ -196,7 +198,7 @@ class SiteManager: return SiteManager.registry[typ]() if typ in SiteManager.registry else None @staticmethod - def get_site_by_url(url: str) -> AbstractSite: + def get_site_by_url(url: str) -> AbstractSite | None: if not url: return None cls = next( @@ -229,4 +231,21 @@ class SiteManager: ExternalResource.get_site = lambda resource: SiteManager.get_site_by_id_type( resource.id_type ) -# ExternalResource.get_site = SiteManager.get_site_by_resource + + +def crawl_related_resources_task(resource_pk): + resource = ExternalResource.objects.get(pk=resource_pk) + links = resource.related_resources + for w in links: + try: + item = None + site = SiteManager.get_site_by_url(w["url"]) + if site: + site.get_resource_ready(ignore_existing_content=False, auto_link=True) + item = site.get_item() + if item: + _logger.info(f"crawled {w['url']} {item}") + else: + _logger.warn(f"crawl {w['url']} failed") + except Exception as e: + _logger.warn(f"crawl {w['url']} error {e}") diff --git a/catalog/models.py b/catalog/models.py index ff405708..cac6107d 100644 --- a/catalog/models.py +++ b/catalog/models.py @@ -9,7 +9,9 @@ from .performance.models import Performance from .collection.models import Collection as CatalogCollection from django.contrib.contenttypes.models import ContentType from django.conf import settings +import logging +_logger = logging.getLogger(__name__) if settings.SEARCH_BACKEND == "MEILISEARCH": from .search.meilisearch import Indexer @@ -71,6 +73,9 @@ def all_categories(): def init_catalog_search_models(): if settings.DISABLE_MODEL_SIGNAL: + _logger.warn( + "Catalog models are not being indexed with DISABLE_MODEL_SIGNAL configuration" + ) return Indexer.update_model_indexable(Edition) Indexer.update_model_indexable(Work) diff --git a/catalog/search/typesense.py b/catalog/search/typesense.py index 0570a433..2caacb9f 100644 --- a/catalog/search/typesense.py +++ b/catalog/search/typesense.py @@ -194,6 +194,8 @@ class Indexer: @classmethod def replace_item(cls, obj): + if obj.is_deleted or obj.merged_to_item_id: + return cls.delete_item(obj) try: cls.instance().collections[INDEX_NAME].documents.upsert( cls.obj_to_dict(obj), {"dirty_values": "coerce_or_drop"} @@ -212,7 +214,7 @@ class Indexer: @classmethod def delete_item(cls, obj): - pk = f"{obj.__class__.__name__}-{obj.id}" + pk = obj.uuid try: cls.instance().collections[INDEX_NAME].documents[pk].delete() except Exception as e: @@ -259,6 +261,5 @@ class Indexer: try: return Item.get_by_url(item["id"]) except Exception as e: - print(e) - logger.error(f"unable to load search result item from db:\n{item}") + logger.error(f"unable to load search result item from db:{item}\n{e}") return None diff --git a/catalog/search/views.py b/catalog/search/views.py index cf9ea995..3762068d 100644 --- a/catalog/search/views.py +++ b/catalog/search/views.py @@ -107,7 +107,7 @@ def search(request): if request.user.is_authenticated and keywords.find("://") > 0: site = SiteManager.get_site_by_url(keywords) if site: - return fetch(request, keywords, site) + return fetch(request, keywords, False, site) if settings.SEARCH_BACKEND is None: # return limited results if no SEARCH_BACKEND result = { @@ -191,5 +191,6 @@ def fetch_task(url, is_refetch): if item: _logger.info(f"fetched {url} {item.url} {item}") item_url = item.url - finally: - return item_url + except Exception as e: + _logger.info(f"fetch error {e}") + return item_url diff --git a/catalog/sites/douban_movie.py b/catalog/sites/douban_movie.py index 1a246887..d413d894 100644 --- a/catalog/sites/douban_movie.py +++ b/catalog/sites/douban_movie.py @@ -5,7 +5,7 @@ from catalog.tv.models import * import logging from django.db import models from django.utils.translation import gettext_lazy as _ -from .tmdb import TMDB_TV, search_tmdb_by_imdb_id, query_tmdb_tv_episode +from .tmdb import TMDB_TV, TMDB_TVSeason, search_tmdb_by_imdb_id, query_tmdb_tv_episode _logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class DoubanMovie(AbstractSite): # no DEFAULT_MODEL as it may be either TV Season and Movie @classmethod - def id_to_url(self, id_value): + def id_to_url(cls, id_value): return "https://movie.douban.com/subject/" + id_value + "/" def scrape(self): @@ -218,51 +218,77 @@ class DoubanMovie(AbstractSite): pd.metadata["preferred_model"] = ( ("TVSeason" if season else "TVShow") if is_series else "Movie" ) - + tmdb_season_id = None if imdb_code: res_data = search_tmdb_by_imdb_id(imdb_code) tmdb_show_id = None if "movie_results" in res_data and len(res_data["movie_results"]) > 0: pd.metadata["preferred_model"] = "Movie" elif "tv_results" in res_data and len(res_data["tv_results"]) > 0: - pd.metadata["preferred_model"] = "TVShow" + if pd.metadata["preferred_model"] == "TVSeason": + """ + determine if this Douban Movie item should map to + a single season tv show, or + first season of multi-season show + """ + tmdb_show_id = res_data["tv_results"][0]["id"] + tmdb_season_id = f"{tmdb_show_id}-1" + site = TMDB_TVSeason(TMDB_TVSeason.id_to_url(tmdb_season_id)) + tmdb_tvseason = site.get_resource_ready().item + tmdb_tv = tmdb_tvseason.show + if tmdb_tv.season_count == 1: + pd.metadata["preferred_model"] = "TVShow" + # else: + # pd.metadata["preferred_model"] = "TVSeason" + # resp = query_tmdb_tv_episode(tmdb_show_id, 1, 1) + # imdb_code = resp["external_ids"]["imdb_id"] + # _logger.warning( + # f"Douban Movie {self.url} re-mapped to imdb episode {imdb_code}" + # ) elif ( "tv_season_results" in res_data and len(res_data["tv_season_results"]) > 0 ): pd.metadata["preferred_model"] = "TVSeason" tmdb_show_id = res_data["tv_season_results"][0]["show_id"] + tmdb_season_id = f"{tmdb_show_id}-{season}" elif ( "tv_episode_results" in res_data and len(res_data["tv_episode_results"]) > 0 ): pd.metadata["preferred_model"] = "TVSeason" tmdb_show_id = res_data["tv_episode_results"][0]["show_id"] - if res_data["tv_episode_results"][0]["episode_number"] != 1: - _logger.warning( - f"Douban Movie {self.url} mapping to unexpected imdb episode {imdb_code}" - ) - resp = query_tmdb_tv_episode( - tmdb_show_id, - res_data["tv_episode_results"][0]["season_number"], - 1, - ) - imdb_code = resp["external_ids"]["imdb_id"] - _logger.warning( - f"Douban Movie {self.url} re-mapped to imdb episode {imdb_code}" - ) + tmdb_season_id = f"{tmdb_show_id}-{season}" + # if res_data["tv_episode_results"][0]["episode_number"] != 1: + # _logger.warning( + # f"Douban Movie {self.url} mapping to unexpected imdb episode {imdb_code}" + # ) + # resp = query_tmdb_tv_episode( + # tmdb_show_id, + # res_data["tv_episode_results"][0]["season_number"], + # 1, + # ) + # imdb_code = resp["external_ids"]["imdb_id"] + # _logger.warning( + # f"Douban Movie {self.url} re-mapped to imdb episode {imdb_code}" + # ) pd.lookup_ids[IdType.IMDB] = imdb_code - if tmdb_show_id: - pd.metadata["required_resources"] = [ - { - "model": "TVShow", - "id_type": IdType.TMDB_TV, - "id_value": tmdb_show_id, - "title": title, - "url": TMDB_TV.id_to_url(tmdb_show_id), - } - ] + if pd.metadata["preferred_model"] == "TVSeason": + pd.lookup_ids[IdType.TMDB_TVSeason] = tmdb_season_id + elif pd.metadata["preferred_model"] == "TVShow": + pd.lookup_ids[IdType.TMDB_TV] = tmdb_show_id + + # if tmdb_show_id: + # pd.metadata["required_resources"] = [ + # { + # "model": "TVShow", + # "id_type": IdType.TMDB_TV, + # "id_value": tmdb_show_id, + # "title": title, + # "url": TMDB_TV.id_to_url(tmdb_show_id), + # } + # ] # TODO parse sister seasons # pd.metadata['related_resources'] = [] if pd.metadata["cover_image_url"]: diff --git a/catalog/templates/fetch_pending.html b/catalog/templates/fetch_pending.html index ff757d67..09af22cc 100644 --- a/catalog/templates/fetch_pending.html +++ b/catalog/templates/fetch_pending.html @@ -58,11 +58,9 @@ -
-
+ + {% include "search_sidebar.html" %} -
-
diff --git a/catalog/templates/search_results.html b/catalog/templates/search_results.html index 1e20db3a..ea98acc4 100644 --- a/catalog/templates/search_results.html +++ b/catalog/templates/search_results.html @@ -103,45 +103,7 @@ -
-
- -
-
-
- {% trans '没有想要的结果？' %} -
-

- 如果在 - {% for site in sites %} - {{ site }} - {% if not forloop.last %}/{% endif %} - {% endfor %} - 找到了条目，可以在搜索栏中输入完整链接提交。 -

-

- 当然也可以手工创建条目。 -

- - - - - - - - - - - - - - - -
-
- -
-
+ {% include "search_sidebar.html" %} diff --git a/catalog/templates/search_sidebar.html b/catalog/templates/search_sidebar.html new file mode 100644 index 00000000..a53d685c --- /dev/null +++ b/catalog/templates/search_sidebar.html @@ -0,0 +1,43 @@ +{% load static %} +{% load i18n %} +{% load l10n %} + +
+
+ +
+
+
+ {% trans '没有想要的结果？' %} +
+

+ 如果在 + {% for site in sites %} + {{ site }} + {% if not forloop.last %}/{% endif %} + {% endfor %} + 找到了条目，可以在搜索栏中输入完整链接提交。 +

+

+ 当然也可以手工创建条目。 +

+ + + + + + + + + + + + + + + +
+
+ +
+
\ No newline at end of file diff --git a/catalog/templates/tvseason.html b/catalog/templates/tvseason.html index 4420a4d9..38534712 100644 --- a/catalog/templates/tvseason.html +++ b/catalog/templates/tvseason.html @@ -13,7 +13,7 @@ {% block title %}
{% if item.season_number %} - {{ item.title }} {% trans '第' %}{{ item.season_number|apnumber }}{% trans '季' %} {{ item.orig_title }} Season {{ item.season_number }} + {{ item.title }} {{ item.orig_title }} Season {{ item.season_number }} {% if item.year %}({{ item.year }}){% endif %} @@ -125,10 +125,24 @@
-
{% if item.duration %}{% trans '片长：' %}{{ item.duration }}{% endif %}
-
{% if item.season_count %}{% trans '季数：' %}{{ item.season_count }}{% endif %}
-
{% if item.episode_count %}{% trans '集数：' %}{{ item.episode_count }}{% endif %}
+
{% if item.season_number %}{% trans '本季序号：' %}{{ item.season_number }}{% endif %}
+
{% if item.episode_count %}{% trans '本季集数：' %}{{ item.episode_count }}{% endif %}
+
+
{% if item.show %}{% trans '所属剧集：' %}{{ item.show.title }}{% endif %}
+
{% if item.season_count %}{% trans '总季数：' %}{{ item.season_count }}{% endif %}
{% if item.single_episode_length %}{% trans '单集长度：' %}{{ item.single_episode_length }}{% endif %}
+ {% with item.all_seasons as seasons %} + {% if seasons %} +
+ {% trans '本剧所有季：' %} + {% for s in seasons %} + + {{ s.season_number }} + + {% endfor %} +
+ {% endif %} + {% endwith %}
{% if item.showtime %}{% trans '上映时间：' %} {% for showtime in item.showtime %} @@ -160,7 +174,7 @@ {% endif %}
- {% trans '编辑这部剧集' %} + {% trans '编辑' %}{{ item.demonstrative }} {% if user.is_staff %} / {% trans '删除' %} {% endif %} diff --git a/catalog/templates/tvshow.html b/catalog/templates/tvshow.html index 4a45a1ee..8d7d9375 100644 --- a/catalog/templates/tvshow.html +++ b/catalog/templates/tvshow.html @@ -125,12 +125,24 @@
-
{% if item.duration %}{% trans '片长：' %}{{ item.duration }}{% endif %}
{% if item.season_count %}{% trans '季数：' %}{{ item.season_count }}{% endif %}
{% if item.episode_count %}{% trans '集数：' %}{{ item.episode_count }}{% endif %}
{% if item.single_episode_length %}{% trans '单集长度：' %}{{ item.single_episode_length }}{% endif %}
-
{% if item.showtime %}{% trans '上映时间：' %} + {% with item.all_seasons as seasons %} + {% if seasons %} +
+ {% trans '本剧所有季：' %} + {% for s in seasons %} + + {{ s.season_number }} + + {% endfor %} +
+ {% endif %} + {% endwith %} + +
{% if item.showtime %}{% trans '播出时间：' %} {% for showtime in item.showtime %} {% for time, region in showtime.items %} {{ time }}{% if region != '' %}({{ region }}){% endif %} diff --git a/catalog/tv/models.py b/catalog/tv/models.py index e1d27581..2b7fb381 100644 --- a/catalog/tv/models.py +++ b/catalog/tv/models.py @@ -24,9 +24,11 @@ tv specials are are shown as movies For now, we follow Douban convention, but keep an eye on it in case it breaks its own rules... """ +from simple_history.models import cached_property from catalog.common import * from django.db import models from django.utils.translation import gettext_lazy as _ +import re class TVShow(Item): @@ -150,6 +152,10 @@ class TVShow(Item): ] return [(i.value, i.label) for i in id_types] + @cached_property + def all_seasons(self): + return self.seasons.all().order_by("season_number") + class TVSeason(Item): category = ItemCategory.TV @@ -269,6 +275,15 @@ class TVSeason(Item): ] return [(i.value, i.label) for i in id_types] + def is_partial_title(self): + return re.match("^(第.+季|特别篇)$", self.title) is not None + + def get_full_title(self): + if self.is_partial_title() and self.show: + return f"{self.show.title} {self.title}" + else: + return self.title + def update_linked_items_from_external_resource(self, resource): """add Work from resource.metadata['work'] if not yet""" links = resource.required_resources + resource.related_resources @@ -277,8 +292,12 @@ class TVSeason(Item): p = ExternalResource.objects.filter( id_type=w["id_type"], id_value=w["id_value"] ).first() - if p and p.item and self.show != p.item: + if p and p.item and w in resource.required_resources: self.show = p.item + self.title = self.get_full_title() + + def all_seasons(self): + return self.show.all_seasons if self.show else [] class TVEpisode(Item): diff --git a/catalog/urls.py b/catalog/urls.py index 36567fbb..b2595f96 100644 --- a/catalog/urls.py +++ b/catalog/urls.py @@ -44,6 +44,13 @@ urlpatterns = [ delete, name="delete", ), + re_path( + r"^(?P" + + _get_all_url_paths() + + ")/(?P[A-Za-z0-9]{21,22})/merge$", + merge, + name="merge", + ), re_path( r"^(?P" + _get_all_url_paths() diff --git a/catalog/views.py b/catalog/views.py index bd5ebcb5..963394c6 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -25,7 +25,11 @@ from .models import * from django.conf import settings from django.utils.baseconv import base62 from journal.models import Mark, ShelfMember, Review -from journal.models import query_visible, query_following +from journal.models import ( + query_visible, + query_following, + update_journal_for_merged_item, +) from common.utils import PageLinksGenerator from common.config import PAGE_LINK_NUMBER from journal.models import ShelfTypeNames @@ -169,9 +173,28 @@ def edit(request, item_path, item_uuid): @login_required def delete(request, item_path, item_uuid): + if request.method != "POST": + return HttpResponseBadRequest() + if not request.user.is_staff: + raise PermissionDenied() return HttpResponseBadRequest() +@login_required +def merge(request, item_path, item_uuid): + if request.method != "POST": + return HttpResponseBadRequest() + if not request.user.is_staff: + raise PermissionDenied() + item = get_object_or_404(Item, uid=base62.decode(item_uuid)) + new_item = Item.get_by_url(request.POST.get("new_item_url")) + if not new_item or new_item.is_deleted or new_item.merged_to_item_id: + return HttpResponseBadRequest(b"invalid new item") + item.merge_to(new_item) + update_journal_for_merged_item(item_uuid) + return redirect(new_item.url) + + @login_required def mark_list(request, item_path, item_uuid, following_only=False): item = get_object_or_404(Item, uid=base62.decode(item_uuid)) diff --git a/common/templates/partial/_navbar.html b/common/templates/partial/_navbar.html index 8d387103..d2456c7e 100644 --- a/common/templates/partial/_navbar.html +++ b/common/templates/partial/_navbar.html @@ -17,6 +17,7 @@ + diff --git a/journal/models.py b/journal/models.py index 7ab6852a..275a60da 100644 --- a/journal/models.py +++ b/journal/models.py @@ -4,7 +4,6 @@ from users.models import User from catalog.common.models import Item, ItemCategory from .mixins import UserOwnedObjectMixin from catalog.collection.models import Collection as CatalogCollection -from enum import Enum from markdownx.models import MarkdownxField from django.utils import timezone from django.conf import settings @@ -15,7 +14,6 @@ from functools import cached_property from django.db.models import Count, Avg from django.contrib.contenttypes.models import ContentType import django.dispatch -import math import uuid import re from catalog.common.utils import DEFAULT_ITEM_COVER, item_cover_path @@ -27,6 +25,8 @@ from django.contrib.contenttypes.models import ContentType from markdown import markdown from catalog.common import jsondata +_logger = logging.getLogger(__name__) + class VisibilityType(models.IntegerChoices): Public = 0, _("公开") @@ -248,6 +248,9 @@ class Review(Content): class Rating(Content): + class Meta: + unique_together = [["owner", "item"]] + grade = models.PositiveSmallIntegerField( default=0, validators=[MaxValueValidator(10), MinValueValidator(1)], null=True ) @@ -932,3 +935,20 @@ def remove_data_by_user(user: User): Comment.objects.filter(owner=user).delete() Rating.objects.filter(owner=user).delete() Review.objects.filter(owner=user).delete() + + +def update_journal_for_merged_item(legacy_item_uuid): + legacy_item = Item.get_by_url(legacy_item_uuid) + if not legacy_item: + _logger.error("update_journal_for_merged_item: unable to find item") + return + new_item = legacy_item.merged_to_item + for cls in Content.__subclasses__ + ListMember.__subclasses__: + _logger.info(f"update {cls.__name__}: {legacy_item} -> {new_item}") + for p in cls.objects.filter(item=legacy_item): + try: + p.item = new_item + p.save(update_fields=["item_id"]) + except: + _logger.info(f"delete duplicated piece {p}") + p.delete() diff --git a/legacy/management/commands/migrate_catalog.py b/legacy/management/commands/migrate_catalog.py index eb01e8c6..ee75b375 100644 --- a/legacy/management/commands/migrate_catalog.py +++ b/legacy/management/commands/migrate_catalog.py @@ -50,7 +50,7 @@ def _book_convert(entity): if t: content.lookup_ids[t] = v if entity.other_info and entity.other_info.get("统一书号"): - content.lookup_ids[IdType.CUBN] = entity.other_info.get("统一书号") + content.lookup_ids[IdType.CUBN] = entity.other_info.get("统一书号").strip() return content @@ -77,12 +77,13 @@ def _album_convert(entity): else None, } ) - if entity.other_info and entity.other_info.get("ISRC"): - content.lookup_ids[IdType.ISRC] = entity.other_info.get("ISRC") - if entity.other_info and entity.other_info.get("条形码"): - content.lookup_ids[IdType.GTIN] = entity.other_info.get("条形码") - if entity.other_info and entity.other_info.get("UPC"): - content.lookup_ids[IdType.GTIN] = entity.other_info.get("UPC") + if entity.other_info: + if entity.other_info.get("ISRC"): + content.lookup_ids[IdType.ISRC] = entity.other_info.get("ISRC") + if entity.other_info.get("条形码") and entity.other_info.get("条形码") != "none": + content.lookup_ids[IdType.GTIN] = entity.other_info.get("条形码") + if entity.other_info.get("UPC") and entity.other_info.get("UPC") != "none": + content.lookup_ids[IdType.GTIN] = entity.other_info.get("UPC") return content diff --git a/legacy/management/commands/refetch_tmdb.py b/legacy/management/commands/refetch_tmdb.py new file mode 100644 index 00000000..633e137b --- /dev/null +++ b/legacy/management/commands/refetch_tmdb.py @@ -0,0 +1,31 @@ +from catalog.common import * +from catalog.models import * +from catalog.sites import * +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator +import pprint +from tqdm import tqdm +import logging + +_logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Refetch TMDB TV Shows" + + def add_arguments(self, parser): + + parser.add_argument("--minid", help="min id to start") + + def handle(self, *args, **options): + qs = ExternalResource.objects.all().filter(id_type="tmdb_tv").order_by("id") + if options["minid"]: + qs = qs.filter(id__gte=int(options["minid"])) + for res in tqdm(qs): + if res: + try: + site = SiteManager.get_site_by_url(res.url) + site.get_resource_ready(ignore_existing_content=True) + _logger.info(f"fetch {res.url} success {site.get_item().title}") + except Exception as e: + _logger.error(f"fetch {res.url} error {e}") diff --git a/social/models.py b/social/models.py index b9fb820e..9cf563be 100644 --- a/social/models.py +++ b/social/models.py @@ -91,9 +91,13 @@ class DataSignalManager: @staticmethod def add_handler_for_model(model): - if not settings.DISABLE_MODEL_SIGNAL: - post_save.connect(DataSignalManager.save_handler, sender=model) - pre_delete.connect(DataSignalManager.delete_handler, sender=model) + if settings.DISABLE_MODEL_SIGNAL: + _logger.warn( + f"{model.__name__} are not being indexed with DISABLE_MODEL_SIGNAL configuration" + ) + return + post_save.connect(DataSignalManager.save_handler, sender=model) + pre_delete.connect(DataSignalManager.delete_handler, sender=model) @staticmethod def register(processor):