diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75f57e3b..5ffff0c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,21 +25,21 @@ repos: - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.9.1 hooks: - id: ruff - # - id: ruff-format + - id: ruff-format - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - args: ["--profile=black"] + # - repo: https://github.com/pycqa/isort + # rev: 5.13.2 + # hooks: + # - id: isort + # args: ["--profile=black"] - - repo: https://github.com/psf/black - rev: 24.4.2 - hooks: - - id: black + # - repo: https://github.com/psf/black + # rev: 24.4.2 + # hooks: + # - id: black - repo: https://github.com/Riverside-Healthcare/djLint rev: v1.36.4 diff --git a/README.md b/README.md index 9aca018b..879e26c9 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,4 @@ Please see [docs/install.md](docs/install.md) - To translate NeoDB to more languages, please join [our project on Weblate](https://hosted.weblate.org/projects/neodb/neodb/) ## Sponsor -If you like this project, please consider donating to [NeoDB on ko-fi](https://ko-fi.com/neodb), or our friends at [NiceDB](https://patreon.com/tertius) without whom this project won't be possible. +If you like this project, please consider donating to [NeoDB.social on ko-fi](https://ko-fi.com/neodb), or [Takahē](https://www.patreon.com/takahe) and [NiceDB](https://patreon.com/tertius) without whom this project won't be possible. diff --git a/catalog/admin.py b/catalog/admin.py index 8c38f3f3..846f6b40 100644 --- a/catalog/admin.py +++ b/catalog/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/catalog/api.py b/catalog/api.py index 6f667e72..38f383fd 100644 --- a/catalog/api.py +++ b/catalog/api.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Callable, List, Optional, Tuple, Type +from typing import List from django.core.cache import cache from django.http import HttpResponse diff --git a/catalog/apps.py b/catalog/apps.py index 004f317e..53f8547a 100644 --- a/catalog/apps.py +++ b/catalog/apps.py @@ -7,12 +7,9 @@ class CatalogConfig(AppConfig): def ready(self): # load key modules in proper order, make sure class inject and signal works as expected - from catalog import api, models, sites from catalog.models import init_catalog_audit_log, init_catalog_search_models - from journal import models as journal_models # register cron jobs - from catalog.jobs import DiscoverGenerator, PodcastUpdater # isort:skip init_catalog_search_models() init_catalog_audit_log() diff --git a/catalog/book/models.py b/catalog/book/models.py index 0e8c1646..40b94e56 100644 --- a/catalog/book/models.py +++ b/catalog/book/models.py @@ -33,8 +33,6 @@ from catalog.common import ( Item, ItemCategory, ItemInSchema, - ItemSchema, - ItemType, PrimaryLookupIdDescriptor, jsondata, ) @@ -286,7 +284,7 @@ class Edition(Item): logger.warning(f"Unable to find work for {work_res}") else: logger.warning( - f'Unable to find resource for {w["id_type"]}:{w["id_value"]}' + f"Unable to find resource for {w['id_type']}:{w['id_value']}" ) work = Work.objects.filter( primary_lookup_id_type=w["id_type"], @@ -431,7 +429,7 @@ class Work(Item): logger.warning(f"Unable to find edition for {edition_res}") else: logger.warning( - f'Unable to find resource for {e["id_type"]}:{e["id_value"]}' + f"Unable to find resource for {e['id_type']}:{e['id_value']}" ) edition = Edition.objects.filter( primary_lookup_id_type=e["id_type"], diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py index 38801ecf..73a14903 100644 --- a/catalog/common/downloaders.py +++ b/catalog/common/downloaders.py @@ -1,5 +1,4 @@ import json -import logging import re import time from io import BytesIO, StringIO diff --git a/catalog/common/jsondata.py b/catalog/common/jsondata.py index 4765f9b3..316ee571 100644 --- a/catalog/common/jsondata.py +++ b/catalog/common/jsondata.py @@ -1,20 +1,16 @@ # pyright: reportIncompatibleMethodOverride=false import copy -from base64 import b64decode, b64encode +from base64 import b64encode from datetime import date, datetime from functools import partialmethod from hashlib import sha256 -from importlib import import_module -import django -import loguru from cryptography.fernet import Fernet, MultiFernet from django.conf import settings from django.core.exceptions import FieldError from django.db.models import DEFERRED, fields # type:ignore from django.utils import dateparse, timezone from django.utils.encoding import force_bytes -from django.utils.translation import gettext_lazy as _ from django_jsonform.forms.fields import JSONFormField as DJANGO_JSONFormField # from django.db.models import JSONField as DJANGO_JSONField @@ -22,7 +18,6 @@ from django_jsonform.forms.fields import JSONFormField as DJANGO_JSONFormField # from django.contrib.postgres.fields import ArrayField as DJANGO_ArrayField from django_jsonform.models.fields import ArrayField as DJANGO_ArrayField from django_jsonform.models.fields import JSONField as DJANGO_JSONField -from loguru import logger class Patched_DJANGO_JSONField(DJANGO_JSONField): diff --git a/catalog/common/models.py b/catalog/common/models.py index 82380136..bbe32826 100644 --- a/catalog/common/models.py +++ b/catalog/common/models.py @@ -1,19 +1,17 @@ import re import uuid from functools import cached_property -from typing import TYPE_CHECKING, Any, Iterable, Self, Sequence, Type, cast +from typing import TYPE_CHECKING, Any, Self from auditlog.context import disable_auditlog -from auditlog.models import AuditlogHistoryField, LogEntry +from auditlog.models import LogEntry from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.files.uploadedfile import SimpleUploadedFile from django.core.signing import b62_decode, b62_encode from django.db import connection, models -from django.db.models import QuerySet, Value -from django.template.defaultfilters import default +from django.db.models import QuerySet from django.utils import timezone -from django.utils.translation import get_language from django.utils.translation import gettext_lazy as _ from loguru import logger from ninja import Field, Schema @@ -72,36 +70,24 @@ class IdType(models.TextChoices): IMDB = "imdb", _("IMDb") # type:ignore[reportCallIssue] TMDB_TV = "tmdb_tv", _("TMDB TV Serie") # type:ignore[reportCallIssue] TMDB_TVSeason = "tmdb_tvseason", _("TMDB TV Season") # type:ignore[reportCallIssue] - TMDB_TVEpisode = "tmdb_tvepisode", _( - "TMDB TV Episode" - ) # type:ignore[reportCallIssue] + TMDB_TVEpisode = "tmdb_tvepisode", _("TMDB TV Episode") # type:ignore[reportCallIssue] TMDB_Movie = "tmdb_movie", _("TMDB Movie") # type:ignore[reportCallIssue] Goodreads = "goodreads", _("Goodreads") # type:ignore[reportCallIssue] - Goodreads_Work = "goodreads_work", _( - "Goodreads Work" - ) # type:ignore[reportCallIssue] + Goodreads_Work = "goodreads_work", _("Goodreads Work") # type:ignore[reportCallIssue] GoogleBooks = "googlebooks", _("Google Books") # type:ignore[reportCallIssue] DoubanBook = "doubanbook", _("Douban Book") # type:ignore[reportCallIssue] - DoubanBook_Work = "doubanbook_work", _( - "Douban Book Work" - ) # type:ignore[reportCallIssue] + DoubanBook_Work = "doubanbook_work", _("Douban Book Work") # type:ignore[reportCallIssue] DoubanMovie = "doubanmovie", _("Douban Movie") # type:ignore[reportCallIssue] DoubanMusic = "doubanmusic", _("Douban Music") # type:ignore[reportCallIssue] DoubanGame = "doubangame", _("Douban Game") # type:ignore[reportCallIssue] DoubanDrama = "doubandrama", _("Douban Drama") # type:ignore[reportCallIssue] - DoubanDramaVersion = "doubandrama_version", _( - "Douban Drama Version" - ) # type:ignore[reportCallIssue] + DoubanDramaVersion = "doubandrama_version", _("Douban Drama Version") # type:ignore[reportCallIssue] BooksTW = "bookstw", _("BooksTW Book") # type:ignore[reportCallIssue] Bandcamp = "bandcamp", _("Bandcamp") # type:ignore[reportCallIssue] Spotify_Album = "spotify_album", _("Spotify Album") # type:ignore[reportCallIssue] Spotify_Show = "spotify_show", _("Spotify Podcast") # type:ignore[reportCallIssue] - Discogs_Release = "discogs_release", _( - "Discogs Release" - ) # type:ignore[reportCallIssue] - Discogs_Master = "discogs_master", _( - "Discogs Master" - ) # type:ignore[reportCallIssue] + Discogs_Release = "discogs_release", _("Discogs Release") # type:ignore[reportCallIssue] + Discogs_Master = "discogs_master", _("Discogs Master") # type:ignore[reportCallIssue] MusicBrainz = "musicbrainz", _("MusicBrainz ID") # type:ignore[reportCallIssue] # DoubanBook_Author = "doubanbook_author", _("Douban Book Author") # type:ignore[reportCallIssue] # DoubanCelebrity = "doubanmovie_celebrity", _("Douban Movie Celebrity") # type:ignore[reportCallIssue] @@ -142,13 +128,9 @@ class ItemType(models.TextChoices): Album = "music", _("Album") # type:ignore[reportCallIssue] Game = "game", _("Game") # type:ignore[reportCallIssue] Podcast = "podcast", _("Podcast Program") # type:ignore[reportCallIssue] - PodcastEpisode = "podcastepisode", _( - "Podcast Episode" - ) # type:ignore[reportCallIssue] + PodcastEpisode = "podcastepisode", _("Podcast Episode") # type:ignore[reportCallIssue] Performance = "performance", _("Performance") # type:ignore[reportCallIssue] - PerformanceProduction = "production", _( - "Production" - ) # type:ignore[reportCallIssue] + PerformanceProduction = "production", _("Production") # type:ignore[reportCallIssue] FanFic = "fanfic", _("Fanfic") # type:ignore[reportCallIssue] Exhibition = "exhibition", _("Exhibition") # type:ignore[reportCallIssue] Collection = "collection", _("Collection") # type:ignore[reportCallIssue] diff --git a/catalog/common/sites.py b/catalog/common/sites.py index c3acee24..d56451f1 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -10,7 +10,7 @@ ResourceContent persists as an ExternalResource which may link to an Item import json import re from dataclasses import dataclass, field -from typing import Any, Callable, Type, TypeVar +from typing import Type, TypeVar import django_rq import requests diff --git a/catalog/common/tests.py b/catalog/common/tests.py index 0df16a31..ac8498d2 100644 --- a/catalog/common/tests.py +++ b/catalog/common/tests.py @@ -1,10 +1,6 @@ from django.test import TestCase from common.models import ( - LANGUAGE_CHOICES, - LOCALE_CHOICES, - SCRIPT_CHOICES, - SITE_DEFAULT_LANGUAGE, SITE_PREFERRED_LANGUAGES, SITE_PREFERRED_LOCALES, detect_language, diff --git a/catalog/game/models.py b/catalog/game/models.py index b6911563..28f030be 100644 --- a/catalog/game/models.py +++ b/catalog/game/models.py @@ -1,17 +1,14 @@ from datetime import date from django.db import models -from django.template.defaultfilters import default from django.utils.translation import gettext_lazy as _ from catalog.common import ( BaseSchema, - ExternalResource, IdType, Item, ItemCategory, ItemInSchema, - ItemSchema, ItemType, PrimaryLookupIdDescriptor, jsondata, diff --git a/catalog/jobs/__init__.py b/catalog/jobs/__init__.py index a2ebd698..b824d4b9 100644 --- a/catalog/jobs/__init__.py +++ b/catalog/jobs/__init__.py @@ -1,2 +1,7 @@ from .discover import DiscoverGenerator from .podcast import PodcastUpdater + +__all__ = [ + "DiscoverGenerator", + "PodcastUpdater", +] diff --git a/catalog/jobs/discover.py b/catalog/jobs/discover.py index 9a18d603..5e366df8 100644 --- a/catalog/jobs/discover.py +++ b/catalog/jobs/discover.py @@ -7,7 +7,6 @@ from django.db.models import Count, F, Q from django.utils import timezone from loguru import logger -from boofilsic.settings import MIN_MARKS_FOR_DISCOVER from catalog.models import * from common.models import SITE_PREFERRED_LOCALES, BaseJob, JobManager from journal.models import ( @@ -19,7 +18,6 @@ from journal.models import ( q_item_in_category, ) from takahe.utils import Takahe -from users.models import APIdentity MAX_ITEMS_PER_PERIOD = 12 MIN_MARKS = settings.MIN_MARKS_FOR_DISCOVER diff --git a/catalog/jobs/podcast.py b/catalog/jobs/podcast.py index 3b464854..f21e960a 100644 --- a/catalog/jobs/podcast.py +++ b/catalog/jobs/podcast.py @@ -1,6 +1,4 @@ -import pprint from datetime import timedelta -from time import sleep from loguru import logger @@ -30,6 +28,6 @@ class PodcastUpdater(BaseJob): site = RSS(p.feed_url) site.scrape_additional_data() c2 = p.episodes.count() - logger.info(f"updated {p}, {c2-c} new episodes.") + logger.info(f"updated {p}, {c2 - c} new episodes.") count += c2 - c logger.info(f"Podcasts update finished, {count} new episodes total.") diff --git a/catalog/management/commands/cat.py b/catalog/management/commands/cat.py index f2c13cd1..c650930a 100644 --- a/catalog/management/commands/cat.py +++ b/catalog/management/commands/cat.py @@ -46,4 +46,4 @@ class Command(BaseCommand): resource = site.scrape() pprint.pp(resource.metadata) pprint.pp(resource.lookup_ids) - self.stdout.write(self.style.SUCCESS(f"Done.")) + self.stdout.write(self.style.SUCCESS("Done.")) diff --git a/catalog/management/commands/catalog.py b/catalog/management/commands/catalog.py index a5c28bed..c37b3946 100644 --- a/catalog/management/commands/catalog.py +++ b/catalog/management/commands/catalog.py @@ -1,6 +1,3 @@ -import pprint -import re - from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand from django.db.models import Count, F @@ -9,7 +6,6 @@ from tqdm import tqdm from catalog.book.tests import uniq from catalog.models import * from common.models.lang import detect_language -from journal.models import update_journal_for_merged_item class Command(BaseCommand): @@ -49,7 +45,7 @@ class Command(BaseCommand): self.integrity() if options["localize"]: self.localize() - self.stdout.write(self.style.SUCCESS(f"Done.")) + self.stdout.write(self.style.SUCCESS("Done.")) def localize(self): c = Item.objects.all().count() @@ -94,14 +90,14 @@ class Command(BaseCommand): cls.objects.filter(is_deleted=True).delete() def integrity(self): - self.stdout.write(f"Checking circulated merge...") + self.stdout.write("Checking circulated merge...") for i in Item.objects.filter(merged_to_item=F("id")): self.stdout.write(f"! {i} : {i.absolute_url}?skipcheck=1") if self.fix: i.merged_to_item = None i.save() - self.stdout.write(f"Checking chained merge...") + self.stdout.write("Checking chained merge...") for i in ( Item.objects.filter(merged_to_item__isnull=False) .annotate(n=Count("merged_from_items")) @@ -113,14 +109,14 @@ class Command(BaseCommand): j.merged_to_item = i.merged_to_item j.save() - self.stdout.write(f"Checking deleted merge...") + self.stdout.write("Checking deleted merge...") for i in Item.objects.filter(merged_to_item__isnull=False, is_deleted=True): self.stdout.write(f"! {i} : {i.absolute_url}?skipcheck=1") if self.fix: i.is_deleted = False i.save() - self.stdout.write(f"Checking deleted item with external resources...") + self.stdout.write("Checking deleted item with external resources...") for i in ( Item.objects.filter(is_deleted=True) .annotate(n=Count("external_resources")) @@ -132,7 +128,7 @@ class Command(BaseCommand): r.item = None r.save() - self.stdout.write(f"Checking merged item with external resources...") + self.stdout.write("Checking merged item with external resources...") for i in ( Item.objects.filter(merged_to_item__isnull=False) .annotate(n=Count("external_resources")) @@ -145,7 +141,7 @@ class Command(BaseCommand): r.save() tvshow_ct_id = ContentType.objects.get_for_model(TVShow).id - self.stdout.write(f"Checking TVShow merged to other class...") + self.stdout.write("Checking TVShow merged to other class...") for i in ( TVShow.objects.filter(merged_to_item__isnull=False) .filter(merged_to_item__isnull=False) @@ -161,7 +157,7 @@ class Command(BaseCommand): if self.fix: i.recast_to(i.merged_to_item.__class__) # type:ignore - self.stdout.write(f"Checking TVSeason is child of other class...") + self.stdout.write("Checking TVSeason is child of other class...") for i in TVSeason.objects.filter(show__isnull=False).exclude( show__polymorphic_ctype_id=tvshow_ct_id ): @@ -172,7 +168,7 @@ class Command(BaseCommand): i.show = None i.save() - self.stdout.write(f"Checking deleted item with child TV Season...") + self.stdout.write("Checking deleted item with child TV Season...") for i in TVSeason.objects.filter(show__is_deleted=True): if not i.show: continue @@ -181,7 +177,7 @@ class Command(BaseCommand): i.show.is_deleted = False i.show.save() - self.stdout.write(f"Checking merged item with child TV Season...") + self.stdout.write("Checking merged item with child TV Season...") for i in TVSeason.objects.filter(show__merged_to_item__isnull=False): if not i.show: continue diff --git a/catalog/management/commands/index.py b/catalog/management/commands/index.py index 1a07cb59..bbcafb96 100644 --- a/catalog/management/commands/index.py +++ b/catalog/management/commands/index.py @@ -1,12 +1,8 @@ import pprint -from datetime import timedelta from time import sleep -from typing import TYPE_CHECKING -from django.conf import settings from django.core.management.base import BaseCommand from django.core.paginator import Paginator -from django.utils import timezone from tqdm import tqdm from catalog.models import Item diff --git a/catalog/migrations/0001_initial_0_10.py b/catalog/migrations/0001_initial_0_10.py index 98697371..9c8cdc08 100644 --- a/catalog/migrations/0001_initial_0_10.py +++ b/catalog/migrations/0001_initial_0_10.py @@ -3,7 +3,6 @@ import uuid import django.db.models.deletion -from django.conf import settings from django.db import migrations, models import catalog.common.utils diff --git a/catalog/migrations/0011_remove_item_last_editor.py b/catalog/migrations/0011_remove_item_last_editor.py index c5f3948e..e6ffea41 100644 --- a/catalog/migrations/0011_remove_item_last_editor.py +++ b/catalog/migrations/0011_remove_item_last_editor.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("catalog", "0010_alter_item_polymorphic_ctype"), ] diff --git a/catalog/migrations/0012_alter_model_i18n.py b/catalog/migrations/0012_alter_model_i18n.py index bae8c7b9..db7754ed 100644 --- a/catalog/migrations/0012_alter_model_i18n.py +++ b/catalog/migrations/0012_alter_model_i18n.py @@ -6,7 +6,6 @@ import catalog.common.utils class Migration(migrations.Migration): - dependencies = [ ("catalog", "0011_alter_externalresource_id_type_and_more"), ] diff --git a/catalog/models.py b/catalog/models.py index a32bba28..8ebca194 100644 --- a/catalog/models.py +++ b/catalog/models.py @@ -38,7 +38,7 @@ from .tv.models import ( TVShowSchema, ) -from .search.models import Indexer, ExternalSearchResultItem # isort:skip +from .search.models import Indexer # isort:skip # class Exhibition(Item): diff --git a/catalog/movie/models.py b/catalog/movie/models.py index bfe4b66c..db09fff4 100644 --- a/catalog/movie/models.py +++ b/catalog/movie/models.py @@ -3,12 +3,10 @@ from django.utils.translation import gettext_lazy as _ from catalog.common import ( BaseSchema, - ExternalResource, IdType, Item, ItemCategory, ItemInSchema, - ItemSchema, ItemType, PrimaryLookupIdDescriptor, jsondata, diff --git a/catalog/music/models.py b/catalog/music/models.py index 2fe917e3..c629d972 100644 --- a/catalog/music/models.py +++ b/catalog/music/models.py @@ -6,12 +6,10 @@ from django.utils.translation import pgettext_lazy from catalog.common import ( BaseSchema, - ExternalResource, IdType, Item, ItemCategory, ItemInSchema, - ItemSchema, ItemType, PrimaryLookupIdDescriptor, jsondata, diff --git a/catalog/performance/models.py b/catalog/performance/models.py index abdc899e..11ba735d 100644 --- a/catalog/performance/models.py +++ b/catalog/performance/models.py @@ -391,7 +391,9 @@ class PerformanceProduction(Item): return ( self.cover.url # type:ignore if self.cover and self.cover != settings.DEFAULT_ITEM_COVER - else self.show.cover_image_url if self.show else None + else self.show.cover_image_url + if self.show + else None ) def update_linked_items_from_external_resource(self, resource: ExternalResource): diff --git a/catalog/podcast/models.py b/catalog/podcast/models.py index cb5b426c..ed50a9ab 100644 --- a/catalog/podcast/models.py +++ b/catalog/podcast/models.py @@ -6,14 +6,10 @@ from ninja import Field from catalog.common import ( BaseSchema, - ExternalResource, IdType, Item, ItemCategory, ItemInSchema, - ItemSchema, - ItemType, - PrimaryLookupIdDescriptor, jsondata, ) from catalog.common.models import LIST_OF_ONE_PLUS_STR_SCHEMA, LanguageListField diff --git a/catalog/podcast/tests.py b/catalog/podcast/tests.py index 8ab25897..19b1ad58 100644 --- a/catalog/podcast/tests.py +++ b/catalog/podcast/tests.py @@ -62,7 +62,7 @@ class PodcastRSSFeedTestCase(TestCase): site = SiteManager.get_site_by_url(t_url) site.get_resource_ready() self.assertEqual(site.ready, True) - metadata = site.resource.metadata + # metadata = site.resource.metadata self.assertIsNotNone(site.get_item().cover.url) self.assertIsNotNone(site.get_item().recent_episodes[0].title) self.assertIsNotNone(site.get_item().recent_episodes[0].link) @@ -74,7 +74,7 @@ class PodcastRSSFeedTestCase(TestCase): site = SiteManager.get_site_by_url(t_url) site.get_resource_ready() self.assertEqual(site.ready, True) - metadata = site.resource.metadata + # metadata = site.resource.metadata self.assertIsNotNone(site.get_item().recent_episodes[0].title) self.assertIsNotNone(site.get_item().recent_episodes[0].link) self.assertIsNotNone(site.get_item().recent_episodes[0].media_url) diff --git a/catalog/search/external.py b/catalog/search/external.py index 5191322a..ac52529d 100644 --- a/catalog/search/external.py +++ b/catalog/search/external.py @@ -1,6 +1,5 @@ import asyncio import logging -import time from urllib.parse import quote_plus, urlparse import httpx @@ -90,7 +89,7 @@ class GoogleBooks: @classmethod def search(cls, q, page=1): results = [] - api_url = f"https://www.googleapis.com/books/v1/volumes?country=us&q={quote_plus(q)}&startIndex={SEARCH_PAGE_SIZE*(page-1)}&maxResults={SEARCH_PAGE_SIZE}&maxAllowedMaturityRating=MATURE" + api_url = f"https://www.googleapis.com/books/v1/volumes?country=us&q={quote_plus(q)}&startIndex={SEARCH_PAGE_SIZE * (page - 1)}&maxResults={SEARCH_PAGE_SIZE}&maxAllowedMaturityRating=MATURE" try: j = requests.get(api_url, timeout=2).json() if "items" in j: @@ -183,7 +182,7 @@ class Spotify: @classmethod def search(cls, q, page=1): results = [] - api_url = f"https://api.spotify.com/v1/search?q={q}&type=album&limit={SEARCH_PAGE_SIZE}&offset={page*SEARCH_PAGE_SIZE}" + api_url = f"https://api.spotify.com/v1/search?q={q}&type=album&limit={SEARCH_PAGE_SIZE}&offset={page * SEARCH_PAGE_SIZE}" try: headers = {"Authorization": f"Bearer {get_spotify_token()}"} j = requests.get(api_url, headers=headers, timeout=2).json() @@ -255,7 +254,7 @@ class ApplePodcast: @classmethod def search(cls, q, page=1): results = [] - search_url = f"https://itunes.apple.com/search?entity=podcast&limit={page*SEARCH_PAGE_SIZE}&term={quote_plus(q)}" + search_url = f"https://itunes.apple.com/search?entity=podcast&limit={page * SEARCH_PAGE_SIZE}&term={quote_plus(q)}" try: r = requests.get(search_url, timeout=2).json() for p in r["results"][(page - 1) * SEARCH_PAGE_SIZE :]: @@ -291,7 +290,7 @@ class IGDB: class Fediverse: @staticmethod async def search_task(host, q, category=None): - api_url = f"https://{host}/api/catalog/search?query={quote_plus(q)}{'&category='+category if category else ''}" + api_url = f"https://{host}/api/catalog/search?query={quote_plus(q)}{'&category=' + category if category else ''}" async with httpx.AsyncClient() as client: results = [] try: diff --git a/catalog/search/models.py b/catalog/search/models.py index f6279539..68e2c99b 100644 --- a/catalog/search/models.py +++ b/catalog/search/models.py @@ -6,7 +6,6 @@ import django_rq from auditlog.context import set_actor from django.conf import settings from django.core.cache import cache -from django.utils.translation import gettext_lazy as _ from loguru import logger from rq.job import Job diff --git a/catalog/search/typesense.py b/catalog/search/typesense.py index 81ff83b4..715b1441 100644 --- a/catalog/search/typesense.py +++ b/catalog/search/typesense.py @@ -1,6 +1,5 @@ import types from datetime import timedelta -from pprint import pprint from time import sleep import django_rq diff --git a/catalog/search/views.py b/catalog/search/views.py index 7d471559..e7debf38 100644 --- a/catalog/search/views.py +++ b/catalog/search/views.py @@ -1,5 +1,4 @@ import re -from urllib.parse import quote import django_rq from django.conf import settings @@ -7,7 +6,6 @@ from django.contrib.auth.decorators import login_required from django.core.cache import cache from django.core.exceptions import BadRequest from django.shortcuts import redirect, render -from django.urls import reverse from django.utils.translation import gettext as _ from django.views.decorators.http import require_http_methods from rq.job import Job @@ -162,7 +160,7 @@ def external_search(request): keywords = request.GET.get("q", default="").strip() page_number = int_(request.GET.get("page"), 1) items = ExternalSources.search(category, keywords, page_number) if keywords else [] - cache_key = f"search_{category if category!='movietv' else 'movie,tv'}_{keywords}" + cache_key = f"search_{category if category != 'movietv' else 'movie,tv'}_{keywords}" dedupe_urls = cache.get(cache_key, []) items = [i for i in items if i.source_url not in dedupe_urls] diff --git a/catalog/sites/__init__.py b/catalog/sites/__init__.py index 28e1beaf..7688fd69 100644 --- a/catalog/sites/__init__.py +++ b/catalog/sites/__init__.py @@ -25,3 +25,33 @@ from .tmdb import TMDB_Movie from .ypshuo import Ypshuo # from .apple_podcast import ApplePodcast + +__all__ = [ + "SiteManager", + "ArchiveOfOurOwn", + "AppleMusic", + "Bandcamp", + "Bangumi", + "BoardGameGeek", + "BooksTW", + "DiscogsMaster", + "DiscogsRelease", + "DoubanBook", + "DoubanDrama", + "DoubanGame", + "DoubanMovie", + "DoubanMusic", + "FediverseInstance", + "Goodreads", + "GoogleBooks", + "IGDB", + "IMDB", + "JJWXC", + "Qidian", + "RSS", + "Spotify", + "Steam", + "TMDB_Movie", + "Ypshuo", + # "ApplePodcast", +] diff --git a/catalog/sites/ao3.py b/catalog/sites/ao3.py index 1e79c39c..6e5fcfca 100644 --- a/catalog/sites/ao3.py +++ b/catalog/sites/ao3.py @@ -1,5 +1,3 @@ -import logging - from catalog.book.models import * from catalog.common import * diff --git a/catalog/sites/apple_music.py b/catalog/sites/apple_music.py index 3f8ca6bd..b6149138 100644 --- a/catalog/sites/apple_music.py +++ b/catalog/sites/apple_music.py @@ -10,7 +10,6 @@ Scraping the website directly. import json import logging -from threading import local import dateparser diff --git a/catalog/sites/bgg.py b/catalog/sites/bgg.py index 50a032e7..c1c01f25 100644 --- a/catalog/sites/bgg.py +++ b/catalog/sites/bgg.py @@ -6,8 +6,6 @@ ref: https://boardgamegeek.com/wiki/page/BGG_XML_API2 import html -from langdetect import detect -from loguru import logger from catalog.common import * from catalog.game.models import GameReleaseType diff --git a/catalog/sites/bookstw.py b/catalog/sites/bookstw.py index 6049f0f2..67b11bbb 100644 --- a/catalog/sites/bookstw.py +++ b/catalog/sites/bookstw.py @@ -3,7 +3,6 @@ import logging from catalog.book.models import * from catalog.book.utils import * from catalog.common import * -from common.models.lang import detect_language from .douban import * diff --git a/catalog/sites/discogs.py b/catalog/sites/discogs.py index 550bb34e..5a903fa0 100644 --- a/catalog/sites/discogs.py +++ b/catalog/sites/discogs.py @@ -2,10 +2,8 @@ Discogs. """ -import json import logging -import requests from django.conf import settings from catalog.common import * diff --git a/catalog/sites/douban.py b/catalog/sites/douban.py index f228aaa2..f513e1b0 100644 --- a/catalog/sites/douban.py +++ b/catalog/sites/douban.py @@ -37,12 +37,10 @@ class DoubanDownloader(ProxiedDownloader): class DoubanSearcher: @classmethod def search(cls, cat: ItemCategory, c: str, q: str, p: int = 1): - url = f"https://search.douban.com/{c}/subject_search?search_text={q}&start={15*(p-1)}" + url = f"https://search.douban.com/{c}/subject_search?search_text={q}&start={15 * (p - 1)}" content = DoubanDownloader(url).download().html() j = json.loads( - content.xpath( - "//script[text()[contains(.,'window.__DATA__')]]/text()" - )[ # type:ignore + content.xpath("//script[text()[contains(.,'window.__DATA__')]]/text()")[ # type:ignore 0 ] .split("window.__DATA__ = ")[1] # type:ignore diff --git a/catalog/sites/douban_drama.py b/catalog/sites/douban_drama.py index 63d10141..c12022c9 100644 --- a/catalog/sites/douban_drama.py +++ b/catalog/sites/douban_drama.py @@ -7,7 +7,7 @@ from catalog.common import * from catalog.models import * from common.models.lang import detect_language -from .douban import DoubanDownloader, DoubanSearcher +from .douban import DoubanDownloader def _cache_key(url): diff --git a/catalog/sites/douban_game.py b/catalog/sites/douban_game.py index 0fce5479..51c34935 100644 --- a/catalog/sites/douban_game.py +++ b/catalog/sites/douban_game.py @@ -1,5 +1,3 @@ -import logging - import dateparser from catalog.common import * @@ -7,7 +5,7 @@ from catalog.models import * from common.models.lang import detect_language from common.models.misc import uniq -from .douban import DoubanDownloader, DoubanSearcher +from .douban import DoubanDownloader @SiteManager.register diff --git a/catalog/sites/douban_movie.py b/catalog/sites/douban_movie.py index 36ccb87a..bb0e7a8f 100644 --- a/catalog/sites/douban_movie.py +++ b/catalog/sites/douban_movie.py @@ -1,5 +1,5 @@ import json -import logging +import re from loguru import logger @@ -10,7 +10,7 @@ from common.models.lang import detect_language from common.models.misc import int_ from .douban import DoubanDownloader, DoubanSearcher -from .tmdb import TMDB_TV, TMDB_TVSeason, query_tmdb_tv_episode, search_tmdb_by_imdb_id +from .tmdb import TMDB_TV, search_tmdb_by_imdb_id @SiteManager.register diff --git a/catalog/sites/douban_music.py b/catalog/sites/douban_music.py index af57ddf7..af4f41c4 100644 --- a/catalog/sites/douban_music.py +++ b/catalog/sites/douban_music.py @@ -1,5 +1,3 @@ -import logging - import dateparser from catalog.common import * diff --git a/catalog/sites/fedi.py b/catalog/sites/fedi.py index 92cef330..0ea2729f 100644 --- a/catalog/sites/fedi.py +++ b/catalog/sites/fedi.py @@ -1,5 +1,3 @@ -import re - from django.conf import settings from django.core.validators import URLValidator from loguru import logger diff --git a/catalog/sites/goodreads.py b/catalog/sites/goodreads.py index 891fec1d..ca5650f6 100644 --- a/catalog/sites/goodreads.py +++ b/catalog/sites/goodreads.py @@ -1,6 +1,5 @@ import json import logging -import re from datetime import datetime from django.utils.timezone import make_aware diff --git a/catalog/sites/imdb.py b/catalog/sites/imdb.py index d563fdd0..93c64f2a 100644 --- a/catalog/sites/imdb.py +++ b/catalog/sites/imdb.py @@ -1,6 +1,6 @@ import json import logging - +import re from catalog.common import * from catalog.movie.models import * from catalog.tv.models import * diff --git a/catalog/sites/qidian.py b/catalog/sites/qidian.py index a65464d1..eaf7f3db 100644 --- a/catalog/sites/qidian.py +++ b/catalog/sites/qidian.py @@ -1,5 +1,3 @@ -import logging - from catalog.common import * from catalog.models import * diff --git a/catalog/sites/rss.py b/catalog/sites/rss.py index 9456cc50..21d11cb9 100644 --- a/catalog/sites/rss.py +++ b/catalog/sites/rss.py @@ -7,7 +7,6 @@ import bleach import podcastparser from django.conf import settings from django.core.cache import cache -from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.utils.timezone import make_aware diff --git a/catalog/sites/spotify.py b/catalog/sites/spotify.py index 5cea0cc7..ef1456bb 100644 --- a/catalog/sites/spotify.py +++ b/catalog/sites/spotify.py @@ -2,7 +2,6 @@ Spotify """ -import datetime import logging import time diff --git a/catalog/sites/steam.py b/catalog/sites/steam.py index 963434e0..ea0c9f23 100644 --- a/catalog/sites/steam.py +++ b/catalog/sites/steam.py @@ -1,7 +1,5 @@ import logging -import re -import dateparser from django.conf import settings from catalog.common import * diff --git a/catalog/sites/tmdb.py b/catalog/sites/tmdb.py index 760a8155..fdc77275 100644 --- a/catalog/sites/tmdb.py +++ b/catalog/sites/tmdb.py @@ -242,9 +242,9 @@ class TMDB_TV(AbstractSite): lambda s: { "model": "TVSeason", "id_type": IdType.TMDB_TVSeason, - "id_value": f'{self.id_value}-{s["season_number"]}', + "id_value": f"{self.id_value}-{s['season_number']}", "title": s["name"], - "url": f'{self.url}/season/{s["season_number"]}', + "url": f"{self.url}/season/{s['season_number']}", }, res_data["seasons"], ) @@ -359,7 +359,7 @@ class TMDB_TVSeason(AbstractSite): pd.metadata["title"] = ( pd.metadata["title"] if pd.metadata.get("title") - else f'Season {d["season_number"]}' + else f"Season {d['season_number']}" ) pd.metadata["episode_number_list"] = list( map(lambda ep: ep["episode_number"], d["episodes"]) @@ -460,7 +460,7 @@ class TMDB_TVEpisode(AbstractSite): pd.metadata["title"] = ( pd.metadata["title"] if pd.metadata["title"] - else f'S{d["season_number"]} E{d["episode_number"]}' + else f"S{d['season_number']} E{d['episode_number']}" ) if pd.lookup_ids.get(IdType.IMDB): diff --git a/catalog/tv/models.py b/catalog/tv/models.py index 0677a7d2..e5f05a96 100644 --- a/catalog/tv/models.py +++ b/catalog/tv/models.py @@ -25,15 +25,12 @@ For now, we follow Douban convention, but keep an eye on it in case it breaks it """ -import re from functools import cached_property -from typing import TYPE_CHECKING, overload +from typing import TYPE_CHECKING -from auditlog.diff import ForeignKey from auditlog.models import QuerySet from django.db import models from django.utils.translation import gettext_lazy as _ -from typing_extensions import override from catalog.common import ( BaseSchema, diff --git a/catalog/tv/tests.py b/catalog/tv/tests.py index a160ee66..940b8843 100644 --- a/catalog/tv/tests.py +++ b/catalog/tv/tests.py @@ -128,7 +128,7 @@ class DoubanMovieTVTestCase(TestCase): url = "https://movie.douban.com/subject/35597581/" item = SiteManager.get_site_by_url(url).get_resource_ready().item # disable this test to make douban data less disrupted - # self.assertEqual(item.imdb, "tt21599650") + self.assertEqual(item.imdb, "tt21599650") class MultiTVSitesTestCase(TestCase): @@ -186,7 +186,7 @@ class MovieTVModelRecastTestCase(TestCase): @use_local_response def test_recast(self): - from catalog.models import Movie, TVShow + from catalog.models import Movie url2 = "https://www.imdb.com/title/tt0436992/" p2 = SiteManager.get_site_by_url(url2).get_resource_ready() @@ -232,10 +232,10 @@ class IMDBTestCase(TestCase): @use_local_response def test_get_episode_list(self): - l = IMDB.get_episode_list("tt0436992", 4) - self.assertEqual(len(l), 14) - l = IMDB.get_episode_list("tt1205438", 4) - self.assertEqual(len(l), 14) + episodes = IMDB.get_episode_list("tt0436992", 4) + self.assertEqual(len(episodes), 14) + episodes = IMDB.get_episode_list("tt1205438", 4) + self.assertEqual(len(episodes), 14) @use_local_response def test_tvshow(self): diff --git a/catalog/views.py b/catalog/views.py index 661fac17..54bb61d7 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -22,12 +22,10 @@ from journal.models import ( Review, ShelfManager, ShelfMember, - ShelfType, q_piece_in_home_feed_of_user, q_piece_visible_to_user, ) from takahe.utils import Takahe -from users.views import announcements from .forms import * from .models import * diff --git a/common/api.py b/common/api.py index ed4f75a4..d0a29c3c 100644 --- a/common/api.py +++ b/common/api.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, Optional, Tuple, Type +from typing import Any, List from django.conf import settings from django.db.models import QuerySet @@ -91,7 +91,7 @@ class PageNumberPagination(NinjaPageNumberPagination): api = NinjaAPI( auth=OAuthAccessTokenAuth(), - title=f'{settings.SITE_INFO["site_name"]} API', + title=f"{settings.SITE_INFO['site_name']} API", version="1.0.0", description=f"{settings.SITE_INFO['site_name']} API