diff --git a/catalog/management/commands/discover.py b/catalog/management/commands/discover.py deleted file mode 100644 index d259189a..00000000 --- a/catalog/management/commands/discover.py +++ /dev/null @@ -1,104 +0,0 @@ -from datetime import timedelta - -from django.core.cache import cache -from django.core.management.base import BaseCommand -from django.db.models import Count, F -from django.utils import timezone -from loguru import logger - -from catalog.models import * -from journal.models import Comment, ShelfMember, q_item_in_category - -MAX_ITEMS_PER_PERIOD = 12 -MIN_MARKS = 2 -MAX_DAYS_FOR_PERIOD = 96 -MIN_DAYS_FOR_PERIOD = 6 - - -class Command(BaseCommand): - help = "catalog app utilities" - - def add_arguments(self, parser): - parser.add_argument( - "--update", - action="store_true", - help="generate discover data", - ) - - def get_popular_marked_item_ids(self, category, days, exisiting_ids): - item_ids = [ - m["item_id"] - for m in ShelfMember.objects.filter(q_item_in_category(category)) - .filter(created_time__gt=timezone.now() - timedelta(days=days)) - .exclude(item_id__in=exisiting_ids) - .values("item_id") - .annotate(num=Count("item_id")) - .filter(num__gte=MIN_MARKS) - .order_by("-num")[:MAX_ITEMS_PER_PERIOD] - ] - return item_ids - - def get_popular_commented_podcast_ids(self, days, exisiting_ids): - return list( - Comment.objects.filter(q_item_in_category(ItemCategory.Podcast)) - .filter(created_time__gt=timezone.now() - timedelta(days=days)) - .annotate(p=F("item__podcastepisode__program")) - .filter(p__isnull=False) - .exclude(p__in=exisiting_ids) - .values("p") - .annotate(num=Count("p")) - .filter(num__gte=MIN_MARKS) - .order_by("-num") - .values_list("p", flat=True)[:MAX_ITEMS_PER_PERIOD] - ) - - def cleanup_shows(self, items): - seasons = [i for i in items if i.__class__ == TVSeason] - for season in seasons: - if season.show in items: - items.remove(season.show) - return items - - def handle(self, *args, **options): - if options["update"]: - logger.info("Discover data update start.") - cache_key = "public_gallery" - gallery_categories = [ - ItemCategory.Book, - ItemCategory.Movie, - ItemCategory.TV, - ItemCategory.Game, - ItemCategory.Music, - ItemCategory.Podcast, - ] - gallery_list = [] - for category in gallery_categories: - days = MAX_DAYS_FOR_PERIOD - item_ids = [] - while days >= MIN_DAYS_FOR_PERIOD: - ids = self.get_popular_marked_item_ids(category, days, item_ids) - logger.info( - f"Most marked {category} in last {days} days: {len(ids)}" - ) - item_ids = ids + item_ids - days //= 2 - if category == ItemCategory.Podcast: - days = MAX_DAYS_FOR_PERIOD // 4 - extra_ids = self.get_popular_commented_podcast_ids(days, item_ids) - logger.info( - f"Most commented podcast in last {days} days: {len(extra_ids)}" - ) - item_ids = extra_ids + item_ids - items = [Item.objects.get(pk=i) for i in item_ids] - if category == ItemCategory.TV: - items = self.cleanup_shows(items) - gallery_list.append( - { - "name": "popular_" + category.value, - "title": "" - + (category.label if category != ItemCategory.Book else "图书"), - "items": items, - } - ) - cache.set(cache_key, gallery_list, timeout=None) - logger.info("Discover data updated.") diff --git a/catalog/management/commands/podcast.py b/catalog/management/commands/podcast.py deleted file mode 100644 index 206f8a03..00000000 --- a/catalog/management/commands/podcast.py +++ /dev/null @@ -1,58 +0,0 @@ -import pprint -from datetime import timedelta -from time import sleep - -from django.conf import settings -from django.core.management.base import BaseCommand -from django.core.paginator import Paginator -from django.utils import timezone -from loguru import logger -from tqdm import tqdm - -from catalog.common.models import IdType -from catalog.models import * -from catalog.sites import RSS - - -class Command(BaseCommand): - help = "Manage podcast data" - - def add_arguments(self, parser): - parser.add_argument( - "--update", - help="update latest episode", - action="store_true", - ) - parser.add_argument( - "--stat", - action="store_true", - ) - - def stat(self, *args, **options): - qs = Podcast.objects.filter(is_deleted=False, merged_to_item__isnull=True) - for p in qs.order_by("pk"): - self.stdout.write(f"{p.episodes.count()}\t{p.title}\n") - - def update(self): - logger.info("Podcasts update start.") - count = 0 - qs = Podcast.objects.filter(is_deleted=False, merged_to_item__isnull=True) - for p in tqdm(qs.order_by("pk")): - if ( - p.primary_lookup_id_type == IdType.RSS - and p.primary_lookup_id_value is not None - ): - logger.info(f"updating {p}") - c = p.episodes.count() - site = RSS(p.feed_url) - site.scrape_additional_data() - c2 = p.episodes.count() - logger.info(f"updated {p}, {c2-c} new episodes.") - count += c2 - c - logger.info(f"Podcasts update finished, {count} new episodes total.") - - def handle(self, *args, **options): - if options["update"]: - self.update() - if options["stat"]: - self.stat() diff --git a/common/models.py b/common/models.py index f76a1dee..874a2e01 100644 --- a/common/models.py +++ b/common/models.py @@ -7,7 +7,7 @@ from rq.registry import ScheduledJobRegistry class BaseJob: - interval = timedelta(seconds=1) + interval = timedelta(hours=1) @classmethod def cancel(cls): @@ -25,7 +25,7 @@ class BaseJob: @classmethod def schedule(cls): job_id = cls.__name__ - logger.info(f"Scheduling job: {job_id}") + logger.info(f"Scheduling job: {job_id} in {cls.interval}") django_rq.get_queue("cron").enqueue_in( cls.interval, cls._run, job_id=job_id, result_ttl=0, failure_ttl=0 ) diff --git a/users/management/commands/backfill_mastodon.py b/users/management/commands/backfill_mastodon.py deleted file mode 100644 index 5a312b15..00000000 --- a/users/management/commands/backfill_mastodon.py +++ /dev/null @@ -1,22 +0,0 @@ -from django.contrib.sessions.models import Session -from django.core.management.base import BaseCommand - -from users.models import User - - -class Command(BaseCommand): - help = "Backfill Mastodon data if missing" - - def handle(self, *args, **options): - for session in Session.objects.order_by("-expire_date"): - uid = session.get_decoded().get("_auth_user_id") - token = session.get_decoded().get("oauth_token") - if uid and token: - user = User.objects.get(pk=uid) - if user.mastodon_token: - print(f"skip {user}") - continue - user.mastodon_token = token - user.refresh_mastodon_data() - user.save() - print(f"Refreshed {user}")