remove obsoleted task code
This commit is contained in:
Her Email
Henri Dickson
parent
38cf2376b1
commit
cc59e044d5
4 changed files with 2 additions and 186 deletions
|
|
@ -1,104 +0,0 @@
|
|||
from datetime import timedelta
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Count, F
|
||||
from django.utils import timezone
|
||||
from loguru import logger
|
||||
|
||||
from catalog.models import *
|
||||
from journal.models import Comment, ShelfMember, q_item_in_category
|
||||
|
||||
MAX_ITEMS_PER_PERIOD = 12
|
||||
MIN_MARKS = 2
|
||||
MAX_DAYS_FOR_PERIOD = 96
|
||||
MIN_DAYS_FOR_PERIOD = 6
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "catalog app utilities"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--update",
|
||||
action="store_true",
|
||||
help="generate discover data",
|
||||
)
|
||||
|
||||
def get_popular_marked_item_ids(self, category, days, exisiting_ids):
|
||||
item_ids = [
|
||||
m["item_id"]
|
||||
for m in ShelfMember.objects.filter(q_item_in_category(category))
|
||||
.filter(created_time__gt=timezone.now() - timedelta(days=days))
|
||||
.exclude(item_id__in=exisiting_ids)
|
||||
.values("item_id")
|
||||
.annotate(num=Count("item_id"))
|
||||
.filter(num__gte=MIN_MARKS)
|
||||
.order_by("-num")[:MAX_ITEMS_PER_PERIOD]
|
||||
]
|
||||
return item_ids
|
||||
|
||||
def get_popular_commented_podcast_ids(self, days, exisiting_ids):
|
||||
return list(
|
||||
Comment.objects.filter(q_item_in_category(ItemCategory.Podcast))
|
||||
.filter(created_time__gt=timezone.now() - timedelta(days=days))
|
||||
.annotate(p=F("item__podcastepisode__program"))
|
||||
.filter(p__isnull=False)
|
||||
.exclude(p__in=exisiting_ids)
|
||||
.values("p")
|
||||
.annotate(num=Count("p"))
|
||||
.filter(num__gte=MIN_MARKS)
|
||||
.order_by("-num")
|
||||
.values_list("p", flat=True)[:MAX_ITEMS_PER_PERIOD]
|
||||
)
|
||||
|
||||
def cleanup_shows(self, items):
|
||||
seasons = [i for i in items if i.__class__ == TVSeason]
|
||||
for season in seasons:
|
||||
if season.show in items:
|
||||
items.remove(season.show)
|
||||
return items
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options["update"]:
|
||||
logger.info("Discover data update start.")
|
||||
cache_key = "public_gallery"
|
||||
gallery_categories = [
|
||||
ItemCategory.Book,
|
||||
ItemCategory.Movie,
|
||||
ItemCategory.TV,
|
||||
ItemCategory.Game,
|
||||
ItemCategory.Music,
|
||||
ItemCategory.Podcast,
|
||||
]
|
||||
gallery_list = []
|
||||
for category in gallery_categories:
|
||||
days = MAX_DAYS_FOR_PERIOD
|
||||
item_ids = []
|
||||
while days >= MIN_DAYS_FOR_PERIOD:
|
||||
ids = self.get_popular_marked_item_ids(category, days, item_ids)
|
||||
logger.info(
|
||||
f"Most marked {category} in last {days} days: {len(ids)}"
|
||||
)
|
||||
item_ids = ids + item_ids
|
||||
days //= 2
|
||||
if category == ItemCategory.Podcast:
|
||||
days = MAX_DAYS_FOR_PERIOD // 4
|
||||
extra_ids = self.get_popular_commented_podcast_ids(days, item_ids)
|
||||
logger.info(
|
||||
f"Most commented podcast in last {days} days: {len(extra_ids)}"
|
||||
)
|
||||
item_ids = extra_ids + item_ids
|
||||
items = [Item.objects.get(pk=i) for i in item_ids]
|
||||
if category == ItemCategory.TV:
|
||||
items = self.cleanup_shows(items)
|
||||
gallery_list.append(
|
||||
{
|
||||
"name": "popular_" + category.value,
|
||||
"title": ""
|
||||
+ (category.label if category != ItemCategory.Book else "图书"),
|
||||
"items": items,
|
||||
}
|
||||
)
|
||||
cache.set(cache_key, gallery_list, timeout=None)
|
||||
logger.info("Discover data updated.")
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import pprint
|
||||
from datetime import timedelta
|
||||
from time import sleep
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.core.paginator import Paginator
|
||||
from django.utils import timezone
|
||||
from loguru import logger
|
||||
from tqdm import tqdm
|
||||
|
||||
from catalog.common.models import IdType
|
||||
from catalog.models import *
|
||||
from catalog.sites import RSS
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Manage podcast data"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--update",
|
||||
help="update latest episode",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stat",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
def stat(self, *args, **options):
|
||||
qs = Podcast.objects.filter(is_deleted=False, merged_to_item__isnull=True)
|
||||
for p in qs.order_by("pk"):
|
||||
self.stdout.write(f"{p.episodes.count()}\t{p.title}\n")
|
||||
|
||||
def update(self):
|
||||
logger.info("Podcasts update start.")
|
||||
count = 0
|
||||
qs = Podcast.objects.filter(is_deleted=False, merged_to_item__isnull=True)
|
||||
for p in tqdm(qs.order_by("pk")):
|
||||
if (
|
||||
p.primary_lookup_id_type == IdType.RSS
|
||||
and p.primary_lookup_id_value is not None
|
||||
):
|
||||
logger.info(f"updating {p}")
|
||||
c = p.episodes.count()
|
||||
site = RSS(p.feed_url)
|
||||
site.scrape_additional_data()
|
||||
c2 = p.episodes.count()
|
||||
logger.info(f"updated {p}, {c2-c} new episodes.")
|
||||
count += c2 - c
|
||||
logger.info(f"Podcasts update finished, {count} new episodes total.")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options["update"]:
|
||||
self.update()
|
||||
if options["stat"]:
|
||||
self.stat()
|
||||
|
|
@ -7,7 +7,7 @@ from rq.registry import ScheduledJobRegistry
|
|||
|
||||
|
||||
class BaseJob:
|
||||
interval = timedelta(seconds=1)
|
||||
interval = timedelta(hours=1)
|
||||
|
||||
@classmethod
|
||||
def cancel(cls):
|
||||
|
|
@ -25,7 +25,7 @@ class BaseJob:
|
|||
@classmethod
|
||||
def schedule(cls):
|
||||
job_id = cls.__name__
|
||||
logger.info(f"Scheduling job: {job_id}")
|
||||
logger.info(f"Scheduling job: {job_id} in {cls.interval}")
|
||||
django_rq.get_queue("cron").enqueue_in(
|
||||
cls.interval, cls._run, job_id=job_id, result_ttl=0, failure_ttl=0
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
from django.contrib.sessions.models import Session
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from users.models import User
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Backfill Mastodon data if missing"
|
||||
|
||||
def handle(self, *args, **options):
|
||||
for session in Session.objects.order_by("-expire_date"):
|
||||
uid = session.get_decoded().get("_auth_user_id")
|
||||
token = session.get_decoded().get("oauth_token")
|
||||
if uid and token:
|
||||
user = User.objects.get(pk=uid)
|
||||
if user.mastodon_token:
|
||||
print(f"skip {user}")
|
||||
continue
|
||||
user.mastodon_token = token
|
||||
user.refresh_mastodon_data()
|
||||
user.save()
|
||||
print(f"Refreshed {user}")
|
||||
Loading…
Add table
Reference in a new issue