This commit is contained in:
Your Name 2023-12-31 08:32:19 -05:00 committed by Henri Dickson
parent 51398fa40b
commit 53804687b1
18 changed files with 41 additions and 23 deletions

View file

@ -32,7 +32,7 @@ repos:
- id: black
- repo: https://github.com/Riverside-Healthcare/djLint
rev: v1.32.1
rev: v1.34.1
hooks:
- id: djlint-reformat-django
- id: djlint-django

View file

@ -143,7 +143,7 @@ class BasicDownloader:
def get_timeout(self):
return settings.DOWNLOADER_REQUEST_TIMEOUT
def validate_response(self, response):
def validate_response(self, response) -> int:
if response is None:
return RESPONSE_NETWORK_ERROR
elif response.status_code == 200:

View file

@ -1,3 +1,4 @@
# pyright: reportIncompatibleMethodOverride=false, reportFunctionMemberAccess=false
import copy
from datetime import date, datetime
from functools import partialmethod

View file

@ -55,7 +55,7 @@ class AbstractSite:
return u is not None
@classmethod
def validate_url_fallback(cls, url: str):
def validate_url_fallback(cls, url: str) -> bool:
return False
@classmethod
@ -264,12 +264,17 @@ class AbstractSite:
return p
from typing import Any, Callable, Type, TypeVar
T = TypeVar("T")
class SiteManager:
registry = {}
@staticmethod
def register(target) -> Callable:
id_type = target.ID_TYPE
def register(target: Type[T]) -> Type[T]:
id_type = target.ID_TYPE # type: ignore
if id_type in SiteManager.registry:
raise ValueError(f"Site for {id_type} already exists")
SiteManager.registry[id_type] = target
@ -325,7 +330,7 @@ class SiteManager:
@staticmethod
def get_all_sites():
return SiteManager.register.values()
return SiteManager.registry.values()
def crawl_related_resources_task(resource_pk):

View file

@ -1,3 +1,4 @@
# pyright: reportFunctionMemberAccess=false
import hashlib
import logging

View file

@ -25,16 +25,17 @@ class ApplePodcast(AbstractSite):
dl = BasicDownloader(api_url)
resp = dl.download()
r = resp.json()["results"][0]
feed_url = r["feedUrl"]
pd = ResourceContent(
metadata={
"title": r["trackName"],
"feed_url": r["feedUrl"],
"feed_url": feed_url,
"hosts": [r["artistName"]],
"genres": r["genres"],
"cover_image_url": r["artworkUrl600"],
}
)
pd.lookup_ids[IdType.RSS] = RSS.url_to_id(pd.metadata.get("feed_url"))
pd.lookup_ids[IdType.RSS] = RSS.url_to_id(feed_url)
if pd.metadata["cover_image_url"]:
imgdl = BasicImageDownloader(pd.metadata["cover_image_url"], self.url)
try:

View file

@ -45,6 +45,7 @@ class Bandcamp(AbstractSite):
return True
except Exception:
pass
return False
def scrape(self):
content = BasicDownloader(self.url).download().html()

View file

@ -7,7 +7,7 @@ RE_WHITESPACES = re.compile(r"\s+")
class DoubanDownloader(ProxiedDownloader):
def validate_response(self, response):
def validate_response(self, response) -> int:
if response is None:
return RESPONSE_NETWORK_ERROR
elif response.status_code == 204:

View file

@ -47,7 +47,7 @@ class FediverseInstance(AbstractSite):
return "https://" + u[0].lower() + "/" + u[1]
@classmethod
def validate_url_fallback(cls, url):
def validate_url_fallback(cls, url: str):
val = URLValidator()
try:
val(url)

View file

@ -80,9 +80,11 @@ class RSS(AbstractSite):
return False
def scrape(self):
if not self.url:
raise ValueError(f"no url avaialble in RSS site")
feed = self.parse_feed_from_url(self.url)
if not feed:
return None
raise ValueError(f"no feed avaialble in {self.url}")
pd = ResourceContent(
metadata={
"title": feed["title"],

View file

@ -345,6 +345,8 @@ class TMDB_TVSeason(AbstractSite):
season_id = v[1]
site = TMDB_TV(TMDB_TV.id_to_url(show_id))
show_resource = site.get_resource_ready(auto_create=False, auto_link=False)
if not show_resource:
raise ValueError(f"TMDB: failed to get show for season {self.url}")
api_url = f"https://api.themoviedb.org/3/tv/{show_id}/season/{season_id}?api_key={settings.TMDB_API3_KEY}&language=zh-CN&append_to_response=external_ids,credits"
d = BasicDownloader(api_url).download().json()
if not d.get("id"):

View file

@ -41,7 +41,7 @@ class OPMLImporter:
res = RSS(feed.url).get_resource_ready()
except:
res = None
if not res:
if not res or not res.item:
logger.warning(f"{self.user} feed error {feed.url}")
continue
item = res.item

View file

@ -141,7 +141,7 @@ urlpatterns = [
user_calendar_data,
name="user_calendar_data",
),
path("users/<str:id>/feed/reviews/", ReviewFeed(), name="review_feed"),
path("users/<str:username>/feed/reviews/", ReviewFeed(), name="review_feed"),
path("wrapped/<int:year>/", WrappedView.as_view(), name="wrapped"),
path("wrapped/<int:year>/share", WrappedShareView.as_view(), name="wrapped_share"),
]

View file

@ -105,8 +105,8 @@ MAX_ITEM_PER_TYPE = 10
class ReviewFeed(Feed):
def get_object(self, request, id):
return APIdentity.get_by_handler(id)
def get_object(self, request, *args, **kwargs):
return APIdentity.get_by_handler(kwargs["username"])
def title(self, owner):
return "%s的评论" % owner.display_name if owner else "无效链接"
@ -115,7 +115,12 @@ class ReviewFeed(Feed):
return owner.url if owner else settings.SITE_INFO["site_url"]
def description(self, owner):
return "%s的评论合集 - NeoDB" % owner.display_name if owner else "无效链接"
if not owner:
return "无效链接"
elif not owner.anonymous_viewable:
return "该用户已关闭匿名查看"
else:
return "%s的评论合集 - NeoDB" % owner.display_name
def items(self, owner):
if owner is None or not owner.anonymous_viewable:
@ -135,7 +140,7 @@ class ReviewFeed(Feed):
# item_link is only needed if NewsItem has no get_absolute_url method.
def item_link(self, item: Review):
return item.absolute_url
return str(item.absolute_url)
def item_categories(self, item):
return [item.item.category.label]

View file

@ -10,8 +10,10 @@ class OAuth2Backend(ModelBackend):
# a user object that matches those credentials."
# arg request is an interface specification, not used in this implementation
def authenticate(self, request, token=None, username=None, site=None, **kwargs):
def authenticate(self, request, username=None, password=None, **kwargs):
"""when username is provided, assume that token is newly obtained and valid"""
token = kwargs.get("token", None)
site = kwargs.get("site", None)
if token is None or site is None:
return
mastodon_id = None

View file

@ -1,8 +1,6 @@
[tool.pyright]
exclude = [ "media", ".venv", ".git", "playground", "catalog/*/tests.py", "neodb", "**/migrations", "**/sites/douban_*", "neodb-takahe" ]
reportIncompatibleVariableOverride = false
reportFunctionMemberAccess = false
reportIncompatibleMethodOverride = false
[tool.djlint]
ignore="T002,T003,H006,H019,H020,H021,H023,H030,H031"

View file

@ -6,4 +6,4 @@ djlint~=1.34.0
isort~=5.12.0
lxml-stubs
pre-commit
pyright==1.1.337
pyright==1.1.344

View file

@ -485,7 +485,7 @@ class Identity(models.Model):
admin_edit = "{admin}{self.pk}/"
djadmin_edit = "/djadmin/users/identity/{self.id}/change/"
def get_scheme(self, url):
def get_scheme(self, url): # pyright: ignore
return "https"
def get_hostname(self, url):
@ -973,7 +973,7 @@ class Post(models.Model):
action_reply = "/compose/?reply_to={self.id}"
admin_edit = "/djadmin/activities/post/{self.id}/change/"
def get_scheme(self, url):
def get_scheme(self, url): # pyright: ignore
return "https"
def get_hostname(self, url):