fix lint with pyright

This commit is contained in:
Your Name 2023-08-11 01:43:19 -04:00 committed by Henri Dickson
parent 7d1388df30
commit 7cbd759215
31 changed files with 265 additions and 189 deletions

View file

@ -32,12 +32,12 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run Tests
run: |

View file

@ -1,9 +1,26 @@
name: Lint
name: check
on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version: ['3.11']
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Run pre-commit
run: |
python -m pip install pre_commit
SKIP=pyright python -m pre_commit run -a --show-diff-on-failure
type-checker:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
@ -19,6 +36,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install -r requirements-dev.txt
- name: Run pre-commit
python -m pip install -r requirements.txt
- name: Run pyright
run: |
python -m pre_commit run -a --show-diff-on-failure
python -m pyright

View file

@ -37,3 +37,11 @@ repos:
hooks:
- id: djlint-reformat-django
- id: djlint-django
- repo: local
hooks:
- id: pyright
name: pyright
entry: python -m pyright
language: system
pass_filenames: false

View file

@ -155,7 +155,7 @@ AUTHENTICATION_BACKENDS = [
]
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.renderers.render_md"
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md"
# Internationalization

View file

@ -21,7 +21,18 @@ from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
from .utils import *

View file

@ -1,6 +1,6 @@
import re
from .models import IdType
from ..common.models import IdType
def check_digit_10(isbn):

View file

@ -4,6 +4,7 @@ import re
import time
from io import BytesIO, StringIO
from pathlib import Path
from typing import Tuple
from urllib.parse import quote
import filetype
@ -11,6 +12,7 @@ import requests
from django.conf import settings
from lxml import html
from PIL import Image
from requests import Response
from requests.exceptions import RequestException
_logger = logging.getLogger(__name__)
@ -52,6 +54,53 @@ def get_mock_file(url):
return fn
_local_response_path = (
str(Path(__file__).parent.parent.parent.absolute()) + "/test_data/"
)
class MockResponse:
def __init__(self, url):
self.url = url
fn = _local_response_path + get_mock_file(url)
try:
self.content = Path(fn).read_bytes()
self.status_code = 200
_logger.debug(f"use local response for {url} from {fn}")
except Exception:
self.content = b"Error: response file not found"
self.status_code = 404
_logger.debug(f"local response not found for {url} at {fn}")
@property
def text(self):
return self.content.decode("utf-8")
def json(self):
return json.load(StringIO(self.text))
def html(self):
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
self.content.decode("utf-8")
)
@property
def headers(self):
return {
"Content-Type": "image/jpeg" if self.url.endswith("jpg") else "text/html"
}
requests.Response.html = MockResponse.html # type:ignore
class DownloaderResponse(Response):
def html(self):
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
self.content.decode("utf-8")
)
class DownloadError(Exception):
def __init__(self, downloader, msg=None):
self.url = downloader.url
@ -101,7 +150,7 @@ class BasicDownloader:
else:
return RESPONSE_INVALID_CONTENT
def _download(self, url):
def _download(self, url) -> Tuple[DownloaderResponse | MockResponse, int]:
try:
if not _mock_mode:
# TODO cache = get/set from redis
@ -125,12 +174,12 @@ class BasicDownloader:
{"response_type": response_type, "url": url, "exception": None}
)
return resp, response_type
return resp, response_type # type: ignore
except RequestException as e:
self.logs.append(
{"response_type": RESPONSE_NETWORK_ERROR, "url": url, "exception": e}
)
return None, RESPONSE_NETWORK_ERROR
return None, RESPONSE_NETWORK_ERROR # type: ignore
def download(self):
resp, self.response_type = self._download(self.url)
@ -209,9 +258,10 @@ class RetryDownloader(BasicDownloader):
class ImageDownloaderMixin:
def __init__(self, url, referer=None):
self.extention = None
if referer is not None:
self.headers["Referer"] = referer
super().__init__(url)
self.headers["Referer"] = referer # type: ignore
super().__init__(url) # type: ignore
def validate_response(self, response):
if response and response.status_code == 200:
@ -236,12 +286,12 @@ class ImageDownloaderMixin:
@classmethod
def download_image(cls, image_url, page_url, headers=None):
imgdl = cls(image_url, page_url)
imgdl: BasicDownloader = cls(image_url, page_url) # type:ignore
if headers is not None:
imgdl.headers = headers
try:
image = imgdl.download().content
image_extention = imgdl.extention
image_extention = imgdl.extention # type:ignore
return image, image_extention
except Exception:
return None, None
@ -253,43 +303,3 @@ class BasicImageDownloader(ImageDownloaderMixin, BasicDownloader):
class ProxiedImageDownloader(ImageDownloaderMixin, ProxiedDownloader):
pass
_local_response_path = (
str(Path(__file__).parent.parent.parent.absolute()) + "/test_data/"
)
class MockResponse:
def __init__(self, url):
self.url = url
fn = _local_response_path + get_mock_file(url)
try:
self.content = Path(fn).read_bytes()
self.status_code = 200
_logger.debug(f"use local response for {url} from {fn}")
except Exception:
self.content = b"Error: response file not found"
self.status_code = 404
_logger.debug(f"local response not found for {url} at {fn}")
@property
def text(self):
return self.content.decode("utf-8")
def json(self):
return json.load(StringIO(self.text))
def html(self):
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
self.content.decode("utf-8")
)
@property
def headers(self):
return {
"Content-Type": "image/jpeg" if self.url.endswith("jpg") else "text/html"
}
requests.Response.html = MockResponse.html

View file

@ -80,6 +80,9 @@ class JSONFieldDescriptor(object):
setattr(instance, self.field.json_field_name, json_value)
fields.CharField
class JSONFieldMixin(object):
"""
Override django.db.model.fields.Field.contribute_to_class
@ -90,11 +93,11 @@ class JSONFieldMixin(object):
self.json_field_name = kwargs.pop("json_field_name", "metadata")
super(JSONFieldMixin, self).__init__(*args, **kwargs)
def contribute_to_class(self, cls, name, private_only=False):
def contribute_to_class(self: "fields.Field", cls, name, private_only=False): # type: ignore
self.set_attributes_from_name(name)
self.model = cls
self.concrete = False
self.column = self.json_field_name
self.column = self.json_field_name # type: ignore
cls._meta.add_field(self, private=True)
if not getattr(cls, self.attname, None):
@ -131,11 +134,11 @@ class JSONFieldMixin(object):
return transform
json_field = self.model._meta.get_field(self.json_field_name)
transform = json_field.get_transform(self.name)
transform = json_field.get_transform(self.name) # type: ignore
if transform is None:
raise FieldError(
"JSONField '%s' has no support for key '%s' %s lookup"
% (self.json_field_name, self.name, name)
% (self.json_field_name, self.name, name) # type: ignore
)
return TransformFactoryWrapper(json_field, transform, name)
@ -170,10 +173,17 @@ class DateField(JSONFieldMixin, fields.DateField):
class DateTimeField(JSONFieldMixin, fields.DateTimeField):
def to_json(self, value):
def to_json(self, value: datetime | date | str):
if value:
if not isinstance(value, (datetime, date)):
value = dateparse.parse_date(value)
v = dateparse.parse_date(value)
if v is None:
raise ValueError(
"DateTimeField: '{value}' has invalid datatime format"
)
value = v
if isinstance(value, date):
value = datetime.combine(value, datetime.time.min())
if not timezone.is_aware(value):
value = timezone.make_aware(value)
return value.isoformat()

View file

@ -17,6 +17,6 @@ class SoftDeleteMixin:
if soft:
self.clear()
self.is_deleted = True
self.save(using=using)
self.save(using=using) # type: ignore
else:
return super().delete(using=using, *args, **kwargs)
return super().delete(using=using, *args, **kwargs) # type: ignore

View file

@ -2,7 +2,7 @@ import logging
import re
import uuid
from functools import cached_property
from typing import cast
from typing import TYPE_CHECKING, cast
from auditlog.context import disable_auditlog
from auditlog.models import AuditlogHistoryField, LogEntry
@ -22,6 +22,9 @@ from users.models import User
from .mixins import SoftDeleteMixin
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
if TYPE_CHECKING:
from django.utils.functional import _StrOrPromise
_logger = logging.getLogger(__name__)
@ -247,7 +250,7 @@ class Item(SoftDeleteMixin, PolymorphicModel):
type = None # subclass must specify this
parent_class = None # subclass may specify this to allow create child item
category: ItemCategory | None = None # subclass must specify this
demonstrative: str | None = None # subclass must specify this
demonstrative: "_StrOrPromise | None" = None # subclass must specify this
uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
title = models.CharField(_("标题"), max_length=1000, default="")
brief = models.TextField(_("简介"), blank=True, default="")

View file

@ -37,7 +37,7 @@ def _EditForm(item_model):
}
def clean(self):
data = super().clean()
data = super().clean() or {}
t, v = self.Meta.model.lookup_id_cleanup(
data.get("primary_lookup_id_type"), data.get("primary_lookup_id_value")
)

View file

@ -3,7 +3,18 @@ from datetime import date
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
class GameInSchema(ItemInSchema):

View file

@ -8,9 +8,13 @@ from .book.models import Edition, EditionInSchema, EditionSchema, Series, Work
from .collection.models import Collection as CatalogCollection
from .common.models import (
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
SiteName,
item_categories,
item_content_types,
)

View file

@ -1,7 +1,18 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
class MovieInSchema(ItemInSchema):

View file

@ -3,7 +3,18 @@ from datetime import date
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
class AlbumInSchema(ItemInSchema):

View file

@ -1,7 +1,18 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
class PodcastInSchema(ItemInSchema):

View file

@ -65,7 +65,10 @@ class Goodreads:
r = requests.get(search_url)
if r.url.startswith("https://www.goodreads.com/book/show/"):
# Goodreads will 302 if only one result matches ISBN
res = SiteManager.get_site_by_url(r.url).get_resource_ready()
site = SiteManager.get_site_by_url(r.url)
if site:
res = site.get_resource_ready()
if res:
subtitle = f"{res.metadata['pub_year']} {', '.join(res.metadata['author'])} {', '.join(res.metadata['translator'] if res.metadata['translator'] else [])}"
results.append(
SearchResultItem(
@ -80,7 +83,8 @@ class Goodreads:
)
else:
h = html.fromstring(r.content.decode("utf-8"))
for c in h.xpath('//tr[@itemtype="http://schema.org/Book"]'):
books = h.xpath('//tr[@itemtype="http://schema.org/Book"]')
for c in books: # type:ignore
el_cover = c.xpath('.//img[@class="bookCover"]/@src')
cover = el_cover[0] if el_cover else None
el_title = c.xpath('.//a[@class="bookTitle"]//text()')
@ -228,7 +232,8 @@ class Bandcamp:
search_url = f"https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}"
r = requests.get(search_url)
h = html.fromstring(r.content.decode("utf-8"))
for c in h.xpath('//li[@class="searchresult data-search"]'):
albums = h.xpath('//li[@class="searchresult data-search"]')
for c in albums: # type:ignore
el_cover = c.xpath('.//div[@class="art"]/img/@src')
cover = el_cover[0] if el_cover else None
el_title = c.xpath('.//div[@class="heading"]//text()')

View file

@ -24,12 +24,24 @@ tv specials are are shown as movies
For now, we follow Douban convention, but keep an eye on it in case it breaks its own rules...
"""
import re
from functools import cached_property
from django.db import models
from django.utils.translation import gettext_lazy as _
from catalog.common.models import *
from catalog.common import (
BaseSchema,
ExternalResource,
IdType,
Item,
ItemCategory,
ItemInSchema,
ItemSchema,
ItemType,
PrimaryLookupIdDescriptor,
jsondata,
)
class TVShowInSchema(ItemInSchema):

View file

@ -8,7 +8,7 @@ register = template.Library()
class OAuthTokenNode(template.Node):
def render(self, context):
request = context.get("request")
oauth_token = request.user.mastodon_token
oauth_token = request.user.mastodon_token if request else ""
return format_html(oauth_token)

View file

@ -5,7 +5,7 @@ from django.conf import settings
from django.utils.translation import gettext_lazy as _
from openpyxl import Workbook
from catalog.common.models import IdType
from catalog.models import *
from common.utils import GenerateDateUUIDMediaFilePath
from journal.models import *

View file

@ -108,9 +108,12 @@ class GoodreadsImporter:
@classmethod
def get_book(cls, url, user):
site = SiteManager.get_site_by_url(url)
if site:
book = site.get_item()
if not book:
book = site.get_resource_ready().item
resource = site.get_resource_ready()
if resource and resource.item:
book = resource.item
book.last_editor = user
book.save()
return book
@ -129,12 +132,13 @@ class GoodreadsImporter:
if not title_elem:
print(f"Shelf parsing error {url_shelf}")
break
title = title_elem[0].strip()
title = title_elem[0].strip() # type:ignore
print("Shelf title: " + title)
except Exception:
print(f"Shelf loading/parsing error {url_shelf}")
break
for cell in content.xpath("//tbody[@id='booksBody']/tr"):
cells = content.xpath("//tbody[@id='booksBody']/tr")
for cell in cells: # type:ignore
url_book = (
"https://www.goodreads.com"
+ cell.xpath(".//td[@class='field title']//a/@href")[0].strip()
@ -167,10 +171,12 @@ class GoodreadsImporter:
c2 = BasicDownloader(url_review).download().html()
review_elem = c2.xpath("//div[@itemprop='reviewBody']/text()")
review = (
"\n".join(p.strip() for p in review_elem) if review_elem else ""
"\n".join(p.strip() for p in review_elem) # type:ignore
if review_elem
else ""
)
date_elem = c2.xpath("//div[@class='readingTimeline__text']/text()")
for d in date_elem:
for d in date_elem: # type:ignore
date_matched = re.search(r"(\w+)\s+(\d+),\s+(\d+)", d)
if date_matched:
last_updated = make_aware(
@ -201,7 +207,7 @@ class GoodreadsImporter:
pass # likely just download error
next_elem = content.xpath("//a[@class='next_page']/@href")
url_shelf = (
("https://www.goodreads.com" + next_elem[0].strip())
f"https://www.goodreads.com{next_elem[0].strip()}"
if next_elem
else None
)

View file

@ -13,6 +13,7 @@ from .common import (
from .like import Like
from .mark import Mark
from .rating import Rating
from .renderers import render_md
from .review import Review
from .shelf import (
Shelf,

View file

@ -1,3 +1,9 @@
from typing import TYPE_CHECKING, Type
if TYPE_CHECKING:
from .common import Piece
class UserOwnedObjectMixin:
"""
UserOwnedObjectMixin
@ -7,7 +13,7 @@ class UserOwnedObjectMixin:
visibility = models.PositiveSmallIntegerField(default=0)
"""
def is_visible_to(self, viewer):
def is_visible_to(self: "Piece", viewer): # type: ignore
owner = self.owner
if owner == viewer:
return True
@ -24,13 +30,13 @@ class UserOwnedObjectMixin:
else:
return True
def is_editable_by(self, viewer):
def is_editable_by(self: "Piece", viewer): # type: ignore
return viewer.is_authenticated and (
viewer.is_staff or viewer.is_superuser or viewer == self.owner
)
@classmethod
def get_available(cls, entity, request_user, following_only=False):
def get_available(cls: "Type[Piece]", entity, request_user, following_only=False): # type: ignore
# e.g. SongMark.get_available(song, request.user)
query_kwargs = {entity.__class__.__name__.lower(): entity}
all_entities = cls.objects.filter(**query_kwargs).order_by(

View file

@ -4,8 +4,6 @@ from typing import cast
import mistune
from django.utils.html import escape
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.renderers.render_md"
_mistune_plugins = [
"url",
"strikethrough",

View file

@ -3,13 +3,19 @@ from django.shortcuts import get_object_or_404
from django.urls import reverse_lazy
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.views.generic import *
from django.views.generic import (
CreateView,
DeleteView,
DetailView,
ListView,
UpdateView,
)
from django.views.generic.edit import ModelFormMixin
from .models import Announcement
# https://docs.djangoproject.com/en/3.1/topics/class-based-views/intro/
decorators = [login_required, user_passes_test(lambda u: u.is_superuser)]
decorators = [login_required, user_passes_test(lambda u: u.is_superuser)] # type:ignore
class AnnouncementDetailView(DetailView, ModelFormMixin):

View file

@ -1,74 +1 @@
from django.contrib import admin
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import gettext_lazy as _
from requests.exceptions import Timeout
from .api import create_app
from .models import *
# Register your models here.
@admin.register(MastodonApplication)
class MastodonApplicationModelAdmin(admin.ModelAdmin):
def add_view(self, request, form_url="", extra_context=None):
"""
Dirty code here, use POST['domain_name'] to pass error message to user.
"""
if request.method == "POST":
if not request.POST.get("client_id") and not request.POST.get(
"client_secret"
):
# make the post data mutable
request.POST = request.POST.copy()
# (is_proxy xor proxy_to) or (proxy_to!=null and is_proxy=false)
if (
(
bool(request.POST.get("is_proxy"))
or bool(request.POST.get("proxy_to"))
)
and not (
bool(request.POST.get("is_proxy"))
and bool(request.POST.get("proxy_to"))
)
or (
not bool(request.POST.get("is_proxy"))
and bool(request.POST.get("proxy_to"))
)
):
request.POST["domain_name"] = _("请同时填写is_proxy和proxy_to。")
else:
if request.POST.get("is_proxy"):
try:
origin = MastodonApplication.objects.get(
domain_name=request.POST["proxy_to"]
)
# set proxy credentials to those of its original site
request.POST["app_id"] = origin.app_id
request.POST["client_id"] = origin.client_id
request.POST["client_secret"] = origin.client_secret
request.POST["vapid_key"] = origin.vapid_key
except ObjectDoesNotExist:
request.POST["domain_name"] = _("proxy_to所指域名不存在请先添加原站点。")
else:
# create mastodon app
try:
response = create_app(request.POST.get("domain_name"))
except (Timeout, ConnectionError):
request.POST["domain_name"] = _("联邦宇宙请求超时。")
except Exception as e:
request.POST["domain_name"] = str(e)
else:
# fill the form with returned data
data = response.json()
if response.status_code != 200:
request.POST["domain_name"] = str(data)
else:
request.POST["app_id"] = data["id"]
request.POST["client_id"] = data["client_id"]
request.POST["client_secret"] = data["client_secret"]
request.POST["vapid_key"] = data["vapid_key"]
return super().add_view(request, form_url=form_url, extra_context=extra_context)
admin.site.register(CrossSiteUserInfo)

View file

@ -1,5 +1,5 @@
[tool.pyright]
exclude = [ "media", ".venv", ".git" ]
exclude = [ "media", ".venv", ".git", "playground", "**/tests.py", "neodb", "**/migrations", "**/commands", "**/importers", "**/sites" ]
[tool.djlint]
ignore="T002,T003,H006,H019,H020,H021,H023,H030,H031"
@ -17,3 +17,7 @@ plugins = ["mypy_django_plugin.main"]
[tool.django-stubs]
django_settings_module = "boofilsic.settings"
[tool.ruff]
ignore = ['E501']
exclude = [ "media", ".venv", ".git", "playground", "**/tests.py", "neodb", "**/migrations", "**/commands", "**/importers", "**/sites", "legacy" ]

View file

@ -1,8 +1,8 @@
black~=22.12.0
coverage
django-debug-toolbar
django-stubs
djlint~=1.32.1
isort~=5.12.0
pre-commit~=3.3.3
types-dateparser
types-tqdm
pre-commit
pyright

View file

@ -81,10 +81,6 @@ class ActivityManager:
return ActivityManager(user)
User.activity_manager = cached_property(ActivityManager.get_manager_for_user) # type: ignore
User.activity_manager.__set_name__(User, "activity_manager") # type: ignore
class DataSignalManager:
processors = {}

View file

@ -13,7 +13,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
count = 0
for user in tqdm(User.objects.all()):
user.following = user.merge_following_ids()
user.following = user.merged_following_ids()
if user.following:
count += 1
user.save(update_fields=["following"])

View file

@ -4,6 +4,7 @@ from functools import cached_property
from typing import TYPE_CHECKING
from django.contrib.auth.models import AbstractUser
from django.contrib.auth.validators import UnicodeUsernameValidator
from django.core import validators
from django.core.exceptions import ValidationError
from django.db import models
@ -33,7 +34,7 @@ _RESERVED_USERNAMES = [
@deconstructible
class UsernameValidator(validators.RegexValidator):
class UsernameValidator(UnicodeUsernameValidator):
regex = r"^[a-zA-Z0-9_]{2,30}$"
message = _(
"Enter a valid username. This value may contain only unaccented lowercase a-z and uppercase A-Z letters, numbers, and _ characters."
@ -550,6 +551,12 @@ class User(AbstractUser):
return ShelfManager.get_manager_for_user(self)
@cached_property
def activity_manager(self):
from social.models import ActivityManager
return ActivityManager.get_manager_for_user(self)
class Follow(models.Model):
owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="+")