fix lint with pyright
This commit is contained in:
parent
7d1388df30
commit
7cbd759215
31 changed files with 265 additions and 189 deletions
4
.github/workflows/django.yml
vendored
4
.github/workflows/django.yml
vendored
|
@ -32,12 +32,12 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v3
|
uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
cache: pip
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip
|
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
- name: Run Tests
|
- name: Run Tests
|
||||||
run: |
|
run: |
|
||||||
|
|
24
.github/workflows/lint.yml
vendored
24
.github/workflows/lint.yml
vendored
|
@ -1,9 +1,26 @@
|
||||||
name: Lint
|
name: check
|
||||||
|
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
lint:
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
max-parallel: 4
|
||||||
|
matrix:
|
||||||
|
python-version: ['3.11']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
cache: pip
|
||||||
|
- name: Run pre-commit
|
||||||
|
run: |
|
||||||
|
python -m pip install pre_commit
|
||||||
|
SKIP=pyright python -m pre_commit run -a --show-diff-on-failure
|
||||||
|
type-checker:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
max-parallel: 4
|
max-parallel: 4
|
||||||
|
@ -19,6 +36,7 @@ jobs:
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
python -m pip install -r requirements-dev.txt
|
python -m pip install -r requirements-dev.txt
|
||||||
- name: Run pre-commit
|
python -m pip install -r requirements.txt
|
||||||
|
- name: Run pyright
|
||||||
run: |
|
run: |
|
||||||
python -m pre_commit run -a --show-diff-on-failure
|
python -m pyright
|
||||||
|
|
|
@ -37,3 +37,11 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: djlint-reformat-django
|
- id: djlint-reformat-django
|
||||||
- id: djlint-django
|
- id: djlint-django
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: pyright
|
||||||
|
name: pyright
|
||||||
|
entry: python -m pyright
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
|
|
@ -155,7 +155,7 @@ AUTHENTICATION_BACKENDS = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.renderers.render_md"
|
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md"
|
||||||
|
|
||||||
|
|
||||||
# Internationalization
|
# Internationalization
|
||||||
|
|
|
@ -21,7 +21,18 @@ from django.core.validators import MaxValueValidator, MinValueValidator
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
from .utils import *
|
from .utils import *
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .models import IdType
|
from ..common.models import IdType
|
||||||
|
|
||||||
|
|
||||||
def check_digit_10(isbn):
|
def check_digit_10(isbn):
|
||||||
|
|
|
@ -4,6 +4,7 @@ import re
|
||||||
import time
|
import time
|
||||||
from io import BytesIO, StringIO
|
from io import BytesIO, StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Tuple
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
import filetype
|
import filetype
|
||||||
|
@ -11,6 +12,7 @@ import requests
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from lxml import html
|
from lxml import html
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
from requests import Response
|
||||||
from requests.exceptions import RequestException
|
from requests.exceptions import RequestException
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
@ -52,6 +54,53 @@ def get_mock_file(url):
|
||||||
return fn
|
return fn
|
||||||
|
|
||||||
|
|
||||||
|
_local_response_path = (
|
||||||
|
str(Path(__file__).parent.parent.parent.absolute()) + "/test_data/"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MockResponse:
|
||||||
|
def __init__(self, url):
|
||||||
|
self.url = url
|
||||||
|
fn = _local_response_path + get_mock_file(url)
|
||||||
|
try:
|
||||||
|
self.content = Path(fn).read_bytes()
|
||||||
|
self.status_code = 200
|
||||||
|
_logger.debug(f"use local response for {url} from {fn}")
|
||||||
|
except Exception:
|
||||||
|
self.content = b"Error: response file not found"
|
||||||
|
self.status_code = 404
|
||||||
|
_logger.debug(f"local response not found for {url} at {fn}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def text(self):
|
||||||
|
return self.content.decode("utf-8")
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return json.load(StringIO(self.text))
|
||||||
|
|
||||||
|
def html(self):
|
||||||
|
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
|
||||||
|
self.content.decode("utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def headers(self):
|
||||||
|
return {
|
||||||
|
"Content-Type": "image/jpeg" if self.url.endswith("jpg") else "text/html"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
requests.Response.html = MockResponse.html # type:ignore
|
||||||
|
|
||||||
|
|
||||||
|
class DownloaderResponse(Response):
|
||||||
|
def html(self):
|
||||||
|
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
|
||||||
|
self.content.decode("utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DownloadError(Exception):
|
class DownloadError(Exception):
|
||||||
def __init__(self, downloader, msg=None):
|
def __init__(self, downloader, msg=None):
|
||||||
self.url = downloader.url
|
self.url = downloader.url
|
||||||
|
@ -101,7 +150,7 @@ class BasicDownloader:
|
||||||
else:
|
else:
|
||||||
return RESPONSE_INVALID_CONTENT
|
return RESPONSE_INVALID_CONTENT
|
||||||
|
|
||||||
def _download(self, url):
|
def _download(self, url) -> Tuple[DownloaderResponse | MockResponse, int]:
|
||||||
try:
|
try:
|
||||||
if not _mock_mode:
|
if not _mock_mode:
|
||||||
# TODO cache = get/set from redis
|
# TODO cache = get/set from redis
|
||||||
|
@ -125,12 +174,12 @@ class BasicDownloader:
|
||||||
{"response_type": response_type, "url": url, "exception": None}
|
{"response_type": response_type, "url": url, "exception": None}
|
||||||
)
|
)
|
||||||
|
|
||||||
return resp, response_type
|
return resp, response_type # type: ignore
|
||||||
except RequestException as e:
|
except RequestException as e:
|
||||||
self.logs.append(
|
self.logs.append(
|
||||||
{"response_type": RESPONSE_NETWORK_ERROR, "url": url, "exception": e}
|
{"response_type": RESPONSE_NETWORK_ERROR, "url": url, "exception": e}
|
||||||
)
|
)
|
||||||
return None, RESPONSE_NETWORK_ERROR
|
return None, RESPONSE_NETWORK_ERROR # type: ignore
|
||||||
|
|
||||||
def download(self):
|
def download(self):
|
||||||
resp, self.response_type = self._download(self.url)
|
resp, self.response_type = self._download(self.url)
|
||||||
|
@ -209,9 +258,10 @@ class RetryDownloader(BasicDownloader):
|
||||||
|
|
||||||
class ImageDownloaderMixin:
|
class ImageDownloaderMixin:
|
||||||
def __init__(self, url, referer=None):
|
def __init__(self, url, referer=None):
|
||||||
|
self.extention = None
|
||||||
if referer is not None:
|
if referer is not None:
|
||||||
self.headers["Referer"] = referer
|
self.headers["Referer"] = referer # type: ignore
|
||||||
super().__init__(url)
|
super().__init__(url) # type: ignore
|
||||||
|
|
||||||
def validate_response(self, response):
|
def validate_response(self, response):
|
||||||
if response and response.status_code == 200:
|
if response and response.status_code == 200:
|
||||||
|
@ -236,12 +286,12 @@ class ImageDownloaderMixin:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def download_image(cls, image_url, page_url, headers=None):
|
def download_image(cls, image_url, page_url, headers=None):
|
||||||
imgdl = cls(image_url, page_url)
|
imgdl: BasicDownloader = cls(image_url, page_url) # type:ignore
|
||||||
if headers is not None:
|
if headers is not None:
|
||||||
imgdl.headers = headers
|
imgdl.headers = headers
|
||||||
try:
|
try:
|
||||||
image = imgdl.download().content
|
image = imgdl.download().content
|
||||||
image_extention = imgdl.extention
|
image_extention = imgdl.extention # type:ignore
|
||||||
return image, image_extention
|
return image, image_extention
|
||||||
except Exception:
|
except Exception:
|
||||||
return None, None
|
return None, None
|
||||||
|
@ -253,43 +303,3 @@ class BasicImageDownloader(ImageDownloaderMixin, BasicDownloader):
|
||||||
|
|
||||||
class ProxiedImageDownloader(ImageDownloaderMixin, ProxiedDownloader):
|
class ProxiedImageDownloader(ImageDownloaderMixin, ProxiedDownloader):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
_local_response_path = (
|
|
||||||
str(Path(__file__).parent.parent.parent.absolute()) + "/test_data/"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MockResponse:
|
|
||||||
def __init__(self, url):
|
|
||||||
self.url = url
|
|
||||||
fn = _local_response_path + get_mock_file(url)
|
|
||||||
try:
|
|
||||||
self.content = Path(fn).read_bytes()
|
|
||||||
self.status_code = 200
|
|
||||||
_logger.debug(f"use local response for {url} from {fn}")
|
|
||||||
except Exception:
|
|
||||||
self.content = b"Error: response file not found"
|
|
||||||
self.status_code = 404
|
|
||||||
_logger.debug(f"local response not found for {url} at {fn}")
|
|
||||||
|
|
||||||
@property
|
|
||||||
def text(self):
|
|
||||||
return self.content.decode("utf-8")
|
|
||||||
|
|
||||||
def json(self):
|
|
||||||
return json.load(StringIO(self.text))
|
|
||||||
|
|
||||||
def html(self):
|
|
||||||
return html.fromstring( # may throw exception unexpectedly due to OS bug, see https://github.com/neodb-social/neodb/issues/5
|
|
||||||
self.content.decode("utf-8")
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def headers(self):
|
|
||||||
return {
|
|
||||||
"Content-Type": "image/jpeg" if self.url.endswith("jpg") else "text/html"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
requests.Response.html = MockResponse.html
|
|
||||||
|
|
|
@ -80,6 +80,9 @@ class JSONFieldDescriptor(object):
|
||||||
setattr(instance, self.field.json_field_name, json_value)
|
setattr(instance, self.field.json_field_name, json_value)
|
||||||
|
|
||||||
|
|
||||||
|
fields.CharField
|
||||||
|
|
||||||
|
|
||||||
class JSONFieldMixin(object):
|
class JSONFieldMixin(object):
|
||||||
"""
|
"""
|
||||||
Override django.db.model.fields.Field.contribute_to_class
|
Override django.db.model.fields.Field.contribute_to_class
|
||||||
|
@ -90,11 +93,11 @@ class JSONFieldMixin(object):
|
||||||
self.json_field_name = kwargs.pop("json_field_name", "metadata")
|
self.json_field_name = kwargs.pop("json_field_name", "metadata")
|
||||||
super(JSONFieldMixin, self).__init__(*args, **kwargs)
|
super(JSONFieldMixin, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
def contribute_to_class(self, cls, name, private_only=False):
|
def contribute_to_class(self: "fields.Field", cls, name, private_only=False): # type: ignore
|
||||||
self.set_attributes_from_name(name)
|
self.set_attributes_from_name(name)
|
||||||
self.model = cls
|
self.model = cls
|
||||||
self.concrete = False
|
self.concrete = False
|
||||||
self.column = self.json_field_name
|
self.column = self.json_field_name # type: ignore
|
||||||
cls._meta.add_field(self, private=True)
|
cls._meta.add_field(self, private=True)
|
||||||
|
|
||||||
if not getattr(cls, self.attname, None):
|
if not getattr(cls, self.attname, None):
|
||||||
|
@ -131,11 +134,11 @@ class JSONFieldMixin(object):
|
||||||
return transform
|
return transform
|
||||||
|
|
||||||
json_field = self.model._meta.get_field(self.json_field_name)
|
json_field = self.model._meta.get_field(self.json_field_name)
|
||||||
transform = json_field.get_transform(self.name)
|
transform = json_field.get_transform(self.name) # type: ignore
|
||||||
if transform is None:
|
if transform is None:
|
||||||
raise FieldError(
|
raise FieldError(
|
||||||
"JSONField '%s' has no support for key '%s' %s lookup"
|
"JSONField '%s' has no support for key '%s' %s lookup"
|
||||||
% (self.json_field_name, self.name, name)
|
% (self.json_field_name, self.name, name) # type: ignore
|
||||||
)
|
)
|
||||||
|
|
||||||
return TransformFactoryWrapper(json_field, transform, name)
|
return TransformFactoryWrapper(json_field, transform, name)
|
||||||
|
@ -170,10 +173,17 @@ class DateField(JSONFieldMixin, fields.DateField):
|
||||||
|
|
||||||
|
|
||||||
class DateTimeField(JSONFieldMixin, fields.DateTimeField):
|
class DateTimeField(JSONFieldMixin, fields.DateTimeField):
|
||||||
def to_json(self, value):
|
def to_json(self, value: datetime | date | str):
|
||||||
if value:
|
if value:
|
||||||
if not isinstance(value, (datetime, date)):
|
if not isinstance(value, (datetime, date)):
|
||||||
value = dateparse.parse_date(value)
|
v = dateparse.parse_date(value)
|
||||||
|
if v is None:
|
||||||
|
raise ValueError(
|
||||||
|
"DateTimeField: '{value}' has invalid datatime format"
|
||||||
|
)
|
||||||
|
value = v
|
||||||
|
if isinstance(value, date):
|
||||||
|
value = datetime.combine(value, datetime.time.min())
|
||||||
if not timezone.is_aware(value):
|
if not timezone.is_aware(value):
|
||||||
value = timezone.make_aware(value)
|
value = timezone.make_aware(value)
|
||||||
return value.isoformat()
|
return value.isoformat()
|
||||||
|
|
|
@ -17,6 +17,6 @@ class SoftDeleteMixin:
|
||||||
if soft:
|
if soft:
|
||||||
self.clear()
|
self.clear()
|
||||||
self.is_deleted = True
|
self.is_deleted = True
|
||||||
self.save(using=using)
|
self.save(using=using) # type: ignore
|
||||||
else:
|
else:
|
||||||
return super().delete(using=using, *args, **kwargs)
|
return super().delete(using=using, *args, **kwargs) # type: ignore
|
||||||
|
|
|
@ -2,7 +2,7 @@ import logging
|
||||||
import re
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
from typing import cast
|
from typing import TYPE_CHECKING, cast
|
||||||
|
|
||||||
from auditlog.context import disable_auditlog
|
from auditlog.context import disable_auditlog
|
||||||
from auditlog.models import AuditlogHistoryField, LogEntry
|
from auditlog.models import AuditlogHistoryField, LogEntry
|
||||||
|
@ -22,6 +22,9 @@ from users.models import User
|
||||||
from .mixins import SoftDeleteMixin
|
from .mixins import SoftDeleteMixin
|
||||||
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
|
from .utils import DEFAULT_ITEM_COVER, item_cover_path, resource_cover_path
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from django.utils.functional import _StrOrPromise
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@ -247,7 +250,7 @@ class Item(SoftDeleteMixin, PolymorphicModel):
|
||||||
type = None # subclass must specify this
|
type = None # subclass must specify this
|
||||||
parent_class = None # subclass may specify this to allow create child item
|
parent_class = None # subclass may specify this to allow create child item
|
||||||
category: ItemCategory | None = None # subclass must specify this
|
category: ItemCategory | None = None # subclass must specify this
|
||||||
demonstrative: str | None = None # subclass must specify this
|
demonstrative: "_StrOrPromise | None" = None # subclass must specify this
|
||||||
uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
|
uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
|
||||||
title = models.CharField(_("标题"), max_length=1000, default="")
|
title = models.CharField(_("标题"), max_length=1000, default="")
|
||||||
brief = models.TextField(_("简介"), blank=True, default="")
|
brief = models.TextField(_("简介"), blank=True, default="")
|
||||||
|
|
|
@ -37,7 +37,7 @@ def _EditForm(item_model):
|
||||||
}
|
}
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
data = super().clean()
|
data = super().clean() or {}
|
||||||
t, v = self.Meta.model.lookup_id_cleanup(
|
t, v = self.Meta.model.lookup_id_cleanup(
|
||||||
data.get("primary_lookup_id_type"), data.get("primary_lookup_id_value")
|
data.get("primary_lookup_id_type"), data.get("primary_lookup_id_value")
|
||||||
)
|
)
|
||||||
|
|
|
@ -3,7 +3,18 @@ from datetime import date
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class GameInSchema(ItemInSchema):
|
class GameInSchema(ItemInSchema):
|
||||||
|
|
|
@ -8,9 +8,13 @@ from .book.models import Edition, EditionInSchema, EditionSchema, Series, Work
|
||||||
from .collection.models import Collection as CatalogCollection
|
from .collection.models import Collection as CatalogCollection
|
||||||
from .common.models import (
|
from .common.models import (
|
||||||
ExternalResource,
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
Item,
|
Item,
|
||||||
ItemCategory,
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
ItemSchema,
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
SiteName,
|
||||||
item_categories,
|
item_categories,
|
||||||
item_content_types,
|
item_content_types,
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,7 +1,18 @@
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class MovieInSchema(ItemInSchema):
|
class MovieInSchema(ItemInSchema):
|
||||||
|
|
|
@ -3,7 +3,18 @@ from datetime import date
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AlbumInSchema(ItemInSchema):
|
class AlbumInSchema(ItemInSchema):
|
||||||
|
|
|
@ -1,7 +1,18 @@
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PodcastInSchema(ItemInSchema):
|
class PodcastInSchema(ItemInSchema):
|
||||||
|
|
|
@ -65,22 +65,26 @@ class Goodreads:
|
||||||
r = requests.get(search_url)
|
r = requests.get(search_url)
|
||||||
if r.url.startswith("https://www.goodreads.com/book/show/"):
|
if r.url.startswith("https://www.goodreads.com/book/show/"):
|
||||||
# Goodreads will 302 if only one result matches ISBN
|
# Goodreads will 302 if only one result matches ISBN
|
||||||
res = SiteManager.get_site_by_url(r.url).get_resource_ready()
|
site = SiteManager.get_site_by_url(r.url)
|
||||||
subtitle = f"{res.metadata['pub_year']} {', '.join(res.metadata['author'])} {', '.join(res.metadata['translator'] if res.metadata['translator'] else [])}"
|
if site:
|
||||||
results.append(
|
res = site.get_resource_ready()
|
||||||
SearchResultItem(
|
if res:
|
||||||
ItemCategory.Book,
|
subtitle = f"{res.metadata['pub_year']} {', '.join(res.metadata['author'])} {', '.join(res.metadata['translator'] if res.metadata['translator'] else [])}"
|
||||||
SiteName.Goodreads,
|
results.append(
|
||||||
res.url,
|
SearchResultItem(
|
||||||
res.metadata["title"],
|
ItemCategory.Book,
|
||||||
subtitle,
|
SiteName.Goodreads,
|
||||||
res.metadata["brief"],
|
res.url,
|
||||||
res.metadata["cover_image_url"],
|
res.metadata["title"],
|
||||||
)
|
subtitle,
|
||||||
)
|
res.metadata["brief"],
|
||||||
|
res.metadata["cover_image_url"],
|
||||||
|
)
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
h = html.fromstring(r.content.decode("utf-8"))
|
h = html.fromstring(r.content.decode("utf-8"))
|
||||||
for c in h.xpath('//tr[@itemtype="http://schema.org/Book"]'):
|
books = h.xpath('//tr[@itemtype="http://schema.org/Book"]')
|
||||||
|
for c in books: # type:ignore
|
||||||
el_cover = c.xpath('.//img[@class="bookCover"]/@src')
|
el_cover = c.xpath('.//img[@class="bookCover"]/@src')
|
||||||
cover = el_cover[0] if el_cover else None
|
cover = el_cover[0] if el_cover else None
|
||||||
el_title = c.xpath('.//a[@class="bookTitle"]//text()')
|
el_title = c.xpath('.//a[@class="bookTitle"]//text()')
|
||||||
|
@ -228,7 +232,8 @@ class Bandcamp:
|
||||||
search_url = f"https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}"
|
search_url = f"https://bandcamp.com/search?from=results&item_type=a&page={page}&q={quote_plus(q)}"
|
||||||
r = requests.get(search_url)
|
r = requests.get(search_url)
|
||||||
h = html.fromstring(r.content.decode("utf-8"))
|
h = html.fromstring(r.content.decode("utf-8"))
|
||||||
for c in h.xpath('//li[@class="searchresult data-search"]'):
|
albums = h.xpath('//li[@class="searchresult data-search"]')
|
||||||
|
for c in albums: # type:ignore
|
||||||
el_cover = c.xpath('.//div[@class="art"]/img/@src')
|
el_cover = c.xpath('.//div[@class="art"]/img/@src')
|
||||||
cover = el_cover[0] if el_cover else None
|
cover = el_cover[0] if el_cover else None
|
||||||
el_title = c.xpath('.//div[@class="heading"]//text()')
|
el_title = c.xpath('.//div[@class="heading"]//text()')
|
||||||
|
|
|
@ -24,12 +24,24 @@ tv specials are are shown as movies
|
||||||
For now, we follow Douban convention, but keep an eye on it in case it breaks its own rules...
|
For now, we follow Douban convention, but keep an eye on it in case it breaks its own rules...
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
import re
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
|
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from catalog.common.models import *
|
from catalog.common import (
|
||||||
|
BaseSchema,
|
||||||
|
ExternalResource,
|
||||||
|
IdType,
|
||||||
|
Item,
|
||||||
|
ItemCategory,
|
||||||
|
ItemInSchema,
|
||||||
|
ItemSchema,
|
||||||
|
ItemType,
|
||||||
|
PrimaryLookupIdDescriptor,
|
||||||
|
jsondata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TVShowInSchema(ItemInSchema):
|
class TVShowInSchema(ItemInSchema):
|
||||||
|
|
|
@ -8,7 +8,7 @@ register = template.Library()
|
||||||
class OAuthTokenNode(template.Node):
|
class OAuthTokenNode(template.Node):
|
||||||
def render(self, context):
|
def render(self, context):
|
||||||
request = context.get("request")
|
request = context.get("request")
|
||||||
oauth_token = request.user.mastodon_token
|
oauth_token = request.user.mastodon_token if request else ""
|
||||||
return format_html(oauth_token)
|
return format_html(oauth_token)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ from django.conf import settings
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
from openpyxl import Workbook
|
from openpyxl import Workbook
|
||||||
|
|
||||||
from catalog.common.models import IdType
|
from catalog.models import *
|
||||||
from common.utils import GenerateDateUUIDMediaFilePath
|
from common.utils import GenerateDateUUIDMediaFilePath
|
||||||
from journal.models import *
|
from journal.models import *
|
||||||
|
|
||||||
|
|
|
@ -108,12 +108,15 @@ class GoodreadsImporter:
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_book(cls, url, user):
|
def get_book(cls, url, user):
|
||||||
site = SiteManager.get_site_by_url(url)
|
site = SiteManager.get_site_by_url(url)
|
||||||
book = site.get_item()
|
if site:
|
||||||
if not book:
|
book = site.get_item()
|
||||||
book = site.get_resource_ready().item
|
if not book:
|
||||||
book.last_editor = user
|
resource = site.get_resource_ready()
|
||||||
book.save()
|
if resource and resource.item:
|
||||||
return book
|
book = resource.item
|
||||||
|
book.last_editor = user
|
||||||
|
book.save()
|
||||||
|
return book
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def parse_shelf(cls, url, user):
|
def parse_shelf(cls, url, user):
|
||||||
|
@ -129,12 +132,13 @@ class GoodreadsImporter:
|
||||||
if not title_elem:
|
if not title_elem:
|
||||||
print(f"Shelf parsing error {url_shelf}")
|
print(f"Shelf parsing error {url_shelf}")
|
||||||
break
|
break
|
||||||
title = title_elem[0].strip()
|
title = title_elem[0].strip() # type:ignore
|
||||||
print("Shelf title: " + title)
|
print("Shelf title: " + title)
|
||||||
except Exception:
|
except Exception:
|
||||||
print(f"Shelf loading/parsing error {url_shelf}")
|
print(f"Shelf loading/parsing error {url_shelf}")
|
||||||
break
|
break
|
||||||
for cell in content.xpath("//tbody[@id='booksBody']/tr"):
|
cells = content.xpath("//tbody[@id='booksBody']/tr")
|
||||||
|
for cell in cells: # type:ignore
|
||||||
url_book = (
|
url_book = (
|
||||||
"https://www.goodreads.com"
|
"https://www.goodreads.com"
|
||||||
+ cell.xpath(".//td[@class='field title']//a/@href")[0].strip()
|
+ cell.xpath(".//td[@class='field title']//a/@href")[0].strip()
|
||||||
|
@ -167,10 +171,12 @@ class GoodreadsImporter:
|
||||||
c2 = BasicDownloader(url_review).download().html()
|
c2 = BasicDownloader(url_review).download().html()
|
||||||
review_elem = c2.xpath("//div[@itemprop='reviewBody']/text()")
|
review_elem = c2.xpath("//div[@itemprop='reviewBody']/text()")
|
||||||
review = (
|
review = (
|
||||||
"\n".join(p.strip() for p in review_elem) if review_elem else ""
|
"\n".join(p.strip() for p in review_elem) # type:ignore
|
||||||
|
if review_elem
|
||||||
|
else ""
|
||||||
)
|
)
|
||||||
date_elem = c2.xpath("//div[@class='readingTimeline__text']/text()")
|
date_elem = c2.xpath("//div[@class='readingTimeline__text']/text()")
|
||||||
for d in date_elem:
|
for d in date_elem: # type:ignore
|
||||||
date_matched = re.search(r"(\w+)\s+(\d+),\s+(\d+)", d)
|
date_matched = re.search(r"(\w+)\s+(\d+),\s+(\d+)", d)
|
||||||
if date_matched:
|
if date_matched:
|
||||||
last_updated = make_aware(
|
last_updated = make_aware(
|
||||||
|
@ -201,7 +207,7 @@ class GoodreadsImporter:
|
||||||
pass # likely just download error
|
pass # likely just download error
|
||||||
next_elem = content.xpath("//a[@class='next_page']/@href")
|
next_elem = content.xpath("//a[@class='next_page']/@href")
|
||||||
url_shelf = (
|
url_shelf = (
|
||||||
("https://www.goodreads.com" + next_elem[0].strip())
|
f"https://www.goodreads.com{next_elem[0].strip()}"
|
||||||
if next_elem
|
if next_elem
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
|
@ -13,6 +13,7 @@ from .common import (
|
||||||
from .like import Like
|
from .like import Like
|
||||||
from .mark import Mark
|
from .mark import Mark
|
||||||
from .rating import Rating
|
from .rating import Rating
|
||||||
|
from .renderers import render_md
|
||||||
from .review import Review
|
from .review import Review
|
||||||
from .shelf import (
|
from .shelf import (
|
||||||
Shelf,
|
Shelf,
|
||||||
|
|
|
@ -1,3 +1,9 @@
|
||||||
|
from typing import TYPE_CHECKING, Type
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from .common import Piece
|
||||||
|
|
||||||
|
|
||||||
class UserOwnedObjectMixin:
|
class UserOwnedObjectMixin:
|
||||||
"""
|
"""
|
||||||
UserOwnedObjectMixin
|
UserOwnedObjectMixin
|
||||||
|
@ -7,7 +13,7 @@ class UserOwnedObjectMixin:
|
||||||
visibility = models.PositiveSmallIntegerField(default=0)
|
visibility = models.PositiveSmallIntegerField(default=0)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def is_visible_to(self, viewer):
|
def is_visible_to(self: "Piece", viewer): # type: ignore
|
||||||
owner = self.owner
|
owner = self.owner
|
||||||
if owner == viewer:
|
if owner == viewer:
|
||||||
return True
|
return True
|
||||||
|
@ -24,13 +30,13 @@ class UserOwnedObjectMixin:
|
||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def is_editable_by(self, viewer):
|
def is_editable_by(self: "Piece", viewer): # type: ignore
|
||||||
return viewer.is_authenticated and (
|
return viewer.is_authenticated and (
|
||||||
viewer.is_staff or viewer.is_superuser or viewer == self.owner
|
viewer.is_staff or viewer.is_superuser or viewer == self.owner
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_available(cls, entity, request_user, following_only=False):
|
def get_available(cls: "Type[Piece]", entity, request_user, following_only=False): # type: ignore
|
||||||
# e.g. SongMark.get_available(song, request.user)
|
# e.g. SongMark.get_available(song, request.user)
|
||||||
query_kwargs = {entity.__class__.__name__.lower(): entity}
|
query_kwargs = {entity.__class__.__name__.lower(): entity}
|
||||||
all_entities = cls.objects.filter(**query_kwargs).order_by(
|
all_entities = cls.objects.filter(**query_kwargs).order_by(
|
||||||
|
|
|
@ -4,8 +4,6 @@ from typing import cast
|
||||||
import mistune
|
import mistune
|
||||||
from django.utils.html import escape
|
from django.utils.html import escape
|
||||||
|
|
||||||
MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.renderers.render_md"
|
|
||||||
|
|
||||||
_mistune_plugins = [
|
_mistune_plugins = [
|
||||||
"url",
|
"url",
|
||||||
"strikethrough",
|
"strikethrough",
|
||||||
|
|
|
@ -3,13 +3,19 @@ from django.shortcuts import get_object_or_404
|
||||||
from django.urls import reverse_lazy
|
from django.urls import reverse_lazy
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.utils.decorators import method_decorator
|
from django.utils.decorators import method_decorator
|
||||||
from django.views.generic import *
|
from django.views.generic import (
|
||||||
|
CreateView,
|
||||||
|
DeleteView,
|
||||||
|
DetailView,
|
||||||
|
ListView,
|
||||||
|
UpdateView,
|
||||||
|
)
|
||||||
from django.views.generic.edit import ModelFormMixin
|
from django.views.generic.edit import ModelFormMixin
|
||||||
|
|
||||||
from .models import Announcement
|
from .models import Announcement
|
||||||
|
|
||||||
# https://docs.djangoproject.com/en/3.1/topics/class-based-views/intro/
|
# https://docs.djangoproject.com/en/3.1/topics/class-based-views/intro/
|
||||||
decorators = [login_required, user_passes_test(lambda u: u.is_superuser)]
|
decorators = [login_required, user_passes_test(lambda u: u.is_superuser)] # type:ignore
|
||||||
|
|
||||||
|
|
||||||
class AnnouncementDetailView(DetailView, ModelFormMixin):
|
class AnnouncementDetailView(DetailView, ModelFormMixin):
|
||||||
|
|
|
@ -1,74 +1 @@
|
||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.core.exceptions import ObjectDoesNotExist
|
|
||||||
from django.utils.translation import gettext_lazy as _
|
|
||||||
from requests.exceptions import Timeout
|
|
||||||
|
|
||||||
from .api import create_app
|
|
||||||
from .models import *
|
|
||||||
|
|
||||||
|
|
||||||
# Register your models here.
|
|
||||||
@admin.register(MastodonApplication)
|
|
||||||
class MastodonApplicationModelAdmin(admin.ModelAdmin):
|
|
||||||
def add_view(self, request, form_url="", extra_context=None):
|
|
||||||
"""
|
|
||||||
Dirty code here, use POST['domain_name'] to pass error message to user.
|
|
||||||
"""
|
|
||||||
if request.method == "POST":
|
|
||||||
if not request.POST.get("client_id") and not request.POST.get(
|
|
||||||
"client_secret"
|
|
||||||
):
|
|
||||||
# make the post data mutable
|
|
||||||
request.POST = request.POST.copy()
|
|
||||||
# (is_proxy xor proxy_to) or (proxy_to!=null and is_proxy=false)
|
|
||||||
if (
|
|
||||||
(
|
|
||||||
bool(request.POST.get("is_proxy"))
|
|
||||||
or bool(request.POST.get("proxy_to"))
|
|
||||||
)
|
|
||||||
and not (
|
|
||||||
bool(request.POST.get("is_proxy"))
|
|
||||||
and bool(request.POST.get("proxy_to"))
|
|
||||||
)
|
|
||||||
or (
|
|
||||||
not bool(request.POST.get("is_proxy"))
|
|
||||||
and bool(request.POST.get("proxy_to"))
|
|
||||||
)
|
|
||||||
):
|
|
||||||
request.POST["domain_name"] = _("请同时填写is_proxy和proxy_to。")
|
|
||||||
else:
|
|
||||||
if request.POST.get("is_proxy"):
|
|
||||||
try:
|
|
||||||
origin = MastodonApplication.objects.get(
|
|
||||||
domain_name=request.POST["proxy_to"]
|
|
||||||
)
|
|
||||||
# set proxy credentials to those of its original site
|
|
||||||
request.POST["app_id"] = origin.app_id
|
|
||||||
request.POST["client_id"] = origin.client_id
|
|
||||||
request.POST["client_secret"] = origin.client_secret
|
|
||||||
request.POST["vapid_key"] = origin.vapid_key
|
|
||||||
except ObjectDoesNotExist:
|
|
||||||
request.POST["domain_name"] = _("proxy_to所指域名不存在,请先添加原站点。")
|
|
||||||
else:
|
|
||||||
# create mastodon app
|
|
||||||
try:
|
|
||||||
response = create_app(request.POST.get("domain_name"))
|
|
||||||
except (Timeout, ConnectionError):
|
|
||||||
request.POST["domain_name"] = _("联邦宇宙请求超时。")
|
|
||||||
except Exception as e:
|
|
||||||
request.POST["domain_name"] = str(e)
|
|
||||||
else:
|
|
||||||
# fill the form with returned data
|
|
||||||
data = response.json()
|
|
||||||
if response.status_code != 200:
|
|
||||||
request.POST["domain_name"] = str(data)
|
|
||||||
else:
|
|
||||||
request.POST["app_id"] = data["id"]
|
|
||||||
request.POST["client_id"] = data["client_id"]
|
|
||||||
request.POST["client_secret"] = data["client_secret"]
|
|
||||||
request.POST["vapid_key"] = data["vapid_key"]
|
|
||||||
|
|
||||||
return super().add_view(request, form_url=form_url, extra_context=extra_context)
|
|
||||||
|
|
||||||
|
|
||||||
admin.site.register(CrossSiteUserInfo)
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
[tool.pyright]
|
[tool.pyright]
|
||||||
exclude = [ "media", ".venv", ".git" ]
|
exclude = [ "media", ".venv", ".git", "playground", "**/tests.py", "neodb", "**/migrations", "**/commands", "**/importers", "**/sites" ]
|
||||||
|
|
||||||
[tool.djlint]
|
[tool.djlint]
|
||||||
ignore="T002,T003,H006,H019,H020,H021,H023,H030,H031"
|
ignore="T002,T003,H006,H019,H020,H021,H023,H030,H031"
|
||||||
|
@ -17,3 +17,7 @@ plugins = ["mypy_django_plugin.main"]
|
||||||
|
|
||||||
[tool.django-stubs]
|
[tool.django-stubs]
|
||||||
django_settings_module = "boofilsic.settings"
|
django_settings_module = "boofilsic.settings"
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
ignore = ['E501']
|
||||||
|
exclude = [ "media", ".venv", ".git", "playground", "**/tests.py", "neodb", "**/migrations", "**/commands", "**/importers", "**/sites", "legacy" ]
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
black~=22.12.0
|
black~=22.12.0
|
||||||
coverage
|
coverage
|
||||||
django-debug-toolbar
|
django-debug-toolbar
|
||||||
|
django-stubs
|
||||||
djlint~=1.32.1
|
djlint~=1.32.1
|
||||||
isort~=5.12.0
|
isort~=5.12.0
|
||||||
pre-commit~=3.3.3
|
pre-commit
|
||||||
types-dateparser
|
pyright
|
||||||
types-tqdm
|
|
||||||
|
|
|
@ -81,10 +81,6 @@ class ActivityManager:
|
||||||
return ActivityManager(user)
|
return ActivityManager(user)
|
||||||
|
|
||||||
|
|
||||||
User.activity_manager = cached_property(ActivityManager.get_manager_for_user) # type: ignore
|
|
||||||
User.activity_manager.__set_name__(User, "activity_manager") # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
class DataSignalManager:
|
class DataSignalManager:
|
||||||
processors = {}
|
processors = {}
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@ class Command(BaseCommand):
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
count = 0
|
count = 0
|
||||||
for user in tqdm(User.objects.all()):
|
for user in tqdm(User.objects.all()):
|
||||||
user.following = user.merge_following_ids()
|
user.following = user.merged_following_ids()
|
||||||
if user.following:
|
if user.following:
|
||||||
count += 1
|
count += 1
|
||||||
user.save(update_fields=["following"])
|
user.save(update_fields=["following"])
|
||||||
|
|
|
@ -4,6 +4,7 @@ from functools import cached_property
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from django.contrib.auth.models import AbstractUser
|
from django.contrib.auth.models import AbstractUser
|
||||||
|
from django.contrib.auth.validators import UnicodeUsernameValidator
|
||||||
from django.core import validators
|
from django.core import validators
|
||||||
from django.core.exceptions import ValidationError
|
from django.core.exceptions import ValidationError
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
@ -33,7 +34,7 @@ _RESERVED_USERNAMES = [
|
||||||
|
|
||||||
|
|
||||||
@deconstructible
|
@deconstructible
|
||||||
class UsernameValidator(validators.RegexValidator):
|
class UsernameValidator(UnicodeUsernameValidator):
|
||||||
regex = r"^[a-zA-Z0-9_]{2,30}$"
|
regex = r"^[a-zA-Z0-9_]{2,30}$"
|
||||||
message = _(
|
message = _(
|
||||||
"Enter a valid username. This value may contain only unaccented lowercase a-z and uppercase A-Z letters, numbers, and _ characters."
|
"Enter a valid username. This value may contain only unaccented lowercase a-z and uppercase A-Z letters, numbers, and _ characters."
|
||||||
|
@ -550,6 +551,12 @@ class User(AbstractUser):
|
||||||
|
|
||||||
return ShelfManager.get_manager_for_user(self)
|
return ShelfManager.get_manager_for_user(self)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def activity_manager(self):
|
||||||
|
from social.models import ActivityManager
|
||||||
|
|
||||||
|
return ActivityManager.get_manager_for_user(self)
|
||||||
|
|
||||||
|
|
||||||
class Follow(models.Model):
|
class Follow(models.Model):
|
||||||
owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="+")
|
owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="+")
|
||||||
|
|
Loading…
Add table
Reference in a new issue