refactor note footer strip and add test
This commit is contained in:
parent
43a7bf5174
commit
f6229c7b31
9 changed files with 136 additions and 31 deletions
|
@ -14,6 +14,7 @@ try:
|
||||||
except Exception:
|
except Exception:
|
||||||
NEODB_VERSION = __version__ + "-unknown"
|
NEODB_VERSION = __version__ + "-unknown"
|
||||||
|
|
||||||
|
TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
|
||||||
|
|
||||||
# Parse configuration from:
|
# Parse configuration from:
|
||||||
# - environment variables
|
# - environment variables
|
||||||
|
|
|
@ -519,6 +519,14 @@ class Item(PolymorphicModel):
|
||||||
item = None
|
item = None
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_by_remote_url(cls, url: str) -> "Self | None":
|
||||||
|
url_ = url.replace("/~neodb~/", "/")
|
||||||
|
if url_.startswith(settings.SITE_INFO["site_url"]):
|
||||||
|
return cls.get_by_url(url_, True)
|
||||||
|
er = ExternalResource.objects.filter(url=url_).first()
|
||||||
|
return er.item if er else None
|
||||||
|
|
||||||
# def get_lookup_id(self, id_type: str) -> str:
|
# def get_lookup_id(self, id_type: str) -> str:
|
||||||
# prefix = id_type.strip().lower() + ':'
|
# prefix = id_type.strip().lower() + ':'
|
||||||
# return next((x[len(prefix):] for x in self.lookup_ids if x.startswith(prefix)), None)
|
# return next((x[len(prefix):] for x in self.lookup_ids if x.startswith(prefix)), None)
|
||||||
|
|
|
@ -12,6 +12,8 @@ class CommonConfig(AppConfig):
|
||||||
def setup(self, **kwargs):
|
def setup(self, **kwargs):
|
||||||
from .setup import Setup
|
from .setup import Setup
|
||||||
|
|
||||||
|
if kwargs.get("using", "") == "default":
|
||||||
|
# only run setup on the default database, not on takahe
|
||||||
Setup().run()
|
Setup().run()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -121,7 +121,15 @@ class Setup:
|
||||||
)
|
)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
if settings.TESTING:
|
||||||
|
# Only do necessary initialization when testing
|
||||||
|
logger.info("Running minimal post-migration setup for testing...")
|
||||||
|
self.sync_site_config()
|
||||||
|
Indexer.init()
|
||||||
|
return
|
||||||
|
|
||||||
logger.info("Running post-migration setup...")
|
logger.info("Running post-migration setup...")
|
||||||
|
|
||||||
# Update site name if changed
|
# Update site name if changed
|
||||||
self.sync_site_config()
|
self.sync_site_config()
|
||||||
|
|
||||||
|
|
|
@ -252,19 +252,23 @@ class Piece(PolymorphicModel, UserOwnedObjectMixin):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def params_from_ap_object(cls, post, obj, piece):
|
def params_from_ap_object(
|
||||||
|
cls, post: "Post", obj: dict[str, Any], piece: Self | None
|
||||||
|
) -> dict[str, Any]:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def to_post_params(self):
|
def to_post_params(self) -> dict[str, Any]:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def to_mastodon_params(self):
|
def to_mastodon_params(self) -> dict[str, Any]:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def update_by_ap_object(cls, owner: APIdentity, item: Item, obj, post: "Post"):
|
def update_by_ap_object(
|
||||||
|
cls, owner: APIdentity, item: Item, obj, post: "Post"
|
||||||
|
) -> Self | None:
|
||||||
"""
|
"""
|
||||||
Create or update a content piece with related AP message
|
Create or update a content piece with related AP message
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -16,12 +16,12 @@ from .renderers import render_text
|
||||||
from .shelf import ShelfMember
|
from .shelf import ShelfMember
|
||||||
|
|
||||||
_progress = re.compile(
|
_progress = re.compile(
|
||||||
r"(.*\s)?(?P<prefix>(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P<value>(\d[\d\:\.\-]*\d|\d))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
|
r"(.*\s)?(?P<prefix>(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P<value>([\d\:\.\-]+))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|
||||||
_progress2 = re.compile(
|
_progress2 = re.compile(
|
||||||
r"(.*\s)?(?P<value>(\d[\d\:\.\-]*\d|\d))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
|
r"(.*\s)?(?P<value>([\d\:\.\-]+))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -106,7 +106,7 @@ class Note(Content):
|
||||||
}
|
}
|
||||||
if self.progress_value:
|
if self.progress_value:
|
||||||
d["progress"] = {
|
d["progress"] = {
|
||||||
"type": self.progress_type,
|
"type": self.progress_type or "",
|
||||||
"value": self.progress_value,
|
"value": self.progress_value,
|
||||||
}
|
}
|
||||||
return d
|
return d
|
||||||
|
@ -114,33 +114,32 @@ class Note(Content):
|
||||||
@override
|
@override
|
||||||
@classmethod
|
@classmethod
|
||||||
def params_from_ap_object(cls, post, obj, piece):
|
def params_from_ap_object(cls, post, obj, piece):
|
||||||
content = obj.get("content", "").strip()
|
|
||||||
footer = []
|
|
||||||
if post.local:
|
|
||||||
# strip footer from local post if detected
|
|
||||||
lines = content.splitlines()
|
|
||||||
if len(lines) > 2 and lines[-2].strip() in _separaters:
|
|
||||||
content = "\n".join(lines[:-2])
|
|
||||||
footer = lines[-2:]
|
|
||||||
params = {
|
params = {
|
||||||
"title": obj.get("title", post.summary),
|
"title": obj.get("title", post.summary),
|
||||||
"content": content,
|
"content": obj.get("content", "").strip(),
|
||||||
"sensitive": obj.get("sensitive", post.sensitive),
|
"sensitive": obj.get("sensitive", post.sensitive),
|
||||||
"attachments": [],
|
"attachments": [],
|
||||||
}
|
}
|
||||||
progress = obj.get("progress", {})
|
if post.local:
|
||||||
if progress.get("type"):
|
# for local post, strip footer and detect progress from content
|
||||||
params["progress_type"] = progress.get("type")
|
# if not detected, keep default/original value by not including it in return val
|
||||||
if progress.get("value"):
|
params["content"], progress_type, progress_value = cls.strip_footer(
|
||||||
params["progress_value"] = progress.get("value")
|
params["content"]
|
||||||
if post.local and len(footer) == 2:
|
)
|
||||||
progress_type, progress_value = cls.extract_progress(footer[1])
|
if progress_value is not None:
|
||||||
if progress_value:
|
|
||||||
params["progress_type"] = progress_type
|
params["progress_type"] = progress_type
|
||||||
params["progress_value"] = progress_value
|
params["progress_value"] = progress_value
|
||||||
elif not footer[1].startswith("https://"):
|
else:
|
||||||
# add footer back if unable to regconize correct patterns
|
# for remote post, progress is always in "progress" field
|
||||||
params["content"] += "\n" + "\n".join(footer)
|
progress = obj.get("progress", {})
|
||||||
|
params["progress_value"] = progress.get("value", None)
|
||||||
|
params["progress_type"] = None
|
||||||
|
if params["progress_value"]:
|
||||||
|
t = progress.get("type", None)
|
||||||
|
try:
|
||||||
|
params["progress_type"] = Note.ProgressType(t)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
if post:
|
if post:
|
||||||
for atta in post.attachments.all():
|
for atta in post.attachments.all():
|
||||||
params["attachments"].append(
|
params["attachments"].append(
|
||||||
|
@ -205,11 +204,24 @@ class Note(Content):
|
||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def extract_progress(cls, content):
|
def strip_footer(cls, content: str) -> tuple[str, str | None, str | None]:
|
||||||
|
"""strip footer if 2nd last line is "-" or similar characters"""
|
||||||
|
lines = content.splitlines()
|
||||||
|
if len(lines) < 3 or lines[-2].strip() not in _separaters:
|
||||||
|
return content, None, None
|
||||||
|
progress_type, progress_value = cls.extract_progress(lines[-1])
|
||||||
|
# if progress_value is None and not lines[-2].startswith("https://"):
|
||||||
|
# return content, None, None
|
||||||
|
return "\n".join(lines[:-2]), progress_type, progress_value
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def extract_progress(cls, content) -> tuple[str | None, str | None]:
|
||||||
m = _progress.match(content)
|
m = _progress.match(content)
|
||||||
if not m:
|
if not m:
|
||||||
m = _progress2.match(content)
|
m = _progress2.match(content)
|
||||||
if m and m["value"]:
|
if m and m["value"]:
|
||||||
|
if m["value"] == "-":
|
||||||
|
return None, ""
|
||||||
m = m.groupdict()
|
m = m.groupdict()
|
||||||
typ_ = "percentage" if m["postfix"] == "%" else m.get("prefix", "")
|
typ_ = "percentage" if m["postfix"] == "%" else m.get("prefix", "")
|
||||||
match typ_:
|
match typ_:
|
||||||
|
|
|
@ -236,3 +236,60 @@ class DebrisTest(TestCase):
|
||||||
update_journal_for_merged_item(self.book3.uuid, delete_duplicated=True)
|
update_journal_for_merged_item(self.book3.uuid, delete_duplicated=True)
|
||||||
cnt = Debris.objects.all().count()
|
cnt = Debris.objects.all().count()
|
||||||
self.assertEqual(cnt, 4) # Rating, Shelf, 2x TagMember
|
self.assertEqual(cnt, 4) # Rating, Shelf, 2x TagMember
|
||||||
|
|
||||||
|
|
||||||
|
class NoteTest(TestCase):
|
||||||
|
databases = "__all__"
|
||||||
|
|
||||||
|
# def setUp(self):
|
||||||
|
# self.book1 = Edition.objects.create(title="Hyperion")
|
||||||
|
# self.user1 = User.register(email="test@test", username="test")
|
||||||
|
|
||||||
|
def test_parse(self):
|
||||||
|
c0 = "test \n - \n"
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, c0)
|
||||||
|
self.assertEqual(t, None)
|
||||||
|
self.assertEqual(v, None)
|
||||||
|
|
||||||
|
c0 = "test\n \n - \nhttps://xyz"
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test\n ")
|
||||||
|
self.assertEqual(t, None)
|
||||||
|
self.assertEqual(v, None)
|
||||||
|
|
||||||
|
c0 = "test \n - \np1"
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.PAGE)
|
||||||
|
self.assertEqual(v, "1")
|
||||||
|
|
||||||
|
c0 = "test \n - \n pt 1 "
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.PART)
|
||||||
|
self.assertEqual(v, "1")
|
||||||
|
|
||||||
|
c0 = "test \n - \nx chapter 1.1 \n"
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.CHAPTER)
|
||||||
|
self.assertEqual(v, "1.1")
|
||||||
|
|
||||||
|
c0 = "test \n - \n book pg 1.1% "
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.PERCENTAGE)
|
||||||
|
self.assertEqual(v, "1.1")
|
||||||
|
|
||||||
|
c0 = "test \n - \n show e 1. "
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.EPISODE)
|
||||||
|
self.assertEqual(v, "1.")
|
||||||
|
|
||||||
|
c0 = "test \n - \nch 2"
|
||||||
|
c, t, v = Note.strip_footer(c0)
|
||||||
|
self.assertEqual(c, "test ")
|
||||||
|
self.assertEqual(t, Note.ProgressType.CHAPTER)
|
||||||
|
self.assertEqual(v, "2")
|
||||||
|
|
|
@ -57,8 +57,12 @@ class NoteForm(NeoModelForm):
|
||||||
self.fields["content"].required = False
|
self.fields["content"].required = False
|
||||||
# get the corresponding progress types for the item
|
# get the corresponding progress types for the item
|
||||||
types = Note.get_progress_types_by_item(item)
|
types = Note.get_progress_types_by_item(item)
|
||||||
if self.instance.progress_type and self.instance.progress_type not in types:
|
pt = self.instance.progress_type
|
||||||
types.append(self.instance.progress_type)
|
if pt and pt not in types:
|
||||||
|
try:
|
||||||
|
types.append(Note.ProgressType(pt))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
choices = [("", _("Progress Type (optional)"))] + [(x, x.label) for x in types]
|
choices = [("", _("Progress Type (optional)"))] + [(x, x.label) for x in types]
|
||||||
self.fields["progress_type"].choices = choices # type: ignore
|
self.fields["progress_type"].choices = choices # type: ignore
|
||||||
|
|
||||||
|
|
|
@ -1080,6 +1080,15 @@ class Post(models.Model):
|
||||||
},
|
},
|
||||||
)[0]
|
)[0]
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def piece(self):
|
||||||
|
from journal.models import Piece, ShelfMember
|
||||||
|
|
||||||
|
pcs = Piece.objects.filter(post_id=self.pk)
|
||||||
|
if len(pcs) == 1:
|
||||||
|
return pcs[0]
|
||||||
|
return next((p for p in pcs if p.__class__ == ShelfMember), None)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def create_local(
|
def create_local(
|
||||||
cls,
|
cls,
|
||||||
|
|
Loading…
Add table
Reference in a new issue