diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 626eb12f..9d937ba0 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -14,6 +14,7 @@ try: except Exception: NEODB_VERSION = __version__ + "-unknown" +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" # Parse configuration from: # - environment variables diff --git a/catalog/common/models.py b/catalog/common/models.py index 95c81a73..cfd62969 100644 --- a/catalog/common/models.py +++ b/catalog/common/models.py @@ -519,6 +519,14 @@ class Item(PolymorphicModel): item = None return item + @classmethod + def get_by_remote_url(cls, url: str) -> "Self | None": + url_ = url.replace("/~neodb~/", "/") + if url_.startswith(settings.SITE_INFO["site_url"]): + return cls.get_by_url(url_, True) + er = ExternalResource.objects.filter(url=url_).first() + return er.item if er else None + # def get_lookup_id(self, id_type: str) -> str: # prefix = id_type.strip().lower() + ':' # return next((x[len(prefix):] for x in self.lookup_ids if x.startswith(prefix)), None) diff --git a/common/apps.py b/common/apps.py index 67cd0df5..658bdf96 100644 --- a/common/apps.py +++ b/common/apps.py @@ -12,7 +12,9 @@ class CommonConfig(AppConfig): def setup(self, **kwargs): from .setup import Setup - Setup().run() + if kwargs.get("using", "") == "default": + # only run setup on the default database, not on takahe + Setup().run() @register(Tags.admin, deploy=True) diff --git a/common/setup.py b/common/setup.py index da211c08..6fecee50 100644 --- a/common/setup.py +++ b/common/setup.py @@ -121,7 +121,15 @@ class Setup: ) def run(self): + if settings.TESTING: + # Only do necessary initialization when testing + logger.info("Running minimal post-migration setup for testing...") + self.sync_site_config() + Indexer.init() + return + logger.info("Running post-migration setup...") + # Update site name if changed self.sync_site_config() diff --git a/journal/models/common.py b/journal/models/common.py index 2ad07a8e..d9e415d4 100644 --- a/journal/models/common.py +++ b/journal/models/common.py @@ -252,19 +252,23 @@ class Piece(PolymorphicModel, UserOwnedObjectMixin): @classmethod @abstractmethod - def params_from_ap_object(cls, post, obj, piece): + def params_from_ap_object( + cls, post: "Post", obj: dict[str, Any], piece: Self | None + ) -> dict[str, Any]: return {} @abstractmethod - def to_post_params(self): + def to_post_params(self) -> dict[str, Any]: return {} @abstractmethod - def to_mastodon_params(self): + def to_mastodon_params(self) -> dict[str, Any]: return {} @classmethod - def update_by_ap_object(cls, owner: APIdentity, item: Item, obj, post: "Post"): + def update_by_ap_object( + cls, owner: APIdentity, item: Item, obj, post: "Post" + ) -> Self | None: """ Create or update a content piece with related AP message """ diff --git a/journal/models/note.py b/journal/models/note.py index 54b10ca7..5b445e00 100644 --- a/journal/models/note.py +++ b/journal/models/note.py @@ -16,12 +16,12 @@ from .renderers import render_text from .shelf import ShelfMember _progress = re.compile( - r"(.*\s)?(?P(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P(\d[\d\:\.\-]*\d|\d))\s*(?P(%))?(\s|\n|\.|。)?$", + r"(.*\s)?(?P(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P([\d\:\.\-]+))\s*(?P(%))?(\s|\n|\.|。)?$", re.IGNORECASE, ) _progress2 = re.compile( - r"(.*\s)?(?P(\d[\d\:\.\-]*\d|\d))\s*(?P(%))?(\s|\n|\.|。)?$", + r"(.*\s)?(?P([\d\:\.\-]+))\s*(?P(%))?(\s|\n|\.|。)?$", re.IGNORECASE, ) @@ -106,7 +106,7 @@ class Note(Content): } if self.progress_value: d["progress"] = { - "type": self.progress_type, + "type": self.progress_type or "", "value": self.progress_value, } return d @@ -114,33 +114,32 @@ class Note(Content): @override @classmethod def params_from_ap_object(cls, post, obj, piece): - content = obj.get("content", "").strip() - footer = [] - if post.local: - # strip footer from local post if detected - lines = content.splitlines() - if len(lines) > 2 and lines[-2].strip() in _separaters: - content = "\n".join(lines[:-2]) - footer = lines[-2:] params = { "title": obj.get("title", post.summary), - "content": content, + "content": obj.get("content", "").strip(), "sensitive": obj.get("sensitive", post.sensitive), "attachments": [], } - progress = obj.get("progress", {}) - if progress.get("type"): - params["progress_type"] = progress.get("type") - if progress.get("value"): - params["progress_value"] = progress.get("value") - if post.local and len(footer) == 2: - progress_type, progress_value = cls.extract_progress(footer[1]) - if progress_value: + if post.local: + # for local post, strip footer and detect progress from content + # if not detected, keep default/original value by not including it in return val + params["content"], progress_type, progress_value = cls.strip_footer( + params["content"] + ) + if progress_value is not None: params["progress_type"] = progress_type params["progress_value"] = progress_value - elif not footer[1].startswith("https://"): - # add footer back if unable to regconize correct patterns - params["content"] += "\n" + "\n".join(footer) + else: + # for remote post, progress is always in "progress" field + progress = obj.get("progress", {}) + params["progress_value"] = progress.get("value", None) + params["progress_type"] = None + if params["progress_value"]: + t = progress.get("type", None) + try: + params["progress_type"] = Note.ProgressType(t) + except ValueError: + pass if post: for atta in post.attachments.all(): params["attachments"].append( @@ -205,11 +204,24 @@ class Note(Content): } @classmethod - def extract_progress(cls, content): + def strip_footer(cls, content: str) -> tuple[str, str | None, str | None]: + """strip footer if 2nd last line is "-" or similar characters""" + lines = content.splitlines() + if len(lines) < 3 or lines[-2].strip() not in _separaters: + return content, None, None + progress_type, progress_value = cls.extract_progress(lines[-1]) + # if progress_value is None and not lines[-2].startswith("https://"): + # return content, None, None + return "\n".join(lines[:-2]), progress_type, progress_value + + @classmethod + def extract_progress(cls, content) -> tuple[str | None, str | None]: m = _progress.match(content) if not m: m = _progress2.match(content) if m and m["value"]: + if m["value"] == "-": + return None, "" m = m.groupdict() typ_ = "percentage" if m["postfix"] == "%" else m.get("prefix", "") match typ_: diff --git a/journal/tests.py b/journal/tests.py index 7b7927a3..3134c998 100644 --- a/journal/tests.py +++ b/journal/tests.py @@ -236,3 +236,60 @@ class DebrisTest(TestCase): update_journal_for_merged_item(self.book3.uuid, delete_duplicated=True) cnt = Debris.objects.all().count() self.assertEqual(cnt, 4) # Rating, Shelf, 2x TagMember + + +class NoteTest(TestCase): + databases = "__all__" + + # def setUp(self): + # self.book1 = Edition.objects.create(title="Hyperion") + # self.user1 = User.register(email="test@test", username="test") + + def test_parse(self): + c0 = "test \n - \n" + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, c0) + self.assertEqual(t, None) + self.assertEqual(v, None) + + c0 = "test\n \n - \nhttps://xyz" + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test\n ") + self.assertEqual(t, None) + self.assertEqual(v, None) + + c0 = "test \n - \np1" + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.PAGE) + self.assertEqual(v, "1") + + c0 = "test \n - \n pt 1 " + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.PART) + self.assertEqual(v, "1") + + c0 = "test \n - \nx chapter 1.1 \n" + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.CHAPTER) + self.assertEqual(v, "1.1") + + c0 = "test \n - \n book pg 1.1% " + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.PERCENTAGE) + self.assertEqual(v, "1.1") + + c0 = "test \n - \n show e 1. " + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.EPISODE) + self.assertEqual(v, "1.") + + c0 = "test \n - \nch 2" + c, t, v = Note.strip_footer(c0) + self.assertEqual(c, "test ") + self.assertEqual(t, Note.ProgressType.CHAPTER) + self.assertEqual(v, "2") diff --git a/journal/views/note.py b/journal/views/note.py index 101fa513..a26e0363 100644 --- a/journal/views/note.py +++ b/journal/views/note.py @@ -57,8 +57,12 @@ class NoteForm(NeoModelForm): self.fields["content"].required = False # get the corresponding progress types for the item types = Note.get_progress_types_by_item(item) - if self.instance.progress_type and self.instance.progress_type not in types: - types.append(self.instance.progress_type) + pt = self.instance.progress_type + if pt and pt not in types: + try: + types.append(Note.ProgressType(pt)) + except ValueError: + pass choices = [("", _("Progress Type (optional)"))] + [(x, x.label) for x in types] self.fields["progress_type"].choices = choices # type: ignore diff --git a/takahe/models.py b/takahe/models.py index dec4c40c..7cacce04 100644 --- a/takahe/models.py +++ b/takahe/models.py @@ -1080,6 +1080,15 @@ class Post(models.Model): }, )[0] + @cached_property + def piece(self): + from journal.models import Piece, ShelfMember + + pcs = Piece.objects.filter(post_id=self.pk) + if len(pcs) == 1: + return pcs[0] + return next((p for p in pcs if p.__class__ == ShelfMember), None) + @classmethod def create_local( cls,