refactor note footer strip and add test

boofilsic/settings.py

@@ -14,6 +14,7 @@ try:
 except Exception:
     NEODB_VERSION = __version__ + "-unknown"
 
+TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
 
 # Parse configuration from:
 # - environment variables

catalog/common/models.py

@@ -519,6 +519,14 @@ class Item(PolymorphicModel):
             item = None
         return item
 
+    @classmethod
+    def get_by_remote_url(cls, url: str) -> "Self | None":
+        url_ = url.replace("/~neodb~/", "/")
+        if url_.startswith(settings.SITE_INFO["site_url"]):
+            return cls.get_by_url(url_, True)
+        er = ExternalResource.objects.filter(url=url_).first()
+        return er.item if er else None
+
     # def get_lookup_id(self, id_type: str) -> str:
     #     prefix = id_type.strip().lower() + ':'
     #     return next((x[len(prefix):] for x in self.lookup_ids if x.startswith(prefix)), None)

common/apps.py

@@ -12,6 +12,8 @@ class CommonConfig(AppConfig):
     def setup(self, **kwargs):
         from .setup import Setup
 
+        if kwargs.get("using", "") == "default":
+            # only run setup on the default database, not on takahe
             Setup().run()
 
 

common/setup.py

@@ -121,7 +121,15 @@ class Setup:
                 )
 
     def run(self):
+        if settings.TESTING:
+            # Only do necessary initialization when testing
+            logger.info("Running minimal post-migration setup for testing...")
+            self.sync_site_config()
+            Indexer.init()
+            return
+
         logger.info("Running post-migration setup...")
+
         # Update site name if changed
         self.sync_site_config()
 

journal/models/common.py

@@ -252,19 +252,23 @@ class Piece(PolymorphicModel, UserOwnedObjectMixin):
 
     @classmethod
     @abstractmethod
-    def params_from_ap_object(cls, post, obj, piece):
+    def params_from_ap_object(
+        cls, post: "Post", obj: dict[str, Any], piece: Self | None
+    ) -> dict[str, Any]:
         return {}
 
     @abstractmethod
-    def to_post_params(self):
+    def to_post_params(self) -> dict[str, Any]:
         return {}
 
     @abstractmethod
-    def to_mastodon_params(self):
+    def to_mastodon_params(self) -> dict[str, Any]:
         return {}
 
     @classmethod
-    def update_by_ap_object(cls, owner: APIdentity, item: Item, obj, post: "Post"):
+    def update_by_ap_object(
+        cls, owner: APIdentity, item: Item, obj, post: "Post"
+    ) -> Self | None:
         """
         Create or update a content piece with related AP message
         """

journal/models/note.py

@@ -16,12 +16,12 @@ from .renderers import render_text
 from .shelf import ShelfMember
 
 _progress = re.compile(
-    r"(.*\s)?(?P<prefix>(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P<value>(\d[\d\:\.\-]*\d|\d))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
+    r"(.*\s)?(?P<prefix>(p|pg|page|ch|chapter|pt|part|e|ep|episode|trk|track|cycle))(\s|\.|#)*(?P<value>([\d\:\.\-]+))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
     re.IGNORECASE,
 )
 
 _progress2 = re.compile(
-    r"(.*\s)?(?P<value>(\d[\d\:\.\-]*\d|\d))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
+    r"(.*\s)?(?P<value>([\d\:\.\-]+))\s*(?P<postfix>(%))?(\s|\n|\.|。)?$",
     re.IGNORECASE,
 )
 
@@ -106,7 +106,7 @@ class Note(Content):
         }
         if self.progress_value:
             d["progress"] = {
-                "type": self.progress_type,
+                "type": self.progress_type or "",
                 "value": self.progress_value,
             }
         return d
@@ -114,33 +114,32 @@ class Note(Content):
     @override
     @classmethod
     def params_from_ap_object(cls, post, obj, piece):
-        content = obj.get("content", "").strip()
-        footer = []
-        if post.local:
-            # strip footer from local post if detected
-            lines = content.splitlines()
-            if len(lines) > 2 and lines[-2].strip() in _separaters:
-                content = "\n".join(lines[:-2])
-                footer = lines[-2:]
         params = {
             "title": obj.get("title", post.summary),
-            "content": content,
+            "content": obj.get("content", "").strip(),
             "sensitive": obj.get("sensitive", post.sensitive),
             "attachments": [],
         }
-        progress = obj.get("progress", {})
+        if post.local:
-        if progress.get("type"):
+            # for local post, strip footer and detect progress from content
-            params["progress_type"] = progress.get("type")
+            # if not detected, keep default/original value by not including it in return val
-        if progress.get("value"):
+            params["content"], progress_type, progress_value = cls.strip_footer(
-            params["progress_value"] = progress.get("value")
+                params["content"]
-        if post.local and len(footer) == 2:
+            )
-            progress_type, progress_value = cls.extract_progress(footer[1])
+            if progress_value is not None:
-            if progress_value:
                 params["progress_type"] = progress_type
                 params["progress_value"] = progress_value
-            elif not footer[1].startswith("https://"):
+        else:
-                # add footer back if unable to regconize correct patterns
+            # for remote post, progress is always in "progress" field
-                params["content"] += "\n" + "\n".join(footer)
+            progress = obj.get("progress", {})
+            params["progress_value"] = progress.get("value", None)
+            params["progress_type"] = None
+            if params["progress_value"]:
+                t = progress.get("type", None)
+                try:
+                    params["progress_type"] = Note.ProgressType(t)
+                except ValueError:
+                    pass
         if post:
             for atta in post.attachments.all():
                 params["attachments"].append(
@@ -205,11 +204,24 @@ class Note(Content):
         }
 
     @classmethod
-    def extract_progress(cls, content):
+    def strip_footer(cls, content: str) -> tuple[str, str | None, str | None]:
+        """strip footer if 2nd last line is "-" or similar characters"""
+        lines = content.splitlines()
+        if len(lines) < 3 or lines[-2].strip() not in _separaters:
+            return content, None, None
+        progress_type, progress_value = cls.extract_progress(lines[-1])
+        # if progress_value is None and not lines[-2].startswith("https://"):
+        #     return content, None, None
+        return "\n".join(lines[:-2]), progress_type, progress_value
+
+    @classmethod
+    def extract_progress(cls, content) -> tuple[str | None, str | None]:
         m = _progress.match(content)
         if not m:
             m = _progress2.match(content)
         if m and m["value"]:
+            if m["value"] == "-":
+                return None, ""
             m = m.groupdict()
             typ_ = "percentage" if m["postfix"] == "%" else m.get("prefix", "")
             match typ_:

journal/tests.py

@@ -236,3 +236,60 @@ class DebrisTest(TestCase):
         update_journal_for_merged_item(self.book3.uuid, delete_duplicated=True)
         cnt = Debris.objects.all().count()
         self.assertEqual(cnt, 4)  # Rating, Shelf, 2x TagMember
+
+
+class NoteTest(TestCase):
+    databases = "__all__"
+
+    # def setUp(self):
+    #     self.book1 = Edition.objects.create(title="Hyperion")
+    #     self.user1 = User.register(email="test@test", username="test")
+
+    def test_parse(self):
+        c0 = "test \n - \n"
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, c0)
+        self.assertEqual(t, None)
+        self.assertEqual(v, None)
+
+        c0 = "test\n \n - \nhttps://xyz"
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test\n ")
+        self.assertEqual(t, None)
+        self.assertEqual(v, None)
+
+        c0 = "test \n - \np1"
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.PAGE)
+        self.assertEqual(v, "1")
+
+        c0 = "test \n - \n pt 1 "
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.PART)
+        self.assertEqual(v, "1")
+
+        c0 = "test \n - \nx chapter 1.1 \n"
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.CHAPTER)
+        self.assertEqual(v, "1.1")
+
+        c0 = "test \n - \n book pg 1.1% "
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.PERCENTAGE)
+        self.assertEqual(v, "1.1")
+
+        c0 = "test \n - \n show e 1. "
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.EPISODE)
+        self.assertEqual(v, "1.")
+
+        c0 = "test \n - \nch 2"
+        c, t, v = Note.strip_footer(c0)
+        self.assertEqual(c, "test ")
+        self.assertEqual(t, Note.ProgressType.CHAPTER)
+        self.assertEqual(v, "2")

journal/views/note.py

@@ -57,8 +57,12 @@ class NoteForm(NeoModelForm):
             self.fields["content"].required = False
         # get the corresponding progress types for the item
         types = Note.get_progress_types_by_item(item)
-        if self.instance.progress_type and self.instance.progress_type not in types:
+        pt = self.instance.progress_type
-            types.append(self.instance.progress_type)
+        if pt and pt not in types:
+            try:
+                types.append(Note.ProgressType(pt))
+            except ValueError:
+                pass
         choices = [("", _("Progress Type (optional)"))] + [(x, x.label) for x in types]
         self.fields["progress_type"].choices = choices  # type: ignore
 

takahe/models.py

@@ -1080,6 +1080,15 @@ class Post(models.Model):
             },
         )[0]
 
+    @cached_property
+    def piece(self):
+        from journal.models import Piece, ShelfMember
+
+        pcs = Piece.objects.filter(post_id=self.pk)
+        if len(pcs) == 1:
+            return pcs[0]
+        return next((p for p in pcs if p.__class__ == ShelfMember), None)
+
     @classmethod
     def create_local(
         cls,