From db7ef5011641471db4db8e4bdbfc1e21740db8d9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 22 Feb 2025 15:35:31 -0500 Subject: [PATCH] use POST for typesense search --- common/models/index.py | 7 ++++-- journal/apis/post.py | 3 +-- journal/models/index.py | 9 ++++++++ journal/tests.py | 31 ++++++++++++++++++++----- journal/views/search.py | 3 +-- pyproject.toml | 1 - requirements-dev.lock | 50 ++++++++++++++++++++--------------------- requirements.lock | 32 +++++++++++++------------- social/views.py | 4 ++-- 9 files changed, 85 insertions(+), 55 deletions(-) diff --git a/common/models/index.py b/common/models/index.py index 994eac7f..bb32d384 100644 --- a/common/models/index.py +++ b/common/models/index.py @@ -282,8 +282,11 @@ class Index: params = query.to_search_params() if settings.DEBUG: logger.debug(f"Typesense: search {self.name} {params}") - r = self.read_collection.documents.search(params) - sr = self.search_result_class(self, r) + # use multi_search as typesense limits query size for normal search + r = self._client.multi_search.perform( + {"searches": [params]}, {"collection": self.read_collection.name} + ) + sr = self.search_result_class(self, r["results"][0]) if settings.DEBUG: logger.debug(f"Typesense: search result {sr}") return sr diff --git a/journal/apis/post.py b/journal/apis/post.py index 556d97f0..0b062225 100644 --- a/journal/apis/post.py +++ b/journal/apis/post.py @@ -132,9 +132,8 @@ def list_posts_for_item(request, item_uuid: str, type: str | None = None): types = [t for t in (type or "").split(",") if t in PostTypes] q = "type:" + ",".join(types or ["comment", "review"]) query = JournalQueryParser(q) + query.filter_by_viewer(request.user.identity) query.filter("item_id", item.pk) - query.filter("visibility", 0) - query.exclude("owner_id", request.user.identity.ignoring) r = JournalIndex.instance().search(query) result = { "data": [ diff --git a/journal/models/index.py b/journal/models/index.py index 6f45d4ec..f9374f79 100644 --- a/journal/models/index.py +++ b/journal/models/index.py @@ -11,6 +11,7 @@ from catalog.models import Item from common.models import Index, QueryParser, SearchResult, int_, uniq from takahe.models import Post from takahe.utils import Takahe +from users.models.apidentity import APIdentity if TYPE_CHECKING: from journal.models import Piece @@ -152,6 +153,14 @@ class JournalQueryParser(QueryParser): except ValueError: return 0, 0 + def filter_by_owner(self, owner: APIdentity): + self.filter("owner_id", owner.pk) + + def filter_by_viewer(self, viewer: APIdentity): + self.filter("visibility", 0) + self.exclude("owner_id", viewer.ignoring) + # TODO support non-public posts + class JournalSearchResult(SearchResult): @cached_property diff --git a/journal/tests.py b/journal/tests.py index 9d505a5f..f8a70aee 100644 --- a/journal/tests.py +++ b/journal/tests.py @@ -2,7 +2,7 @@ import time from django.test import TestCase -from catalog.models import * +from catalog.models import Edition from journal.models.common import Debris from users.models import User @@ -23,7 +23,7 @@ class CollectionTest(TestCase): self.assertEqual(collection.catalog_item.title, "test") member1, _ = collection.append_item(self.book1) self.assertIsNotNone(member1) - member1.note = "my notes" + member1.note = "my notes" # type: ignore member1.save() collection.append_item(self.book2, note="test") self.assertEqual(list(collection.ordered_items), [self.book1, self.book2]) @@ -32,18 +32,18 @@ class CollectionTest(TestCase): collection.move_up_item(self.book2) self.assertEqual(list(collection.ordered_items), [self.book2, self.book1]) members = collection.ordered_members - collection.update_member_order([members[1].id, members[0].id]) + collection.update_member_order([members[1].pk, members[0].pk]) self.assertEqual(list(collection.ordered_items), [self.book1, self.book2]) member1 = collection.get_member_for_item(self.book1) self.assertIsNotNone(member1) if member1 is None: return - self.assertEqual(member1.note, "my notes") + self.assertEqual(member1.note, "my notes") # type: ignore member2 = collection.get_member_for_item(self.book2) self.assertIsNotNone(member2) if member2 is None: return - self.assertEqual(member2.note, "test") + self.assertEqual(member2.note, "test") # type: ignore class ShelfTest(TestCase): @@ -299,3 +299,24 @@ class NoteTest(TestCase): self.assertEqual(c, "test ") self.assertEqual(t, Note.ProgressType.CHAPTER) self.assertEqual(v, "2") + + +class SearchTest(TestCase): + databases = "__all__" + + def setUp(self): + self.book1 = Edition.objects.create(title="Hyperion") + self.book2 = Edition.objects.create(title="Andymion") + self.user1 = User.register(email="x@y.com", username="userx") + self.index = JournalIndex.instance() + self.index.delete_by_owner([self.user1.identity.pk]) + + def test_post(self): + mark = Mark(self.user1.identity, self.book1) + mark.update(ShelfType.WISHLIST, "a gentle comment", 9, ["Sci-Fi", "fic"], 0) + mark = Mark(self.user1.identity, self.book2) + mark.update(ShelfType.WISHLIST, "a gentle comment", None, ["nonfic"], 1) + q = JournalQueryParser("gentle") + q.filter_by_owner(self.user1.identity) + r = self.index.search(q) + self.assertEqual(r.total, 2) diff --git a/journal/views/search.py b/journal/views/search.py index 270d5bf6..e34114a9 100644 --- a/journal/views/search.py +++ b/journal/views/search.py @@ -8,11 +8,10 @@ from journal.models import JournalIndex, JournalQueryParser @login_required def search(request): - identity_id = request.user.identity.pk page = int_(request.GET.get("page"), 1) q = JournalQueryParser(request.GET.get("q", default=""), page) + q.filter_by_owner(request.user.identity) q.filter("item_id", ">0") - q.filter("owner_id", identity_id) if q: index = JournalIndex.instance() r = index.search(q) diff --git a/pyproject.toml b/pyproject.toml index e22d5685..40e56028 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,6 @@ exclude = [ ".git", "playground", "catalog/*/tests.py", - "journal/tests.py", "neodb", "**/migrations", "neodb-takahe", diff --git a/requirements-dev.lock b/requirements-dev.lock index 632f1eb6..8aa2620e 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -9,9 +9,9 @@ # generate-hashes: false # universal: false -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp -aiohttp==3.11.11 +aiohttp==3.11.12 # via discord-py aiosignal==1.3.2 # via aiohttp @@ -28,12 +28,12 @@ attrs==25.1.0 # via aiohttp babel==2.17.0 # via mkdocs-material -beautifulsoup4==4.13.1 +beautifulsoup4==4.13.3 # via markdownify bleach==5.0.1 # via django-bleach blurhash-python==1.2.2 -cachetools==5.5.1 +cachetools==5.5.2 certifi==2025.1.31 # via httpcore # via httpx @@ -56,14 +56,14 @@ colorama==0.4.6 # via mkdocs-material cryptography==43.0.3 # via atproto -cssbeautifier==1.15.1 +cssbeautifier==1.15.3 # via djlint -dateparser==1.2.0 +dateparser==1.2.1 deepmerge==2.0 -discord-py==2.4.0 +discord-py==2.5.0 distlib==0.3.9 # via virtualenv -django==4.2.18 +django==4.2.19 # via django-anymail # via django-appconf # via django-auditlog @@ -82,12 +82,12 @@ django==4.2.18 # via django-tz-detect # via easy-thumbnails django-anymail==12.0 -django-appconf==1.0.6 +django-appconf==1.1.0 # via django-compressor django-auditlog==3.0.0 django-bleach==3.1.0 django-compressor==4.5.1 -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 django-environ==0.12.0 django-hijack==3.7.1 django-jsonform==2.23.2 @@ -98,8 +98,8 @@ django-polymorphic @ git+https://github.com/jazzband/django-polymorphic/@1039f88 django-redis==5.4.0 django-rq==3.0.0 django-sass-processor==1.4.1 -django-stubs==5.1.2 -django-stubs-ext==5.1.2 +django-stubs==5.1.3 +django-stubs-ext==5.1.3 # via django-stubs django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git@03921e05b39d07d143519a435259f66387a088bc django-tz-detect==0.5.0 @@ -128,7 +128,7 @@ httpcore==1.0.7 # via httpx httpx==0.27.2 # via atproto -identify==2.6.6 +identify==2.6.7 # via pre-commit idna==3.10 # via anyio @@ -139,18 +139,18 @@ igdb-api-v4==0.3.3 jinja2==3.1.5 # via mkdocs # via mkdocs-material -jsbeautifier==1.15.1 +jsbeautifier==1.15.3 # via cssbeautifier # via djlint json5==0.10.0 # via djlint langdetect==1.0.9 -libipld==3.0.0 +libipld==3.0.1 # via atproto libsass==0.23.0 listparser==0.20 loguru==0.7.3 -lxml==5.3.0 +lxml==5.3.1 lxml-stubs==0.5.1 markdown==3.7 # via django-markdownx @@ -164,12 +164,12 @@ markupsafe==3.0.2 mergedeep==1.3.4 # via mkdocs # via mkdocs-get-deps -mistune==3.1.1 +mistune==3.1.2 mkdocs==1.6.1 # via mkdocs-material mkdocs-get-deps==0.2.0 # via mkdocs -mkdocs-material==9.6.1 +mkdocs-material==9.6.5 mkdocs-material-extensions==1.3.1 # via mkdocs-material multidict==6.1.0 @@ -196,7 +196,7 @@ platformdirs==4.3.6 # via virtualenv podcastparser==0.6.10 pre-commit==4.1.0 -propcache==0.2.1 +propcache==0.3.0 # via aiohttp # via yarl protobuf==5.29.3 @@ -213,12 +213,12 @@ pygments==2.19.1 # via mkdocs-material pymdown-extensions==10.14.3 # via mkdocs-material -pyright==1.1.393 +pyright==1.1.394 python-dateutil==2.9.0.post0 # via dateparser # via django-auditlog # via ghp-import -python-fsutil==0.14.1 +python-fsutil==0.15.0 # via django-maintenance-mode pytz==2025.1 # via dateparser @@ -251,8 +251,8 @@ rjsmin==1.2.2 # via django-compressor rq==2.1.0 # via django-rq -ruff==0.9.4 -sentry-sdk==2.20.0 +ruff==0.9.7 +sentry-sdk==2.22.0 setproctitle==1.3.4 six==1.17.0 # via bleach @@ -285,7 +285,7 @@ typing-extensions==4.12.2 # via pydantic # via pydantic-core # via pyright -tzlocal==5.2 +tzlocal==5.3 # via dateparser urllib3==2.3.0 # via django-anymail @@ -293,7 +293,7 @@ urllib3==2.3.0 # via sentry-sdk urlman==2.0.2 validators==0.34.0 -virtualenv==20.29.1 +virtualenv==20.29.2 # via pre-commit watchdog==6.0.0 # via mkdocs diff --git a/requirements.lock b/requirements.lock index 24bf4a2d..8a3a9d9b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -9,9 +9,9 @@ # generate-hashes: false # universal: false -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp -aiohttp==3.11.11 +aiohttp==3.11.12 # via discord-py aiosignal==1.3.2 # via aiohttp @@ -25,12 +25,12 @@ asgiref==3.8.1 atproto==0.0.58 attrs==25.1.0 # via aiohttp -beautifulsoup4==4.13.1 +beautifulsoup4==4.13.3 # via markdownify bleach==5.0.1 # via django-bleach blurhash-python==1.2.2 -cachetools==5.5.1 +cachetools==5.5.2 certifi==2025.1.31 # via httpcore # via httpx @@ -46,10 +46,10 @@ click==8.1.8 # via rq cryptography==43.0.3 # via atproto -dateparser==1.2.0 +dateparser==1.2.1 deepmerge==2.0 -discord-py==2.4.0 -django==4.2.18 +discord-py==2.5.0 +django==4.2.19 # via django-anymail # via django-appconf # via django-auditlog @@ -66,12 +66,12 @@ django==4.2.18 # via django-tz-detect # via easy-thumbnails django-anymail==12.0 -django-appconf==1.0.6 +django-appconf==1.1.0 # via django-compressor django-auditlog==3.0.0 django-bleach==3.1.0 django-compressor==4.5.1 -django-cors-headers==4.6.0 +django-cors-headers==4.7.0 django-environ==0.12.0 django-hijack==3.7.1 django-jsonform==2.23.2 @@ -108,16 +108,16 @@ idna==3.10 # via yarl igdb-api-v4==0.3.3 langdetect==1.0.9 -libipld==3.0.0 +libipld==3.0.1 # via atproto libsass==0.23.0 listparser==0.20 loguru==0.7.3 -lxml==5.3.0 +lxml==5.3.1 markdown==3.7 # via django-markdownx markdownify==0.14.1 -mistune==3.1.1 +mistune==3.1.2 multidict==6.1.0 # via aiohttp # via yarl @@ -129,7 +129,7 @@ pillow==11.1.0 # via django-markdownx # via easy-thumbnails podcastparser==0.6.10 -propcache==0.2.1 +propcache==0.3.0 # via aiohttp # via yarl protobuf==5.29.3 @@ -145,7 +145,7 @@ pydantic-core==2.27.2 python-dateutil==2.9.0.post0 # via dateparser # via django-auditlog -python-fsutil==0.14.1 +python-fsutil==0.15.0 # via django-maintenance-mode pytz==2025.1 # via dateparser @@ -166,7 +166,7 @@ rjsmin==1.2.2 # via django-compressor rq==2.1.0 # via django-rq -sentry-sdk==2.20.0 +sentry-sdk==2.22.0 setproctitle==1.3.4 six==1.17.0 # via bleach @@ -191,7 +191,7 @@ typing-extensions==4.12.2 # via beautifulsoup4 # via pydantic # via pydantic-core -tzlocal==5.2 +tzlocal==5.3 # via dateparser urllib3==2.3.0 # via django-anymail diff --git a/social/views.py b/social/views.py index 8ad4311f..65bc068e 100644 --- a/social/views.py +++ b/social/views.py @@ -96,10 +96,10 @@ def search_data(request): identity_id = request.user.identity.pk page = int_(request.GET.get("lastpage")) + 1 q = JournalQueryParser(request.GET.get("q", default=""), page, page_size=PAGE_SIZE) - index = JournalIndex.instance() + q.filter_by_owner(request.user.identity) q.filter("post_id", ">0") - q.filter("owner_id", identity_id) q.sort(["created:desc"]) + index = JournalIndex.instance() if q: r = index.search(q) events = [