Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

use POST for typesense search

Browse source
This commit is contained in:
Your Name 2025-02-22 15:35:31 -05:00 • committed by Henri Dickson
parent 665863e3ca
commit db7ef50116
9 changed files with 85 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
common/models/index.py
View file

@ -282,8 +282,11 @@ class Index:
params = query.to_search_params() params = query.to_search_params()
if settings.DEBUG: if settings.DEBUG:
logger.debug(f"Typesense: search {self.name} {params}") logger.debug(f"Typesense: search {self.name} {params}")
r = self.read_collection.documents.search(params) # use multi_search as typesense limits query size for normal search
sr = self.search_result_class(self, r) r = self._client.multi_search.perform(
{"searches": [params]}, {"collection": self.read_collection.name}
)
sr = self.search_result_class(self, r["results"][0])
if settings.DEBUG: if settings.DEBUG:
logger.debug(f"Typesense: search result {sr}") logger.debug(f"Typesense: search result {sr}")
return sr return sr

3
journal/apis/post.py
View file

@ -132,9 +132,8 @@ def list_posts_for_item(request, item_uuid: str, type: str | None = None):
types = [t for t in (type or "").split(",") if t in PostTypes] types = [t for t in (type or "").split(",") if t in PostTypes]
q = "type:" + ",".join(types or ["comment", "review"]) q = "type:" + ",".join(types or ["comment", "review"])
query = JournalQueryParser(q) query = JournalQueryParser(q)
query.filter_by_viewer(request.user.identity)
query.filter("item_id", item.pk) query.filter("item_id", item.pk)
query.filter("visibility", 0)
query.exclude("owner_id", request.user.identity.ignoring)
r = JournalIndex.instance().search(query) r = JournalIndex.instance().search(query)
result = { result = {
"data": [ "data": [

9
journal/models/index.py
View file

@ -11,6 +11,7 @@ from catalog.models import Item
from common.models import Index, QueryParser, SearchResult, int_, uniq from common.models import Index, QueryParser, SearchResult, int_, uniq
from takahe.models import Post from takahe.models import Post
from takahe.utils import Takahe from takahe.utils import Takahe
from users.models.apidentity import APIdentity
if TYPE_CHECKING: if TYPE_CHECKING:
from journal.models import Piece from journal.models import Piece
@ -152,6 +153,14 @@ class JournalQueryParser(QueryParser):
except ValueError: except ValueError:
return 0, 0 return 0, 0
def filter_by_owner(self, owner: APIdentity):
self.filter("owner_id", owner.pk)
def filter_by_viewer(self, viewer: APIdentity):
self.filter("visibility", 0)
self.exclude("owner_id", viewer.ignoring)
# TODO support non-public posts
class JournalSearchResult(SearchResult): class JournalSearchResult(SearchResult):
@cached_property @cached_property

31
journal/tests.py
View file

@ -2,7 +2,7 @@ import time
from django.test import TestCase from django.test import TestCase
from catalog.models import * from catalog.models import Edition
from journal.models.common import Debris from journal.models.common import Debris
from users.models import User from users.models import User
@ -23,7 +23,7 @@ class CollectionTest(TestCase):
self.assertEqual(collection.catalog_item.title, "test") self.assertEqual(collection.catalog_item.title, "test")
member1, _ = collection.append_item(self.book1) member1, _ = collection.append_item(self.book1)
self.assertIsNotNone(member1) self.assertIsNotNone(member1)
member1.note = "my notes" member1.note = "my notes" # type: ignore
member1.save() member1.save()
collection.append_item(self.book2, note="test") collection.append_item(self.book2, note="test")
self.assertEqual(list(collection.ordered_items), [self.book1, self.book2]) self.assertEqual(list(collection.ordered_items), [self.book1, self.book2])
@ -32,18 +32,18 @@ class CollectionTest(TestCase):
collection.move_up_item(self.book2) collection.move_up_item(self.book2)
self.assertEqual(list(collection.ordered_items), [self.book2, self.book1]) self.assertEqual(list(collection.ordered_items), [self.book2, self.book1])
members = collection.ordered_members members = collection.ordered_members
collection.update_member_order([members[1].id, members[0].id]) collection.update_member_order([members[1].pk, members[0].pk])
self.assertEqual(list(collection.ordered_items), [self.book1, self.book2]) self.assertEqual(list(collection.ordered_items), [self.book1, self.book2])
member1 = collection.get_member_for_item(self.book1) member1 = collection.get_member_for_item(self.book1)
self.assertIsNotNone(member1) self.assertIsNotNone(member1)
if member1 is None: if member1 is None:
return return
self.assertEqual(member1.note, "my notes") self.assertEqual(member1.note, "my notes") # type: ignore
member2 = collection.get_member_for_item(self.book2) member2 = collection.get_member_for_item(self.book2)
self.assertIsNotNone(member2) self.assertIsNotNone(member2)
if member2 is None: if member2 is None:
return return
self.assertEqual(member2.note, "test") self.assertEqual(member2.note, "test") # type: ignore
class ShelfTest(TestCase): class ShelfTest(TestCase):
@ -299,3 +299,24 @@ class NoteTest(TestCase):
self.assertEqual(c, "test ") self.assertEqual(c, "test ")
self.assertEqual(t, Note.ProgressType.CHAPTER) self.assertEqual(t, Note.ProgressType.CHAPTER)
self.assertEqual(v, "2") self.assertEqual(v, "2")
class SearchTest(TestCase):
databases = "__all__"
def setUp(self):
self.book1 = Edition.objects.create(title="Hyperion")
self.book2 = Edition.objects.create(title="Andymion")
self.user1 = User.register(email="x@y.com", username="userx")
self.index = JournalIndex.instance()
self.index.delete_by_owner([self.user1.identity.pk])
def test_post(self):
mark = Mark(self.user1.identity, self.book1)
mark.update(ShelfType.WISHLIST, "a gentle comment", 9, ["Sci-Fi", "fic"], 0)
mark = Mark(self.user1.identity, self.book2)
mark.update(ShelfType.WISHLIST, "a gentle comment", None, ["nonfic"], 1)
q = JournalQueryParser("gentle")
q.filter_by_owner(self.user1.identity)
r = self.index.search(q)
self.assertEqual(r.total, 2)

3
journal/views/search.py
View file

@ -8,11 +8,10 @@ from journal.models import JournalIndex, JournalQueryParser
@login_required @login_required
def search(request): def search(request):
identity_id = request.user.identity.pk
page = int_(request.GET.get("page"), 1) page = int_(request.GET.get("page"), 1)
q = JournalQueryParser(request.GET.get("q", default=""), page) q = JournalQueryParser(request.GET.get("q", default=""), page)
q.filter_by_owner(request.user.identity)
q.filter("item_id", ">0") q.filter("item_id", ">0")
q.filter("owner_id", identity_id)
if q: if q:
index = JournalIndex.instance() index = JournalIndex.instance()
r = index.search(q) r = index.search(q)

1
pyproject.toml
View file

@ -77,7 +77,6 @@ exclude = [
".git", ".git",
"playground", "playground",
"catalog/*/tests.py", "catalog/*/tests.py",
"journal/tests.py",
"neodb", "neodb",
"**/migrations", "**/migrations",
"neodb-takahe", "neodb-takahe",

50
requirements-dev.lock
View file

@ -9,9 +9,9 @@
# generate-hashes: false # generate-hashes: false
# universal: false # universal: false
aiohappyeyeballs==2.4.4 aiohappyeyeballs==2.4.6
# via aiohttp # via aiohttp
aiohttp==3.11.11 aiohttp==3.11.12
# via discord-py # via discord-py
aiosignal==1.3.2 aiosignal==1.3.2
# via aiohttp # via aiohttp
@ -28,12 +28,12 @@ attrs==25.1.0
# via aiohttp # via aiohttp
babel==2.17.0 babel==2.17.0
# via mkdocs-material # via mkdocs-material
beautifulsoup4==4.13.1 beautifulsoup4==4.13.3
# via markdownify # via markdownify
bleach==5.0.1 bleach==5.0.1
# via django-bleach # via django-bleach
blurhash-python==1.2.2 blurhash-python==1.2.2
cachetools==5.5.1 cachetools==5.5.2
certifi==2025.1.31 certifi==2025.1.31
# via httpcore # via httpcore
# via httpx # via httpx
@ -56,14 +56,14 @@ colorama==0.4.6
# via mkdocs-material # via mkdocs-material
cryptography==43.0.3 cryptography==43.0.3
# via atproto # via atproto
cssbeautifier==1.15.1 cssbeautifier==1.15.3
# via djlint # via djlint
dateparser==1.2.0 dateparser==1.2.1
deepmerge==2.0 deepmerge==2.0
discord-py==2.4.0 discord-py==2.5.0
distlib==0.3.9 distlib==0.3.9
# via virtualenv # via virtualenv
django==4.2.18 django==4.2.19
# via django-anymail # via django-anymail
# via django-appconf # via django-appconf
# via django-auditlog # via django-auditlog
@ -82,12 +82,12 @@ django==4.2.18
# via django-tz-detect # via django-tz-detect
# via easy-thumbnails # via easy-thumbnails
django-anymail==12.0 django-anymail==12.0
django-appconf==1.0.6 django-appconf==1.1.0
# via django-compressor # via django-compressor
django-auditlog==3.0.0 django-auditlog==3.0.0
django-bleach==3.1.0 django-bleach==3.1.0
django-compressor==4.5.1 django-compressor==4.5.1
django-cors-headers==4.6.0 django-cors-headers==4.7.0
django-environ==0.12.0 django-environ==0.12.0
django-hijack==3.7.1 django-hijack==3.7.1
django-jsonform==2.23.2 django-jsonform==2.23.2
@ -98,8 +98,8 @@ django-polymorphic @ git+https://github.com/jazzband/django-polymorphic/@1039f88
django-redis==5.4.0 django-redis==5.4.0
django-rq==3.0.0 django-rq==3.0.0
django-sass-processor==1.4.1 django-sass-processor==1.4.1
django-stubs==5.1.2 django-stubs==5.1.3
django-stubs-ext==5.1.2 django-stubs-ext==5.1.3
# via django-stubs # via django-stubs
django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git@03921e05b39d07d143519a435259f66387a088bc django-typed-models @ git+https://github.com/alphatownsman/django-typed-models.git@03921e05b39d07d143519a435259f66387a088bc
django-tz-detect==0.5.0 django-tz-detect==0.5.0
@ -128,7 +128,7 @@ httpcore==1.0.7
# via httpx # via httpx
httpx==0.27.2 httpx==0.27.2
# via atproto # via atproto
identify==2.6.6 identify==2.6.7
# via pre-commit # via pre-commit
idna==3.10 idna==3.10
# via anyio # via anyio
@ -139,18 +139,18 @@ igdb-api-v4==0.3.3
jinja2==3.1.5 jinja2==3.1.5
# via mkdocs # via mkdocs
# via mkdocs-material # via mkdocs-material
jsbeautifier==1.15.1 jsbeautifier==1.15.3
# via cssbeautifier # via cssbeautifier
# via djlint # via djlint
json5==0.10.0 json5==0.10.0
# via djlint # via djlint
langdetect==1.0.9 langdetect==1.0.9
libipld==3.0.0 libipld==3.0.1
# via atproto # via atproto
libsass==0.23.0 libsass==0.23.0
listparser==0.20 listparser==0.20
loguru==0.7.3 loguru==0.7.3
lxml==5.3.0 lxml==5.3.1
lxml-stubs==0.5.1 lxml-stubs==0.5.1
markdown==3.7 markdown==3.7
# via django-markdownx # via django-markdownx
@ -164,12 +164,12 @@ markupsafe==3.0.2
mergedeep==1.3.4 mergedeep==1.3.4
# via mkdocs # via mkdocs
# via mkdocs-get-deps # via mkdocs-get-deps
mistune==3.1.1 mistune==3.1.2
mkdocs==1.6.1 mkdocs==1.6.1
# via mkdocs-material # via mkdocs-material
mkdocs-get-deps==0.2.0 mkdocs-get-deps==0.2.0
# via mkdocs # via mkdocs
mkdocs-material==9.6.1 mkdocs-material==9.6.5
mkdocs-material-extensions==1.3.1 mkdocs-material-extensions==1.3.1
# via mkdocs-material # via mkdocs-material
multidict==6.1.0 multidict==6.1.0
@ -196,7 +196,7 @@ platformdirs==4.3.6
# via virtualenv # via virtualenv
podcastparser==0.6.10 podcastparser==0.6.10
pre-commit==4.1.0 pre-commit==4.1.0
propcache==0.2.1 propcache==0.3.0
# via aiohttp # via aiohttp
# via yarl # via yarl
protobuf==5.29.3 protobuf==5.29.3
@ -213,12 +213,12 @@ pygments==2.19.1
# via mkdocs-material # via mkdocs-material
pymdown-extensions==10.14.3 pymdown-extensions==10.14.3
# via mkdocs-material # via mkdocs-material
pyright==1.1.393 pyright==1.1.394
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
# via dateparser # via dateparser
# via django-auditlog # via django-auditlog
# via ghp-import # via ghp-import
python-fsutil==0.14.1 python-fsutil==0.15.0
# via django-maintenance-mode # via django-maintenance-mode
pytz==2025.1 pytz==2025.1
# via dateparser # via dateparser
@ -251,8 +251,8 @@ rjsmin==1.2.2
# via django-compressor # via django-compressor
rq==2.1.0 rq==2.1.0
# via django-rq # via django-rq
ruff==0.9.4 ruff==0.9.7
sentry-sdk==2.20.0 sentry-sdk==2.22.0
setproctitle==1.3.4 setproctitle==1.3.4
six==1.17.0 six==1.17.0
# via bleach # via bleach
@ -285,7 +285,7 @@ typing-extensions==4.12.2
# via pydantic # via pydantic
# via pydantic-core # via pydantic-core
# via pyright # via pyright
tzlocal==5.2 tzlocal==5.3
# via dateparser # via dateparser
urllib3==2.3.0 urllib3==2.3.0
# via django-anymail # via django-anymail
@ -293,7 +293,7 @@ urllib3==2.3.0
# via sentry-sdk # via sentry-sdk
urlman==2.0.2 urlman==2.0.2
validators==0.34.0 validators==0.34.0
virtualenv==20.29.1 virtualenv==20.29.2
# via pre-commit # via pre-commit
watchdog==6.0.0 watchdog==6.0.0
# via mkdocs # via mkdocs

32
requirements.lock
View file

@ -9,9 +9,9 @@
# generate-hashes: false # generate-hashes: false
# universal: false # universal: false
aiohappyeyeballs==2.4.4 aiohappyeyeballs==2.4.6
# via aiohttp # via aiohttp
aiohttp==3.11.11 aiohttp==3.11.12
# via discord-py # via discord-py
aiosignal==1.3.2 aiosignal==1.3.2
# via aiohttp # via aiohttp
@ -25,12 +25,12 @@ asgiref==3.8.1
atproto==0.0.58 atproto==0.0.58
attrs==25.1.0 attrs==25.1.0
# via aiohttp # via aiohttp
beautifulsoup4==4.13.1 beautifulsoup4==4.13.3
# via markdownify # via markdownify
bleach==5.0.1 bleach==5.0.1
# via django-bleach # via django-bleach
blurhash-python==1.2.2 blurhash-python==1.2.2
cachetools==5.5.1 cachetools==5.5.2
certifi==2025.1.31 certifi==2025.1.31
# via httpcore # via httpcore
# via httpx # via httpx
@ -46,10 +46,10 @@ click==8.1.8
# via rq # via rq
cryptography==43.0.3 cryptography==43.0.3
# via atproto # via atproto
dateparser==1.2.0 dateparser==1.2.1
deepmerge==2.0 deepmerge==2.0
discord-py==2.4.0 discord-py==2.5.0
django==4.2.18 django==4.2.19
# via django-anymail # via django-anymail
# via django-appconf # via django-appconf
# via django-auditlog # via django-auditlog
@ -66,12 +66,12 @@ django==4.2.18
# via django-tz-detect # via django-tz-detect
# via easy-thumbnails # via easy-thumbnails
django-anymail==12.0 django-anymail==12.0
django-appconf==1.0.6 django-appconf==1.1.0
# via django-compressor # via django-compressor
django-auditlog==3.0.0 django-auditlog==3.0.0
django-bleach==3.1.0 django-bleach==3.1.0
django-compressor==4.5.1 django-compressor==4.5.1
django-cors-headers==4.6.0 django-cors-headers==4.7.0
django-environ==0.12.0 django-environ==0.12.0
django-hijack==3.7.1 django-hijack==3.7.1
django-jsonform==2.23.2 django-jsonform==2.23.2
@ -108,16 +108,16 @@ idna==3.10
# via yarl # via yarl
igdb-api-v4==0.3.3 igdb-api-v4==0.3.3
langdetect==1.0.9 langdetect==1.0.9
libipld==3.0.0 libipld==3.0.1
# via atproto # via atproto
libsass==0.23.0 libsass==0.23.0
listparser==0.20 listparser==0.20
loguru==0.7.3 loguru==0.7.3
lxml==5.3.0 lxml==5.3.1
markdown==3.7 markdown==3.7
# via django-markdownx # via django-markdownx
markdownify==0.14.1 markdownify==0.14.1
mistune==3.1.1 mistune==3.1.2
multidict==6.1.0 multidict==6.1.0
# via aiohttp # via aiohttp
# via yarl # via yarl
@ -129,7 +129,7 @@ pillow==11.1.0
# via django-markdownx # via django-markdownx
# via easy-thumbnails # via easy-thumbnails
podcastparser==0.6.10 podcastparser==0.6.10
propcache==0.2.1 propcache==0.3.0
# via aiohttp # via aiohttp
# via yarl # via yarl
protobuf==5.29.3 protobuf==5.29.3
@ -145,7 +145,7 @@ pydantic-core==2.27.2
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
# via dateparser # via dateparser
# via django-auditlog # via django-auditlog
python-fsutil==0.14.1 python-fsutil==0.15.0
# via django-maintenance-mode # via django-maintenance-mode
pytz==2025.1 pytz==2025.1
# via dateparser # via dateparser
@ -166,7 +166,7 @@ rjsmin==1.2.2
# via django-compressor # via django-compressor
rq==2.1.0 rq==2.1.0
# via django-rq # via django-rq
sentry-sdk==2.20.0 sentry-sdk==2.22.0
setproctitle==1.3.4 setproctitle==1.3.4
six==1.17.0 six==1.17.0
# via bleach # via bleach
@ -191,7 +191,7 @@ typing-extensions==4.12.2
# via beautifulsoup4 # via beautifulsoup4
# via pydantic # via pydantic
# via pydantic-core # via pydantic-core
tzlocal==5.2 tzlocal==5.3
# via dateparser # via dateparser
urllib3==2.3.0 urllib3==2.3.0
# via django-anymail # via django-anymail

4
social/views.py
View file

@ -96,10 +96,10 @@ def search_data(request):
identity_id = request.user.identity.pk identity_id = request.user.identity.pk
page = int_(request.GET.get("lastpage")) + 1 page = int_(request.GET.get("lastpage")) + 1
q = JournalQueryParser(request.GET.get("q", default=""), page, page_size=PAGE_SIZE) q = JournalQueryParser(request.GET.get("q", default=""), page, page_size=PAGE_SIZE)
index = JournalIndex.instance() q.filter_by_owner(request.user.identity)
q.filter("post_id", ">0") q.filter("post_id", ">0")
q.filter("owner_id", identity_id)
q.sort(["created:desc"]) q.sort(["created:desc"])
index = JournalIndex.instance()
if q: if q:
r = index.search(q) r = index.search(q)
events = [ events = [