diff --git a/.dockerignore b/.dockerignore index 9ddf87ec..22ed546c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,9 +4,17 @@ .vscode .github .git +.gitignore +.pre-commit-config.yaml __pycache__ +/Dockerfile /doc +/docker-compose.yml /media /static -/docker-compose.yml -/Dockerfile +/test_data +/neodb +/neodb-takahe/doc +/neodb-takahe/docker +/neodb-takahe/static-collected +/neodb-takahe/takahe/local_settings.py diff --git a/Dockerfile b/Dockerfile index fc180bcf..6a6e8d39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN --mount=type=cache,sharing=locked,target=/var/cache/apt-run apt-get update \ && apt-get install -y --no-install-recommends libpq-dev \ busybox \ nginx \ + gettext-base \ opencc RUN busybox --install @@ -42,12 +43,12 @@ COPY --from=build /takahe/.venv .venv RUN pwd && ls RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" .venv/bin/python3 manage.py collectstatic --noinput -COPY misc/nginx.conf.d/* /etc/nginx/conf.d/ +WORKDIR /neodb COPY misc/bin/* /bin/ RUN mkdir -p /www RUN useradd -U app +RUN rm -rf /var/lib/apt/lists/* -WORKDIR /neodb USER app:app # invoke check by default diff --git a/boofilsic/settings.py b/boofilsic/settings.py index f5424cbd..d03d11a8 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -201,6 +201,10 @@ if os.getenv("NEODB_SSL", "") != "": STATIC_URL = "/s/" STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/")) +if DEBUG: + # django-sass-processor will generate neodb.css on-the-fly when DEBUG + # NEODB_STATIC_ROOT is readonly in docker mode, so we give it a writable place + SASS_PROCESSOR_ROOT = "/tmp" STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" STATICFILES_FINDERS = [ @@ -338,42 +342,13 @@ REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379)) REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0)) RQ_QUEUES = { - "mastodon": { + q: { "HOST": REDIS_HOST, "PORT": REDIS_PORT, "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, - }, - "export": { - "HOST": REDIS_HOST, - "PORT": REDIS_PORT, - "DB": REDIS_DB, - "DEFAULT_TIMEOUT": -1, - }, - "import": { - "HOST": REDIS_HOST, - "PORT": REDIS_PORT, - "DB": REDIS_DB, - "DEFAULT_TIMEOUT": -1, - }, - "fetch": { - "HOST": REDIS_HOST, - "PORT": REDIS_PORT, - "DB": REDIS_DB, - "DEFAULT_TIMEOUT": -1, - }, - "crawl": { - "HOST": REDIS_HOST, - "PORT": REDIS_PORT, - "DB": REDIS_DB, - "DEFAULT_TIMEOUT": -1, - }, - "doufen": { - "HOST": REDIS_HOST, - "PORT": REDIS_PORT, - "DB": REDIS_DB, - "DEFAULT_TIMEOUT": -1, - }, + } + for q in ["mastodon", "export", "import", "fetch", "crawl", "ap"] } RQ_SHOW_ADMIN_LINK = True diff --git a/catalog/search/typesense.py b/catalog/search/typesense.py index 57816a12..eb2d83f8 100644 --- a/catalog/search/typesense.py +++ b/catalog/search/typesense.py @@ -2,12 +2,14 @@ import logging import types from datetime import timedelta from pprint import pprint +from time import sleep import django_rq import typesense from django.conf import settings from django.db.models.signals import post_delete, post_save from django_redis import get_redis_connection +from loguru import logger from rq.job import Job from typesense.collection import Collection from typesense.exceptions import ObjectNotFound @@ -51,9 +53,6 @@ SORTING_ATTRIBUTE = None SEARCH_PAGE_SIZE = 20 -logger = logging.getLogger(__name__) - - _PENDING_INDEX_KEY = "pending_index_ids" _PENDING_INDEX_QUEUE = "import" _PENDING_INDEX_JOB_ID = "pending_index_flush" @@ -184,10 +183,30 @@ class Indexer: @classmethod def init(cls): - idx = typesense.Client(settings.TYPESENSE_CONNECTION).collections - if idx: - # idx.delete() - idx.create(cls.config()) + try: + client = typesense.Client(settings.TYPESENSE_CONNECTION) + wait = 5 + while not client.operations.is_healthy() and wait: + logger.warning("Typesense: server not healthy") + sleep(1) + wait -= 1 + idx = client.collections[settings.TYPESENSE_INDEX_NAME] + if idx: + try: + i = idx.retrieve() + logger.debug( + f"Typesense: index {settings.TYPESENSE_INDEX_NAME} has {i['num_documents']} documents" + ) + return + except: + client.collections.create(cls.config()) + logger.info( + f"Typesense: index {settings.TYPESENSE_INDEX_NAME} created" + ) + return + logger.error("Typesense: server unknown error") + except Exception as e: + logger.error(f"Typesense: server error {e}") @classmethod def delete_index(cls): @@ -309,7 +328,7 @@ class Indexer: try: cls.instance().documents[pk].delete() except Exception as e: - logger.warn(f"delete item error: \n{e}") + logger.warning(f"delete item error: \n{e}") @classmethod def search(cls, q, page=1, categories=None, tag=None, sort=None): diff --git a/common/management/commands/setup.py b/common/management/commands/setup.py new file mode 100644 index 00000000..d25b1ecf --- /dev/null +++ b/common/management/commands/setup.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand + +from catalog.search.typesense import Indexer + + +class Command(BaseCommand): + help = "Post-Migration Setup" + + def handle(self, *args, **options): + # Update site name if changed + + # Create/update admin user if configured in env + + # Create basic emoji if not exists + + # Create search index if not exists + Indexer.init() + + # Register cron jobs if not yet diff --git a/common/urls.py b/common/urls.py index e8bc0a15..9686ac1a 100644 --- a/common/urls.py +++ b/common/urls.py @@ -7,5 +7,6 @@ urlpatterns = [ path("", home), path("home/", home, name="home"), path("me/", me, name="me"), + path("nodeinfo/2.0/", nodeinfo2), re_path("^~neodb~(?P.+)", ap_redirect), ] diff --git a/common/views.py b/common/views.py index ce10d644..cbfbb3ac 100644 --- a/common/views.py +++ b/common/views.py @@ -1,8 +1,13 @@ +from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required +from django.db import connection +from django.http import JsonResponse from django.shortcuts import redirect, render from django.urls import reverse +from users.models import User + @login_required def me(request): @@ -26,6 +31,41 @@ def ap_redirect(request, uri): return redirect(uri) +def nodeinfo2(request): + usage = {"users": {"total": User.objects.count()}} + # return estimated number of marks as posts, since count the whole table is slow + # TODO filter local with SQL function in https://wiki.postgresql.org/wiki/Count_estimate + with connection.cursor() as cursor: + cursor.execute( + "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'journal_shelfmember';" + ) + row = cursor.fetchone() + if row: + usage["localPosts"] = row[0] + with connection.cursor() as cursor: + cursor.execute( + "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'journal_comment';" + ) + row = cursor.fetchone() + if row: + usage["localComments"] = row[0] + return JsonResponse( + { + "version": "2.1", + "software": { + "name": "neodb", + "version": settings.NEODB_VERSION, + "repository": "https://github.com/neodb-social/neodb", + "homepage": "https://neodb.net/", + }, + "protocols": ["activitypub", "neodb"], + "services": {"outbound": [], "inbound": []}, + "usage": usage, + "metadata": {"nodeName": settings.SITE_INFO["site_name"]}, + } + ) + + def error_400(request, exception=None): return render( request, diff --git a/docker-compose.yml b/docker-compose.yml index 1e063418..674dc2bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,10 @@ x-shared: build: . image: neodb/neodb:${TAG:-latest} environment: + - NEODB_SITE_NAME + - NEODB_SITE_DOMAIN + - NEODB_DEBUG + - NEODB_SECRET_KEY - NEODB_DB_NAME=neodb - NEODB_DB_USER=neodb - NEODB_DB_PASSWORD=aubergine @@ -29,6 +33,7 @@ x-shared: - NEODB_TYPESENSE_KEY=eggplant - NEODB_FROM_EMAIL=no-reply@${NEODB_SITE_DOMAIN} - NEODB_MEDIA_ROOT=/www/m/ + - NEODB_WEB_SERVER=neodb-web:8000 - TAKAHE_DB_NAME=takahe - TAKAHE_DB_USER=takahe - TAKAHE_DB_PASSWORD=aubergine @@ -46,6 +51,7 @@ x-shared: - TAKAHE_STATOR_CONCURRENCY=4 - TAKAHE_STATOR_CONCURRENCY_PER_MODEL=2 - TAKAHE_DEBUG=${NEODB_DEBUG:-False} + - TAKAHE_WEB_SERVER=takahe-web:8000 restart: "on-failure" volumes: - ${NEODB_DATA:-../data}/neodb-media:/www/m @@ -110,7 +116,7 @@ services: migration: <<: *neodb-service restart: "no" - command: "sh -c '/takahe/.venv/bin/python /takahe/manage.py migrate && /neodb/.venv/bin/python /neodb/manage.py migrate'" + command: /bin/neodb-init depends_on: neodb-db: condition: service_healthy @@ -127,21 +133,21 @@ services: # - "18000:8000" command: /neodb/.venv/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000 healthcheck: - test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/discover/'] + test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/'] depends_on: migration: condition: service_completed_successfully neodb-worker: <<: *neodb-service - command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl + command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl ap depends_on: migration: condition: service_completed_successfully neodb-worker-extra: <<: *neodb-service - command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl + command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl ap depends_on: migration: condition: service_completed_successfully @@ -167,7 +173,7 @@ services: nginx: <<: *neodb-service user: "root:root" - command: nginx -g 'daemon off;' + command: nginx-start depends_on: takahe-web: condition: service_started diff --git a/journal/views/post.py b/journal/views/post.py index cb9bd0eb..96779d7b 100644 --- a/journal/views/post.py +++ b/journal/views/post.py @@ -3,7 +3,6 @@ from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDen from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from httpx import request from catalog.models import * from common.utils import ( diff --git a/misc/bin/neodb-init b/misc/bin/neodb-init new file mode 100755 index 00000000..60048751 --- /dev/null +++ b/misc/bin/neodb-init @@ -0,0 +1,10 @@ +#!/bin/sh +echo '\033[0;35m====== NeoDB ======\033[0m' +echo Initializing ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN} + +[[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON +[[ -z "${NEODB_DEBUG}" ]] || set + +/takahe/.venv/bin/python /takahe/manage.py migrate || exit $? +/neodb/.venv/bin/python /neodb/manage.py migrate || exit $? +/neodb/.venv/bin/python /neodb/manage.py setup || exit $? diff --git a/misc/bin/nginx-start b/misc/bin/nginx-start new file mode 100755 index 00000000..9c93c6fe --- /dev/null +++ b/misc/bin/nginx-start @@ -0,0 +1,3 @@ +#!/bin/sh +envsubst '${NEODB_WEB_SERVER} ${TAKAHE_WEB_SERVER}' < /neodb/misc/nginx.conf.d/neodb.conf > /etc/nginx/conf.d/neodb.conf +nginx -g 'daemon off;' diff --git a/misc/nginx.conf.d/neodb.conf b/misc/nginx.conf.d/neodb.conf index 23703612..4293b2a2 100644 --- a/misc/nginx.conf.d/neodb.conf +++ b/misc/nginx.conf.d/neodb.conf @@ -1,11 +1,11 @@ proxy_cache_path /www/cache levels=1:2 keys_zone=takahe:20m inactive=14d max_size=1g; upstream neodb { - server neodb-web:8000; + server ${NEODB_WEB_SERVER}; } upstream takahe { - server takahe-web:8000; + server ${TAKAHE_WEB_SERVER}; } server { @@ -98,7 +98,7 @@ server { proxy_cache_valid any 72h; add_header X-Cache $upstream_cache_status; } - location ~* ^/(@|\.well-known|actor|inbox|nodeinfo|api/v1|api/v2|auth|oauth|tags|settings|media|proxy|admin|djadmin) { + location ~* ^/(@|\.well-known|actor|inbox|api/v1|api/v2|auth|oauth|tags|settings|media|proxy|admin|djadmin) { proxy_pass http://takahe; } location / { diff --git a/neodb-takahe b/neodb-takahe index 4bf7dd6b..af8880f1 160000 --- a/neodb-takahe +++ b/neodb-takahe @@ -1 +1 @@ -Subproject commit 4bf7dd6b6e6594fdfe2df4e9b3b5383d5aea7063 +Subproject commit af8880f1b61556ae83e1f9970ba3ee6bbfa84292 diff --git a/neodb.env.example b/neodb.env.example index ad1f5f6a..8f1343f2 100644 --- a/neodb.env.example +++ b/neodb.env.example @@ -21,3 +21,6 @@ NEODB_SITE_DOMAIN=example.site # Turn on DEBUG mode, either set this to True or don't set it at all # NEODB_DEBUG=True + +# pull NeoDB Docker image from another tag/branch +# TAG=latest