add init command to ensure settings post migration

.dockerignore

@@ -4,9 +4,17 @@
 .vscode
 .github
 .git
+.gitignore
+.pre-commit-config.yaml
 __pycache__
+/Dockerfile
 /doc
+/docker-compose.yml
 /media
 /static
-/docker-compose.yml
-/Dockerfile
+/test_data
+/neodb
+/neodb-takahe/doc
+/neodb-takahe/docker
+/neodb-takahe/static-collected
+/neodb-takahe/takahe/local_settings.py

Dockerfile

@@ -27,6 +27,7 @@ RUN --mount=type=cache,sharing=locked,target=/var/cache/apt-run apt-get update \
     && apt-get install -y --no-install-recommends libpq-dev \
         busybox \
         nginx \
+        gettext-base \
         opencc
 RUN busybox --install
 
@@ -42,12 +43,12 @@ COPY --from=build /takahe/.venv .venv
 RUN pwd && ls
 RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" .venv/bin/python3 manage.py collectstatic --noinput
 
-COPY misc/nginx.conf.d/* /etc/nginx/conf.d/
+WORKDIR /neodb
 COPY misc/bin/* /bin/
 RUN mkdir -p /www
 RUN useradd -U app
+RUN rm -rf /var/lib/apt/lists/*
 
-WORKDIR /neodb
 USER app:app
 
 # invoke check by default

boofilsic/settings.py

@@ -201,6 +201,10 @@ if os.getenv("NEODB_SSL", "") != "":
 
 STATIC_URL = "/s/"
 STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/"))
+if DEBUG:
+    # django-sass-processor will generate neodb.css on-the-fly when DEBUG
+    # NEODB_STATIC_ROOT is readonly in docker mode, so we give it a writable place
+    SASS_PROCESSOR_ROOT = "/tmp"
 
 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
 STATICFILES_FINDERS = [
@@ -338,42 +342,13 @@ REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379))
 REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0))
 
 RQ_QUEUES = {
-    "mastodon": {
+    q: {
         "HOST": REDIS_HOST,
         "PORT": REDIS_PORT,
         "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
-    },
-    "export": {
-        "HOST": REDIS_HOST,
-        "PORT": REDIS_PORT,
-        "DB": REDIS_DB,
-        "DEFAULT_TIMEOUT": -1,
-    },
-    "import": {
-        "HOST": REDIS_HOST,
-        "PORT": REDIS_PORT,
-        "DB": REDIS_DB,
-        "DEFAULT_TIMEOUT": -1,
-    },
-    "fetch": {
-        "HOST": REDIS_HOST,
-        "PORT": REDIS_PORT,
-        "DB": REDIS_DB,
-        "DEFAULT_TIMEOUT": -1,
-    },
-    "crawl": {
-        "HOST": REDIS_HOST,
-        "PORT": REDIS_PORT,
-        "DB": REDIS_DB,
-        "DEFAULT_TIMEOUT": -1,
-    },
-    "doufen": {
-        "HOST": REDIS_HOST,
-        "PORT": REDIS_PORT,
-        "DB": REDIS_DB,
-        "DEFAULT_TIMEOUT": -1,
-    },
+    }
+    for q in ["mastodon", "export", "import", "fetch", "crawl", "ap"]
 }
 
 RQ_SHOW_ADMIN_LINK = True

catalog/search/typesense.py

@@ -2,12 +2,14 @@ import logging
 import types
 from datetime import timedelta
 from pprint import pprint
+from time import sleep
 
 import django_rq
 import typesense
 from django.conf import settings
 from django.db.models.signals import post_delete, post_save
 from django_redis import get_redis_connection
+from loguru import logger
 from rq.job import Job
 from typesense.collection import Collection
 from typesense.exceptions import ObjectNotFound
@@ -51,9 +53,6 @@ SORTING_ATTRIBUTE = None
 SEARCH_PAGE_SIZE = 20
 
 
-logger = logging.getLogger(__name__)
-
-
 _PENDING_INDEX_KEY = "pending_index_ids"
 _PENDING_INDEX_QUEUE = "import"
 _PENDING_INDEX_JOB_ID = "pending_index_flush"
@@ -184,10 +183,30 @@ class Indexer:
 
     @classmethod
     def init(cls):
-        idx = typesense.Client(settings.TYPESENSE_CONNECTION).collections
-        if idx:
-            # idx.delete()
-            idx.create(cls.config())
+        try:
+            client = typesense.Client(settings.TYPESENSE_CONNECTION)
+            wait = 5
+            while not client.operations.is_healthy() and wait:
+                logger.warning("Typesense: server not healthy")
+                sleep(1)
+                wait -= 1
+            idx = client.collections[settings.TYPESENSE_INDEX_NAME]
+            if idx:
+                try:
+                    i = idx.retrieve()
+                    logger.debug(
+                        f"Typesense: index {settings.TYPESENSE_INDEX_NAME} has {i['num_documents']} documents"
+                    )
+                    return
+                except:
+                    client.collections.create(cls.config())
+                    logger.info(
+                        f"Typesense: index {settings.TYPESENSE_INDEX_NAME} created"
+                    )
+                    return
+            logger.error("Typesense: server unknown error")
+        except Exception as e:
+            logger.error(f"Typesense: server error {e}")
 
     @classmethod
     def delete_index(cls):
@@ -309,7 +328,7 @@ class Indexer:
         try:
             cls.instance().documents[pk].delete()
         except Exception as e:
-            logger.warn(f"delete item error: \n{e}")
+            logger.warning(f"delete item error: \n{e}")
 
     @classmethod
     def search(cls, q, page=1, categories=None, tag=None, sort=None):

common/management/commands/setup.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+
+from catalog.search.typesense import Indexer
+
+
+class Command(BaseCommand):
+    help = "Post-Migration Setup"
+
+    def handle(self, *args, **options):
+        # Update site name if changed
+
+        # Create/update admin user if configured in env
+
+        # Create basic emoji if not exists
+
+        # Create search index if not exists
+        Indexer.init()
+
+        # Register cron jobs if not yet

common/urls.py

@@ -7,5 +7,6 @@ urlpatterns = [
     path("", home),
     path("home/", home, name="home"),
     path("me/", me, name="me"),
+    path("nodeinfo/2.0/", nodeinfo2),
     re_path("^~neodb~(?P<uri>.+)", ap_redirect),
 ]

common/views.py

@@ -1,8 +1,13 @@
+from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
+from django.db import connection
+from django.http import JsonResponse
 from django.shortcuts import redirect, render
 from django.urls import reverse
 
+from users.models import User
+
 
 @login_required
 def me(request):
@@ -26,6 +31,41 @@ def ap_redirect(request, uri):
     return redirect(uri)
 
 
+def nodeinfo2(request):
+    usage = {"users": {"total": User.objects.count()}}
+    # return estimated number of marks as posts, since count the whole table is slow
+    # TODO filter local with SQL function in https://wiki.postgresql.org/wiki/Count_estimate
+    with connection.cursor() as cursor:
+        cursor.execute(
+            "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'journal_shelfmember';"
+        )
+        row = cursor.fetchone()
+        if row:
+            usage["localPosts"] = row[0]
+    with connection.cursor() as cursor:
+        cursor.execute(
+            "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'journal_comment';"
+        )
+        row = cursor.fetchone()
+        if row:
+            usage["localComments"] = row[0]
+    return JsonResponse(
+        {
+            "version": "2.1",
+            "software": {
+                "name": "neodb",
+                "version": settings.NEODB_VERSION,
+                "repository": "https://github.com/neodb-social/neodb",
+                "homepage": "https://neodb.net/",
+            },
+            "protocols": ["activitypub", "neodb"],
+            "services": {"outbound": [], "inbound": []},
+            "usage": usage,
+            "metadata": {"nodeName": settings.SITE_INFO["site_name"]},
+        }
+    )
+
+
 def error_400(request, exception=None):
     return render(
         request,

docker-compose.yml

@@ -15,6 +15,10 @@ x-shared:
     build: .
     image: neodb/neodb:${TAG:-latest}
     environment:
+      - NEODB_SITE_NAME
+      - NEODB_SITE_DOMAIN
+      - NEODB_DEBUG
+      - NEODB_SECRET_KEY
       - NEODB_DB_NAME=neodb
       - NEODB_DB_USER=neodb
       - NEODB_DB_PASSWORD=aubergine
@@ -29,6 +33,7 @@ x-shared:
       - NEODB_TYPESENSE_KEY=eggplant
       - NEODB_FROM_EMAIL=no-reply@${NEODB_SITE_DOMAIN}
       - NEODB_MEDIA_ROOT=/www/m/
+      - NEODB_WEB_SERVER=neodb-web:8000
       - TAKAHE_DB_NAME=takahe
       - TAKAHE_DB_USER=takahe
       - TAKAHE_DB_PASSWORD=aubergine
@@ -46,6 +51,7 @@ x-shared:
       - TAKAHE_STATOR_CONCURRENCY=4
       - TAKAHE_STATOR_CONCURRENCY_PER_MODEL=2
       - TAKAHE_DEBUG=${NEODB_DEBUG:-False}
+      - TAKAHE_WEB_SERVER=takahe-web:8000
     restart: "on-failure"
     volumes:
       - ${NEODB_DATA:-../data}/neodb-media:/www/m
@@ -110,7 +116,7 @@ services:
   migration:
     <<: *neodb-service
     restart: "no"
-    command: "sh -c '/takahe/.venv/bin/python /takahe/manage.py migrate && /neodb/.venv/bin/python /neodb/manage.py migrate'"
+    command: /bin/neodb-init
     depends_on:
       neodb-db:
         condition: service_healthy
@@ -127,21 +133,21 @@ services:
     #   - "18000:8000"
     command: /neodb/.venv/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000
     healthcheck:
-      test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/discover/']
+      test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/']
     depends_on:
       migration:
         condition: service_completed_successfully
 
   neodb-worker:
     <<: *neodb-service
-    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl
+    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl ap
     depends_on:
       migration:
         condition: service_completed_successfully
 
   neodb-worker-extra:
     <<: *neodb-service
-    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl
+    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl ap
     depends_on:
       migration:
         condition: service_completed_successfully
@@ -167,7 +173,7 @@ services:
   nginx:
     <<: *neodb-service
     user: "root:root"
-    command: nginx -g 'daemon off;'
+    command: nginx-start
     depends_on:
       takahe-web:
         condition: service_started

journal/views/post.py

@@ -3,7 +3,6 @@ from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDen
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
-from httpx import request
 
 from catalog.models import *
 from common.utils import (

misc/bin/neodb-init

@@ -0,0 +1,10 @@
+#!/bin/sh
+echo '\033[0;35m====== NeoDB ======\033[0m'
+echo Initializing ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
+
+[[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON
+[[ -z "${NEODB_DEBUG}" ]] || set
+
+/takahe/.venv/bin/python /takahe/manage.py migrate || exit $?
+/neodb/.venv/bin/python /neodb/manage.py migrate || exit $?
+/neodb/.venv/bin/python /neodb/manage.py setup || exit $?

misc/bin/nginx-start

@@ -0,0 +1,3 @@
+#!/bin/sh
+envsubst '${NEODB_WEB_SERVER} ${TAKAHE_WEB_SERVER}' < /neodb/misc/nginx.conf.d/neodb.conf > /etc/nginx/conf.d/neodb.conf
+nginx -g 'daemon off;'

misc/nginx.conf.d/neodb.conf

@@ -1,11 +1,11 @@
 proxy_cache_path /www/cache levels=1:2 keys_zone=takahe:20m inactive=14d max_size=1g;
 
 upstream neodb {
-    server neodb-web:8000;
+    server ${NEODB_WEB_SERVER};
 }
 
 upstream takahe {
-    server takahe-web:8000;
+    server ${TAKAHE_WEB_SERVER};
 }
 
 server {
@@ -98,7 +98,7 @@ server {
         proxy_cache_valid any 72h;
         add_header X-Cache $upstream_cache_status;
     }
-    location ~* ^/(@|\.well-known|actor|inbox|nodeinfo|api/v1|api/v2|auth|oauth|tags|settings|media|proxy|admin|djadmin) {
+    location ~* ^/(@|\.well-known|actor|inbox|api/v1|api/v2|auth|oauth|tags|settings|media|proxy|admin|djadmin) {
         proxy_pass  http://takahe;
     }
     location / {

neodb-takahe

@@ -1 +1 @@
-Subproject commit 4bf7dd6b6e6594fdfe2df4e9b3b5383d5aea7063
+Subproject commit af8880f1b61556ae83e1f9970ba3ee6bbfa84292

neodb.env.example

@@ -21,3 +21,6 @@ NEODB_SITE_DOMAIN=example.site
 
 # Turn on DEBUG mode, either set this to True or don't set it at all
 # NEODB_DEBUG=True
+
+# pull NeoDB Docker image from another tag/branch
+# TAG=latest