add profile(production|dev) to docker setup

2023-08-21 21:46:51 +00:00 · 2023-08-21 21:46:51 +00:00 · 21960224c1
commit 21960224c1
parent 6710e7a950
14 changed files with 298 additions and 69 deletions
--- a/2
+++ b/2
@ -52,4 +52,4 @@ RUN rm -rf /var/lib/apt/lists/*
 USER app:app

 # invoke check by default
-CMD [ "sh", "-c", 'neodb-manage check && TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" takahe-manage check' ]
+CMD [ "neodb-hello"]
--- a/common/management/commands/setup.py
+++ b/common/management/commands/setup.py
@ -15,6 +15,7 @@ class Command(BaseCommand):

    def sync_site_config(self):
        domain = settings.SITE_INFO["site_domain"]
+        icon = settings.SITE_INFO["site_logo"]
        name = settings.SITE_INFO["site_name"]
        service_domain = settings.SITE_INFO.get("site_service_domain")
        TakaheConfig.objects.update_or_create(
@ -31,6 +32,20 @@ class Command(BaseCommand):
            domain_id=domain,
            defaults={"json": name},
        )
+        TakaheConfig.objects.update_or_create(
+            key="site_icon",
+            user=None,
+            identity=None,
+            domain_id=None,
+            defaults={"json": icon},
+        )
+        TakaheConfig.objects.update_or_create(
+            key="site_icon",
+            user=None,
+            identity=None,
+            domain_id=domain,
+            defaults={"json": icon},
+        )

    def sync_admin_user(self):
        users = User.objects.filter(username__in=settings.SETUP_ADMIN_USERNAMES)
--- a/common/views.py
+++ b/common/views.py
@ -51,7 +51,7 @@ def nodeinfo2(request):
            usage["localComments"] = row[0]
    return JsonResponse(
        {
-            "version": "2.1",
+            "version": "2.0",
            "software": {
                "name": "neodb",
                "version": settings.NEODB_VERSION,
@ -59,6 +59,7 @@ def nodeinfo2(request):
                "homepage": "https://neodb.net/",
            },
            "protocols": ["activitypub", "neodb"],
+            "openRegistrations": False,  # settings.SITE_INFO["open_registrations"],
            "services": {"outbound": [], "inbound": []},
            "usage": usage,
            "metadata": {"nodeName": settings.SITE_INFO["site_name"]},
--- a/doc/development.md
+++ b/doc/development.md
@ -1,7 +1,14 @@
 Development
 ===========

-Obviously a working version of local NeoDB instance has to be established first, see [install guide](install.md).
+Overview
+--------
+NeoDB is a Django project, and it runs side by side with a modified version of [Takahe](https://github.com/jointakahe/takahe) (a separate Django project, code in `neodb_takahe` as submodule). They communicate mainly thru database and task queue, the diagram in [Docker Installation](install-docker.md) demostrate a typical architecture. Currently the two are loosely coupled, so you may take either one offline without immediate impact on the other, which makes it very easy to conduct maintenance and troubleshooting separately. In the future, they may get combined but it's not decided and will not be decided very soon.
+
+Before writing code
+-------------------
+
+Obviously a working version of local NeoDB instance has to be established first, see [install guide](install.md). If you are to test anything related with ActivityPub, a setup with externally reachable real domain name might be required, using `localhost` may cause quite a few issues here and there.

 Install development related packages
 ```
@ -36,6 +43,16 @@ OK
 Preserving test database for alias 'default'...
 ```

+Debug in Docker
+---------------
+To debug source code with `docker compose`, add `NEODB_DEBUG=True` in `.env`, and use `--profile dev` instead of `--profile production` in commands. The `dev` profile is different from `production`:
+
+- code in `NEODB_SRC` (default: .) and `TAKAHE_SRC` (default: ./neodb-takahe) will be mounted and used in the container instead of code in the image
+- `runserver` with autoreload will be used instead of `gunicorn` for both neodb and takahe web server
+- /static/ and /s/ url are not map to pre-generated static file path
+- one `rqworker` container will be started, instead of two
+- use `dev-shell` and `dev-root` to invoke shells, instead of `shell` and `root`
+- there's no automatic `migration` container, but it can be triggered manually via `docker compose run dev-shell neodb-init`

 Applications
 ------------
@ -44,5 +61,6 @@ Main django apps for NeoDB:
 - `mastodon` this leverages [Mastodon API](https://docs.joinmastodon.org/client/intro/) and [Twitter API](https://developer.twitter.com/en/docs/twitter-api) for user login and data sync
 - `catalog` manages different types of items user may review, and scrapers to fetch from external resources, see [catalog.md](catalog.md) for more details
 - `journal` manages user created content(review/ratings) and lists(collection/shelf/tag), see [journal.md](journal.md) for more details
- - `social` manages timeline for local users and ActivityStreams for remote servers, see [social.md](social.md) for more details
+ - `social` present timeline for local users, see [social.md](social.md) for more details
+ - `takahe` communicate with Takahe (a separate Django server, run side by side with this server, code in `neodb_takahe` as submodule)
 - `legacy` this is only used by instances upgraded from 0.4.x and earlier, to provide a link mapping from old urls to new ones. If your journey starts with 0.5 and later, feel free to ignore it.
--- a/doc/install-docker.md
+++ b/doc/install-docker.md
@ -2,7 +2,7 @@ Run NeoDB in Docker
 ===================

 ## Overview
-For small and medium NeoDB instances, it's recommended to deploy as a local container cluster with `docker-compose`.
+For small and medium NeoDB instances, it's recommended to deploy as a local container cluster with Docker Compose. If you are running a large instance, please see the bottom of doc for some tips.

 ```mermaid
 flowchart TB
@ -26,13 +26,20 @@ flowchart TB
    end
 ```

-As shown in the diagram, a reverse proxy server (e.g. nginx, or Cloudflare tunnel) will be required, it should have SSL configured and pointing to `http://localhost:8000`; the rest is handled by `docker-compose` and containers.
+As shown in the diagram, a reverse proxy server (e.g. nginx, or Cloudflare tunnel) will be required, it should have SSL configured and pointing to `http://localhost:8000`; the rest is handled by `docker compose` and containers.

 ## Install Docker and add user to docker group
-Create a user (e.g. `neouser`) to run neodb, execute these as *root* :
+
+Follow [official instructions](https://docs.docker.com/compose/install/) to install Docker Compose.
+
+Note: Docker Compose V1 is no longer supported. Please verify its version before next step:
 ```
-# apt install docker.io docker-compose
-# adduser --ingroup docker neouser
+$ docker compose version
+```
+
+To run neodb as your own user (e.g. `neouser`), add them to docker group:
+```
+$ sudo usermod -aG docker neouser
 ```

 ## Get configuration files
@ -48,13 +55,13 @@ Change essential options like `NEODB_SITE_DOMAIN` in `.env` before starting the
 - `NEODB_SECRET_KEY` - encryption key of session data
 - `NEODB_DATA` is the path to store db/media/cache, it's `../data` by default, but can be any path that's writable

-See `configuration.md` for more details
+See `neodb.env.example` and `configuration.md` for more options

 ## Start docker
 in the folder with `docker-compose.yml` and `neodb.env`, execute as the user you just created:
 ```
-$ docker-compose pull
-$ docker-compose up -d
+$ docker compose pull
+$ docker compose --profile production up -d
 ```

 In a minute or so, the site should be up at 127.0.0.1:8000 , you may check it with:
@ -77,25 +84,26 @@ NeoDB requires `https` by default. Although `http` may be technically possible,

 Check the release notes, update `docker-compose.yml` and `.env` as instructed. pull the image
 ```
-docker-compose pull
+docker compose pull
 ```

 If there's no change in `docker-compose.yml`, restart only NeoDB services:
 ```
-$ docker-compose stop neodb-web neodb-worker neodb-worker-extra takahe-web takahe-stator nginx
-$ docker-compose up -d
+$ docker compose stop neodb-web neodb-worker neodb-worker-extra takahe-web takahe-stator nginx
+$ docker compose --profile production up -d
 ```

 Otherwise restart the entire cluster:
 ```
-$ docker-compose down
-$ docker-compose up -d
+$ docker compose down
+$ docker compose --profile production up -d
 ```

 ## Troubleshooting

- - `docker-compose ps` to see if any service is down, (btw it's normal that `migration` is in `Exit 0` state)
- - `docker-compose run shell` to run a shell into the cluster; or `docker-compose run root` for root shell, and `apt` is available if extra package needed
+ - `docker compose ps` to see if any service is down, (btw it's normal that `migration` is in `Exit 0` state)
+ - `docker compose run shell` to run a shell into the cluster; or `docker compose run root` for root shell, and `apt` is available if extra package needed
+ - see `Debug in Docker` in [development doc](development.md) for debugging tips

 ## Scaling

--- a/doc/install.md
+++ b/doc/install.md
@ -22,6 +22,8 @@ This is a very basic guide with limited detail, contributions welcomed

 Recommended, see [Docker Installation](install-docker.md)

+*Manual installation are no longer recommended and the doc below may be outdated*
+
 1 Manual Install
 ----------------
 Install PostgreSQL, Redis and Python (3.10 or above) if not yet
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -13,59 +13,73 @@ version: "3.8"
 x-shared:
  neodb-service: &neodb-service
    build: .
-    image: neodb/neodb:${TAG:-latest}
+    image: neodb/neodb:${NEODB_TAG:-latest}
    environment:
-      - NEODB_SITE_NAME
-      - NEODB_SITE_DOMAIN
-      - NEODB_SITE_LOGO
-      - NEODB_DEBUG
-      - NEODB_SECRET_KEY
-      - NEODB_ADMIN_USERNAMES
-      - NEODB_DB_NAME=neodb
-      - NEODB_DB_USER=neodb
-      - NEODB_DB_PASSWORD=aubergine
-      - NEODB_DB_HOST=neodb-db
-      - NEODB_DB_PORT=5432
-      - NEODB_REDIS_HOST=redis
-      - NEODB_REDIS_PORT=6379
-      - NEODB_REDIS_DB=0
-      - NEODB_TYPESENSE_ENABLE=1
-      - NEODB_TYPESENSE_HOST=typesense
-      - NEODB_TYPESENSE_PORT=8108
-      - NEODB_TYPESENSE_KEY=eggplant
-      - NEODB_FROM_EMAIL=no-reply@${NEODB_SITE_DOMAIN}
-      - NEODB_MEDIA_ROOT=/www/m/
-      - NEODB_WEB_SERVER=neodb-web:8000
-      - TAKAHE_DB_NAME=takahe
-      - TAKAHE_DB_USER=takahe
-      - TAKAHE_DB_PASSWORD=aubergine
-      - TAKAHE_DB_HOST=takahe-db
-      - TAKAHE_DB_PORT=5432
-      - TAKAHE_SECRET_KEY=${NEODB_SECRET_KEY}
-      - TAKAHE_MAIN_DOMAIN=${NEODB_SITE_DOMAIN}
-      - TAKAHE_MEDIA_URL=https://${NEODB_SITE_DOMAIN}/media/
-      - TAKAHE_EMAIL_FROM=no-reply@${NEODB_SITE_DOMAIN}
-      - TAKAHE_DATABASE_SERVER=postgres://takahe:aubergine@takahe-db/takahe
-      - TAKAHE_CACHES_DEFAULT=redis://redis:6379/0
-      - TAKAHE_MEDIA_BACKEND=local://www/media/
-      - TAKAHE_MEDIA_ROOT=/www/media/
-      - TAKAHE_USE_PROXY_HEADERS=true
-      - TAKAHE_STATOR_CONCURRENCY=4
-      - TAKAHE_STATOR_CONCURRENCY_PER_MODEL=2
-      - TAKAHE_DEBUG=${NEODB_DEBUG:-False}
-      - TAKAHE_WEB_SERVER=takahe-web:8000
+      NEODB_SITE_NAME:
+      NEODB_SITE_DOMAIN:
+      NEODB_SITE_LOGO:
+      NEODB_DEBUG:
+      NEODB_SECRET_KEY:
+      NEODB_ADMIN_USERNAMES:
+      NEODB_DB_NAME: neodb
+      NEODB_DB_USER: neodb
+      NEODB_DB_PASSWORD: aubergine
+      NEODB_DB_HOST: neodb-db
+      NEODB_DB_PORT: 5432
+      NEODB_REDIS_HOST: redis
+      NEODB_REDIS_PORT: 6379
+      NEODB_REDIS_DB: 0
+      NEODB_TYPESENSE_ENABLE: 1
+      NEODB_TYPESENSE_HOST: typesense
+      NEODB_TYPESENSE_PORT: 8108
+      NEODB_TYPESENSE_KEY: eggplant
+      NEODB_FROM_EMAIL: no-reply@${NEODB_SITE_DOMAIN}
+      NEODB_MEDIA_ROOT: /www/m/
+      TAKAHE_DB_NAME: takahe
+      TAKAHE_DB_USER: takahe
+      TAKAHE_DB_PASSWORD: aubergine
+      TAKAHE_DB_HOST: takahe-db
+      TAKAHE_DB_PORT: 5432
+      TAKAHE_SECRET_KEY: ${NEODB_SECRET_KEY}
+      TAKAHE_MAIN_DOMAIN: ${NEODB_SITE_DOMAIN}
+      TAKAHE_MEDIA_URL: https://${NEODB_SITE_DOMAIN}/media/
+      TAKAHE_EMAIL_FROM: no-reply@${NEODB_SITE_DOMAIN}
+      TAKAHE_DATABASE_SERVER: postgres://takahe:aubergine@takahe-db/takahe
+      TAKAHE_CACHES_DEFAULT: redis://redis:6379/0
+      TAKAHE_MEDIA_BACKEND: local://www/media/
+      TAKAHE_MEDIA_ROOT: /www/media/
+      TAKAHE_USE_PROXY_HEADERS: true
+      TAKAHE_STATOR_CONCURRENCY: 4
+      TAKAHE_STATOR_CONCURRENCY_PER_MODEL: 2
+      TAKAHE_DEBUG: ${NEODB_DEBUG:-False}
    restart: "on-failure"
    volumes:
      - ${NEODB_DATA:-../data}/neodb-media:/www/m
      - ${NEODB_DATA:-../data}/takahe-media:/www/media
      - ${NEODB_DATA:-../data}/takahe-cache:/www/cache
      - ${NEODB_DATA:-../data}/www-root:/www/root
-      # - ${NEODB_DATA:-../data}/log:/var/log/nginx
    depends_on:
      - redis
      - neodb-db
      - typesense
      - takahe-db
+    profiles:
+      - production
+
+  dev-neodb-service: &dev-neodb-service
+    <<: *neodb-service
+    # environment:
+    #   NEODB_DEBUG: True
+    volumes:
+      - ${NEODB_DATA:-../data}/www-root:/www/root
+      - ${NEODB_DATA:-../data}/neodb-media:/www/m
+      - ${NEODB_DATA:-../data}/takahe-media:/www/media
+      - ${NEODB_DATA:-../data}/takahe-cache:/www/cache
+      - ${NEODB_DATA:-../data}/nginx-log:/var/log/nginx
+      - ${NEODB_SRC:-.}:/neodb
+      - ${TAKAHE_SRC:-./neodb-takahe}:/takahe
+    profiles:
+      - dev

 services:
  redis:
@ -176,6 +190,10 @@ services:
    <<: *neodb-service
    user: "root:root"
    command: nginx-start
+    environment:
+      NEODB_WEB_SERVER: neodb-web:8000
+      TAKAHE_WEB_SERVER: takahe-web:8000
+      NGINX_CONF: /neodb/misc/nginx.conf.d/neodb.conf
    depends_on:
      takahe-web:
        condition: service_started
@ -194,3 +212,50 @@ services:
    command: bash
    profiles: ["tools"]
    user: "root:root"
+
+  dev-neodb-web:
+    <<: *dev-neodb-service
+    ports:
+      - "18000:8000"
+    command: neodb-manage runserver 0.0.0.0:8000
+
+  dev-neodb-worker:
+    <<: *dev-neodb-service
+    command: neodb-manage rqworker --with-scheduler import export mastodon fetch crawl ap
+
+  dev-takahe-web:
+    <<: *dev-neodb-service
+    ports:
+      - "19000:8000"
+    command: takahe-manage runserver 0.0.0.0:8000
+
+  dev-takahe-stator:
+    <<: *dev-neodb-service
+    command: takahe-manage runstator
+
+  dev-nginx:
+    <<: *dev-neodb-service
+    user: "root:root"
+    command: nginx-start
+    environment:
+      NEODB_WEB_SERVER: dev-neodb-web:8000
+      TAKAHE_WEB_SERVER: dev-takahe-web:8000
+      NGINX_CONF: /neodb/misc/nginx.conf.d/neodb-dev.conf
+    depends_on:
+      dev-takahe-web:
+        condition: service_started
+      dev-neodb-web:
+        condition: service_started
+    ports:
+      - "${NEODB_PORT:-8000}:8000"
+
+  dev-shell:
+    <<: *dev-neodb-service
+    command: bash
+    profiles: ["tools"]
+
+  dev-root:
+    <<: *dev-neodb-service
+    command: bash
+    profiles: ["tools"]
+    user: "root:root"
--- a/misc/bin/neodb-hello
+++ b/misc/bin/neodb-hello
@ -0,0 +1,18 @@
+#!/bin/sh
+echo '\033[0;35m====== Welcome to NeoDB ======\033[0m'
+echo Your configuration is for ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
+[[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON
+[[ -z "${NEODB_DEBUG}" ]] || env
+echo Running some basic checks...
+neodb-manage check
+TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" takahe-manage check
+cat <<EOF
+check complete
+
+check Docker Compose version:   docker compose version
+start NeoDB (root) shell:       docker compose run -it <shell|root>
+start NeoDB instance:           docker compose --profile <production|dev> up -d
+
+Please follow latest documentations on https://neodb.net to configure your instance before continuing.
+
+EOF
--- a/misc/bin/neodb-init
+++ b/misc/bin/neodb-init
@ -1,10 +1,13 @@
 #!/bin/sh
-echo '\033[0;35m====== NeoDB ======\033[0m'
-echo Initializing ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
-
-[[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON
-[[ -z "${NEODB_DEBUG}" ]] || set
+echo '\033[0;35m====== Welcome to NeoDB ======\033[0m'
+echo Your configuration is for ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
+[[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON, show environment:
+[[ -z "${NEODB_DEBUG}" ]] || env
+echo
+echo NeoDB initializing...

 /takahe/.venv/bin/python /takahe/manage.py migrate || exit $?
 /neodb/.venv/bin/python /neodb/manage.py migrate || exit $?
 /neodb/.venv/bin/python /neodb/manage.py setup || exit $?
+
+echo NeoDB initialization complete.
--- a/misc/bin/neodb-manage
+++ b/misc/bin/neodb-manage
@ -1,2 +1,2 @@
 #!/bin/sh
-/neodb/.venv/bin/python /neodb/manage.py $@
+cd /neodb && .venv/bin/python manage.py $@
--- a/misc/bin/nginx-start
+++ b/misc/bin/nginx-start
@ -1,3 +1,3 @@
 #!/bin/sh
-envsubst '${NEODB_WEB_SERVER} ${TAKAHE_WEB_SERVER}' < /neodb/misc/nginx.conf.d/neodb.conf > /etc/nginx/conf.d/neodb.conf
+envsubst '${NEODB_WEB_SERVER} ${TAKAHE_WEB_SERVER}' < $NGINX_CONF > /etc/nginx/conf.d/neodb.conf
 nginx -g 'daemon off;'
--- a/misc/bin/takahe-manage
+++ b/misc/bin/takahe-manage
@ -1,2 +1,2 @@
 #!/bin/sh
-/takahe/.venv/bin/python /takahe/manage.py $@
+cd /takahe && .venv/bin/python manage.py $@
--- a/misc/nginx.conf.d/neodb-dev.conf
+++ b/misc/nginx.conf.d/neodb-dev.conf
@ -0,0 +1,99 @@
+proxy_cache_path /www/cache levels=1:2 keys_zone=takahe:20m inactive=14d max_size=1g;
+
+upstream neodb {
+    server ${NEODB_WEB_SERVER};
+}
+
+upstream takahe {
+    server ${TAKAHE_WEB_SERVER};
+}
+
+server {
+    listen         8000;
+
+    charset utf-8;
+    ignore_invalid_headers on;
+    client_max_body_size 100M;
+    client_body_buffer_size 128k;
+    proxy_connect_timeout 900;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_http_version 1.1;
+    proxy_hide_header X-Takahe-User;
+    proxy_hide_header X-Takahe-Identity;
+
+    # allow admin to serv their own robots.txt/favicon.ico/...
+    location ~ ^/\w+\.\w+$ {
+        root /www/root;
+        access_log off;
+        log_not_found off;
+    }
+    location /m/ {
+        alias /www/m/;
+        add_header Cache-Control "public, max-age=604800, immutable";
+    }
+    # Proxies media and remote media with caching
+    location ~* ^/(media|proxy) {
+        # Cache media and proxied resources
+        proxy_cache takahe;
+        proxy_cache_key $host$uri;
+        proxy_cache_valid 200 304 4h;
+        proxy_cache_valid 301 307 4h;
+        proxy_cache_valid 500 502 503 504 0s;
+        proxy_cache_valid any 1h;
+        add_header X-Cache $upstream_cache_status;
+
+        # Signal to Takahē that we support full URI accel proxying
+        proxy_set_header X-Takahe-Accel true;
+        proxy_pass http://takahe;
+    }
+    # Internal target for X-Accel redirects that stashes the URI in a var
+    location /__takahe_accel__/ {
+        internal;
+        set $takahe_realuri $upstream_http_x_takahe_realuri;
+        rewrite ^/(.+) /__takahe_accel__/real/;
+    }
+    # Real internal-only target for X-Accel redirects
+    location /__takahe_accel__/real/ {
+        # Only allow internal redirects
+        internal;
+
+        # # Reconstruct the remote URL
+        resolver 9.9.9.9 8.8.8.8 valid=300s;
+
+        # Unset Authorization and Cookie for security reasons.
+        proxy_set_header Authorization '';
+        proxy_set_header Cookie '';
+        proxy_set_header User-Agent 'takahe/nginx';
+        proxy_set_header Host $proxy_host;
+        proxy_set_header X-Forwarded-For '';
+        proxy_set_header X-Forwarded-Host '';
+        proxy_set_header X-Forwarded-Server '';
+        proxy_set_header X-Real-Ip '';
+
+        # Stops the local disk from being written to (just forwards data through)
+        proxy_max_temp_file_size 0;
+
+        # Proxy the remote file through to the client
+        proxy_pass $takahe_realuri;
+        proxy_ssl_server_name on;
+        add_header X-Takahe-Accel "HIT";
+
+        # Cache these responses too
+        proxy_cache takahe;
+        # Cache after a single request
+        proxy_cache_min_uses 1;
+        proxy_cache_key $takahe_realuri;
+        proxy_cache_valid 200 304 720h;
+        proxy_cache_valid 301 307 12h;
+        proxy_cache_valid 500 502 503 504 0s;
+        proxy_cache_valid any 72h;
+        add_header X-Cache $upstream_cache_status;
+    }
+    location ~* ^/(static|@|\.well-known|actor|inbox|api/v1|api/v2|auth|oauth|tags|settings|media|proxy|admin|djadmin) {
+        proxy_pass  http://takahe;
+    }
+    location / {
+        proxy_pass  http://neodb;
+    }
+}
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit af8880f1b61556ae83e1f9970ba3ee6bbfa84292
+Subproject commit 83727018414294e950cf9976ee2a53aa767b5cba