make venv path configurable in docker

Dockerfile

@@ -8,13 +8,13 @@ RUN --mount=type=cache,sharing=locked,target=/var/cache/apt apt-get update \
 
 COPY requirements.txt /neodb/
 WORKDIR /neodb
-RUN python -m venv .venv
-RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt
+RUN python -m venv /neodb-venv
+RUN --mount=type=cache,sharing=locked,target=/root/.cache /neodb-venv/bin/python3 -m pip install --upgrade -r requirements.txt
 
 COPY neodb-takahe/requirements.txt /takahe/
 WORKDIR /takahe
-RUN python -m venv /takahe/.venv
-RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt
+RUN python -m venv /takahe-venv
+RUN --mount=type=cache,sharing=locked,target=/root/.cache /takahe-venv/bin/python3 -m pip install --upgrade -r requirements.txt
 
 RUN apt-get purge -y --auto-remove build-essential && rm -rf /var/lib/apt/lists/*
 
@@ -33,15 +33,15 @@ RUN busybox --install
 
 COPY . /neodb
 WORKDIR /neodb
-COPY --from=build /neodb/.venv .venv
-RUN .venv/bin/python3 manage.py compilescss
-RUN .venv/bin/python3 manage.py collectstatic --noinput
+COPY --from=build /neodb-venv /neodb-venv
+RUN /neodb-venv/bin/python3 manage.py compilescss
+RUN /neodb-venv/bin/python3 manage.py collectstatic --noinput
 
 RUN mv /neodb/neodb-takahe /takahe
 WORKDIR /takahe
-COPY --from=build /takahe/.venv .venv
+COPY --from=build /takahe-venv /takahe-venv
 RUN pwd && ls
-RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" .venv/bin/python3 manage.py collectstatic --noinput
+RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" /takahe-venv/bin/python3 manage.py collectstatic --noinput
 
 WORKDIR /neodb
 COPY misc/bin/* /bin/

doc/development.md

@@ -54,7 +54,13 @@ To debug source code with `docker compose`, add `NEODB_DEBUG=True` in `.env`, an
 - use `dev-shell` and `dev-root` to invoke shells, instead of `shell` and `root`
 - there's no automatic `migration` container, but it can be triggered manually via `docker compose run dev-shell neodb-init`
 
-Also note: debugging in this way requires `${NEODB_SRC}/.venv` and `${TAKAHE_SRC}/.venv` both ready with all the requirements installed, and python binary pointing to `/usr/local/bin/python` (because that's where python is in the docker base image).
+Note:
+- Python virtual environments inside docker image, which are `/neodb-venv` and `/takahe-venv`, will be used by default. They can be changed to different locations with `TAKAHE_VENV` and `NEODB_VENV` if needed, usually in a case of the local code using a package not in docker venv.
+- Some packages inside python virtual environments are platform dependent, so mount venv from macOS into the Linux container will likely not work.
+- Python servers are launched as `app` user, who has no write access to anywhere except /tmp and media path, that's by design.
+- Database/redis used in the container cluster are not accessible outside, which is by design. Querying them can be done by either apt update/install client packages in `dev-root` or `root` container, or a modified `docker-compose.yml` with `ports` section uncommented.
+
+ requires `${NEODB_SRC}/.venv` and `${TAKAHE_SRC}/.venv` both ready with all the requirements installed, and python binary pointing to `/usr/local/bin/python` (because that's where python is in the docker base image).
 
 Applications
 ------------

docker-compose.yml

@@ -35,6 +35,7 @@ x-shared:
       NEODB_TYPESENSE_KEY: eggplant
       NEODB_FROM_EMAIL: no-reply@${NEODB_SITE_DOMAIN}
       NEODB_MEDIA_ROOT: /www/m
+      NEODB_VENV: /neodb-venv
       TAKAHE_DB_NAME: takahe
       TAKAHE_DB_USER: takahe
       TAKAHE_DB_PASSWORD: aubergine
@@ -52,6 +53,7 @@ x-shared:
       TAKAHE_STATOR_CONCURRENCY: 4
       TAKAHE_STATOR_CONCURRENCY_PER_MODEL: 2
       TAKAHE_DEBUG: ${NEODB_DEBUG:-False}
+      TAKAHE_VENV: /takahe-venv
     restart: "on-failure"
     volumes:
       - ${NEODB_DATA:-../data}/neodb-media:/www/m
@@ -147,7 +149,7 @@ services:
     <<: *neodb-service
     # ports:
     #   - "18000:8000"
-    command: /neodb/.venv/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload  --max-requests 1000 -b 0.0.0.0:8000
+    command: ${NEODB_VENV:-/neodb-venv}/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload  --max-requests 1000 -b 0.0.0.0:8000
     healthcheck:
       test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/']
     depends_on:
@@ -156,14 +158,14 @@ services:
 
   neodb-worker:
     <<: *neodb-service
-    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl ap
+    command: neodb-manage rqworker --with-scheduler import export mastodon fetch crawl ap
     depends_on:
       migration:
         condition: service_completed_successfully
 
   neodb-worker-extra:
     <<: *neodb-service
-    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl ap
+    command: neodb-manage rqworker --with-scheduler fetch crawl ap
     depends_on:
       migration:
         condition: service_completed_successfully
@@ -172,7 +174,7 @@ services:
     <<: *neodb-service
     # ports:
     #   - "19000:8000"
-    command: /takahe/.venv/bin/gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8}  --max-requests 1000 --preload -b 0.0.0.0:8000
+    command: ${TAKAHE_VENV:-/takahe-venv}/bin/gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8}  --max-requests 1000 --preload -b 0.0.0.0:8000
     healthcheck:
       test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/']
     depends_on:
@@ -181,7 +183,7 @@ services:
 
   takahe-stator:
     <<: *neodb-service
-    command: /takahe/.venv/bin/python /takahe/manage.py runstator
+    command: takahe-manage runstator
     depends_on:
       migration:
         condition: service_completed_successfully

misc/bin/neodb-init

@@ -6,8 +6,8 @@ echo Your configuration is for ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
 echo
 echo NeoDB initializing...
 
-/takahe/.venv/bin/python /takahe/manage.py migrate || exit $?
-/neodb/.venv/bin/python /neodb/manage.py migrate || exit $?
-/neodb/.venv/bin/python /neodb/manage.py setup || exit $?
+takahe-manage migrate || exit $?
+neodb-manage migrate || exit $?
+neodb-manage setup || exit $?
 
 echo NeoDB initialization complete.

misc/bin/neodb-manage

@@ -1,2 +1,2 @@
 #!/bin/sh
-cd /neodb && .venv/bin/python manage.py $@
+cd /neodb && ${NEODB_VENV}/bin/python manage.py $@

misc/bin/takahe-manage

@@ -1,2 +1,2 @@
 #!/bin/sh
-cd /takahe && .venv/bin/python manage.py $@
+cd /takahe && ${TAKAHE_VENV}/bin/python manage.py $@