diff --git a/Dockerfile b/Dockerfile index eb183c63..8917ffa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,13 @@ RUN --mount=type=cache,sharing=locked,target=/var/cache/apt apt-get update \ COPY requirements.txt /neodb/ WORKDIR /neodb -RUN python -m venv .venv -RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt +RUN python -m venv /neodb-venv +RUN --mount=type=cache,sharing=locked,target=/root/.cache /neodb-venv/bin/python3 -m pip install --upgrade -r requirements.txt COPY neodb-takahe/requirements.txt /takahe/ WORKDIR /takahe -RUN python -m venv /takahe/.venv -RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt +RUN python -m venv /takahe-venv +RUN --mount=type=cache,sharing=locked,target=/root/.cache /takahe-venv/bin/python3 -m pip install --upgrade -r requirements.txt RUN apt-get purge -y --auto-remove build-essential && rm -rf /var/lib/apt/lists/* @@ -33,15 +33,15 @@ RUN busybox --install COPY . /neodb WORKDIR /neodb -COPY --from=build /neodb/.venv .venv -RUN .venv/bin/python3 manage.py compilescss -RUN .venv/bin/python3 manage.py collectstatic --noinput +COPY --from=build /neodb-venv /neodb-venv +RUN /neodb-venv/bin/python3 manage.py compilescss +RUN /neodb-venv/bin/python3 manage.py collectstatic --noinput RUN mv /neodb/neodb-takahe /takahe WORKDIR /takahe -COPY --from=build /takahe/.venv .venv +COPY --from=build /takahe-venv /takahe-venv RUN pwd && ls -RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" .venv/bin/python3 manage.py collectstatic --noinput +RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" /takahe-venv/bin/python3 manage.py collectstatic --noinput WORKDIR /neodb COPY misc/bin/* /bin/ diff --git a/doc/development.md b/doc/development.md index faa06714..691af236 100644 --- a/doc/development.md +++ b/doc/development.md @@ -54,7 +54,13 @@ To debug source code with `docker compose`, add `NEODB_DEBUG=True` in `.env`, an - use `dev-shell` and `dev-root` to invoke shells, instead of `shell` and `root` - there's no automatic `migration` container, but it can be triggered manually via `docker compose run dev-shell neodb-init` -Also note: debugging in this way requires `${NEODB_SRC}/.venv` and `${TAKAHE_SRC}/.venv` both ready with all the requirements installed, and python binary pointing to `/usr/local/bin/python` (because that's where python is in the docker base image). +Note: +- Python virtual environments inside docker image, which are `/neodb-venv` and `/takahe-venv`, will be used by default. They can be changed to different locations with `TAKAHE_VENV` and `NEODB_VENV` if needed, usually in a case of the local code using a package not in docker venv. +- Some packages inside python virtual environments are platform dependent, so mount venv from macOS into the Linux container will likely not work. +- Python servers are launched as `app` user, who has no write access to anywhere except /tmp and media path, that's by design. +- Database/redis used in the container cluster are not accessible outside, which is by design. Querying them can be done by either apt update/install client packages in `dev-root` or `root` container, or a modified `docker-compose.yml` with `ports` section uncommented. + + requires `${NEODB_SRC}/.venv` and `${TAKAHE_SRC}/.venv` both ready with all the requirements installed, and python binary pointing to `/usr/local/bin/python` (because that's where python is in the docker base image). Applications ------------ diff --git a/docker-compose.yml b/docker-compose.yml index 912aa7e8..abd6ccf1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ x-shared: NEODB_TYPESENSE_KEY: eggplant NEODB_FROM_EMAIL: no-reply@${NEODB_SITE_DOMAIN} NEODB_MEDIA_ROOT: /www/m + NEODB_VENV: /neodb-venv TAKAHE_DB_NAME: takahe TAKAHE_DB_USER: takahe TAKAHE_DB_PASSWORD: aubergine @@ -52,6 +53,7 @@ x-shared: TAKAHE_STATOR_CONCURRENCY: 4 TAKAHE_STATOR_CONCURRENCY_PER_MODEL: 2 TAKAHE_DEBUG: ${NEODB_DEBUG:-False} + TAKAHE_VENV: /takahe-venv restart: "on-failure" volumes: - ${NEODB_DATA:-../data}/neodb-media:/www/m @@ -147,7 +149,7 @@ services: <<: *neodb-service # ports: # - "18000:8000" - command: /neodb/.venv/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload --max-requests 1000 -b 0.0.0.0:8000 + command: ${NEODB_VENV:-/neodb-venv}/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload --max-requests 1000 -b 0.0.0.0:8000 healthcheck: test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/'] depends_on: @@ -156,14 +158,14 @@ services: neodb-worker: <<: *neodb-service - command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl ap + command: neodb-manage rqworker --with-scheduler import export mastodon fetch crawl ap depends_on: migration: condition: service_completed_successfully neodb-worker-extra: <<: *neodb-service - command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl ap + command: neodb-manage rqworker --with-scheduler fetch crawl ap depends_on: migration: condition: service_completed_successfully @@ -172,7 +174,7 @@ services: <<: *neodb-service # ports: # - "19000:8000" - command: /takahe/.venv/bin/gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8} --max-requests 1000 --preload -b 0.0.0.0:8000 + command: ${TAKAHE_VENV:-/takahe-venv}/bin/gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8} --max-requests 1000 --preload -b 0.0.0.0:8000 healthcheck: test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/'] depends_on: @@ -181,7 +183,7 @@ services: takahe-stator: <<: *neodb-service - command: /takahe/.venv/bin/python /takahe/manage.py runstator + command: takahe-manage runstator depends_on: migration: condition: service_completed_successfully diff --git a/misc/bin/neodb-init b/misc/bin/neodb-init index 59cfa8de..36e5fefe 100755 --- a/misc/bin/neodb-init +++ b/misc/bin/neodb-init @@ -6,8 +6,8 @@ echo Your configuration is for ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN} echo echo NeoDB initializing... -/takahe/.venv/bin/python /takahe/manage.py migrate || exit $? -/neodb/.venv/bin/python /neodb/manage.py migrate || exit $? -/neodb/.venv/bin/python /neodb/manage.py setup || exit $? +takahe-manage migrate || exit $? +neodb-manage migrate || exit $? +neodb-manage setup || exit $? echo NeoDB initialization complete. diff --git a/misc/bin/neodb-manage b/misc/bin/neodb-manage index b00db1e1..86a946f2 100755 --- a/misc/bin/neodb-manage +++ b/misc/bin/neodb-manage @@ -1,2 +1,2 @@ #!/bin/sh -cd /neodb && .venv/bin/python manage.py $@ +cd /neodb && ${NEODB_VENV}/bin/python manage.py $@ diff --git a/misc/bin/takahe-manage b/misc/bin/takahe-manage index ec31f01c..79dd4922 100755 --- a/misc/bin/takahe-manage +++ b/misc/bin/takahe-manage @@ -1,2 +1,2 @@ #!/bin/sh -cd /takahe && .venv/bin/python manage.py $@ +cd /takahe && ${TAKAHE_VENV}/bin/python manage.py $@