multi-staged docker build

.dockerignore

@@ -8,3 +8,5 @@ __pycache__
 /doc
 /media
 /static
+/docker-compose.yml
+/Dockerfile

.github/workflows/docker-dev.yml

@@ -0,0 +1,42 @@
+name: publish
+
+on: [push, pull_request]
+
+jobs:
+  push_to_docker_hub:
+    name: Push image to Docker Hub
+    if: github.repository_owner == 'alphatownsman'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: neodb/neodb
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: "linux/amd64,linux/arm64"
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,40 +1,54 @@
 # syntax=docker/dockerfile:1
-FROM python:3.11-slim
+FROM python:3.11-slim as build
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
-RUN useradd -U app
-COPY . /neodb
-RUN mkdir -p /www
+
+RUN --mount=type=cache,sharing=locked,target=/var/cache/apt apt-get update \
+    && apt-get install -y --no-install-recommends build-essential libpq-dev python3-venv opencc git
+
+COPY requirements.txt /neodb/
 WORKDIR /neodb
-RUN mv neodb-takahe /takahe
-RUN cp misc/neodb-manage misc/takahe-manage /bin
-RUN --mount=type=cache,target=/var/cache/apt apt-get update \
-    && apt-get install -y --no-install-recommends \
-        build-essential \
-        libpq-dev \
+RUN python -m venv .venv
+RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt
+
+COPY neodb-takahe/requirements.txt /takahe/
+WORKDIR /takahe
+RUN python -m venv /takahe/.venv
+RUN --mount=type=cache,sharing=locked,target=/root/.cache .venv/bin/python3 -m pip install --upgrade -r requirements.txt
+
+RUN apt-get purge -y --auto-remove build-essential && rm -rf /var/lib/apt/lists/*
+
+# runtime stage
+FROM python:3.11-slim as runtime
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+RUN --mount=type=cache,sharing=locked,target=/var/cache/apt-run apt-get update \
+    && apt-get install -y --no-install-recommends libpq-dev \
         busybox \
-        postgresql-client \
         nginx \
-        opencc \
-        git
+        opencc
 RUN busybox --install
+
+COPY . /neodb
+WORKDIR /neodb
+COPY --from=build /neodb/.venv .venv
+RUN .venv/bin/python3 manage.py compilescss
+RUN .venv/bin/python3 manage.py collectstatic --noinput
+
+RUN mv /neodb/neodb-takahe /takahe
+WORKDIR /takahe
+COPY --from=build /takahe/.venv .venv
+RUN pwd && ls
+RUN TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" .venv/bin/python3 manage.py collectstatic --noinput
+
 COPY misc/nginx.conf.d/* /etc/nginx/conf.d/
+COPY misc/bin/* /bin/
+RUN mkdir -p /www
+RUN useradd -U app
 
-RUN --mount=type=cache,target=/root/.cache python3 -m pip install --upgrade -r requirements.txt
-
-RUN --mount=type=cache,target=/root/.cache cd /takahe && python3 -m pip install --upgrade -r requirements.txt
-
-RUN apt-get purge -y --auto-remove \
-        build-essential \
-        libpq-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN python3 manage.py compilescss \
-    && python3 manage.py collectstatic --noinput
-
-RUN cd /takahe && TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" python3 manage.py collectstatic --noinput
-
+WORKDIR /neodb
 USER app:app
 
 # invoke check by default
-CMD [ "sh", "-c", 'python3 /neodb/manage.py check && TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" python3 manage.py collectstatic --noinput python3 /takahe/manage.py check' ]
+CMD [ "sh", "-c", 'neodb-manage check && TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" takahe-manage check' ]

docker-compose.yml

@@ -13,7 +13,7 @@ version: "3.8"
 x-shared:
   neodb-service: &neodb-service
     build: .
-    image: nerodb/neodb:latest
+    image: neodb/neodb:${TAG:-latest}
     environment:
       - NEODB_DB_NAME=neodb
       - NEODB_DB_USER=neodb
@@ -110,7 +110,7 @@ services:
   migration:
     <<: *neodb-service
     restart: "no"
-    command: "sh -c 'python /takahe/manage.py migrate && python /neodb/manage.py migrate'"
+    command: "sh -c '/takahe/.venv/bin/python /takahe/manage.py migrate && /neodb/.venv/bin/python /neodb/manage.py migrate'"
     depends_on:
       neodb-db:
         condition: service_healthy
@@ -125,7 +125,7 @@ services:
     <<: *neodb-service
     # ports:
     #   - "18000:8000"
-    command: gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000
+    command: /neodb/.venv/bin/gunicorn boofilsic.wsgi -w ${NEODB_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000
     healthcheck:
       test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/discover/']
     depends_on:
@@ -134,14 +134,14 @@ services:
 
   neodb-worker:
     <<: *neodb-service
-    command: python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl
+    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl
     depends_on:
       migration:
         condition: service_completed_successfully
 
   neodb-worker-extra:
     <<: *neodb-service
-    command: python /neodb/manage.py rqworker --with-scheduler fetch crawl
+    command: /neodb/.venv/bin/python /neodb/manage.py rqworker --with-scheduler fetch crawl
     depends_on:
       migration:
         condition: service_completed_successfully
@@ -150,7 +150,7 @@ services:
     <<: *neodb-service
     # ports:
     #   - "19000:8000"
-    command: gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000
+    command: /takahe/.venv/bin/gunicorn --chdir /takahe takahe.wsgi -w ${TAKAHE_WEB_WORKER_NUM:-8} --preload -b 0.0.0.0:8000
     healthcheck:
       test: ['CMD', 'wget', '-qO/tmp/test', 'http://127.0.0.1:8000/nodeinfo/2.0/']
     depends_on:
@@ -159,7 +159,7 @@ services:
 
   takahe-stator:
     <<: *neodb-service
-    command: python /takahe/manage.py runstator
+    command: /takahe/.venv/bin/python /takahe/manage.py runstator
     depends_on:
       migration:
         condition: service_completed_successfully

misc/bin/neodb-manage

@@ -0,0 +1,2 @@
+#!/bin/sh
+/neodb/.venv/bin/python /neodb/manage.py $@

misc/bin/takahe-manage

@@ -0,0 +1,2 @@
+#!/bin/sh
+/takahe/.venv/bin/python /takahe/manage.py $@

misc/neodb-manage

@@ -1,2 +0,0 @@
-#!/bin/sh
-python /neodb/manage.py $@

misc/takahe-manage

@@ -1,2 +0,0 @@
-#!/bin/sh
-python /takahe/manage.py $@