update docker configuration; pin pyright to a working version for now

2023-08-17 02:09:36 -04:00 · 2023-08-17 02:09:36 -04:00 · f552ae64cc
commit f552ae64cc
parent 7a00159e69
11 changed files with 248 additions and 120 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,5 +1,7 @@
 .DS_Store
 .venv
+/.env
+/neodb.env

 # Byte-compiled / optimized / DLL files
 __pycache__/
--- a/40
+++ b/40
@ -1,23 +1,31 @@
 # syntax=docker/dockerfile:1
-FROM python:3.8-slim
+FROM python:3.11-slim-bullseye
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
+COPY . /neodb
+WORKDIR /neodb
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends build-essential libpq-dev git \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        libpq-dev \
+        busybox \
+        postgresql-client \
+        nginx \
+        opencc \
+        git
+COPY misc/nginx.conf.d/* /etc/nginx/conf.d/
+RUN echo >> /etc/nginx/nginx.conf
+RUN echo 'daemon off;' >> /etc/nginx/nginx.conf
+RUN python3 -m pip install --no-cache-dir --upgrade -r requirements.txt
+RUN apt-get purge -y --auto-remove \
+        build-essential \
+        libpq-dev \
    && rm -rf /var/lib/apt/lists/*
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt \
-    && rm -rf /tmp/requirements.txt \
-    && useradd -U app_user \
-    && install -d -m 0755 -o app_user -g app_user /app/static

-ENV DJANGO_SETTINGS_MODULE=yoursettings.dev
-WORKDIR /app
-USER app_user:app_user
-COPY --chown=app_user:app_user . .
-RUN chmod +x docker/*.sh
+RUN python3 manage.py compilescss \
+    && python3 manage.py collectstatic --noinput
+RUN cp -R misc/www /www
+RUN mv static /www/static

-# Section 6- Docker Run Checks and Configurations
-ENTRYPOINT [ "docker/entrypoint.sh" ]
-
-CMD [ "docker/start.sh", "server" ]
+# invoke check by default
+CMD [ "python3", "/neodb/manage.py", "check" ]
--- a/boofilsic/settings.py
+++ b/boofilsic/settings.py
@ -7,17 +7,19 @@ PROJECT_ROOT = os.path.abspath(os.path.dirname(__name__))
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

-# https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys
-DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+# for legacy deployment:
+# DEFAULT_AUTO_FIELD = "django.db.models.AutoField"

 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/

-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "nbv58c^&b8-095(^)&_BV98596v)&CX#^$&%*^V5"
+# SECURITY WARNING: use your own secret key and keep it!
+SECRET_KEY = os.environ.get("NEODB_SECRET_KEY", "insecure")
+

 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = os.environ.get("NEODB_DEBUG", "") != ""

 ALLOWED_HOSTS = ["*"]

@ -117,34 +119,20 @@ CACHES = {
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases

-if DEBUG:
-    DATABASES = {
+DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql",
-            "NAME": os.environ.get("DB_NAME", "test"),
-            "USER": os.environ.get("DB_USER", "postgres"),
-            "PASSWORD": os.environ.get("DB_PASSWORD", "admin123"),
-            "HOST": os.environ.get("DB_HOST", "127.0.0.1"),
+        "NAME": os.environ.get("NEODB_DB_NAME", "test"),
+        "USER": os.environ.get("NEODB_DB_USER", "postgres"),
+        "PASSWORD": os.environ.get("NEODB_DB_PASSWORD", "admin123"),
+        "HOST": os.environ.get("NEODB_DB_HOST", "127.0.0.1"),
+        "PORT": int(os.environ.get("NEODB_DB_PORT", 5432)),
        "OPTIONS": {
            "client_encoding": "UTF8",
            # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
        },
    }
-    }
-else:
-    DATABASES = {
-        "default": {
-            "ENGINE": "django.db.backends.postgresql",
-            "NAME": "boofilsic",
-            "USER": "doubaniux",
-            "PASSWORD": "password",
-            "HOST": "localhost",
-            "OPTIONS": {
-                "client_encoding": "UTF8",
-                # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
-            },
-        }
-    }
+}

 # Customized auth backend, glue OAuth2 and Django User model together
 # https://docs.djangoproject.com/en/3.0/topics/auth/customizing/#authentication-backends
@ -172,14 +160,19 @@ USE_L10N = True
 USE_TZ = True


-if not DEBUG:
-    SESSION_COOKIE_SECURE = True
-    CSRF_COOKIE_SECURE = True
+USE_X_FORWARDED_HOST = True
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024
+CSRF_COOKIE_SECURE = True
+SESSION_COOKIE_SECURE = True
+
+if os.getenv("NEODB_SSL", "") != "":
    SECURE_SSL_REDIRECT = True
    SECURE_HSTS_PRELOAD = True
    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
    SECURE_HSTS_SECONDS = 31536000

+if not DEBUG:
    LOGGING = {
        "version": 1,
        "disable_existing_loggers": False,
@ -208,7 +201,7 @@ if not DEBUG:
 # https://docs.djangoproject.com/en/3.0/howto/static-files/

 STATIC_URL = "/static/"
-STATIC_ROOT = os.path.join(BASE_DIR, "static/")
+STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/"))

 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
 STATICFILES_FINDERS = [
@ -224,23 +217,22 @@ SILENCED_SYSTEM_CHECKS = [
 ]

 MEDIA_URL = "/media/"
-MEDIA_ROOT = os.path.join(BASE_DIR, "media/")
+MEDIA_ROOT = os.environ.get("NEODB_MEDIA_ROOT", os.path.join(BASE_DIR, "media/"))

+SITE_DOMAIN = os.environ.get("NEODB_SITE_DOMAIN", "nicedb.org")
 SITE_INFO = {
-    "site_name": os.environ.get("APP_NAME", "NiceDB"),
-    "site_url": os.environ.get("APP_URL", "https://nicedb.org"),
+    "site_name": os.environ.get("NEODB_SITE_NAME", "NiceDB"),
+    "site_domain": SITE_DOMAIN,
+    "site_url": os.environ.get("NEODB_SITE_URL", "https://" + SITE_DOMAIN),
    "support_link": "https://github.com/doubaniux/boofilsic/issues",
    "social_link": "https://donotban.com/@testie",
    "donation_link": "https://patreon.com/tertius",
-    "version_hash": None,
    "settings_module": os.getenv("DJANGO_SETTINGS_MODULE"),
-    "sentry_dsn": None,
 }

-REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/'
-# if you are creating new site, use
-# REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth"
-
+REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth"
+# for sites migrated from previous version, either wipe mastodon client ids or use:
+# REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/'

 # Path to save report related images, ends with slash
 REPORT_MEDIA_PATH_ROOT = "report/"
@ -261,7 +253,7 @@ SYNC_FILE_PATH_ROOT = "sync/"
 EXPORT_FILE_PATH_ROOT = "export/"

 # Allow user to login via any Mastodon/Pleroma sites
-MASTODON_ALLOW_ANY_SITE = False
+MASTODON_ALLOW_ANY_SITE = True

 # Allow user to create account with email (and link to Mastodon account later)
 ALLOW_EMAIL_ONLY_ACCOUNT = False
@ -312,7 +304,7 @@ DISCOGS_API_KEY = "***REMOVED***"

 # IGDB
 IGDB_CLIENT_ID = "deadbeef"
-IGDB_CLIENT_SECRET = "deadbeef"
+IGDB_CLIENT_SECRET = ""

 BLEACH_STRIP_COMMENTS = True
 BLEACH_STRIP_TAGS = True
@ -335,43 +327,45 @@ if DEBUG:
 # https://django-debug-toolbar.readthedocs.io/en/latest/
 # maybe benchmarking before deployment

-REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1")
+REDIS_HOST = os.environ.get("NEODB_REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379))
+REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0))

 RQ_QUEUES = {
    "mastodon": {
        "HOST": REDIS_HOST,
-        "PORT": 6379,
-        "DB": 0,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
    "export": {
        "HOST": REDIS_HOST,
-        "PORT": 6379,
-        "DB": 0,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
    "import": {
-        "HOST": "localhost",
-        "PORT": 6379,
-        "DB": 0,
+        "HOST": REDIS_HOST,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
    "fetch": {
-        "HOST": "localhost",
-        "PORT": 6379,
-        "DB": 0,
+        "HOST": REDIS_HOST,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
    "crawl": {
-        "HOST": "localhost",
-        "PORT": 6379,
-        "DB": 0,
+        "HOST": REDIS_HOST,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
    "doufen": {
        "HOST": REDIS_HOST,
-        "PORT": 6379,
-        "DB": 0,
+        "PORT": REDIS_PORT,
+        "DB": REDIS_DB,
        "DEFAULT_TIMEOUT": -1,
    },
 }
@ -380,19 +374,27 @@ RQ_SHOW_ADMIN_LINK = True

 SEARCH_INDEX_NEW_ONLY = False

+SEARCH_BACKEND = None

 # SEARCH_BACKEND = 'MEILISEARCH'
 # MEILISEARCH_SERVER = 'http://127.0.0.1:7700'
 # MEILISEARCH_KEY = 'deadbeef'

-# SEARCH_BACKEND = "TYPESENSE"
+if os.environ.get("NEODB_TYPESENSE_ENABLE", ""):
+    SEARCH_BACKEND = "TYPESENSE"
+
 TYPESENSE_CONNECTION = {
-    "api_key": "xyz",
-    "nodes": [{"host": "localhost", "port": "8108", "protocol": "http"}],
+    "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"),
+    "nodes": [
+        {
+            "host": os.environ.get("NEODB_TYPESENSE_HOST", "127.0.0.1"),
+            "port": os.environ.get("NEODB_TYPESENSE_PORT", "8108"),
+            "protocol": "http",
+        }
+    ],
    "connection_timeout_seconds": 2,
 }

-SEARCH_BACKEND = None

 DOWNLOADER_RETRIES = 3
 DOWNLOADER_SAVEDIR = None
--- a/catalog/sites/igdb.py
+++ b/catalog/sites/igdb.py
@ -19,6 +19,8 @@ _logger = logging.getLogger(__name__)


 def _igdb_access_token():
+    if not settings.IGDB_CLIENT_SECRET:
+        return "<missing>"
    try:
        token = requests.post(
            f"https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials"
--- a/doc/install.md
+++ b/doc/install.md
@ -3,7 +3,8 @@ NiceDB / NeoDB - Getting Start
 This is a very basic guide with limited detail, contributions welcomed

 ## Table of Contents
- [1 Install](#1-install)
+- [Run in Docker](#0-run-in-docker)
+- [1 Install](#1-manual-install)
  * [1.1 Database](#11-database)
  * [1.2 Configuration](#12-configuration)
  * [1.3 Packages and Build](#13-packages-and-build)
@ -15,8 +16,18 @@ This is a very basic guide with limited detail, contributions welcomed
 - [7 Frequently Asked Questions](#7-frequently-asked-questions)


-1 Install
-------
+
+0 Run in Docker
+---------------
+
+```
+cp neodb.env.dist neodb.env  # update this configuration
+
+docker-compose up
+```
+
+1 Manual Install
+----------------
 Install PostgreSQL, Redis and Python (3.10 or above) if not yet

 ### 1.1 Database
@ -123,12 +134,6 @@ Requeue failed import jobs
 rq requeue --all --queue import
 ```

-Run in Docker
-```
-docker-compose build
-docker-compose up
-```
-
 Run Tests
 ```
 coverage run --source='.' manage.py test
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,32 +1,110 @@
-version: '3'
+version: "3.4"
+
+# NEODB Docker Compose File
+#
+# Note: configuration here may not be secure for production usage
+
+x-shared:
+  neodb-service: &neodb-service
+    build: .
+    image: neodb:latest
+    env_file:
+      - neodb.env
+    environment:
+      - NEODB_DB_NAME=neodb
+      - NEODB_DB_USER=neodb
+      - NEODB_DB_PASSWORD=aubergine
+      - NEODB_DB_HOST=neodb-db
+      - NEODB_DB_PORT=5432
+      - NEODB_REDIS_HOST=neodb-redis
+      - NEODB_REDIS_PORT=6379
+      - NEODB_REDIS_DB=0
+      - NEODB_TYPESENSE_ENABLE=1
+      - NEODB_TYPESENSE_HOST=neodb-search
+      - NEODB_TYPESENSE_PORT=8108
+      - NEODB_TYPESENSE_KEY=eggplant
+      - NEODB_STATIC_ROOT=/www/static/
+      - NEODB_MEDIA_ROOT=/www/media/
+    restart: "on-failure"
+    volumes:
+      - ${NEODB_DATA:-../data}/neodb-media:/www/media
+    depends_on:
+      - neodb-redis
+      - neodb-db
+      - neodb-search

 services:
-  redis:
+  neodb-redis:
    image: redis:alpine
-
-  db:
-    image: postgres:14-alpine
-    volumes:
-      - /tmp/data/db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-
-  web:
-    build: .
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - .:/code
    ports:
-      - "8000:8000"
+      - "16379:6379"
+
+  neodb-search:
+    image: typesense/typesense:0.25.0
+    restart: "on-failure"
+    healthcheck:
+      test: ['CMD', 'curl', '-vf', 'http://127.0.0.1:8108/health']
+    # ports:
+    #   - "18108:8108"
    environment:
-      - DB_HOST=db
-      - DB_NAME=postgres
-      - DB_USER=postgres
-      - DB_PASSWORD=postgres
-      - REDIS_HOST=redis
-      - DJANGO_SETTINGS_MODULE=yoursettings.dev
+      GLOG_minloglevel: 2
+    volumes:
+      - ${NEODB_DATA:-../data}/typesense:/data
+    command: '--data-dir /data --api-key=eggplant'
+
+  neodb-db:
+    image: postgres:14-alpine
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'neodb']
+    volumes:
+      - ${NEODB_DATA:-../data}/neodb-data:/var/lib/postgresql/data
+    # ports:
+    #   - "15432:5432"
+    environment:
+      - POSTGRES_DB=neodb
+      - POSTGRES_USER=neodb
+      - POSTGRES_PASSWORD=aubergine
+
+  migration:
+    <<: *neodb-service
+    restart: "no"
+    command: python /neodb/manage.py migrate
    depends_on:
-      - db
-      - redis
+      neodb-db:
+        condition: service_healthy
+      neodb-search:
+        condition: service_started
+      neodb-redis:
+        condition: service_started
+
+  neodb-web:
+    <<: *neodb-service
+    # ports:
+    #   - "18000:8000"
+    command: gunicorn boofilsic.wsgi -w 8 --preload -b 0.0.0.0:8000
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-worker:
+    <<: *neodb-service
+    command: python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-worker-secondary:
+    <<: *neodb-service
+    command: python /neodb/manage.py rqworker --with-scheduler fetch crawl
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-nginx:
+    <<: *neodb-service
+    command: nginx
+    depends_on:
+      neodb-web:
+        condition: service_started
+    ports:
+      - "${NEODB_PORT:-8000}:8000"
--- a/misc/nginx.conf.d/neodb.conf
+++ b/misc/nginx.conf.d/neodb.conf
@ -0,0 +1,22 @@
+server {
+    server_name    neodb.social;
+    listen         8000;
+    location = /favicon.ico {
+        root /www;
+        access_log off; log_not_found off;
+    }
+    location / {
+        client_max_body_size 100M;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_pass http://neodb-web:8000;
+    }
+
+    location /static/ {
+        root /www;
+    }
+
+    location /media/ {
+        root /www;
+    }
+}
--- a/misc/www/robots.txt
+++ b/misc/www/robots.txt
@ -0,0 +1,2 @@
+User-agent: GPTBot
+Disallow: /review/
--- a/neodb.env.dist
+++ b/neodb.env.dist
@ -0,0 +1,6 @@
+NEODB_SECRET_KEY=change_me
+NEODB_SITE_NAME=Example Site
+NEODB_SITE_DOMAIN=example.site
+#NEODB_PORT=8000
+#NEODB_SSL=1
+#NEODB_DATA=/var/lib/neodb
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -5,4 +5,4 @@ django-stubs
 djlint~=1.32.1
 isort~=5.12.0
 pre-commit
-pyright
+pyright==1.1.322
--- a/requirements.txt
+++ b/requirements.txt
@ -24,6 +24,7 @@ dnspython
 easy-thumbnails
 filetype
 fontawesomefree
+gunicorn
 igdb-api-v4
 libsass
 listparser