update docker configuration; pin pyright to a working version for now

.gitignore

@@ -1,5 +1,7 @@
 .DS_Store
 .venv
+/.env
+/neodb.env
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,23 +1,31 @@
 # syntax=docker/dockerfile:1
-FROM python:3.8-slim
+FROM python:3.11-slim-bullseye
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
+COPY . /neodb
+WORKDIR /neodb
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends build-essential libpq-dev git \
+    && apt-get install -y --no-install-recommends \
-  && rm -rf /var/lib/apt/lists/*
+        build-essential \
-COPY requirements.txt /tmp/requirements.txt
+        libpq-dev \
-RUN pip install --no-cache-dir -r /tmp/requirements.txt \
+        busybox \
-    && rm -rf /tmp/requirements.txt \
+        postgresql-client \
-    && useradd -U app_user \
+        nginx \
-    && install -d -m 0755 -o app_user -g app_user /app/static
+        opencc \
+        git
+COPY misc/nginx.conf.d/* /etc/nginx/conf.d/
+RUN echo >> /etc/nginx/nginx.conf
+RUN echo 'daemon off;' >> /etc/nginx/nginx.conf
+RUN python3 -m pip install --no-cache-dir --upgrade -r requirements.txt
+RUN apt-get purge -y --auto-remove \
+        build-essential \
+        libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
 
-ENV DJANGO_SETTINGS_MODULE=yoursettings.dev
+RUN python3 manage.py compilescss \
-WORKDIR /app
+    && python3 manage.py collectstatic --noinput
-USER app_user:app_user
+RUN cp -R misc/www /www
-COPY --chown=app_user:app_user . .
+RUN mv static /www/static
-RUN chmod +x docker/*.sh
 
-# Section 6- Docker Run Checks and Configurations
+# invoke check by default
-ENTRYPOINT [ "docker/entrypoint.sh" ]
+CMD [ "python3", "/neodb/manage.py", "check" ]
-
-CMD [ "docker/start.sh", "server" ]

boofilsic/settings.py

@@ -7,17 +7,19 @@ PROJECT_ROOT = os.path.abspath(os.path.dirname(__name__))
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-# https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
-DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+# for legacy deployment:
+# DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
+# SECURITY WARNING: use your own secret key and keep it!
-SECRET_KEY = "nbv58c^&b8-095(^)&_BV98596v)&CX#^$&%*^V5"
+SECRET_KEY = os.environ.get("NEODB_SECRET_KEY", "insecure")
+
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = os.environ.get("NEODB_DEBUG", "") != ""
 
 ALLOWED_HOSTS = ["*"]
 
@@ -117,34 +119,20 @@ CACHES = {
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
-if DEBUG:
+DATABASES = {
-    DATABASES = {
+    "default": {
-        "default": {
+        "ENGINE": "django.db.backends.postgresql",
-            "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.environ.get("NEODB_DB_NAME", "test"),
-            "NAME": os.environ.get("DB_NAME", "test"),
+        "USER": os.environ.get("NEODB_DB_USER", "postgres"),
-            "USER": os.environ.get("DB_USER", "postgres"),
+        "PASSWORD": os.environ.get("NEODB_DB_PASSWORD", "admin123"),
-            "PASSWORD": os.environ.get("DB_PASSWORD", "admin123"),
+        "HOST": os.environ.get("NEODB_DB_HOST", "127.0.0.1"),
-            "HOST": os.environ.get("DB_HOST", "127.0.0.1"),
+        "PORT": int(os.environ.get("NEODB_DB_PORT", 5432)),
-            "OPTIONS": {
+        "OPTIONS": {
-                "client_encoding": "UTF8",
+            "client_encoding": "UTF8",
-                # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
+            # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
-            },
+        },
-        }
-    }
-else:
-    DATABASES = {
-        "default": {
-            "ENGINE": "django.db.backends.postgresql",
-            "NAME": "boofilsic",
-            "USER": "doubaniux",
-            "PASSWORD": "password",
-            "HOST": "localhost",
-            "OPTIONS": {
-                "client_encoding": "UTF8",
-                # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
-            },
-        }
     }
+}
 
 # Customized auth backend, glue OAuth2 and Django User model together
 # https://docs.djangoproject.com/en/3.0/topics/auth/customizing/#authentication-backends
@@ -172,14 +160,19 @@ USE_L10N = True
 USE_TZ = True
 
 
-if not DEBUG:
+USE_X_FORWARDED_HOST = True
-    SESSION_COOKIE_SECURE = True
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
-    CSRF_COOKIE_SECURE = True
+DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024
+CSRF_COOKIE_SECURE = True
+SESSION_COOKIE_SECURE = True
+
+if os.getenv("NEODB_SSL", "") != "":
     SECURE_SSL_REDIRECT = True
     SECURE_HSTS_PRELOAD = True
     SECURE_HSTS_INCLUDE_SUBDOMAINS = True
     SECURE_HSTS_SECONDS = 31536000
 
+if not DEBUG:
     LOGGING = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -208,7 +201,7 @@ if not DEBUG:
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
 STATIC_URL = "/static/"
-STATIC_ROOT = os.path.join(BASE_DIR, "static/")
+STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/"))
 
 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
 STATICFILES_FINDERS = [
@@ -224,23 +217,22 @@ SILENCED_SYSTEM_CHECKS = [
 ]
 
 MEDIA_URL = "/media/"
-MEDIA_ROOT = os.path.join(BASE_DIR, "media/")
+MEDIA_ROOT = os.environ.get("NEODB_MEDIA_ROOT", os.path.join(BASE_DIR, "media/"))
 
+SITE_DOMAIN = os.environ.get("NEODB_SITE_DOMAIN", "nicedb.org")
 SITE_INFO = {
-    "site_name": os.environ.get("APP_NAME", "NiceDB"),
+    "site_name": os.environ.get("NEODB_SITE_NAME", "NiceDB"),
-    "site_url": os.environ.get("APP_URL", "https://nicedb.org"),
+    "site_domain": SITE_DOMAIN,
+    "site_url": os.environ.get("NEODB_SITE_URL", "https://" + SITE_DOMAIN),
     "support_link": "https://github.com/doubaniux/boofilsic/issues",
     "social_link": "https://donotban.com/@testie",
     "donation_link": "https://patreon.com/tertius",
-    "version_hash": None,
     "settings_module": os.getenv("DJANGO_SETTINGS_MODULE"),
-    "sentry_dsn": None,
 }
 
-REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/'
+REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth"
-# if you are creating new site, use
+# for sites migrated from previous version, either wipe mastodon client ids or use:
-# REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth"
+# REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/'
-
 
 # Path to save report related images, ends with slash
 REPORT_MEDIA_PATH_ROOT = "report/"
@@ -261,7 +253,7 @@ SYNC_FILE_PATH_ROOT = "sync/"
 EXPORT_FILE_PATH_ROOT = "export/"
 
 # Allow user to login via any Mastodon/Pleroma sites
-MASTODON_ALLOW_ANY_SITE = False
+MASTODON_ALLOW_ANY_SITE = True
 
 # Allow user to create account with email (and link to Mastodon account later)
 ALLOW_EMAIL_ONLY_ACCOUNT = False
@@ -312,7 +304,7 @@ DISCOGS_API_KEY = "***REMOVED***"
 
 # IGDB
 IGDB_CLIENT_ID = "deadbeef"
-IGDB_CLIENT_SECRET = "deadbeef"
+IGDB_CLIENT_SECRET = ""
 
 BLEACH_STRIP_COMMENTS = True
 BLEACH_STRIP_TAGS = True
@@ -335,43 +327,45 @@ if DEBUG:
 # https://django-debug-toolbar.readthedocs.io/en/latest/
 # maybe benchmarking before deployment
 
-REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1")
+REDIS_HOST = os.environ.get("NEODB_REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379))
+REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0))
 
 RQ_QUEUES = {
     "mastodon": {
         "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
     "export": {
         "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
     "import": {
-        "HOST": "localhost",
+        "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
     "fetch": {
-        "HOST": "localhost",
+        "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
     "crawl": {
-        "HOST": "localhost",
+        "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
     "doufen": {
         "HOST": REDIS_HOST,
-        "PORT": 6379,
+        "PORT": REDIS_PORT,
-        "DB": 0,
+        "DB": REDIS_DB,
         "DEFAULT_TIMEOUT": -1,
     },
 }
@@ -380,19 +374,27 @@ RQ_SHOW_ADMIN_LINK = True
 
 SEARCH_INDEX_NEW_ONLY = False
 
+SEARCH_BACKEND = None
 
 # SEARCH_BACKEND = 'MEILISEARCH'
 # MEILISEARCH_SERVER = 'http://127.0.0.1:7700'
 # MEILISEARCH_KEY = 'deadbeef'
 
-# SEARCH_BACKEND = "TYPESENSE"
+if os.environ.get("NEODB_TYPESENSE_ENABLE", ""):
+    SEARCH_BACKEND = "TYPESENSE"
+
 TYPESENSE_CONNECTION = {
-    "api_key": "xyz",
+    "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"),
-    "nodes": [{"host": "localhost", "port": "8108", "protocol": "http"}],
+    "nodes": [
+        {
+            "host": os.environ.get("NEODB_TYPESENSE_HOST", "127.0.0.1"),
+            "port": os.environ.get("NEODB_TYPESENSE_PORT", "8108"),
+            "protocol": "http",
+        }
+    ],
     "connection_timeout_seconds": 2,
 }
 
-SEARCH_BACKEND = None
 
 DOWNLOADER_RETRIES = 3
 DOWNLOADER_SAVEDIR = None

catalog/sites/igdb.py

@@ -19,6 +19,8 @@ _logger = logging.getLogger(__name__)
 
 
 def _igdb_access_token():
+    if not settings.IGDB_CLIENT_SECRET:
+        return "<missing>"
     try:
         token = requests.post(
             f"https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials"

doc/install.md

@@ -3,7 +3,8 @@ NiceDB / NeoDB - Getting Start
 This is a very basic guide with limited detail, contributions welcomed
 
 ## Table of Contents
-- [1 Install](#1-install)
+- [Run in Docker](#0-run-in-docker)
+- [1 Install](#1-manual-install)
   * [1.1 Database](#11-database)
   * [1.2 Configuration](#12-configuration)
   * [1.3 Packages and Build](#13-packages-and-build)
@@ -15,8 +16,18 @@ This is a very basic guide with limited detail, contributions welcomed
 - [7 Frequently Asked Questions](#7-frequently-asked-questions)
 
 
-1 Install
+
--------
+0 Run in Docker
+---------------
+
+```
+cp neodb.env.dist neodb.env  # update this configuration
+
+docker-compose up
+```
+
+1 Manual Install
+----------------
 Install PostgreSQL, Redis and Python (3.10 or above) if not yet
 
 ### 1.1 Database
@@ -123,12 +134,6 @@ Requeue failed import jobs
 rq requeue --all --queue import
 ```
 
-Run in Docker
-```
-docker-compose build
-docker-compose up
-```
-
 Run Tests
 ```
 coverage run --source='.' manage.py test

docker-compose.yml

@@ -1,32 +1,110 @@
-version: '3'
+version: "3.4"
+
+# NEODB Docker Compose File
+#
+# Note: configuration here may not be secure for production usage
+
+x-shared:
+  neodb-service: &neodb-service
+    build: .
+    image: neodb:latest
+    env_file:
+      - neodb.env
+    environment:
+      - NEODB_DB_NAME=neodb
+      - NEODB_DB_USER=neodb
+      - NEODB_DB_PASSWORD=aubergine
+      - NEODB_DB_HOST=neodb-db
+      - NEODB_DB_PORT=5432
+      - NEODB_REDIS_HOST=neodb-redis
+      - NEODB_REDIS_PORT=6379
+      - NEODB_REDIS_DB=0
+      - NEODB_TYPESENSE_ENABLE=1
+      - NEODB_TYPESENSE_HOST=neodb-search
+      - NEODB_TYPESENSE_PORT=8108
+      - NEODB_TYPESENSE_KEY=eggplant
+      - NEODB_STATIC_ROOT=/www/static/
+      - NEODB_MEDIA_ROOT=/www/media/
+    restart: "on-failure"
+    volumes:
+      - ${NEODB_DATA:-../data}/neodb-media:/www/media
+    depends_on:
+      - neodb-redis
+      - neodb-db
+      - neodb-search
 
 services:
-  redis:
+  neodb-redis:
     image: redis:alpine
-
-  db:
-    image: postgres:14-alpine
-    volumes:
-      - /tmp/data/db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-
-  web:
-    build: .
-    command: python manage.py runserver 0.0.0.0:8000
-    volumes:
-      - .:/code
     ports:
-      - "8000:8000"
+      - "16379:6379"
+
+  neodb-search:
+    image: typesense/typesense:0.25.0
+    restart: "on-failure"
+    healthcheck:
+      test: ['CMD', 'curl', '-vf', 'http://127.0.0.1:8108/health']
+    # ports:
+    #   - "18108:8108"
     environment:
-      - DB_HOST=db
+      GLOG_minloglevel: 2
-      - DB_NAME=postgres
+    volumes:
-      - DB_USER=postgres
+      - ${NEODB_DATA:-../data}/typesense:/data
-      - DB_PASSWORD=postgres
+    command: '--data-dir /data --api-key=eggplant'
-      - REDIS_HOST=redis
+
-      - DJANGO_SETTINGS_MODULE=yoursettings.dev
+  neodb-db:
+    image: postgres:14-alpine
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'neodb']
+    volumes:
+      - ${NEODB_DATA:-../data}/neodb-data:/var/lib/postgresql/data
+    # ports:
+    #   - "15432:5432"
+    environment:
+      - POSTGRES_DB=neodb
+      - POSTGRES_USER=neodb
+      - POSTGRES_PASSWORD=aubergine
+
+  migration:
+    <<: *neodb-service
+    restart: "no"
+    command: python /neodb/manage.py migrate
     depends_on:
-      - db
+      neodb-db:
-      - redis
+        condition: service_healthy
+      neodb-search:
+        condition: service_started
+      neodb-redis:
+        condition: service_started
+
+  neodb-web:
+    <<: *neodb-service
+    # ports:
+    #   - "18000:8000"
+    command: gunicorn boofilsic.wsgi -w 8 --preload -b 0.0.0.0:8000
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-worker:
+    <<: *neodb-service
+    command: python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-worker-secondary:
+    <<: *neodb-service
+    command: python /neodb/manage.py rqworker --with-scheduler fetch crawl
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+
+  neodb-nginx:
+    <<: *neodb-service
+    command: nginx
+    depends_on:
+      neodb-web:
+        condition: service_started
+    ports:
+      - "${NEODB_PORT:-8000}:8000"

misc/nginx.conf.d/neodb.conf

@@ -0,0 +1,22 @@
+server {
+    server_name    neodb.social;
+    listen         8000;
+    location = /favicon.ico {
+        root /www;
+        access_log off; log_not_found off;
+    }
+    location / {
+        client_max_body_size 100M;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_pass http://neodb-web:8000;
+    }
+
+    location /static/ {
+        root /www;
+    }
+
+    location /media/ {
+        root /www;
+    }
+}

misc/www/robots.txt

@@ -0,0 +1,2 @@
+User-agent: GPTBot
+Disallow: /review/

neodb.env.dist

@@ -0,0 +1,6 @@
+NEODB_SECRET_KEY=change_me
+NEODB_SITE_NAME=Example Site
+NEODB_SITE_DOMAIN=example.site
+#NEODB_PORT=8000
+#NEODB_SSL=1
+#NEODB_DATA=/var/lib/neodb

requirements-dev.txt

@@ -5,4 +5,4 @@ django-stubs
 djlint~=1.32.1
 isort~=5.12.0
 pre-commit
-pyright
+pyright==1.1.322

requirements.txt

@@ -24,6 +24,7 @@ dnspython
 easy-thumbnails
 filetype
 fontawesomefree
+gunicorn
 igdb-api-v4
 libsass
 listparser