add checks when manage.py check --deploy

boofilsic/settings.py

@@ -85,6 +85,8 @@ env = environ.FileAwareEnv(
     DISCORD_WEBHOOKS=(dict, {"user-report": None}),
     # Slack API token, for sending exceptions to Slack, may deprecate in future
     SLACK_API_TOKEN=(str, ""),
+    # SSL only, better be True for production security
+    SSL_ONLY=(bool, False),
     NEODB_SENTRY_DSN=(str, ""),
     NEODB_FANOUT_LIMIT_DAYS=(int, 9),
 )
@@ -101,8 +103,8 @@ DATABASES["default"]["OPTIONS"] = {"client_encoding": "UTF8"}
 DATABASES["default"]["TEST"] = {"DEPENDENCIES": ["takahe"]}
 DATABASES["takahe"]["OPTIONS"] = {"client_encoding": "UTF8"}
 DATABASES["takahe"]["TEST"] = {"DEPENDENCIES": []}
+REDIS_URL = env("NEODB_REDIS_URL")
 CACHES = {"default": env.cache_url("NEODB_REDIS_URL")}
-
 _parsed_redis_url = env.url("NEODB_REDIS_URL")
 RQ_QUEUES = {
     q: {
@@ -375,13 +377,11 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024
 CSRF_COOKIE_SECURE = True
 SESSION_COOKIE_SECURE = True
-
-if env("NEODB_SSL", default="") != "":  # type: ignore
-    # FIXME: remove this since user may enforce SSL in reverse proxy
-    SECURE_SSL_REDIRECT = True
-    SECURE_HSTS_PRELOAD = True
-    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
-    SECURE_HSTS_SECONDS = 31536000
+SSL_ONLY = env("SSL_ONLY")
+SECURE_SSL_REDIRECT = SSL_ONLY
+SECURE_HSTS_PRELOAD = SSL_ONLY
+SECURE_HSTS_INCLUDE_SUBDOMAINS = SSL_ONLY
+SECURE_HSTS_SECONDS = 2592000 if SSL_ONLY else 0
 
 STATIC_URL = "/s/"
 STATIC_ROOT = env("NEODB_STATIC_ROOT", default=os.path.join(BASE_DIR, "static/"))  # type: ignore

catalog/search/models.py

@@ -19,6 +19,10 @@ _logger = logging.getLogger(__name__)
 
 
 class DbIndexer:
+    @classmethod
+    def check(cls):
+        pass
+
     @classmethod
     def init(cls):
         pass

catalog/search/typesense.py

@@ -181,6 +181,17 @@ class Indexer:
             # "default_sorting_field": "rating_count",
         }
 
+    @classmethod
+    def check(cls):
+        client = typesense.Client(settings.TYPESENSE_CONNECTION)
+        wait = 5
+        if not client.operations.is_healthy():
+            raise ValueError("Typesense: server not healthy")
+        idx = client.collections[settings.TYPESENSE_INDEX_NAME]
+        if not idx:
+            raise ValueError("Typesense: index not found")
+        return idx.retrieve()
+
     @classmethod
     def init(cls):
         try:

common/apps.py

@@ -1,4 +1,5 @@
 from django.apps import AppConfig
+from django.core.checks import Tags, register
 from django.db.models.signals import post_migrate
 
 
@@ -12,3 +13,10 @@ class CommonConfig(AppConfig):
         from .setup import Setup
 
         Setup().run()
+
+
+@register(Tags.admin, deploy=True)
+def setup_check(app_configs, **kwargs):
+    from .setup import Setup
+
+    return Setup().check()

common/setup.py

@@ -1,4 +1,6 @@
+import django
 from django.conf import settings
+from django.core.checks import Error, Warning
 from loguru import logger
 
 from catalog.search.models import Indexer
@@ -158,3 +160,62 @@ class Setup:
             JobManager.schedule_all()
 
         logger.info("Finished post-migration setup.")
+
+    def check(self):
+        from redis import Redis
+
+        errors = []
+        # check env
+        domain = settings.SITE_INFO.get("site_domain")
+        if not domain:
+            errors.append(
+                Error(
+                    "SITE DOMAIN is not specified",
+                    hint="Check NEODB_SITE_DOMAIN in .env",
+                    id="neodb.E001",
+                )
+            )
+        # check redis
+        try:
+            redis = Redis.from_url(settings.REDIS_URL)
+            if not redis:
+                raise Exception("Redis unavailable")
+            redis.ping()
+        except Exception as e:
+            errors.append(
+                Error(
+                    f"Error while connecting to redis: {e}",
+                    hint="Check NEODB_REDIS_URL in .env",
+                    id="neodb.E002",
+                )
+            )
+        # check indexer
+        try:
+            Indexer.check()
+        except Exception as e:
+            errors.append(
+                Error(
+                    f"Error while connecting to elasticsearch: {e}",
+                    hint="Check ELASTICSEARCH_URL in .env",
+                    id="neodb.E003",
+                )
+            )
+        # check takahe
+        try:
+            if not TakaheDomain.objects.filter(domain=domain).exists():
+                errors.append(
+                    Warning(
+                        f"Domain {domain} not found in takahe database",
+                        hint="Run migration once to create the domain",
+                        id="neodb.W001",
+                    )
+                )
+        except Exception as e:
+            errors.append(
+                Error(
+                    f"Error while querying Takahe database: {e}",
+                    hint="Check TAKAHE_DB_URL in .env",
+                    id="neodb.E004",
+                )
+            )
+        return errors

compose.yml

@@ -65,6 +65,7 @@ x-shared:
       IGDB_API_CLIENT_SECRET:
       DISCORD_WEBHOOKS:
       SLACK_API_TOKEN:
+      SSL_ONLY:
     restart: "on-failure"
     volumes:
       - ${NEODB_DATA:-../data}/neodb-media:/www/m

misc/bin/neodb-hello

@@ -5,7 +5,7 @@ echo Your configuration is for ${NEODB_SITE_NAME} on ${NEODB_SITE_DOMAIN}
 [[ -z "${NEODB_DEBUG}" ]] || echo DEBUG is ON, showing environment variables:
 [[ -z "${NEODB_DEBUG}" ]] || env
 [[ -z "${NEODB_DEBUG}" ]] || echo Running some basic checks...
-[[ -z "${NEODB_DEBUG}" ]] || neodb-manage check
+[[ -z "${NEODB_DEBUG}" ]] || neodb-manage check --database default --database takahe --deploy
 [[ -z "${NEODB_DEBUG}" ]] || TAKAHE_DATABASE_SERVER="postgres://x@y/z" TAKAHE_SECRET_KEY="t" TAKAHE_MAIN_DOMAIN="x.y" takahe-manage check
 [[ -z "${NEODB_DEBUG}" ]] || echo check complete.
 cat <<EOF