diff --git a/.gitignore b/.gitignore index cf53e58f..d33a3b7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .DS_Store .venv +/.env +/neodb.env # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Dockerfile b/Dockerfile index 96162a1c..8c18f1be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,31 @@ # syntax=docker/dockerfile:1 -FROM python:3.8-slim +FROM python:3.11-slim-bullseye ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 +COPY . /neodb +WORKDIR /neodb RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential libpq-dev git \ - && rm -rf /var/lib/apt/lists/* -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt \ - && rm -rf /tmp/requirements.txt \ - && useradd -U app_user \ - && install -d -m 0755 -o app_user -g app_user /app/static + && apt-get install -y --no-install-recommends \ + build-essential \ + libpq-dev \ + busybox \ + postgresql-client \ + nginx \ + opencc \ + git +COPY misc/nginx.conf.d/* /etc/nginx/conf.d/ +RUN echo >> /etc/nginx/nginx.conf +RUN echo 'daemon off;' >> /etc/nginx/nginx.conf +RUN python3 -m pip install --no-cache-dir --upgrade -r requirements.txt +RUN apt-get purge -y --auto-remove \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* -ENV DJANGO_SETTINGS_MODULE=yoursettings.dev -WORKDIR /app -USER app_user:app_user -COPY --chown=app_user:app_user . . -RUN chmod +x docker/*.sh +RUN python3 manage.py compilescss \ + && python3 manage.py collectstatic --noinput +RUN cp -R misc/www /www +RUN mv static /www/static -# Section 6- Docker Run Checks and Configurations -ENTRYPOINT [ "docker/entrypoint.sh" ] - -CMD [ "docker/start.sh", "server" ] +# invoke check by default +CMD [ "python3", "/neodb/manage.py", "check" ] diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 2904b9b0..7449bb02 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -7,17 +7,19 @@ PROJECT_ROOT = os.path.abspath(os.path.dirname(__name__)) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -# https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys -DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" +# for legacy deployment: +# DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "nbv58c^&b8-095(^)&_BV98596v)&CX#^$&%*^V5" +# SECURITY WARNING: use your own secret key and keep it! +SECRET_KEY = os.environ.get("NEODB_SECRET_KEY", "insecure") + # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = os.environ.get("NEODB_DEBUG", "") != "" ALLOWED_HOSTS = ["*"] @@ -117,34 +119,20 @@ CACHES = { # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases -if DEBUG: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("DB_NAME", "test"), - "USER": os.environ.get("DB_USER", "postgres"), - "PASSWORD": os.environ.get("DB_PASSWORD", "admin123"), - "HOST": os.environ.get("DB_HOST", "127.0.0.1"), - "OPTIONS": { - "client_encoding": "UTF8", - # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, - }, - } - } -else: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "boofilsic", - "USER": "doubaniux", - "PASSWORD": "password", - "HOST": "localhost", - "OPTIONS": { - "client_encoding": "UTF8", - # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, - }, - } +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("NEODB_DB_NAME", "test"), + "USER": os.environ.get("NEODB_DB_USER", "postgres"), + "PASSWORD": os.environ.get("NEODB_DB_PASSWORD", "admin123"), + "HOST": os.environ.get("NEODB_DB_HOST", "127.0.0.1"), + "PORT": int(os.environ.get("NEODB_DB_PORT", 5432)), + "OPTIONS": { + "client_encoding": "UTF8", + # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, + }, } +} # Customized auth backend, glue OAuth2 and Django User model together # https://docs.djangoproject.com/en/3.0/topics/auth/customizing/#authentication-backends @@ -172,14 +160,19 @@ USE_L10N = True USE_TZ = True -if not DEBUG: - SESSION_COOKIE_SECURE = True - CSRF_COOKIE_SECURE = True +USE_X_FORWARDED_HOST = True +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 +CSRF_COOKIE_SECURE = True +SESSION_COOKIE_SECURE = True + +if os.getenv("NEODB_SSL", "") != "": SECURE_SSL_REDIRECT = True SECURE_HSTS_PRELOAD = True SECURE_HSTS_INCLUDE_SUBDOMAINS = True SECURE_HSTS_SECONDS = 31536000 +if not DEBUG: LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -208,7 +201,7 @@ if not DEBUG: # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_URL = "/static/" -STATIC_ROOT = os.path.join(BASE_DIR, "static/") +STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/")) STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" STATICFILES_FINDERS = [ @@ -224,23 +217,22 @@ SILENCED_SYSTEM_CHECKS = [ ] MEDIA_URL = "/media/" -MEDIA_ROOT = os.path.join(BASE_DIR, "media/") +MEDIA_ROOT = os.environ.get("NEODB_MEDIA_ROOT", os.path.join(BASE_DIR, "media/")) +SITE_DOMAIN = os.environ.get("NEODB_SITE_DOMAIN", "nicedb.org") SITE_INFO = { - "site_name": os.environ.get("APP_NAME", "NiceDB"), - "site_url": os.environ.get("APP_URL", "https://nicedb.org"), + "site_name": os.environ.get("NEODB_SITE_NAME", "NiceDB"), + "site_domain": SITE_DOMAIN, + "site_url": os.environ.get("NEODB_SITE_URL", "https://" + SITE_DOMAIN), "support_link": "https://github.com/doubaniux/boofilsic/issues", "social_link": "https://donotban.com/@testie", "donation_link": "https://patreon.com/tertius", - "version_hash": None, "settings_module": os.getenv("DJANGO_SETTINGS_MODULE"), - "sentry_dsn": None, } -REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/' -# if you are creating new site, use -# REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth" - +REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth" +# for sites migrated from previous version, either wipe mastodon client ids or use: +# REDIRECT_URIS = f'{SITE_INFO["site_url"]}/users/OAuth2_login/' # Path to save report related images, ends with slash REPORT_MEDIA_PATH_ROOT = "report/" @@ -261,7 +253,7 @@ SYNC_FILE_PATH_ROOT = "sync/" EXPORT_FILE_PATH_ROOT = "export/" # Allow user to login via any Mastodon/Pleroma sites -MASTODON_ALLOW_ANY_SITE = False +MASTODON_ALLOW_ANY_SITE = True # Allow user to create account with email (and link to Mastodon account later) ALLOW_EMAIL_ONLY_ACCOUNT = False @@ -312,7 +304,7 @@ DISCOGS_API_KEY = "***REMOVED***" # IGDB IGDB_CLIENT_ID = "deadbeef" -IGDB_CLIENT_SECRET = "deadbeef" +IGDB_CLIENT_SECRET = "" BLEACH_STRIP_COMMENTS = True BLEACH_STRIP_TAGS = True @@ -335,43 +327,45 @@ if DEBUG: # https://django-debug-toolbar.readthedocs.io/en/latest/ # maybe benchmarking before deployment -REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1") +REDIS_HOST = os.environ.get("NEODB_REDIS_HOST", "127.0.0.1") +REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379)) +REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0)) RQ_QUEUES = { "mastodon": { "HOST": REDIS_HOST, - "PORT": 6379, - "DB": 0, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, "export": { "HOST": REDIS_HOST, - "PORT": 6379, - "DB": 0, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, "import": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, + "HOST": REDIS_HOST, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, "fetch": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, + "HOST": REDIS_HOST, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, "crawl": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, + "HOST": REDIS_HOST, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, "doufen": { "HOST": REDIS_HOST, - "PORT": 6379, - "DB": 0, + "PORT": REDIS_PORT, + "DB": REDIS_DB, "DEFAULT_TIMEOUT": -1, }, } @@ -380,19 +374,27 @@ RQ_SHOW_ADMIN_LINK = True SEARCH_INDEX_NEW_ONLY = False +SEARCH_BACKEND = None # SEARCH_BACKEND = 'MEILISEARCH' # MEILISEARCH_SERVER = 'http://127.0.0.1:7700' # MEILISEARCH_KEY = 'deadbeef' -# SEARCH_BACKEND = "TYPESENSE" +if os.environ.get("NEODB_TYPESENSE_ENABLE", ""): + SEARCH_BACKEND = "TYPESENSE" + TYPESENSE_CONNECTION = { - "api_key": "xyz", - "nodes": [{"host": "localhost", "port": "8108", "protocol": "http"}], + "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"), + "nodes": [ + { + "host": os.environ.get("NEODB_TYPESENSE_HOST", "127.0.0.1"), + "port": os.environ.get("NEODB_TYPESENSE_PORT", "8108"), + "protocol": "http", + } + ], "connection_timeout_seconds": 2, } -SEARCH_BACKEND = None DOWNLOADER_RETRIES = 3 DOWNLOADER_SAVEDIR = None diff --git a/catalog/sites/igdb.py b/catalog/sites/igdb.py index e1eaef97..7c64b572 100644 --- a/catalog/sites/igdb.py +++ b/catalog/sites/igdb.py @@ -19,6 +19,8 @@ _logger = logging.getLogger(__name__) def _igdb_access_token(): + if not settings.IGDB_CLIENT_SECRET: + return "" try: token = requests.post( f"https://id.twitch.tv/oauth2/token?client_id={settings.IGDB_CLIENT_ID}&client_secret={settings.IGDB_CLIENT_SECRET}&grant_type=client_credentials" diff --git a/doc/install.md b/doc/install.md index 917cbf03..87d8bc59 100644 --- a/doc/install.md +++ b/doc/install.md @@ -3,7 +3,8 @@ NiceDB / NeoDB - Getting Start This is a very basic guide with limited detail, contributions welcomed ## Table of Contents -- [1 Install](#1-install) +- [Run in Docker](#0-run-in-docker) +- [1 Install](#1-manual-install) * [1.1 Database](#11-database) * [1.2 Configuration](#12-configuration) * [1.3 Packages and Build](#13-packages-and-build) @@ -15,8 +16,18 @@ This is a very basic guide with limited detail, contributions welcomed - [7 Frequently Asked Questions](#7-frequently-asked-questions) -1 Install -------- + +0 Run in Docker +--------------- + +``` +cp neodb.env.dist neodb.env # update this configuration + +docker-compose up +``` + +1 Manual Install +---------------- Install PostgreSQL, Redis and Python (3.10 or above) if not yet ### 1.1 Database @@ -123,12 +134,6 @@ Requeue failed import jobs rq requeue --all --queue import ``` -Run in Docker -``` -docker-compose build -docker-compose up -``` - Run Tests ``` coverage run --source='.' manage.py test diff --git a/docker-compose.yml b/docker-compose.yml index 01ebbb32..18caaa3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,110 @@ -version: '3' +version: "3.4" + +# NEODB Docker Compose File +# +# Note: configuration here may not be secure for production usage + +x-shared: + neodb-service: &neodb-service + build: . + image: neodb:latest + env_file: + - neodb.env + environment: + - NEODB_DB_NAME=neodb + - NEODB_DB_USER=neodb + - NEODB_DB_PASSWORD=aubergine + - NEODB_DB_HOST=neodb-db + - NEODB_DB_PORT=5432 + - NEODB_REDIS_HOST=neodb-redis + - NEODB_REDIS_PORT=6379 + - NEODB_REDIS_DB=0 + - NEODB_TYPESENSE_ENABLE=1 + - NEODB_TYPESENSE_HOST=neodb-search + - NEODB_TYPESENSE_PORT=8108 + - NEODB_TYPESENSE_KEY=eggplant + - NEODB_STATIC_ROOT=/www/static/ + - NEODB_MEDIA_ROOT=/www/media/ + restart: "on-failure" + volumes: + - ${NEODB_DATA:-../data}/neodb-media:/www/media + depends_on: + - neodb-redis + - neodb-db + - neodb-search services: - redis: + neodb-redis: image: redis:alpine - - db: - image: postgres:14-alpine - volumes: - - /tmp/data/db:/var/lib/postgresql/data - environment: - - POSTGRES_DB=postgres - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - web: - build: . - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/code ports: - - "8000:8000" + - "16379:6379" + + neodb-search: + image: typesense/typesense:0.25.0 + restart: "on-failure" + healthcheck: + test: ['CMD', 'curl', '-vf', 'http://127.0.0.1:8108/health'] + # ports: + # - "18108:8108" environment: - - DB_HOST=db - - DB_NAME=postgres - - DB_USER=postgres - - DB_PASSWORD=postgres - - REDIS_HOST=redis - - DJANGO_SETTINGS_MODULE=yoursettings.dev + GLOG_minloglevel: 2 + volumes: + - ${NEODB_DATA:-../data}/typesense:/data + command: '--data-dir /data --api-key=eggplant' + + neodb-db: + image: postgres:14-alpine + healthcheck: + test: ['CMD', 'pg_isready', '-U', 'neodb'] + volumes: + - ${NEODB_DATA:-../data}/neodb-data:/var/lib/postgresql/data + # ports: + # - "15432:5432" + environment: + - POSTGRES_DB=neodb + - POSTGRES_USER=neodb + - POSTGRES_PASSWORD=aubergine + + migration: + <<: *neodb-service + restart: "no" + command: python /neodb/manage.py migrate depends_on: - - db - - redis + neodb-db: + condition: service_healthy + neodb-search: + condition: service_started + neodb-redis: + condition: service_started + + neodb-web: + <<: *neodb-service + # ports: + # - "18000:8000" + command: gunicorn boofilsic.wsgi -w 8 --preload -b 0.0.0.0:8000 + depends_on: + migration: + condition: service_completed_successfully + + neodb-worker: + <<: *neodb-service + command: python /neodb/manage.py rqworker --with-scheduler import export mastodon fetch crawl + depends_on: + migration: + condition: service_completed_successfully + + neodb-worker-secondary: + <<: *neodb-service + command: python /neodb/manage.py rqworker --with-scheduler fetch crawl + depends_on: + migration: + condition: service_completed_successfully + + neodb-nginx: + <<: *neodb-service + command: nginx + depends_on: + neodb-web: + condition: service_started + ports: + - "${NEODB_PORT:-8000}:8000" diff --git a/misc/nginx.conf.d/neodb.conf b/misc/nginx.conf.d/neodb.conf new file mode 100644 index 00000000..42e8e001 --- /dev/null +++ b/misc/nginx.conf.d/neodb.conf @@ -0,0 +1,22 @@ +server { + server_name neodb.social; + listen 8000; + location = /favicon.ico { + root /www; + access_log off; log_not_found off; + } + location / { + client_max_body_size 100M; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto https; + proxy_pass http://neodb-web:8000; + } + + location /static/ { + root /www; + } + + location /media/ { + root /www; + } +} diff --git a/misc/www/robots.txt b/misc/www/robots.txt new file mode 100644 index 00000000..80eef49a --- /dev/null +++ b/misc/www/robots.txt @@ -0,0 +1,2 @@ +User-agent: GPTBot +Disallow: /review/ diff --git a/neodb.env.dist b/neodb.env.dist new file mode 100644 index 00000000..ecb44ed3 --- /dev/null +++ b/neodb.env.dist @@ -0,0 +1,6 @@ +NEODB_SECRET_KEY=change_me +NEODB_SITE_NAME=Example Site +NEODB_SITE_DOMAIN=example.site +#NEODB_PORT=8000 +#NEODB_SSL=1 +#NEODB_DATA=/var/lib/neodb diff --git a/requirements-dev.txt b/requirements-dev.txt index 381a219a..b146c739 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ django-stubs djlint~=1.32.1 isort~=5.12.0 pre-commit -pyright +pyright==1.1.322 diff --git a/requirements.txt b/requirements.txt index c24b2e6a..98d75b85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ dnspython easy-thumbnails filetype fontawesomefree +gunicorn igdb-api-v4 libsass listparser