From 65a2b823834c0f2df4f76b13206f31dca14707b9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Aug 2023 05:48:14 +0000 Subject: [PATCH] more configurable --- .github/workflows/docker-dev.yml | 2 +- boofilsic/settings.py | 305 +++++++++++++++--------------- catalog/common/downloaders.py | 23 +-- common/templates/_header.html | 2 +- common/templates/common_libs.html | 2 +- doc/development.md | 12 +- doc/install-docker.md | 8 +- docker-compose.yml | 4 +- neodb.env.example | 11 +- users/models/apidentity.py | 3 +- users/models/user.py | 4 +- 11 files changed, 186 insertions(+), 190 deletions(-) diff --git a/.github/workflows/docker-dev.yml b/.github/workflows/docker-dev.yml index d659eb9d..0290863d 100644 --- a/.github/workflows/docker-dev.yml +++ b/.github/workflows/docker-dev.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: push_to_docker_hub: name: build image and push to Docker Hub - if: github.repository_owner == 'neodb-social' + # if: github.repository_owner == 'neodb-social' runs-on: ubuntu-latest steps: - name: Check out the repo diff --git a/boofilsic/settings.py b/boofilsic/settings.py index edbefdcb..32318f24 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -6,6 +6,149 @@ env = environ.Env( # set casting, default value DEBUG=(bool, False) ) +# TODO: use django-environ or similar package for env parsing + +# ====== USER CONFIGUTRATION START ====== + +# SECURITY WARNING: use your own secret key and keep it! +SECRET_KEY = os.environ.get("NEODB_SECRET_KEY", "insecure") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = os.environ.get("NEODB_DEBUG", "") != "" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("NEODB_DB_NAME", "test_neodb"), + "USER": os.environ.get("NEODB_DB_USER", "testuser"), + "PASSWORD": os.environ.get("NEODB_DB_PASSWORD", "testpass"), + "HOST": os.environ.get("NEODB_DB_HOST", "127.0.0.1"), + "PORT": int(os.environ.get("NEODB_DB_PORT", 5432)), + "OPTIONS": { + "client_encoding": "UTF8", + # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, + }, + "TEST": { + "DEPENDENCIES": ["takahe"], + }, + }, + "takahe": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("TAKAHE_DB_NAME", "test_neodb_takahe"), + "USER": os.environ.get("TAKAHE_DB_USER", "testuser"), + "PASSWORD": os.environ.get("TAKAHE_DB_PASSWORD", "testpass"), + "HOST": os.environ.get("TAKAHE_DB_HOST", "127.0.0.1"), + "PORT": os.environ.get("TAKAHE_DB_PORT", 15432), + "OPTIONS": { + "client_encoding": "UTF8", + # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, + }, + "TEST": { + "DEPENDENCIES": [], + }, + }, +} + +REDIS_HOST = os.environ.get("NEODB_REDIS_HOST", "127.0.0.1") +REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379)) +REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0)) + +if os.environ.get("NEODB_TYPESENSE_ENABLE", ""): + SEARCH_BACKEND = "TYPESENSE" + +SEARCH_BACKEND = None + +# SEARCH_BACKEND = 'MEILISEARCH' +# MEILISEARCH_SERVER = 'http://127.0.0.1:7700' +# MEILISEARCH_KEY = 'deadbeef' + +TYPESENSE_CONNECTION = { + "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"), + "nodes": [ + { + "host": os.environ.get("NEODB_TYPESENSE_HOST", "127.0.0.1"), + "port": os.environ.get("NEODB_TYPESENSE_PORT", "8108"), + "protocol": "http", + } + ], + "connection_timeout_seconds": 2, +} + +SITE_DOMAIN = os.environ.get("NEODB_SITE_DOMAIN", "nicedb.org") +SITE_INFO = { + "site_name": os.environ.get("NEODB_SITE_NAME", "NiceDB"), + "site_domain": SITE_DOMAIN, + "site_url": os.environ.get("NEODB_SITE_URL", "https://" + SITE_DOMAIN), + "site_logo": os.environ.get("NEODB_SITE_LOGO", "/s/img/logo.svg"), + "site_icon": os.environ.get("NEODB_SITE_ICON", "/s/img/logo.svg"), + "user_icon": os.environ.get("NEODB_USER_ICON", "/s/img/avatar.svg"), + "social_link": "https://donotban.com/@testie", + "support_link": "https://github.com/doubaniux/boofilsic/issues", + "donation_link": "https://patreon.com/tertius", +} + +SETUP_ADMIN_USERNAMES = [ + u for u in os.environ.get("NEODB_ADMIN_USERNAMES", "").split(",") if u +] + + +# Mastodon/Pleroma instance allowed to login, keep empty to allow any instance to login +MASTODON_ALLOWED_SITES = [] + +# Allow user to create account with email (and link to Mastodon account later) +ALLOW_EMAIL_ONLY_ACCOUNT = False + +# Timeout of requests to Mastodon, in seconds +MASTODON_TIMEOUT = 30 + +MASTODON_CLIENT_SCOPE = "read:accounts read:follows read:search read:blocks read:mutes write:statuses write:media" +# use the following to avoid re-authorize when migrating to a future version with more features +# MASTODON_CLIENT_SCOPE = "read write follow" + +# some Mastodon-compatible software like Pixelfed does not support granular scopes +MASTODON_LEGACY_CLIENT_SCOPE = "read write follow" + +# Emoji code in mastodon +STAR_SOLID = ":star_solid:" +STAR_HALF = ":star_half:" +STAR_EMPTY = ":star_empty:" + +DISCORD_WEBHOOKS = {"user-report": None} + +# Spotify credentials +SPOTIFY_CREDENTIAL = "***REMOVED***" + +# The Movie Database (TMDB) API Keys +TMDB_API3_KEY = "***REMOVED***" +# TMDB_API4_KEY = "deadbeef.deadbeef.deadbeef" + +# Google Books API Key +GOOGLE_API_KEY = "***REMOVED***" + +# Discogs API Key +# How to get: a personal access token from https://www.discogs.com/settings/developers +DISCOGS_API_KEY = "***REMOVED***" + +# IGDB +IGDB_CLIENT_ID = "deadbeef" +IGDB_CLIENT_SECRET = "" + +# List of available proxies for proxy downloader, in format of ["http://x.y:port?url=__URL__", ...] +DOWNLOADER_PROXY_LIST = [ + u for u in os.environ.get("NEODB_DOWNLOADER_PROXY_LIST", "").split(",") if u +] + +# Backup proxy for proxy downloader, in format of "http://xyz:port?url=__URL__" +DOWNLOADER_BACKUP_PROXY = os.environ.get("NEODB_DOWNLOADER_BACKUP_PROXY", "") + +# Timeout of downloader requests, in seconds +DOWNLOADER_REQUEST_TIMEOUT = 90 +# Timeout of downloader cache, in seconds +DOWNLOADER_CACHE_TIMEOUT = 300 +# Number of retries of downloader, when site is using RetryDownloader +DOWNLOADER_RETRIES = 3 + +# ====== USER CONFIGUTRATION END ====== NEODB_VERSION = "0.8" DATABASE_ROUTERS = ["takahe.db_routes.TakaheRouter"] @@ -17,16 +160,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" # for legacy deployment: # DEFAULT_AUTO_FIELD = "django.db.models.AutoField" -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ - -# SECURITY WARNING: use your own secret key and keep it! -SECRET_KEY = os.environ.get("NEODB_SECRET_KEY", "insecure") - - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.environ.get("NEODB_DEBUG", "") != "" - ALLOWED_HOSTS = ["*"] # To allow debug in template context @@ -118,51 +251,6 @@ WSGI_APPLICATION = "boofilsic.wsgi.application" SESSION_COOKIE_NAME = "neodbsid" -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - } -} - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("NEODB_DB_NAME", "test_neodb"), - "USER": os.environ.get("NEODB_DB_USER", "testuser"), - "PASSWORD": os.environ.get("NEODB_DB_PASSWORD", "testpass"), - "HOST": os.environ.get("NEODB_DB_HOST", "127.0.0.1"), - "PORT": int(os.environ.get("NEODB_DB_PORT", 5432)), - "OPTIONS": { - "client_encoding": "UTF8", - # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, - }, - "TEST": { - "DEPENDENCIES": ["takahe"], - }, - }, - "takahe": { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("TAKAHE_DB_NAME", "test_neodb_takahe"), - "USER": os.environ.get("TAKAHE_DB_USER", "testuser"), - "PASSWORD": os.environ.get("TAKAHE_DB_PASSWORD", "testpass"), - "HOST": os.environ.get("TAKAHE_DB_HOST", "127.0.0.1"), - "PORT": os.environ.get("TAKAHE_DB_PORT", 15432), - "OPTIONS": { - "client_encoding": "UTF8", - # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT, - }, - "TEST": { - "DEPENDENCIES": [], - }, - }, -} - -# Customized auth backend, glue OAuth2 and Django User model together -# https://docs.djangoproject.com/en/3.0/topics/auth/customizing/#authentication-backends - AUTHENTICATION_BACKENDS = [ "mastodon.auth.OAuth2Backend", "oauth2_provider.backends.OAuth2Backend", @@ -171,7 +259,6 @@ AUTHENTICATION_BACKENDS = [ MARKDOWNX_MARKDOWNIFY_FUNCTION = "journal.models.render_md" - # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ @@ -185,21 +272,19 @@ USE_L10N = True USE_TZ = True - USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 CSRF_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True + if os.getenv("NEODB_SSL", "") != "": + # FIXME: remove this since user may enforce SSL in reverse proxy SECURE_SSL_REDIRECT = True SECURE_HSTS_PRELOAD = True SECURE_HSTS_INCLUDE_SUBDOMAINS = True SECURE_HSTS_SECONDS = 31536000 -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ - STATIC_URL = "/s/" STATIC_ROOT = os.environ.get("NEODB_STATIC_ROOT", os.path.join(BASE_DIR, "static/")) if DEBUG: @@ -240,16 +325,9 @@ STORAGES = { # TODO: support S3 }, }, } -SITE_DOMAIN = os.environ.get("NEODB_SITE_DOMAIN", "nicedb.org") -SITE_INFO = { - "site_name": os.environ.get("NEODB_SITE_NAME", "NiceDB"), - "site_domain": SITE_DOMAIN, - "site_url": os.environ.get("NEODB_SITE_URL", "https://" + SITE_DOMAIN), - "support_link": "https://github.com/doubaniux/boofilsic/issues", - "social_link": "https://donotban.com/@testie", - "donation_link": "https://patreon.com/tertius", - "settings_module": os.getenv("DJANGO_SETTINGS_MODULE"), -} + +# Allow user to login via any Mastodon/Pleroma sites +MASTODON_ALLOW_ANY_SITE = False if MASTODON_ALLOWED_SITES else True REDIRECT_URIS = SITE_INFO["site_url"] + "/account/login/oauth" # for sites migrated from previous version, either wipe mastodon client ids or use: @@ -277,60 +355,12 @@ DEFAULT_COLLECTION_IMAGE = os.path.join(COLLECTION_MEDIA_PATH_ROOT, "default.svg SYNC_FILE_PATH_ROOT = "sync/" EXPORT_FILE_PATH_ROOT = "export/" -# Allow user to login via any Mastodon/Pleroma sites -MASTODON_ALLOW_ANY_SITE = True - -# Allow user to create account with email (and link to Mastodon account later) -ALLOW_EMAIL_ONLY_ACCOUNT = False - -# Timeout of requests to Mastodon, in seconds -MASTODON_TIMEOUT = 30 - -MASTODON_CLIENT_SCOPE = "read write follow" -# use the following if it's a new site -# MASTODON_CLIENT_SCOPE = 'read:accounts read:follows read:search read:blocks read:mutes write:statuses write:media' - -MASTODON_LEGACY_CLIENT_SCOPE = "read write follow" - -# Emoji code in mastodon -STAR_SOLID = ":star_solid:" -STAR_HALF = ":star_half:" -STAR_EMPTY = ":star_empty:" - # Default redirect loaction when access login required view LOGIN_URL = "/account/login" # Admin site root url ADMIN_URL = "tertqX7256n7ej8nbv5cwvsegdse6w7ne5rHd" -SCRAPING_TIMEOUT = 90 - -# ScraperAPI api key -SCRAPERAPI_KEY = "***REMOVED***" -PROXYCRAWL_KEY = None -SCRAPESTACK_KEY = None - -# Spotify credentials -SPOTIFY_CREDENTIAL = "***REMOVED***" - -# IMDb API service https://imdb-api.com/ -IMDB_API_KEY = "***REMOVED***" - -# The Movie Database (TMDB) API Keys -TMDB_API3_KEY = "***REMOVED***" -# TMDB_API4_KEY = "deadbeef.deadbeef.deadbeef" - -# Google Books API Key -GOOGLE_API_KEY = "***REMOVED***" - -# Discogs API Key -# How to get: a personal access token from https://www.discogs.com/settings/developers -DISCOGS_API_KEY = "***REMOVED***" - -# IGDB -IGDB_CLIENT_ID = "deadbeef" -IGDB_CLIENT_SECRET = "" - BLEACH_STRIP_COMMENTS = True BLEACH_STRIP_TAGS = True @@ -349,12 +379,6 @@ THUMBNAIL_ALIASES = { if DEBUG: THUMBNAIL_DEBUG = True -# https://django-debug-toolbar.readthedocs.io/en/latest/ -# maybe benchmarking before deployment - -REDIS_HOST = os.environ.get("NEODB_REDIS_HOST", "127.0.0.1") -REDIS_PORT = int(os.environ.get("NEODB_REDIS_PORT", 6379)) -REDIS_DB = int(os.environ.get("NEODB_REDIS_DB", 0)) CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", @@ -379,32 +403,10 @@ RQ_SHOW_ADMIN_LINK = True SEARCH_INDEX_NEW_ONLY = False -SEARCH_BACKEND = None - -# SEARCH_BACKEND = 'MEILISEARCH' -# MEILISEARCH_SERVER = 'http://127.0.0.1:7700' -# MEILISEARCH_KEY = 'deadbeef' - -if os.environ.get("NEODB_TYPESENSE_ENABLE", ""): - SEARCH_BACKEND = "TYPESENSE" - TYPESENSE_INDEX_NAME = "catalog" -TYPESENSE_CONNECTION = { - "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"), - "nodes": [ - { - "host": os.environ.get("NEODB_TYPESENSE_HOST", "127.0.0.1"), - "port": os.environ.get("NEODB_TYPESENSE_PORT", "8108"), - "protocol": "http", - } - ], - "connection_timeout_seconds": 2, -} - -DOWNLOADER_CACHE_TIMEOUT = 300 -DOWNLOADER_RETRIES = 3 DOWNLOADER_SAVEDIR = None + DISABLE_MODEL_SIGNAL = False # disable index and social feeds during importing/etc # MAINTENANCE_MODE = False @@ -419,9 +421,6 @@ DISABLE_MODEL_SIGNAL = False # disable index and social feeds during importing/ # SILKY_MAX_RESPONSE_BODY_SIZE = 1024 # If response body>1024 bytes, ignore # SILKY_INTERCEPT_PERCENT = 10 -DISCORD_WEBHOOKS = {"user-report": None} - - NINJA_PAGINATION_PER_PAGE = 20 OAUTH2_PROVIDER = { "ACCESS_TOKEN_EXPIRE_SECONDS": 3600 * 24 * 365, @@ -430,9 +429,3 @@ OAUTH2_PROVIDER = { OAUTH2_PROVIDER_APPLICATION_MODEL = "developer.Application" DEVELOPER_CONSOLE_APPLICATION_CLIENT_ID = "NEODB_DEVELOPER_CONSOLE" - -SETUP_ADMIN_USERNAMES = [ - u for u in os.environ.get("NEODB_ADMIN_USERNAMES", "").split(",") if u -] - -SITE_INFO["site_logo"] = os.environ.get("NEODB_SITE_LOGO", "/s/img/logo.svg") diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py index 31d0507d..0ecbb208 100644 --- a/catalog/common/downloaders.py +++ b/catalog/common/downloaders.py @@ -141,7 +141,7 @@ class BasicDownloader: self.headers = headers def get_timeout(self): - return settings.SCRAPING_TIMEOUT + return settings.DOWNLOADER_REQUEST_TIMEOUT def validate_response(self, response): if response is None: @@ -191,26 +191,17 @@ class BasicDownloader: class ProxiedDownloader(BasicDownloader): def get_proxied_urls(self): + if not settings.DOWNLOADER_PROXY_LIST: + return [self.url] urls = [] - if settings.SCRAPESTACK_KEY is not None: - # urls.append(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={self.url}') - urls.append( - f"http://api.scrapestack.com/scrape?keep_headers=1&access_key={settings.SCRAPESTACK_KEY}&url={quote(self.url)}" - ) - if settings.PROXYCRAWL_KEY is not None: - urls.append( - f"https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={quote(self.url)}" - ) - if settings.SCRAPERAPI_KEY is not None: - urls.append( - f"http://api.scraperapi.com/?api_key={settings.SCRAPERAPI_KEY}&url={quote(self.url)}" - ) + for p in settings.DOWNLOADER_PROXY_LIST: + urls.append(p.replace("__URL__", quote(self.url))) return urls def get_special_proxied_url(self): return ( - f"{settings.LOCAL_PROXY}?url={quote(self.url)}" - if settings.LOCAL_PROXY is not None + settings.DOWNLOADER_BACKUP_PROXY.replace("__URL__", quote(self.url)) + if settings.DOWNLOADER_BACKUP_PROXY else None ) diff --git a/common/templates/_header.html b/common/templates/_header.html index 58418f76..2a9d17d6 100644 --- a/common/templates/_header.html +++ b/common/templates/_header.html @@ -73,7 +73,7 @@ + src="{% if request.user.is_authenticated %}{{ request.user.avatar }}{% else %}{{ user_icon }}{% endif %}" />