From 85b59aab1a4710251c37cc0f07d1aa21123e0eb6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 5 Jun 2024 21:53:02 -0400 Subject: [PATCH] mkdocs --- .dockerignore | 5 +- .github/workflows/check.yml | 18 +++- .pre-commit-config.yaml | 1 + README.md | 6 +- catalog/jobs/discover.py | 2 +- common/templatetags/duration.py | 16 ++-- doc/install-docker.md | 147 ----------------------------- doc/install.md | 6 -- docs/assets/logo.svg | 140 +++++++++++++++++++++++++++ {doc => docs}/configuration.md | 32 +++++-- {doc => docs}/development.md | 12 +-- docs/features.md | 60 ++++++++++++ docs/index.md | 43 +++++++++ docs/install.md | 97 +++++++++++++++++++ {doc => docs/internals}/catalog.md | 2 +- docs/internals/federation.md | 0 {doc => docs/internals}/journal.md | 0 {doc => docs/internals}/social.md | 0 docs/stylesheets/extra.css | 8 ++ docs/troubleshooting.md | 56 +++++++++++ docs/upgrade.md | 21 +++++ mkdocs.yml | 46 +++++++++ pyproject.toml | 1 + requirements-dev.lock | 45 +++++++++ 24 files changed, 579 insertions(+), 185 deletions(-) delete mode 100644 doc/install-docker.md delete mode 100644 doc/install.md create mode 100644 docs/assets/logo.svg rename {doc => docs}/configuration.md (51%) rename {doc => docs}/development.md (90%) create mode 100644 docs/features.md create mode 100644 docs/index.md create mode 100644 docs/install.md rename {doc => docs/internals}/catalog.md (94%) create mode 100644 docs/internals/federation.md rename {doc => docs/internals}/journal.md (100%) rename {doc => docs/internals}/social.md (100%) create mode 100644 docs/stylesheets/extra.css create mode 100644 docs/troubleshooting.md create mode 100644 docs/upgrade.md create mode 100644 mkdocs.yml diff --git a/.dockerignore b/.dockerignore index 386c62ba..7bbffd49 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,12 +9,13 @@ __pycache__ /compose.yml /compose.override.yml /Dockerfile -/doc +/docs +/mkdocs.yml /media /static /test_data /neodb -/neodb-takahe/doc +/neodb-takahe/docs /neodb-takahe/docker /neodb-takahe/static-collected /neodb-takahe/takahe/local_settings.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2b797d33..b01e5b85 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,10 +3,25 @@ name: code check on: [push, pull_request] jobs: + doc: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.12'] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Run pre-commit + run: | + python -m pip install mkdocs-material + mkdocs build -s -d /tmp/neodb-docs lint: runs-on: ubuntu-latest strategy: - max-parallel: 4 matrix: python-version: ['3.12'] steps: @@ -23,7 +38,6 @@ jobs: type-checker: runs-on: ubuntu-latest strategy: - max-parallel: 4 matrix: python-version: ['3.12'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c080d7e..d0e468ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: rev: v4.6.0 hooks: - id: check-yaml + args: [--unsafe] - id: check-json - id: check-xml - id: check-toml diff --git a/README.md b/README.md index cbebd6bd..da7f4b86 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ # 🧩 NeoDB _mark the things you love._ -NeoDB (fka boofilsic) is an open source project and free service to help users manage, share and discover collections, reviews and ratings for culture products (e.g. books, movies, music, podcasts, games and performances) in Fediverse. +[NeoDB](https://neodb.net) (fka boofilsic) is an open source project and free service to help users manage, share and discover collections, reviews and ratings for culture products (e.g. books, movies, music, podcasts, games and performances) in Fediverse. [NeoDB.social](https://neodb.social) and [NiceDB](https://nicedb.org) are free instances hosted by volunteers. Your support is essential to keep these services free and open-sourced. @@ -63,11 +63,11 @@ Follow us on [Fediverse](https://mastodon.social/@neodb) or join our [Discord co + i18n/language support is being worked on ## Host your own instance -Please see [doc/install.md](doc/install.md) +Please see [docs/install.md](docs/install.md) ## Contribution - To build application with NeoDB API, documentation is available in [NeoDB API Developer Console](https://neodb.social/developer/) - - To help develop NeoDB, please see [doc/development.md](doc/development.md) for some basics to start with + - To help develop NeoDB, please see [docs/development.md](docs/development.md) for some basics to start with ## Sponsor If you like this project, please consider sponsoring diff --git a/catalog/jobs/discover.py b/catalog/jobs/discover.py index 9d1a7164..a5ce8de0 100644 --- a/catalog/jobs/discover.py +++ b/catalog/jobs/discover.py @@ -123,7 +123,7 @@ class DiscoverGenerator(BaseJob): "provider_name": str(i.category.label), "history": [ { - "day": str(int(time.time() / 38400 - 3) * 38400), + "day": str(int(time.time() / 86400 - 3) * 86400), "accounts": str(cnt), "uses": str(cnt), } diff --git a/common/templatetags/duration.py b/common/templatetags/duration.py index bee41481..0c5bb0e8 100644 --- a/common/templatetags/duration.py +++ b/common/templatetags/duration.py @@ -49,15 +49,15 @@ def naturaldelta(v: datetime | None): return _("just now") if d < 3600: return f"{d//60}m" - if d < 38400: + if d < 86400: return f"{d//3600}h" - if d < 38400 * 14: - return f"{d//38400}d" - if d < 38400 * 56: - return f"{d//38400//7}w" - if d < 38400 * 30 * 18: - return f"{d//38400//30}mo" - return f"{d//38400//365}yr" + if d < 86400 * 14: + return f"{d//86400}d" + if d < 86400 * 56: + return f"{d//86400//7}w" + if d < 86400 * 30 * 18: + return f"{d//86400//30}mo" + return f"{d//86400//365}yr" @register.filter(is_safe=True) diff --git a/doc/install-docker.md b/doc/install-docker.md deleted file mode 100644 index b853b624..00000000 --- a/doc/install-docker.md +++ /dev/null @@ -1,147 +0,0 @@ -Run NeoDB in Docker -=================== - -## Overview -For small and medium NeoDB instances, it's recommended to deploy as a local container cluster with Docker Compose. To run a large instance, please see the bottom of doc for some tips. - -```mermaid -flowchart TB - web[[Your reverse proxy server with SSL]] --- neodb-nginx[nginx listening on localhost:8000] - subgraph Containers managed by compose.yml - neodb-nginx --- neodb-web - neodb-nginx --- takahe-web - neodb-worker --- typesense[(typesense)] - neodb-worker --- neodb-db[(neodb-db)] - neodb-worker --- redis[(redis)] - neodb-web --- typesense - neodb-web --- neodb-db - neodb-web --- redis - neodb-web --- takahe-db[(takahe-db)] - migration([migration]) --- neodb-db - migration --- takahe-db - takahe-web --- takahe-db - takahe-web --- redis - takahe-stator --- takahe-db - takahe-stator --- redis - end -``` - -As shown in the diagram, a reverse proxy server (e.g. nginx, or Cloudflare tunnel) will be required, it should have SSL configured and pointing to `http://localhost:8000`; the rest is handled by `docker compose` and containers. - -## Install Docker and add user to docker group - -Follow [official instructions](https://docs.docker.com/compose/install/) to install Docker Compose. - -Note: Docker Compose V1 is no longer supported. Please verify its version before next step: -``` -$ docker compose version -``` - -To run neodb as your own user (e.g. `neouser`), add them to docker group: -``` -$ sudo usermod -aG docker neouser -``` - -## Get configuration files - - create a folder for configuration, eg ~/mysite/config - - grab `compose.yml` and `neodb.env.example` from source code - - rename `neodb.env.example` to `.env` - -## Set up .env file and www root -Change essential options like `NEODB_SITE_DOMAIN` in `.env` before starting the cluster for the first time. Changing them later may have unintended consequences, please make sure they are correct before exposing the service externally. - -- `NEODB_SITE_NAME` - name of your site -- `NEODB_SITE_DOMAIN` - domain name of your site -- `NEODB_SECRET_KEY` - encryption key of session data -- `NEODB_DATA` is the path to store db/media/cache, it's `../data` by default, but can be any path that's writable -- `NEODB_DEBUG` - set to `False` for production deployment - -Optionally, `robots.txt` and `logo.png` may be placed under `$NEODB_DATA/www-root/`. - -See `neodb.env.example` and `configuration.md` for more options - -## Start docker -in the folder with `compose.yml` and `.env`, execute as the user you just created: -``` -$ docker compose pull -$ docker compose --profile production up -d -``` - -Starting up for the first time might take a few minutes, depending on download speed, use the following commands for status and logs: -``` -$ docker compose ps -$ docker compose --profile production logs -f -``` - -In a few seconds, the site should be up at 127.0.0.1:8000 , you may check it with: -``` -$ curl http://localhost:8000/nodeinfo/2.0/ -``` - -JSON response will be returned if the server is up and running: -``` -{"version": "2.0", "software": {"name": "neodb", "version": "0.8-dev"}, "protocols": ["activitypub", "neodb"], "services": {"outbound": [], "inbound": []}, "usage": {"users": {"total": 1}, "localPosts": 0}, "openRegistrations": true, "metadata": {}} -``` - - - -## Make the site available publicly - -Next step is to expose `127.0.0.1:8000` to external network as `https://yourdomain.tld` . There are many ways to do it, you may use nginx as a reverse proxy with a ssl cert, or configure a CDN provider to handle the SSL. There's no detailed instruction yet but contributions are welcomed. - -NeoDB requires `https` by default. Although `http` may be technically possible, it's tedious to set up and not secure, hence not recommended. - -## Update NeoDB - -Check the release notes, update `compose.yml` and `.env` as instructed. pull the image -``` -docker compose pull -``` - -If there's no change in `compose.yml`, restart only NeoDB services: -``` -$ docker compose stop neodb-web neodb-worker neodb-worker-extra takahe-web takahe-stator nginx -$ docker compose --profile production up -d -``` - -Otherwise restart the entire cluster (including database/etc, hence slower): -``` -$ docker compose down -$ docker compose --profile production up -d -``` - -If there is `compose.override.yml` in the directory, make sure it's compatible with the updated `compose.yml`. - -## Folders explained -a typical neodb folder after starting up should look like: -``` -mysite -β”œβ”€β”€ data # neodb data folder, location can be changed via NEODB_DATA in .env -β”‚ β”œβ”€β”€ neodb-db # neodb database -β”‚ β”œβ”€β”€ neodb-media # uid must be 1000 (app user in docker image), chmod if not so -β”‚ β”œβ”€β”€ redis # neodb/takahe cache -β”‚ β”œβ”€β”€ takahe-cache # uid must be 33 (www-data user in docker image), chmod if not so -β”‚ β”œβ”€β”€ takahe-db # neodb database -β”‚ β”œβ”€β”€ takahe-media # uid must be 1000 (app user in docker image), chmod if not so -β”‚ β”œβ”€β”€ typesense # neodb search index -β”‚ └── www-root # neodb web root for robots.txt, logo.png and etc -└── config - β”œβ”€β”€ compose.yml # copied from neodb release - └── .env # your configuration, see neodb.env.example -``` - -## Troubleshooting - - - `docker compose ps` to see if any service is down, (btw it's normal that `migration` is in `Exit 0` state) - - `docker compose run shell` to run a shell into the cluster; or `docker compose run root` for root shell, and `apt` is available if extra package needed - - see `Debug in Docker` in [development doc](development.md) for debugging tips - -## Multiple instance - -It's possible to run multiple clusters in one host server, as long as `NEODB_SITE_DOMAIN`, `NEODB_PORT` and `NEODB_DATA` are different. - -## Scaling - -For high-traffic instance, spin up `NEODB_WEB_WORKER_NUM`, `NEODB_API_WORKER_NUM`, `NEODB_RQ_WORKER_NUM`, `TAKAHE_WEB_WORKER_NUM`, `TAKAHE_STATOR_CONCURRENCY` and `TAKAHE_STATOR_CONCURRENCY_PER_MODEL` as long as the host server can handle them. - -Further scaling up with multiple nodes (e.g. via Kubernetes) is beyond the scope of this document, but consider run db/redis/typesense separately, and then duplicate web/worker/stator containers as long as connections and mounts are properly configured; `migration` only runs once when start or upgrade, it should be kept that way. diff --git a/doc/install.md b/doc/install.md deleted file mode 100644 index 9332a661..00000000 --- a/doc/install.md +++ /dev/null @@ -1,6 +0,0 @@ -NiceDB / NeoDB - Getting Start -============================== - -see [Docker Installation](install-docker.md) - -*Manual installation are no longer supported* diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 00000000..8a1418e3 --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/configuration.md b/docs/configuration.md similarity index 51% rename from doc/configuration.md rename to docs/configuration.md index fc98052c..f1a97088 100644 --- a/doc/configuration.md +++ b/docs/configuration.md @@ -1,10 +1,7 @@ -Configuration -============= +# Configuration -Settings you may want to change -------------------------------- -most settings resides in `settings.py`, a few notable ones: +## Settings you may want to change absolutely set these before start the instance for the first time: @@ -20,14 +17,12 @@ if you are doing debug or development: - NEODB_IMAGE - the docker image to use, `neodb/neodb:edge` for the main branch -Settings for Scrapers ---------------------- +## Settings for Scrapers TBA -Other maintenance tasks ------------------------ +## Other maintenance tasks Add alias to your shell for easier access @@ -40,3 +35,22 @@ Enable Developer Console ``` neodb-manage createapplication --client-id NEODB_DEVELOPER_CONSOLE --skip-authorization --name 'NeoDB Developer Console' --redirect-uris 'https://example.org/lol' confidential authorization-code ``` + + +## Multiple instances on one server + +It's possible to run multiple clusters in one host server, as long as `NEODB_SITE_DOMAIN`, `NEODB_PORT` and `NEODB_DATA` are different. + + +## Scaling up + +For high-traffic instance, spin up these configurations to a higher number, as long as the host server can handle them: + + - `NEODB_WEB_WORKER_NUM` + - `NEODB_API_WORKER_NUM` + - `NEODB_RQ_WORKER_NUM` + - `TAKAHE_WEB_WORKER_NUM` + - `TAKAHE_STATOR_CONCURRENCY` + - `TAKAHE_STATOR_CONCURRENCY_PER_MODEL` + +Further scaling up with multiple nodes (e.g. via Kubernetes) is beyond the scope of this document, but consider run db/redis/typesense separately, and then duplicate web/worker/stator containers as long as connections and mounts are properly configured; `migration` only runs once when start or upgrade, it should be kept that way. diff --git a/doc/development.md b/docs/development.md similarity index 90% rename from doc/development.md rename to docs/development.md index 12802ed0..b24d9f4a 100644 --- a/doc/development.md +++ b/docs/development.md @@ -3,7 +3,7 @@ Development Overview -------- -NeoDB is a Django project, and it runs side by side with a [modified version](https://github.com/neodb-social/neodb-takahe) of [TakahΔ“](https://github.com/jointakahe/takahe) (a separate Django project, code in `neodb_takahe` folder of this repo as submodule). They communicate with each other mainly thru database and task queue, the diagram in [Docker Installation](install-docker.md) demonstrates a typical architecture. Currently the two are loosely coupled, so you may take either one offline without immediate impact on the other, which makes it very easy to conduct maintenance and troubleshooting separately. In the future, they may get combined but it's not decided and will not be decided very soon. +NeoDB is a Django project, and it runs side by side with a [modified version](https://github.com/neodb-social/neodb-takahe) of [TakahΔ“](https://github.com/jointakahe/takahe) (a separate Django project, code in `neodb_takahe` folder of this repo as submodule). They communicate with each other mainly thru database and task queue, [the diagram](troubleshooting.md#containers) demonstrates a typical architecture. Currently the two are loosely coupled, so you may take either one offline without immediate impact on the other, which makes it very easy to conduct maintenance and troubleshooting separately. In the future, they may get combined but it's not decided and will not be decided very soon. Prerequisite @@ -36,7 +36,7 @@ Note: the virtual environments and packages installed in this step are mostly fo Start local instance for development ------------------------------------ -Follow [install guide](install-docker.md) to create `.env` in the root folder of NeoDB code, including at least these configuration: +Follow [install guide](install.md) to create `.env` in the root folder of NeoDB code, including at least these configuration: ``` NEODB_SITE_NAME="My Test" NEODB_SITE_DOMAIN=mydomain.dev @@ -161,10 +161,10 @@ Applications Main Django apps for NeoDB: - `users` manages user in typical Django fashion - `mastodon` this leverages [Mastodon API](https://docs.joinmastodon.org/client/intro/) ~and [Twitter API](https://developer.twitter.com/en/docs/twitter-api)~ for user login and data sync - - `catalog` manages different types of items user may collect, and scrapers to fetch from external resources, see [catalog.md](catalog.md) for more details - - `journal` manages user created content(review/ratings) and lists(collection/shelf/tag), see [journal.md](journal.md) for more details - - `social` present timeline and notification for local users, see [social.md](social.md) for more details - - `takahe` communicate with Takahe (a separate Django server, run side by side with this server, code in `neodb_takahe` as submodule) + - `catalog` manages different types of items user may collect, and scrapers to fetch from external resources, see [catalog.md](internals/catalog.md) for more details + - `journal` manages user created content(review/ratings) and lists(collection/shelf/tag), see [journal.md](internals/journal.md) for more details + - `social` present timeline and notification for local users, see [social.md](internals/social.md) for more details + - `takahe` communicate with Takahe (a separate Django server, run side by side with this server, code in `neodb_takahe` as submodule), see [federation.md](internals/federation.md) for customization of ActivityPub protocol - `legacy` this is only used by instances upgraded from 0.4.x and earlier, to provide a link mapping from old urls to new ones. If your journey starts with 0.5 and later, feel free to ignore it. diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 00000000..9d2b15cf --- /dev/null +++ b/docs/features.md @@ -0,0 +1,60 @@ +# Features + +NeoDB has [various features](features.md), and you may image it as a mix of Mastodon, Goodreads, Letterboxd, RateYourMusic, Podchaser, and more. + +## Public catalog + + - a shared catalog of books/movies/tv shows/music album/games/podcasts/performances + - search or create catalog items in each category + - one click create item with links to 3rd party sites: + - Goodreads + - IMDB + - The Movie Database + - Douban + - Google Books + - Discogs + - Spotify + - Apple Music + - Bandcamp + - Steam + - IGDB + - Bangumi + - Board Game Geek + - any RSS link to a podcast + + +## Personal collections + + - mark an item as wishlist/in progress/complete/dropped + - rate and write reviews for an item + - create tags for an item, either privately or publicly + - create and share list of items + - tracking progress of a list (e.g. personal reading challenges) + - Import and export full user data archive + - import list or archives from some 3rd party sites: + - Goodreads reading list + - Letterboxd watch list + - Douban archive (via [Doufen](https://doufen.org/)) + + +## Social + + - view home feed with friends' activities + - every activity can be set as viewable to self/follower-only/public + - eligible items, e.g. podcasts and albums, are playable in feed + - login with other Fediverse server account and import social graph + - supported servers: Mastodon/Pleroma/Firefish/GoToSocial/Pixelfed/friendica/TakahΔ“ + - share collections and reviews to Fediverse ~~and Twitter~~ feed + - ActivityPub support + - NeoDB users can follow and interact with users on other ActivityPub services like Mastodon and Pleroma + - NeoDB instances communicate with each other via an extended version of ActivityPub + - NeoDB instances may share public rating and reviews with a default relay + - implementation is based on [TakahΔ“](https://jointakahe.org/) server + + +## Languages + + - English + - Simplified Chinese + - Traditional Chinese + - more to come and your contributions are welcomed! diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..defb37ed --- /dev/null +++ b/docs/index.md @@ -0,0 +1,43 @@ +NeoDB is both a free software and a global community. You can manage and discover collections, reviews and ratings for culture products (e.g. books, movies, music, podcasts, games and performances) in NeoDB; you can also share your collections and publish micro blogs, follow and communicate with each others in the Fediverse. You can image it as a mix of Mastodon, Goodreads, Letterboxd, RateYourMusic, Podchaser, and more. + +NeoDB is an open source project licensed in AGPLv3. + +## How to Join + +There are free instances hosted by volunteers, you may choose any to join: + + - [NeoDB.social](https://neodb.social) - flagship instance + - [NeoDB experimental](https://eggplant.place) - an instance running edge version of NeoDB software, may have more features and bugs + - [NiceDB](https://nicedb.org) - the original one, running legacy version and registration limited. + +There are no mobile app for NeoDB yet, but the web version works well in mobile devices, and you can also use [Mastodon compatible apps](https://joinmastodon.org/apps) to login and use the micro blogging features. + + +## Host your own instance + +And you can [install and run your own instance](install.md). Feel free to let us know once your server is ready to share with public, so that we can add it to the list above. + +All instances are interacted with each other in Fediverse via ActivityPub, and you can follow users from other NeoDB instances or instances running other type of ActivityPub software like Mastodon. + + +## Contribution + + - NeoDB has API to manage user's collections, swagger-based documentation is available in [NeoDB API Developer Console](https://neodb.social/developer/) + - [Mastodon client compatible API](https://docs.joinmastodon.org/client/) is also available + - To help develop NeoDB, please see [development](development.md) for some basics to start with + + +## Donation + + If you like this project, please consider sponsoring us. Your support is essential to keep these services free and open-sourced. + + [![Kofi](https://img.shields.io/badge/Ko--Fi-Donate-orange?label=Support%20this%20project%20on%20Ko-fi&style=for-the-badge&color=ff5f5f&logo=ko-fi)](https://ko-fi.com/neodb) + + +## Feedback + +Follow us on [Fediverse](https://mastodon.social/@neodb) or join our [Discord community](https://discord.gg/QBHkrV8bxK) to share your ideas/questions/creations + +[![Mastodon](https://img.shields.io/mastodon/follow/106935434138389355?domain=https%3A%2F%2Fmastodon.online&style=for-the-badge&logo=mastodon&logoColor=fff&label=%40neodb%40mastodon.online&color=6D75D2)](https://mastodon.online/@neodb) +[![Mastodon](https://img.shields.io/mastodon/follow/106919732872456302?style=for-the-badge&logo=mastodon&logoColor=fff&label=%40neodb%40mastodon.social&color=6D75D2)](https://mastodon.social/@neodb) +[![Discord](https://img.shields.io/discord/1041738638364528710?label=Discord%20Community&logo=discord&logoColor=fff&color=6D75D2&style=for-the-badge&cacheSeconds=21600)](https://discord.gg/QBHkrV8bxK) diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 00000000..8d4b83d1 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,97 @@ +Install +======= + +For small and medium NeoDB instances, it's recommended to deploy as a container cluster with Docker Compose. To run a large instance, please see [scaling up](configuration.md#scaling-up) for some tips. + +## Install docker compose + +Follow [official instructions](https://docs.docker.com/compose/install/) to install Docker Compose if not yet. + +Please verify its version is 2.x or higher before next step: + +``` +docker compose version +``` + +The rest of this doc assumes you can run docker commands without `sudo`, to verify that: + +``` +docker run --rm hello-world +``` + +Follow [official instructions](https://docs.docker.com/engine/install/linux-postinstall/) if it's not enabled, or use `sudo` to run commands in this doc. + + +## Prepare configuration files + - create a folder for configuration, eg ~/mysite/config + - grab [compose.yml](https://raw.githubusercontent.com/neodb-social/neodb/main/compose.yml) and [neodb.env.example](https://raw.githubusercontent.com/neodb-social/neodb/main/neodb.env.example) from source code + - rename `neodb.env.example` to `.env` + + +## Set up .env file and web root +Change essential options like `NEODB_SITE_DOMAIN` in `.env` before starting the cluster for the first time. Changing them later may have unintended consequences, please make sure they are correct before exposing the service externally. + +- `NEODB_SITE_NAME` - name of your site +- `NEODB_SITE_DOMAIN` - domain name of your site +- `NEODB_SECRET_KEY` - encryption key of session data +- `NEODB_DATA` is the path to store db/media/cache, it's `../data` by default, but can be any path that's writable +- `NEODB_DEBUG` - set to `False` for production deployment + +Optionally, `robots.txt` and `logo.png` may be placed under `$NEODB_DATA/www-root/`. + +See [neodb.env.example](https://raw.githubusercontent.com/neodb-social/neodb/main/neodb.env.example) and [configuration](configuration.md) for more options + + +## Start container + +in the folder with `compose.yml` and `.env`, execute as the user you just created: +``` +docker compose --profile production pull +docker compose --profile production up -d +``` + +Starting up for the first time might take a few minutes, depending on download speed, use the following commands for status and logs: +``` +docker compose ps +docker compose --profile production logs -f +``` + +In a few seconds, the site should be up at 127.0.0.1:8000 , you may check it with: +``` +curl http://localhost:8000/nodeinfo/2.0/ +``` + +JSON response will be returned if the server is up and running: +``` +{"version": "2.0", "software": {"name": "neodb", "version": "0.8-dev"}, "protocols": ["activitypub", "neodb"], "services": {"outbound": [], "inbound": []}, "usage": {"users": {"total": 1}, "localPosts": 0}, "openRegistrations": true, "metadata": {}} +``` + + +## Make the site available publicly + +Next step is to expose `http://127.0.0.1:8000` to external network as `https://yourdomain.tld` (NeoDB requires `https`). There are many ways to do it, you may use nginx or caddy as a reverse proxy server with an SSL cert configured, or configure a tunnel provider like cloudflared to do the same. Once done, you may check it with: + +``` +curl https://yourdomain.tld/nodeinfo/2.0/ +``` + +You should see the same JSON response as above, and the site is now accessible to the public. + + +## Register an account and make it admin + +Now open `https://yourdomain.tld` in your browser and register an account, assuming username `admin` + +add the following line to `.env` to make it an admin account: + +``` +NEODB_ADMIN_USERS=admin +``` + +now restart the cluster to make it effective: + +```bash +docker compose --profile production up -d +``` + +Now your instance should be ready to serve. More tweaks are available, see [configuration](configuration.md) for options. diff --git a/doc/catalog.md b/docs/internals/catalog.md similarity index 94% rename from doc/catalog.md rename to docs/internals/catalog.md index 83cc7e41..a0c0a5a1 100644 --- a/doc/catalog.md +++ b/docs/internals/catalog.md @@ -109,6 +109,6 @@ Add a new site + move captured response file to `test_data/`, except large/images files. Or if have to, use a smallest version (e.g. 1x1 pixel / 1s audio) + add `@use_local_response` decorator to test methods that should pick up these responses (if `BasicDownloader` or `ProxiedDownloader` is used) - run all the tests and make sure they pass - - Command: `python3 manage.py test [--keepdb]`. More detailed for `test`, please check [this doc](https://github.com/neodb-social/neodb/blob/main/doc/development.md#run-test) + - Command: `python3 manage.py test [--keepdb]`. - See [this issue](https://github.com/neodb-social/neodb/issues/5) if `lxml.etree.ParserError` occurs on macOS. - add a site UI label to `common/static/scss/_sitelabel.scss` diff --git a/docs/internals/federation.md b/docs/internals/federation.md new file mode 100644 index 00000000..e69de29b diff --git a/doc/journal.md b/docs/internals/journal.md similarity index 100% rename from doc/journal.md rename to docs/internals/journal.md diff --git a/doc/social.md b/docs/internals/social.md similarity index 100% rename from doc/social.md rename to docs/internals/social.md diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 00000000..d5976887 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,8 @@ +:root { + --md-primary-fg-color: #0172ad; +/* --md-primary-fg-color--light: #0172ad;*/ +/* --md-primary-fg-color--dark: #01aaff;*/ + --md-accent-fg-color: #01aaff; +/* --md-accent-fg-color--light: #01aaff;*/ + font-family:system-ui, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, Helvetica, Arial, "Helvetica Neue", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; +} diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..585d68e2 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,56 @@ +# Troubleshooting + + +## Tips + + - `docker compose ps` to see if any service is down, (btw it's normal that `migration` is in `Exit 0` state) + - `docker compose run shell` to run a shell into the cluster; or `docker compose run root` for root shell, and `apt` is available if extra package needed + - see `Debug in Docker` in [development doc](development.md) for debugging tips + + +## Containers + +a typical neodb cluster looks like: + +```mermaid +flowchart TB + web[[Your reverse proxy server with SSL]] --- neodb-nginx[nginx listening on localhost:8000] + subgraph Containers managed by compose.yml + neodb-nginx --- neodb-web + neodb-nginx --- takahe-web + neodb-worker --- typesense[(typesense)] + neodb-worker --- neodb-db[(neodb-db)] + neodb-worker --- redis[(redis)] + neodb-web --- typesense + neodb-web --- neodb-db + neodb-web --- redis + neodb-web --- takahe-db[(takahe-db)] + migration([migration]) --- neodb-db + migration --- takahe-db + takahe-web --- takahe-db + takahe-web --- redis + takahe-stator --- takahe-db + takahe-stator --- redis + end +``` + + +## Data Folders + +a typical neodb folder after starting up should look like: + +``` +mysite +β”œβ”€β”€ data # neodb data folder, location can be changed via NEODB_DATA in .env +β”‚ β”œβ”€β”€ neodb-db # neodb database +β”‚ β”œβ”€β”€ neodb-media # uid must be 1000 (app user in docker image), chmod if not so +β”‚ β”œβ”€β”€ redis # neodb/takahe cache +β”‚ β”œβ”€β”€ takahe-cache # uid must be 33 (www-data user in docker image), chmod if not so +β”‚ β”œβ”€β”€ takahe-db # neodb database +β”‚ β”œβ”€β”€ takahe-media # uid must be 1000 (app user in docker image), chmod if not so +β”‚ β”œβ”€β”€ typesense # neodb search index +β”‚ └── www-root # neodb web root for robots.txt, logo.png and etc +└── config + β”œβ”€β”€ compose.yml # copied from neodb release + └── .env # your configuration, see neodb.env.example +``` diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 00000000..0b6e9d63 --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,21 @@ +Upgrade +------- + +Check the [release notes](https://github.com/neodb-social/neodb/releases), update `compose.yml` and `.env` as instructed. + +If there is `compose.override.yml`, make sure it's compatible with the updated `compose.yml`. + +Pull the latest container image +```bash +docker compose --profile production pull +``` + +Restart the entire cluster: +```bash +docker compose --profile production up -d +``` + +Optionally, clean up old images: +```bash +docker system prune -af --volumes +``` diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..4d4fbf7c --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,46 @@ +site_name: 🧩 NeoDB +site_url: https://neodb.net +repo_url: http://github.com/neodb-social/neodb +edit_uri: blob/main/docs/ +site_description: Mark the things you love. +nav: + - Introduction: 'index.md' + - features.md + - install.md + - upgrade.md + - configuration.md + - troubleshooting.md + - development.md +theme: + logo: assets/logo.svg + name: material + palette: + primary: custom + accent: custom + features: + # - navigation.tabs + # - navigation.footer +extra: + social: + - icon: fontawesome/brands/mastodon + link: https://mastodon.online/@neodb/ + - icon: fontawesome/brands/x-twitter + link: https://twitter.com/NeoDBsocial + - icon: fontawesome/brands/threads + link: https://www.threads.net/@neodb.social + - icon: fontawesome/brands/bluesky + link: https://bsky.app/profile/neodb.bsky.social + - icon: fontawesome/brands/discord + link: https://discord.gg/QBHkrV8bxK + - icon: fontawesome/brands/github + link: http://github.com/neodb-social/neodb + - icon: fontawesome/brands/docker + link: https://hub.docker.com/u/neodb +extra_css: + - stylesheets/extra.css +markdown_extensions: + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format diff --git a/pyproject.toml b/pyproject.toml index 23f9d8c7..cdf2985a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ dev-dependencies = [ "lxml-stubs", "pyright==1.1.365", "ruff", + "mkdocs-material>=9.5.25", ] [tool.pyright] diff --git a/requirements-dev.lock b/requirements-dev.lock index d20dd673..a335e0f1 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -22,6 +22,8 @@ asgiref==3.8.1 # via django-stubs attrs==23.2.0 # via aiohttp +babel==2.15.0 + # via mkdocs-material beautifulsoup4==4.12.3 # via markdownify black==24.4.2 @@ -44,9 +46,11 @@ charset-normalizer==3.3.2 click==8.1.7 # via black # via djlint + # via mkdocs # via rq colorama==0.4.6 # via djlint + # via mkdocs-material cryptography==42.0.8 # via jwcrypto cssbeautifier==1.15.1 @@ -114,6 +118,8 @@ filetype==1.2.0 frozenlist==1.4.1 # via aiohttp # via aiosignal +ghp-import==2.1.0 + # via mkdocs gunicorn==22.0.0 h11==0.14.0 # via httpcore @@ -133,6 +139,9 @@ idna==3.7 # via yarl igdb-api-v4==0.3.2 isort==5.13.2 +jinja2==3.1.4 + # via mkdocs + # via mkdocs-material jsbeautifier==1.15.1 # via cssbeautifier # via djlint @@ -148,8 +157,24 @@ lxml==5.2.2 lxml-stubs==0.5.1 markdown==3.6 # via django-markdownx + # via mkdocs + # via mkdocs-material + # via pymdown-extensions markdownify==0.12.1 +markupsafe==2.1.5 + # via jinja2 + # via mkdocs +mergedeep==1.3.4 + # via mkdocs + # via mkdocs-get-deps mistune==3.0.2 +mkdocs==1.6.0 + # via mkdocs-material +mkdocs-get-deps==0.2.0 + # via mkdocs +mkdocs-material==9.5.25 +mkdocs-material-extensions==1.3.1 + # via mkdocs-material multidict==6.0.5 # via aiohttp # via yarl @@ -164,15 +189,20 @@ openpyxl==3.1.3 packaging==24.0 # via black # via gunicorn + # via mkdocs +paginate==0.5.6 + # via mkdocs-material pathspec==0.12.1 # via black # via djlint + # via mkdocs pillow==10.3.0 # via blurhash-python # via django-markdownx # via easy-thumbnails platformdirs==4.2.2 # via black + # via mkdocs-get-deps # via virtualenv podcastparser==0.6.10 pre-commit==3.7.1 @@ -185,10 +215,15 @@ pydantic==2.7.3 # via django-ninja pydantic-core==2.18.4 # via pydantic +pygments==2.18.0 + # via mkdocs-material +pymdown-extensions==10.8.1 + # via mkdocs-material pyright==1.1.365 python-dateutil==2.9.0.post0 # via dateparser # via django-auditlog + # via ghp-import python-fsutil==0.14.1 # via django-maintenance-mode pytz==2024.1 @@ -196,7 +231,13 @@ pytz==2024.1 # via django-tz-detect pyyaml==6.0.1 # via djlint + # via mkdocs + # via mkdocs-get-deps # via pre-commit + # via pymdown-extensions + # via pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs rcssmin==1.1.1 # via django-compressor redis==5.0.4 @@ -206,11 +247,13 @@ redis==5.0.4 regex==2023.12.25 # via dateparser # via djlint + # via mkdocs-material requests==2.32.3 # via django-anymail # via django-oauth-toolkit # via django-slack # via igdb-api-v4 + # via mkdocs-material # via typesense rjsmin==1.2.1 # via django-compressor @@ -257,6 +300,8 @@ urlman==2.0.2 validators==0.28.3 virtualenv==20.26.2 # via pre-commit +watchdog==4.0.1 + # via mkdocs webencodings==0.5.1 # via bleach # via tinycss2