diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 01e5ffd6..2b2443b2 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -3,22 +3,6 @@ name: code check
on: [push, pull_request]
jobs:
- doc:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: ['3.12']
- steps:
- - uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: pip
- - name: Run pre-commit
- run: |
- python -m pip install mkdocs-material
- mkdocs build -s -d /tmp/neodb-docs
lint:
runs-on: ubuntu-latest
strategy:
diff --git a/FEDERATION.md b/FEDERATION.md
index 8f93139f..22289e5b 100644
--- a/FEDERATION.md
+++ b/FEDERATION.md
@@ -1,3 +1,3 @@
# Federation
-see [doc](docs/internals/federation.md) for FEP-67ff related information.
+see [doc](https://neodb.net/internals/federation.md) for FEP-67ff related information.
diff --git a/README.md b/README.md
index cfc1269c..5ad6b863 100644
--- a/README.md
+++ b/README.md
@@ -4,19 +4,21 @@
[](https://hub.docker.com/r/neodb/neodb)
[](https://github.com/neodb-social/neodb/blob/main/LICENSE)
+
# 🧩 NeoDB
_mark the things you love._
[NeoDB](https://neodb.net) (fka boofilsic) is an open source project and free service to help users manage, share and discover collections, reviews and ratings for culture products (e.g. books, movies, music, podcasts, games and performances) in Fediverse.
-[NeoDB.social](https://neodb.social) is a free instance hosted by NeoDB developers, there are more [servers](https://neodb.net/servers/) and [apps](https://neodb.net/apps/) available. Your support is essential to keep them free and open-sourced.
+[NeoDB.social](https://neodb.social) is a free instance hosted by NeoDB developers, there are more [servers](https://neodb.net/servers/) and [apps](https://neodb.net/apps/) available.
-Follow us on [Fediverse](https://mastodon.online/@neodb), [Bluesky](https://bsky.app/profile/neodb.net) or join our [Discord community](https://discord.gg/QBHkrV8bxK) to share your ideas/questions/creations.
+Follow us on [Fediverse](https://mastodon.online/@neodb), [Bluesky](https://bsky.app/profile/neodb.net) or join our [Discord community](https://discord.gg/QBHkrV8bxK) to share your ideas/questions/creations. Your support is essential to keep the services free and open-sourced.
[](https://mastodon.social/@neodb)
[](https://discord.gg/QBHkrV8bxK)
[](https://ko-fi.com/neodb)
+
## Features
- Manage a shared catalog of books/movies/tv shows/music album/games/podcasts/performances
+ search or create catalog items in each category
@@ -66,13 +68,17 @@ Follow us on [Fediverse](https://mastodon.online/@neodb), [Bluesky](https://bsky
- Other
+ i18n: English, Danish and Simp/Trad Chinese available; contribution for more languages welcomed
+
## Host your own instance
-Please see [docs/install.md](docs/install.md)
+Please see [installation guide](https://neodb.net/install/).
+
## Contribution
- To build application with NeoDB API, documentation is available in [NeoDB API Developer Console](https://neodb.social/developer/)
- - To help develop NeoDB, please see [docs/development.md](docs/development.md) for some basics to start with
+ - To help develop NeoDB, please see [development guide](https://neodb.net/development/) for some basics to start with
- To translate NeoDB to more languages, please join [our project on Weblate](https://hosted.weblate.org/projects/neodb/neodb/)
+ - Source code for [NeoDB documentation](https://neodb.net) can be found [here](https://github.com/neodb-social/neodb-doc)
+
## Sponsor
If you like this project, please consider donating to [NeoDB.social on ko-fi](https://ko-fi.com/neodb), or [Takahē](https://www.patreon.com/takahe) and [NiceDB](https://patreon.com/tertius) without whom this project won't be possible.
diff --git a/boofilsic/settings.py b/boofilsic/settings.py
index 04b1df5f..3f74a9cb 100644
--- a/boofilsic/settings.py
+++ b/boofilsic/settings.py
@@ -449,6 +449,7 @@ LANGUAGE_CODE, PREFERRED_LANGUAGES = _init_language_settings(
if TESTING: # force en if testing
LANGUAGE_CODE = "en"
+ PREFERRED_LANGUAGES = ["en"]
LOCALE_PATHS = [os.path.join(BASE_DIR, "locale")]
@@ -580,7 +581,7 @@ SEARCH_INDEX_NEW_ONLY = False
INDEX_ALIASES = env("INDEX_ALIASES")
-DOWNLOADER_SAVEDIR = env("NEODB_DOWNLOADER_SAVE_DIR", default="/tmp") # type: ignore
+DOWNLOADER_SAVEDIR = env("NEODB_DOWNLOADER_SAVE_DIR", default="") # type: ignore
DISABLE_MODEL_SIGNAL = False # disable index and social feeds during importing/etc
diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py
index c4da7244..4a1c1b15 100644
--- a/catalog/common/downloaders.py
+++ b/catalog/common/downloaders.py
@@ -205,6 +205,7 @@ class BasicDownloader:
)
return resp, response_type
except RequestException as e:
+ # logger.debug(f"RequestException: {e}")
self.logs.append(
{"response_type": RESPONSE_NETWORK_ERROR, "url": url, "exception": e}
)
@@ -340,16 +341,19 @@ class ImageDownloaderMixin:
def validate_response(self, response):
if response and response.status_code == 200:
try:
- raw_img = response.content
- img = Image.open(BytesIO(raw_img))
- img.load() # corrupted image will trigger exception
- content_type = response.headers.get("Content-Type")
+ content_type = response.headers["content-type"]
+ if content_type.startswith("image/svg+xml"):
+ self.extention = "svg"
+ return RESPONSE_OK
file_type = filetype.get_type(
mime=content_type.partition(";")[0].strip()
)
if file_type is None:
return RESPONSE_NETWORK_ERROR
self.extention = file_type.extension
+ raw_img = response.content
+ img = Image.open(BytesIO(raw_img))
+ img.load() # corrupted image will trigger exception
return RESPONSE_OK
except Exception:
return RESPONSE_NETWORK_ERROR
diff --git a/catalog/common/models.py b/catalog/common/models.py
index a6201474..07d02f72 100644
--- a/catalog/common/models.py
+++ b/catalog/common/models.py
@@ -771,22 +771,22 @@ class Item(PolymorphicModel):
return not self.is_deleted and self.merged_to_item is None
@cached_property
- def rating(self):
+ def rating_info(self):
from journal.models import Rating
- return Rating.get_rating_for_item(self)
+ return Rating.get_info_for_item(self)
+
+ @property
+ def rating(self):
+ return self.rating_info.get("average")
@cached_property
def rating_count(self):
- from journal.models import Rating
-
- return Rating.get_rating_count_for_item(self)
+ return self.rating_info.get("count")
@cached_property
def rating_distribution(self):
- from journal.models import Rating
-
- return Rating.get_rating_distribution_for_item(self)
+ return self.rating_info.get("distribution")
@cached_property
def tags(self):
diff --git a/catalog/music/tests.py b/catalog/music/tests.py
index 0bfd8a98..d489fe7a 100644
--- a/catalog/music/tests.py
+++ b/catalog/music/tests.py
@@ -6,6 +6,8 @@ from catalog.music.utils import *
class BasicMusicTest(TestCase):
+ databases = "__all__"
+
def test_gtin(self):
self.assertIsNone(upc_to_gtin_13("018771208112X"))
self.assertIsNone(upc_to_gtin_13("999018771208112"))
@@ -15,6 +17,8 @@ class BasicMusicTest(TestCase):
class SpotifyTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.Spotify_Album
t_id_value = "65KwtzkJXw7oT819NFWmEP"
@@ -48,6 +52,8 @@ class SpotifyTestCase(TestCase):
class DoubanMusicTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.DoubanMusic
t_id_value = "33551231"
@@ -74,6 +80,8 @@ class DoubanMusicTestCase(TestCase):
class MultiMusicSitesTestCase(TestCase):
+ databases = "__all__"
+
@use_local_response
def test_albums(self):
url1 = "https://music.douban.com/subject/33551231/"
@@ -92,6 +100,8 @@ class MultiMusicSitesTestCase(TestCase):
class BandcampTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.Bandcamp
t_id_value = "intlanthem.bandcamp.com/album/in-these-times"
@@ -119,6 +129,8 @@ class BandcampTestCase(TestCase):
class DiscogsReleaseTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.Discogs_Release
t_id_value = "25829341"
@@ -155,6 +167,8 @@ class DiscogsReleaseTestCase(TestCase):
class DiscogsMasterTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.Discogs_Master
t_id_value = "469004"
@@ -182,6 +196,8 @@ class DiscogsMasterTestCase(TestCase):
class AppleMusicTestCase(TestCase):
+ databases = "__all__"
+
def test_parse(self):
t_id_type = IdType.AppleMusic
t_id_value = "1284391545"
@@ -201,8 +217,10 @@ class AppleMusicTestCase(TestCase):
self.assertEqual(site.ready, False)
site.get_resource_ready()
self.assertEqual(site.ready, True)
- self.assertEqual(site.resource.metadata["title"], "Kids Only")
+ self.assertEqual(
+ site.resource.metadata["localized_title"][0]["text"], "Kids Only"
+ )
self.assertEqual(site.resource.metadata["artist"], ["Leah Dou"])
self.assertIsInstance(site.resource.item, Album)
- self.assertEqual(site.resource.item.genre, ["Pop"])
- self.assertEqual(site.resource.item.duration, 2371628)
+ self.assertEqual(site.resource.item.genre, ["Pop", "Music"])
+ self.assertEqual(site.resource.item.duration, 2368000)
diff --git a/catalog/sites/apple_music.py b/catalog/sites/apple_music.py
index 19b89d98..194944d8 100644
--- a/catalog/sites/apple_music.py
+++ b/catalog/sites/apple_music.py
@@ -9,8 +9,9 @@ Scraping the website directly.
"""
import json
+from datetime import timedelta
-import dateparser
+from django.utils.dateparse import parse_duration
from loguru import logger
from catalog.common import *
@@ -18,7 +19,6 @@ from catalog.models import *
from common.models.lang import (
SITE_DEFAULT_LANGUAGE,
SITE_PREFERRED_LANGUAGES,
- detect_language,
)
from common.models.misc import uniq
@@ -39,7 +39,6 @@ class AppleMusic(AbstractSite):
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Language": BasicDownloader.get_accept_language(),
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"DNT": "1",
@@ -70,80 +69,63 @@ class AppleMusic(AbstractSite):
return locales
def scrape(self):
- matched_content = None
+ matched_schema_data = None
localized_title = []
localized_desc = []
for lang, locales in self.get_locales().items():
for loc in locales: # waterfall thru all locales
url = f"https://music.apple.com/{loc}/album/{self.id_value}"
try:
+ tl = f"{lang}-{loc}" if lang == "zh" else lang
+ headers = {
+ "Accept-Language": tl,
+ }
+ headers.update(self.headers)
content = (
BasicDownloader(url, headers=self.headers).download().html()
)
- logger.info(f"got localized content from {url}")
- elem = content.xpath(
- "//script[@id='serialized-server-data']/text()"
- )
- txt: str = elem[0] # type:ignore
- page_data = json.loads(txt)[0]
- album_data = page_data["data"]["sections"][0]["items"][0]
- title = album_data["title"]
- brief = album_data.get("modalPresentationDescriptor", {}).get(
- "paragraphText", ""
- )
- tl = detect_language(title + " " + brief)
- localized_title.append({"lang": tl, "text": title})
- if brief:
- localized_desc.append({"lang": tl, "text": brief})
- if lang == SITE_DEFAULT_LANGUAGE or not matched_content:
- matched_content = content
+ logger.debug(f"got localized content from {url}")
+ txt: str = content.xpath(
+ "//script[@id='schema:music-album']/text()"
+ )[0] # type:ignore
+ schema_data = json.loads(txt)
+ title = schema_data["name"]
+ if title:
+ localized_title.append({"lang": tl, "text": title})
+ try:
+ txt: str = content.xpath(
+ "//script[@id='serialized-server-data']/text()"
+ )[0] # type:ignore
+ server_data = json.loads(txt)
+ brief = server_data[0]["data"]["sections"][0]["items"][0][
+ "modalPresentationDescriptor"
+ ]["paragraphText"]
+ if brief:
+ localized_desc.append({"lang": tl, "text": brief})
+ except Exception:
+ server_data = brief = None
+ if lang == SITE_DEFAULT_LANGUAGE or not matched_schema_data:
+ matched_schema_data = schema_data
break
except Exception:
pass
- if matched_content is None:
+ if matched_schema_data is None: # no schema data found
raise ParseError(self, f"localized content for {self.url}")
- elem = matched_content.xpath("//script[@id='serialized-server-data']/text()")
- txt: str = elem[0] # type:ignore
- page_data = json.loads(txt)[0]
- album_data = page_data["data"]["sections"][0]["items"][0]
- title = album_data["title"]
- brief = album_data.get("modalPresentationDescriptor")
- brief = brief.get("paragraphText") if brief else None
- artist_list = album_data["subtitleLinks"]
- artist = [item["title"] for item in artist_list]
-
- track_data = page_data["data"]["seoData"]
- date_elem = track_data.get("musicReleaseDate")
- release_datetime = dateparser.parse(date_elem.strip()) if date_elem else None
- release_date = (
- release_datetime.strftime("%Y-%m-%d") if release_datetime else None
+ artist = [a["name"] for a in matched_schema_data.get("byArtist", [])]
+ release_date = matched_schema_data.get("datePublished", None)
+ genre = matched_schema_data.get("genre", [])
+ image_url = matched_schema_data.get("image", None)
+ track_list = [t["name"] for t in matched_schema_data.get("tracks", [])]
+ duration = round(
+ sum(
+ (parse_duration(t["duration"]) or timedelta()).total_seconds() * 1000
+ for t in matched_schema_data.get("tracks", [])
+ )
)
-
- track_list = [
- f"{i}. {track['attributes']['name']}"
- for i, track in enumerate(track_data["ogSongs"], 1)
- ]
- duration_list = [
- track["attributes"].get("durationInMillis", 0)
- for track in track_data["ogSongs"]
- ]
- duration = int(sum(duration_list))
- genre = track_data["schemaContent"].get("genre")
- if genre:
- genre = [
- genre[0]
- ] # apple treat "Music" as a genre. Thus, only the first genre is obtained.
-
- images = matched_content.xpath("//source[@type='image/jpeg']/@srcset")
- image_elem: str = images[0] if images else "" # type:ignore
- image_url = image_elem.split(" ")[0] if image_elem else None
-
pd = ResourceContent(
metadata={
"localized_title": uniq(localized_title),
"localized_description": uniq(localized_desc),
- "title": title,
- "brief": brief,
"artist": artist,
"genre": genre,
"release_date": release_date,
diff --git a/catalog/tv/models.py b/catalog/tv/models.py
index c4aec7d9..a9422f9d 100644
--- a/catalog/tv/models.py
+++ b/catalog/tv/models.py
@@ -44,6 +44,7 @@ from catalog.common import (
)
from catalog.common.models import LANGUAGE_CHOICES_JSONFORM, LanguageListField
from common.models.lang import RE_LOCALIZED_SEASON_NUMBERS, localize_number
+from common.models.misc import uniq
class TVShowInSchema(ItemInSchema):
@@ -414,30 +415,35 @@ class TVSeason(Item):
- "Show Title Season X" with some localization
"""
s = super().display_title
- if self.parent_item and (
- RE_LOCALIZED_SEASON_NUMBERS.sub("", s) == ""
- or s == self.parent_item.display_title
- ):
- if self.parent_item.get_season_count() == 1:
- return self.parent_item.display_title
- elif self.season_number:
- return _("{show_title} Season {season_number}").format(
- show_title=self.parent_item.display_title,
- season_number=localize_number(self.season_number),
- )
- else:
- return f"{self.parent_item.display_title} {s}"
+ if self.parent_item:
+ if (
+ RE_LOCALIZED_SEASON_NUMBERS.sub("", s) == ""
+ or s == self.parent_item.display_title
+ ):
+ if self.parent_item.get_season_count() == 1:
+ return self.parent_item.display_title
+ elif self.season_number:
+ return _("{show_title} Season {season_number}").format(
+ show_title=self.parent_item.display_title,
+ season_number=localize_number(self.season_number),
+ )
+ else:
+ return f"{self.parent_item.display_title} {s}"
+ elif self.parent_item.display_title not in s:
+ return f"{self.parent_item.display_title} ({s})"
return s
@cached_property
def additional_title(self) -> list[str]:
title = self.display_title
- return [
- t["text"]
- for t in self.localized_title
- if t["text"] != title
- and RE_LOCALIZED_SEASON_NUMBERS.sub("", t["text"]) != ""
- ]
+ return uniq(
+ [
+ t["text"]
+ for t in self.localized_title
+ if t["text"] != title
+ and RE_LOCALIZED_SEASON_NUMBERS.sub("", t["text"]) != ""
+ ]
+ )
def to_indexable_titles(self) -> list[str]:
titles = [t["text"] for t in self.localized_title if t["text"]]
diff --git a/catalog/views.py b/catalog/views.py
index 1c8345f2..a8dd5d5e 100644
--- a/catalog/views.py
+++ b/catalog/views.py
@@ -86,7 +86,7 @@ def retrieve(request, item_path, item_uuid):
if request.method == "HEAD":
return HttpResponse()
if request.headers.get("Accept", "").endswith("json"):
- return JsonResponse(item.ap_object)
+ return JsonResponse(item.ap_object, content_type="application/activity+json")
focus_item = None
if request.GET.get("focus"):
focus_item = get_object_or_404(
diff --git a/common/views.py b/common/views.py
index 3334a293..c9b827d0 100644
--- a/common/views.py
+++ b/common/views.py
@@ -2,7 +2,7 @@ from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.core.exceptions import DisallowedHost
-from django.http import HttpRequest, JsonResponse
+from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import redirect, render
from django.urls import reverse
@@ -81,36 +81,41 @@ def nodeinfo2(request):
)
-def _is_json_request(request) -> bool:
- return request.headers.get("HTTP_ACCEPT", "").endswith("json")
+def _error_response(request, status: int, exception=None, default_message=""):
+ message = str(exception) if exception else default_message
+ if request.headers.get("HTTP_ACCEPT", "").endswith("json"):
+ return JsonResponse({"error": message}, status=status)
+ if (
+ request.headers.get("HTTP_HX_REQUEST") is not None
+ and request.headers.get("HTTP_HX_BOOSTED") is None
+ ):
+ return HttpResponse(message, status=status)
+ return render(
+ request,
+ f"{status}.html",
+ status=status,
+ context={"message": message, "exception": exception},
+ )
def error_400(request, exception=None):
if isinstance(exception, DisallowedHost):
url = settings.SITE_INFO["site_url"] + request.get_full_path()
return redirect(url, permanent=True)
- if _is_json_request(request):
- return JsonResponse({"error": "invalid request"}, status=400)
- return render(request, "400.html", status=400, context={"exception": exception})
+ return _error_response(request, 400, exception, "invalid request")
def error_403(request, exception=None):
- if _is_json_request(request):
- return JsonResponse({"error": "forbidden"}, status=403)
- return render(request, "403.html", status=403, context={"exception": exception})
+ return _error_response(request, 403, exception, "forbidden")
def error_404(request, exception=None):
- if _is_json_request(request):
- return JsonResponse({"error": "not found"}, status=404)
request.session.pop("next_url", None)
- return render(request, "404.html", status=404, context={"exception": exception})
+ return _error_response(request, 404, exception, "not found")
def error_500(request, exception=None):
- if _is_json_request(request):
- return JsonResponse({"error": "something wrong"}, status=500)
- return render(request, "500.html", status=500, context={"exception": exception})
+ return _error_response(request, 500, exception, "something wrong")
def console(request):
diff --git a/docs/api.md b/docs/api.md
deleted file mode 100644
index 06d65e2f..00000000
--- a/docs/api.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# API
-
-## Endpoints
-
-NeoDB has a set of API endpoints mapping to its functions like marking a book or listing collections, they can be found in swagger based API documentation at `/developer/` of your running instance, [a version of it](https://neodb.social/developer/) is available on our flagship instance.
-
-NeoDB also supports a subset of Mastodon API, details can be found in [Mastodon API documentation](https://docs.joinmastodon.org/api/).
-
-Both set of APIs can be accessed by the same access token.
-
-## How to authorize
-
-### Create an application
-
-you must have at least one URL included in the Redirect URIs field, e.g. `https://example.org/callback`, or use `urn:ietf:wg:oauth:2.0:oob` if you don't have a callback URL.
-
-```
-curl https://neodb.social/api/v1/apps \
- -d client_name=MyApp \
- -d redirect_uris=https://example.org/callback \
- -d website=https://my.site
-```
-
-and save of the `client_id` and `client_secret` returned in the response:
-
-```
-{
- "client_id": "CLIENT_ID",
- "client_secret": "CLIENT_SECRET",
- "name": "MyApp",
- "redirect_uri": "https://example.org/callback",
- "vapid_key": "PUSH_KEY",
- "website": "https://my.site"
-}
-```
-
-
-### Guide your user to open this URL
-
-```
-https://neodb.social/oauth/authorize?response_type=code&client_id=CLIENT_ID&redirect_uri=https://example.org/callback&scope=read+write
-```
-
-### Once authorizated by user, it will redirect to `https://example.org/callback` with a `code` parameter:
-
-```
-https://example.org/callback?code=AUTH_CODE
-```
-
-### Obtain access token with the following POST request:
-
-```
-curl https://neodb.social/oauth/token \
- -d "client_id=CLIENT_ID" \
- -d "client_secret=CLIENT_SECRET" \
- -d "code=AUTH_CODE" \
- -d "redirect_uri=https://example.org/callback" \
- -d "grant_type=authorization_code"
-```
-
-and access token will be returned in the response:
-
-```
-{
- "access_token": "ACCESS_TOKEN",
- "token_type": "Bearer",
- "scope": "read write"
-}
-```
-
-### Use the access token to access protected endpoints like `/api/me`
-
-```
-curl -H "Authorization: Bearer ACCESS_TOKEN" -X GET https://neodb.social/api/me
-```
-
-and response will be returned accordingly:
-
-```
-{"url": "https://neodb.social/users/xxx/", "external_acct": "xxx@yyy.zzz", "display_name": "XYZ", "avatar": "https://yyy.zzz/xxx.gif"}
-```
diff --git a/docs/apps.md b/docs/apps.md
deleted file mode 100644
index e5175a28..00000000
--- a/docs/apps.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# Apps
-
-NeoDB web version will provide the most features and experience, while some third-party apps are also available below.
-
-## Apps for NeoDB
-
-A few apps for NeoDB are being actively developed:
-
- - [Piecelet](https://piecelet.app) by `@piecelet@mastodon.social` - [App Store](https://apps.apple.com/app/piecelet-for-neodb/id6739444863) / [Source Code](https://github.com/lcandy2/neodb-app)
- - [Chihu](https://chihu.app) by `@chihu@mastodon.social` - [Test Flight](https://testflight.apple.com/join/WmbnP9Vx)
-
-These apps are not affiliated with NeoDB, but they are being developed with the support of this community. If you are also developing an app for NeoDB, and wish to share that with the community, please [edit this file](https://github.com/neodb-social/neodb/edit/main/docs/apps.md) and submit a pull request.
-
-
-## Mastodon apps
-
-[Mastodon compatible mobile and native apps](https://joinmastodon.org/apps) can be used to log in and utilize the micro-blogging features in NeoDB server.
-
-In addition to micro-blogging, Mastodon compatible can also be used to take note on book you are currently reading. Just head to bookmark section in your app, your currently reading books are listed there as bookmarked posts, replying any of them will make a note for that book.
diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg
deleted file mode 100644
index 8a1418e3..00000000
--- a/docs/assets/logo.svg
+++ /dev/null
@@ -1,140 +0,0 @@
-
-
diff --git a/docs/configuration.md b/docs/configuration.md
deleted file mode 100644
index b52a85af..00000000
--- a/docs/configuration.md
+++ /dev/null
@@ -1,210 +0,0 @@
-# Configuration
-
-
-## Important settings you may want to change first
-
-absolutely set these in `.env` before start the instance for the first time:
-
- - `NEODB_SECRET_KEY` - 50 characters of random string, no white space
- - `NEODB_SITE_NAME` - the name of your site
- - `NEODB_SITE_DOMAIN` - the domain name of your site
-
-**`NEODB_SECRET_KEY` and `NEODB_SITE_DOMAIN` must not be changed later.**
-
-if you are doing debug or development:
-
- - `NEODB_DEBUG` - True will turn on debug for both neodb and takahe, turn off relay, and reveal self as debug mode in nodeinfo (so peers won't try to run fedi search on this node)
- - `NEODB_IMAGE` - the docker image to use, `neodb/neodb:edge` for the main branch
-
-## Settings for customization
-
- - `NEODB_SITE_LOGO`
- - `NEODB_SITE_ICON`
- - `NEODB_USER_ICON`
- - `NEODB_SITE_COLOR` - one of [these color schemes](https://picocss.com/docs/colors)
- - `NEODB_SITE_INTRO`
- - `NEODB_SITE_HEAD`
- - `NEODB_SITE_DESCRIPTION`
- - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default. It can includes languages with no UI translations yet, e.g. if set to `ja,en,zh`, NeoDB scraper will fetch catalog metadata in three languages if they are available from third party sites, Japanese users (= whose browser language set to ja-JP) will see English UI with Japanese metadata.
- - `NEODB_DISCOVER_FILTER_LANGUAGE` - `False` by default; when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`.
- - `NEODB_DISCOVER_SHOW_LOCAL_ONLY` - `False` by default; when set to `True`, only show items marked by local users rather than entire network on `/discover/`
- - `NEODB_DISCOVER_UPDATE_INTERVAL` - minutes between each update for popular items on `/discover/`
- - `NEODB_SITE_LINKS` - a list of title and links to show in the footer, comma separated, e.g. `Feedback=https://discord.gg/8KweCuApaK,ToS=/pages/rules/`
- - `NEODB_INVITE_ONLY` - `False` by default, set to `True` to require invite code(generated by `neodb-manage invite --create`) to register
- - `NEODB_ENABLE_LOCAL_ONLY` - `False` by default, set to `True` to allow user to post marks as "local public"
- - `NEODB_LOGIN_MASTODON_WHITELIST` - a list of Mastodon instances to allow login from, comma separated
- - `NEODB_EMAIL_FROM` - the email address to send email from
- - `NEODB_EMAIL_URL` - email sender configuration, e.g.
- - `smtp://:@:`
- - `smtp+tls://:@:`
- - `smtp+ssl://:@:`
- - `anymail://?`, to send email via email service providers, see [anymail doc](https://anymail.dev/)
-
-## Settings for administration
- - `DISCORD_WEBHOOKS` - Discord channel to send notification about user submitted suggestion and changes, e.g. `suggest=https://discord.com/api/webhooks/123/abc,audit=https://discord.com/api/webhooks/123/def`. Both suggest and audit channels must be in forum mode.
- - `NEODB_SENTRY_DSN` , `TAKAHE_SENTRY_DSN` - [Sentry](https://sentry.io/) DSN to log errors.
-
-## Settings for Federation
-
- - `NEODB_SEARCH_PEERS` is empty by default, which means NeoDB will search all known peers running production version of NeoDB when user look for items. This can be set to a comma-separated list of host names, so that NeoDB will only search those servers; or search no other peers if set to just `-`.
-
- - `NEODB_DISABLE_DEFAULT_RELAY` is set to `False` by default, the server will send and receive public posts from `relay.neodb.net`.
-
- `relay.neodb.net` is [open sourced](https://github.com/neodb-social/neodb-relay) and operated by NeoDB developers, it works like most ActivityPub relays except it only relays between NeoDB instances, it helps public information like catalogs and trends flow between NeoDB instances. You may set it to `True` if you don't want to relay public posts with other NeoDB instances.
-
-
-## Settings for external item sources
-
-- `SPOTIFY_API_KEY` - base64('CLIENT_ID:SECRET'), see [spotify doc](https://developer.spotify.com/documentation/web-api/tutorials/client-credentials-flow)
-- `TMDB_API_V3_KEY` - API v3 key from [TMDB](https://developer.themoviedb.org/)
-- `GOOGLE_API_KEY` - API key for [Google Books](https://developers.google.com/books/docs/v1/using)
-- `DISCOGS_API_KEY` - personal access token from [Discogs](https://www.discogs.com/settings/developers)
-- `IGDB_API_CLIENT_ID`, `IGDB_API_CLIENT_SECRET` - IGDB [keys](https://api-docs.igdb.com/)
-- `NEODB_SEARCH_SITES` is empty by default, which means NeoDB will search all available sources. This can be set to a comma-separated list of site names (e.g. `goodreads,googlebooks,spotify,tmdb,igdb,bandcamp,apple_podcast`), so that NeoDB will only search those sites; or not search any of them if set to just `-`.
-
-
-## Other maintenance tasks
-
-Add alias to your shell for easier access
-
-```
-alias neodb-manage='docker-compose --profile production run --rm shell neodb-manage'
-```
-
-Toggle user's active, staff and super user status
-
-```
-neodb-manage user --active
-neodb-manage user --staff
-neodb-manage user --super
-```
-
-create a super user; delete a user / remote identity (`takahe-stator` and `neodb-worker` containers must be running to complete the deletion)
-```
-neodb-manage createsuperuser
-neodb-manage user --delete username
-neodb-manage user --delete username@remote.instance
-```
-
-Create an invite link
-
-```
-neodb-manage invite --create
-```
-
-Manage user tasks and cron jobs
-
-```
-neodb-manage task --list
-neodb-manage cron --list
-```
-
-Manage search index
-```
-neo-manage index --reindex
-```
-
-Crawl links
-```
-neodb-manage cat [--save] # parse / save a supported link
-neodb-manage crawl # crawl all recognizable links from a page
-```
-
-
-## Run PostgresQL/Redis/Typesense without Docker
-
-It's currently possible but quite cumbersome to run without Docker, hence not recommended. However it's possible to only use docker to run neodb server but reuse existing PostgresQL/Redis/Typesense servers with `compose.override.yml`, an example for reference:
-
-```
-services:
- redis:
- profiles: ['disabled']
- typesense:
- profiles: ['disabled']
- neodb-db:
- profiles: ['disabled']
- takahe-db:
- profiles: ['disabled']
- migration:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- neodb-web:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- healthcheck: !reset {}
- neodb-web-api:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- healthcheck: !reset {}
- neodb-worker:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- neodb-worker-extra:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- takahe-web:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- takahe-stator:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- shell:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- root:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-neodb-web:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-neodb-worker:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-takahe-web:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-takahe-stator:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-shell:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
- dev-root:
- extra_hosts:
- - "host.docker.internal:host-gateway"
- depends_on: !reset []
-```
-(`extra_hosts` is only needed if PostgresQL/Redis/Typesense is on your host server)
-
-
-## Multiple instances on one server
-
-It's possible to run multiple clusters in one host server with docker compose, as long as `NEODB_SITE_DOMAIN`, `NEODB_PORT` and `NEODB_DATA` are different.
-
-
-## Scaling up
-
-For high-traffic instance, spin up these configurations to a higher number, as long as the host server can handle them:
-
- - `NEODB_WEB_WORKER_NUM`
- - `NEODB_API_WORKER_NUM`
- - `NEODB_RQ_WORKER_NUM`
- - `TAKAHE_WEB_WORKER_NUM`
- - `TAKAHE_STATOR_CONCURRENCY`
- - `TAKAHE_STATOR_CONCURRENCY_PER_MODEL`
-
-Further scaling up with multiple nodes (e.g. via Kubernetes) is beyond the scope of this document, but consider run db/redis/typesense separately, and then duplicate web/worker/stator containers as long as connections and mounts are properly configured; `migration` only runs once when start or upgrade, it should be kept that way.
diff --git a/docs/development.md b/docs/development.md
deleted file mode 100644
index 95d3f973..00000000
--- a/docs/development.md
+++ /dev/null
@@ -1,187 +0,0 @@
-Development
-===========
-
-Overview
---------
-NeoDB is a Django project, and it runs side by side with a [modified version](https://github.com/neodb-social/neodb-takahe) of [Takahē](https://github.com/jointakahe/takahe) (a separate Django project, code in `neodb_takahe` folder of this repo as submodule). They communicate with each other mainly thru database and task queue, [the diagram](troubleshooting.md#containers) demonstrates a typical architecture. Currently the two are loosely coupled, so you may take either one offline without immediate impact on the other, which makes it very easy to conduct maintenance and troubleshooting separately. In the future, they may get combined but it's not decided and will not be decided very soon.
-
-
-Prerequisite
-------------
-- Python 3.12.x
-- Docker Compose v2 or newer
-
-
-Prepare the code
-----------------
-When checking out NeoDB source code, make sure submodules are also checked out:
-```
-git clone https://github.com/neodb-social/neodb.git
-cd neodb
-git submodule update --init
-```
-
-Install [rye](http://rye.astral.sh) package manager, packages and pre-commit hooks:
-```
-curl -sSf https://rye.astral.sh/get | bash
-rye sync
-. .venv/bin/activate
-pre-commit install
-```
-
-To develop Takahe, install requirements(-dev) and pre-commit hooks for `neodb-takahe` project as well, preferably using a different virtual environment.
-
-Note: the virtual environments and packages installed in this step are mostly for linting, the actual virtual environments and packages are from NeoDB docker image, and they can be configured differently, more on this later in this document.
-
-
-Start local instance for development
-------------------------------------
-Follow [install guide](install.md) to create `.env` in the root folder of NeoDB code, including at least these configuration:
-```
-NEODB_SITE_NAME="My Test"
-NEODB_SITE_DOMAIN=mydomain.dev
-NEODB_SECRET_KEY=_random_string__50_characters_of_length__no_whitespaces_
-NEODB_IMAGE=neodb/neodb:edge
-NEODB_DEBUG=True
-```
-
-Download docker images and start pgsql/redis/typesense before initializing database schema:
-```
-docker compose --profile dev pull
-docker compose up -d
-```
-
-Initialize database schema:
-```
-docker compose --profile dev run --rm dev-shell takahe-manage collectstatic --noinput
-docker compose --profile dev run --rm dev-shell neodb-init
-```
-
-Start the cluster:
-```
-docker compose --profile dev up -d
-```
-
-Watch the logs:
-```
-docker compose --profile dev logs -f
-```
-
-Now the local development instance is ready to serve at `http://localhost:8000`, but to develop or test anything related with ActivityPub, reverse proxying it from externally reachable https://`NEODB_SITE_DOMAIN`/ is required; https is optional theoretically but in reality required for various compatibility reasons.
-
-Note: `dev` profile is for development only, and quite different from `production`, so always use `--profile dev` instead of `--profile production`, more on those differences later in this document.
-
-
-Common development tasks
-------------------------
-Shutdown the cluster:
-```
-docker compose --profile dev down
-```
-
-Restart background tasks (unlike web servers, background tasks don't auto reload after code change):
-```
-docker-compose --profile dev restart dev-neodb-worker dev-takahe-stator
-```
-
-When updating code, always update submodules:
-```
-git pull
-git submodule update --init
-```
-With newer git 2.15+, you main use `git pull --recurse-submodules` or `git config --global submodule.recurse true` to make it automatic.
-
-
-To save some typing, consider adding some aliases to `~/.profile`:
-```
-alias neodb-logs='docker compose --profile dev logs'
-alias neodb-shell='docker compose --profile dev run --rm dev-shell'
-alias neodb-manage='docker compose --profile dev run --rm dev-shell neodb-manage'
-```
-
-Always use `neodb-init`, not `python3 manage.py migrate`, to update db schema after updating code:
-```
-neodb-shell neodb-init
-```
-
-Run unit test:
-```
-neodb-manage test
-```
-
-Update translations:
-```
-django-admin makemessages --no-wrap --no-obsolete -i .venv -i neodb-takahe --keep-pot -l zh_Hans -l zh_Hant
-
-# edit .po files, run the following to make sure edits are correct
-
-django-admin makemessages --no-wrap --no-obsolete -i .venv -i neodb-takahe --keep-pot -l zh_Hans -l zh_Hant
-django-admin compilemessages -i .venv -i neodb-takahe
-```
-
-Preview documentation:
-```
-python -m mkdocs serve
-```
-
-Development in Docker Compose
------------------------------
-The `dev` profile is different from `production`:
-
-- code in `NEODB_SRC` (default: .) and `TAKAHE_SRC` (default: ./neodb-takahe) will be mounted and used in the container instead of code in the image
-- `runserver` with autoreload will be used instead of `gunicorn` for both neodb and takahe web server
-- /static/ and /s/ url are not map to pre-generated/collected static file path, `NEODB_DEBUG=True` is required locate static files from source code
-- one `rqworker` container will be started, instead of two
-- use `dev-shell` and `dev-root` to invoke shells, instead of `shell` and `root`
-- there's no automatic `migration` container, but it can be triggered manually via `docker compose run dev-shell neodb-init`
-
-Note:
-
-- Python virtual environments inside docker image, which are `/neodb-venv` and `/takahe-venv`, will be used by default. They can be changed to different locations with `TAKAHE_VENV` and `NEODB_VENV` if needed, usually in a case of development code using a package not in docker venv.
-- Some packages inside python virtual environments are platform dependent, so mount venv built by macOS host into the Linux container will likely not work.
-- Python servers are launched as `app` user, who has no write access to anywhere except /tmp and media path, that's by design.
-- Database/redis/typesense used in the container cluster are not accessible from host directly, which is by design. Querying them can be done by one of the following:
- - `neodb-manage dbshell`
- - `neodb-shell redis-cli -h redis`
- - or create `compose.override.yml` to uncomment `ports` section.
-- To expose the neodb and takahe web server directly, in the folder for configuration, create `compose.override.yml` with the following content:
-
-```
-services:
- dev-neodb-web:
- ports:
- - "8001:8000"
-
- dev-takahe-web:
- ports:
- - "8002:8000"
-```
-
-
-Development with Github Codespace
----------------------------------
-At the time of writing, docker compose will work in Github Codespace by adding this in `.env`:
-
-```
-NEODB_SITE_DOMAIN=${CODESPACE_NAME}-8000.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}
-```
-
-
-Applications
-------------
-Main Django apps for NeoDB:
-
- - `users` manages user in typical Django fashion
- - `mastodon` this leverages [Mastodon API](https://docs.joinmastodon.org/client/intro/), [Threads API](https://developers.facebook.com/docs/threads/) and [ATProto}(https://atproto.com) for user login and data sync
- - `catalog` manages different types of items user may collect, and scrapers to fetch from external resources, see [catalog.md](internals/catalog.md) for more details
- - `journal` manages user created content(review/ratings) and lists(collection/shelf/tag/note), see [journal.md](internals/journal.md) for more details
- - `social` present timeline and notification for local users
- - `takahe` communicate with Takahe (a separate Django server, run side by side with this server, code in `neodb_takahe` as submodule), see [federation.md](internals/federation.md) for customization of ActivityPub protocol
- - `legacy` this is only used by instances upgraded from 0.4.x and earlier, to provide a link mapping from old urls to new ones. If your journey starts with 0.5 and later, feel free to ignore it.
-
-
-Miscellaneous notes
--------------------
-If models in `takahe/models.py` are changed, instead of adding incremental migrations, just regenerate `takahe/migrations/0001_initial.py` instead, because these migrations will never be applied except for constructing a test database.
-
-A `libsass` wheel is stored in the repo to speed up docker image building process in Github Action.
diff --git a/docs/features.md b/docs/features.md
deleted file mode 100644
index d054e7cc..00000000
--- a/docs/features.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# Features
-
-NeoDB has [various features](features.md), and you may image it as a mix of Mastodon, Goodreads, Letterboxd, RateYourMusic, Podchaser, and more.
-
-## Public catalog
-
- - a shared catalog of books/movies/tv shows/music album/games/podcasts/performances
- - search or create catalog items in each category
- - one click create item with links to 3rd party sites:
- - Goodreads
- - IMDB
- - The Movie Database
- - Douban
- - Google Books
- - Discogs
- - Spotify
- - Apple Music
- - Bandcamp
- - Steam
- - IGDB
- - Bangumi
- - Board Game Geek
- - any RSS link to a podcast
- - ...[full list](sites.md)
-
-
-## Personal collections
-
- - mark an item as wishlist/in progress/complete/dropped
- - rate and write reviews for an item
- - write notes for an item with progress (e.g. reading notes at page 42)
- - create tags for an item, either privately or publicly
- - create and share list of items
- - tracking progress of a list (e.g. personal reading challenges)
- - Import and export full user data archive
- - import list or archives from some 3rd party sites:
- - Goodreads reading list
- - Letterboxd watch list
- - Douban archive (via [Doufen](https://doufen.org/))
-
-
-## Social
-
- - view home feed with friends' activities
- - every activity can be set as viewable to self/follower-only/public
- - eligible items, e.g. podcasts and albums, are playable in feed
- - login with other Fediverse identity and import social graph
- - supported servers: Mastodon/Pleroma/Firefish/GoToSocial/Pixelfed/friendica/Takahē
- - login with Bluesky / ATProto identity and import social graph
- - login with threads.net (requires app verification by Meta)
- - share collections and reviews to Fediverse/Bluesky/Threads
- - ActivityPub support
- - NeoDB users can follow and interact with users on other ActivityPub services like Mastodon and Pleroma
- - NeoDB instances communicate with each other via an extended version of ActivityPub
- - NeoDB instances may share public rating and reviews with a default relay
- - implementation is based on [Takahē](https://jointakahe.org/) server
-
-
-## API
- - Mastodon compatible API
- - most mastodon compatible apps are compatible with NeoDB
- - NeoDB API to manage reviews and collections
-
-## Languages
-
- - English
- - Simplified Chinese
- - Traditional Chinese
- - more to come and your contributions are welcomed!
diff --git a/docs/index.md b/docs/index.md
deleted file mode 100644
index 7b504e75..00000000
--- a/docs/index.md
+++ /dev/null
@@ -1,47 +0,0 @@
-NeoDB is an open-source software and global community platform [since 2021](origin.md). It helps users to manage and explore collections, reviews, and ratings for various cultural products, including books, movies, music, podcasts, games, and performances. Additionally, users can share their collections, publish microblogs, and engage with others in the Fediverse. NeoDB integrates the functionalities of platforms like Goodreads, Letterboxd, RateYourMusic, and Podchaser, among others. It also supports self-hosting and interconnection through containerized deployment and the ActivityPub protocol.
-
-This project is licensed under AGPLv3.
-
-
-## How to use
-
-Please [find a server](servers.md) to register with your email, Mastodon or Bluesky ID.
-
-Besides the web version of your chosen server, you may also use [various apps](apps.md) to login and manage your collections.
-
-
-## Host your own instance
-
-You can [install and run your own instance of NeoDB](install.md). If you decide to share your server with the public, please inform us so that we can add it to the list above.
-
-All instances interact with each other in the Fediverse via ActivityPub, allowing you to follow users from other NeoDB instances or instances running other ActivityPub software, such as Mastodon.
-
-
-## API, Development and Translation
-
- - NeoDB offers [APIs to manage user collections](api.md), and [Mastodon client compatible API](https://docs.joinmastodon.org/client/) to manage user posts.
- - For those interested in developing for NeoDB, please refer to the [development](development.md) section for basic instructions to get started.
- - To help translate NeoDB to more languages, please join [our project on Weblate](https://hosted.weblate.org/projects/neodb/)
-
-
-## Copyleft
-
- - NeoDB software code is licensed under AGPL, please check it if you plan to provide a service with modified code.
- - If you publish a work (e.g. application, website or service) utilizing NeoDB software or API, you may use "NeoDB" and its visual identity in your work, but please refrain from any usage that may imply the affiliation with NeoDB software and team.
- - If you publish such work to serve others, we do appreciate you clearly mention the usage of NeoDB in your work, provide descriptions and links to NeoDB software (neodb.net) and service providers (e.g. neodb.social), and guide your users to support them directly.
-
-
-## Donation
-
-If you appreciate this project, please help spread the words, and consider sponsoring the service providers. Your support is essential to keep these services free, open-sourced and stay committed to their mission. You may donate to the flagship instance, NeoDB.social, through the link below; or check the instance you use for their details about donations.
-
- [](https://ko-fi.com/neodb)
-
-
-## Feedback
-
-Follow us on the Fediverse or join our Discord community to share your ideas, questions, or creations.
-
-[](https://mastodon.online/@neodb)
-[](https://mastodon.social/@neodb)
-[](https://discord.gg/QBHkrV8bxK)
diff --git a/docs/install.md b/docs/install.md
deleted file mode 100644
index 3ee14c42..00000000
--- a/docs/install.md
+++ /dev/null
@@ -1,90 +0,0 @@
-Install
-=======
-
-For small and medium NeoDB instances, it's recommended to deploy as a container cluster with Docker Compose. To run a large instance, please see [scaling up](configuration.md#scaling-up) for some tips.
-
-## Install docker compose
-
-Follow [official instructions](https://docs.docker.com/compose/install/) to install Docker Compose if not yet.
-
-Please verify its version is 2.x or higher before next step:
-
-```
-docker compose version
-```
-
-The rest of this doc assumes you can run docker commands without `sudo`, to verify that:
-
-```
-docker run --rm hello-world
-```
-
-Follow [official instructions](https://docs.docker.com/engine/install/linux-postinstall/) if it's not enabled, or use `sudo` to run commands in this doc.
-
-
-## Prepare configuration files
- - create a folder for configuration, eg ~/mysite/config
- - grab `compose.yml` and `neodb.env.example` from [latest release](https://github.com/neodb-social/neodb/releases)
- - rename `neodb.env.example` to `.env`
-
-
-## Set up .env file and web root
-Change essential options like `NEODB_SITE_DOMAIN` in `.env` before starting the cluster for the first time. Changing them later may have unintended consequences, please make sure they are correct before exposing the service externally.
-
-- `NEODB_SITE_NAME` - name of your site
-- `NEODB_SITE_DOMAIN` - domain name of your site
-- `NEODB_SECRET_KEY` - encryption key of session data
-- `NEODB_DATA` is the path to store db/media/cache, it's `../data` by default, but can be any path that's writable
-- `NEODB_DEBUG` - set to `False` for production deployment
-- `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, 'en,zh' by default.
-
-Optionally, `robots.txt` and `logo.png` may be placed under `$NEODB_DATA/www-root/`.
-
-See [neodb.env.example](https://raw.githubusercontent.com/neodb-social/neodb/main/neodb.env.example) and [configuration](configuration.md) for more options
-
-
-## Start container
-
-in the folder with `compose.yml` and `.env`, execute as the user you just created:
-```
-docker compose --profile production pull
-docker compose --profile production up -d
-```
-
-Starting up for the first time might take a few minutes, depending on download speed, use the following commands for status and logs:
-```
-docker compose ps
-docker compose --profile production logs -f
-```
-
-In a few seconds, the site should be up at 127.0.0.1:8000 , you may check it with:
-```
-curl http://localhost:8000/nodeinfo/2.0/
-```
-
-JSON response will be returned if the server is up and running:
-```
-{"version": "2.0", "software": {"name": "neodb", "version": "0.8-dev"}, "protocols": ["activitypub", "neodb"], "services": {"outbound": [], "inbound": []}, "usage": {"users": {"total": 1}, "localPosts": 0}, "openRegistrations": true, "metadata": {}}
-```
-
-
-## Make the site available publicly
-
-Next step is to expose `http://127.0.0.1:8000` to external network as `https://yourdomain.tld` (NeoDB requires `https`). There are many ways to do it, you may use nginx or caddy as a reverse proxy server with an SSL cert configured, or configure a tunnel provider like cloudflared to do the same. Once done, you may check it with:
-
-```
-curl https://yourdomain.tld/nodeinfo/2.0/
-```
-
-You should see the same JSON response as above, and the site is now accessible to the public.
-
-
-## Register an account and make it admin
-
-Open `https://yourdomain.tld` in your browser and register an account, assuming username `admin`, run the following command to make it super user
-
-```
-docker compose --profile production run --rm shell neodb-manage user --super admin
-```
-
-Now your instance should be ready to serve. More tweaks are available, see [configuration](configuration.md) for options.
diff --git a/docs/internals/catalog.md b/docs/internals/catalog.md
deleted file mode 100644
index 5d25a6c4..00000000
--- a/docs/internals/catalog.md
+++ /dev/null
@@ -1,114 +0,0 @@
-Catalog
-=======
-
-Data Models
------------
-all types of catalog items inherits from `Item` which stores as multi-table django model.
-one `Item` may have multiple `ExternalResource`s, each represents one page on an external site
-
-```mermaid
-classDiagram
- class Item {
- <>
- }
- Item <|-- Album
- class Album {
- +String barcode
- +String Douban_ID
- +String Spotify_ID
- }
- Item <|-- Game
- class Game {
- +String Steam_ID
- }
- Item <|-- Podcast
- class Podcast {
- +String feed_url
- +String Apple_ID
- }
- Item <|-- Performance
- Item <|-- Work
- class Work {
- +String Douban_Work_ID
- +String Goodreads_Work_ID
- }
- Item <|-- Edition
- Item <|-- Series
-
- Series *-- Work
- Work *-- Edition
-
- class Series {
- +String Goodreads_Series_ID
- }
- class Work {
- +String Douban_ID
- +String Goodreads_ID
- }
- class Edition{
- +String ISBN
- +String Douban_ID
- +String Goodreads_ID
- +String GoogleBooks_ID
- }
-
- Item <|-- Movie
- Item <|-- TVShow
- Item <|-- TVSeason
- Item <|-- TVEpisode
- TVShow *-- TVSeason
- TVSeason *-- TVEpisode
-
- class TVShow{
- +String IMDB_ID
- +String TMDB_ID
- }
- class TVSeason{
- +String Douban_ID
- +String TMDB_ID
- }
- class TVEpisode{
- +String IMDB_ID
- +String TMDB_ID
- }
- class Movie{
- +String Douban_ID
- +String IMDB_ID
- +String TMDB_ID
- }
-
- Item <|-- Collection
-
- ExternalResource --* Item
- class ExternalResource {
- +enum site
- +url: string
- }
-```
-
-Add a new site
---------------
-
- - If official API is available for the site, it should be the preferred way to get data.
- - add a new value to `IdType` and `SiteName` in `catalog/common/models.py`
- - add a new file in `catalog/sites/`, a new class inherits `AbstractSite`, with:
- * `SITE_NAME`
- * `ID_TYPE`
- * `URL_PATTERNS`
- * `WIKI_PROPERTY_ID` (not used now)
- * `DEFAULT_MODEL` (unless specified in `scrape()` return val)
- * a classmethod `id_to_url()`
- * a method `scrape()` returns a `ResourceContent` object
- * `BasicDownloader` or `ProxiedDownloader` can used to download website content or API data. e.g. `content = BasicDownloader(url).download().html()`
- * check out existing files in `catalog/sites/` for more examples
- - add an import in `catalog/sites/__init__.py`
- - add some tests to `catalog//tests.py` according to site type
- + add `DOWNLOADER_SAVEDIR = '/tmp'` to `settings.py` can save all response to /tmp
- + run `neodb-manage cat ` for debugging or saving response file to `/tmp`. Detailed code of `cat` is in `catalog/management/commands/cat.py`
- + move captured response file to `test_data/`, except large/images files. Or if have to, replace it with a smallest version (e.g. 1x1 pixel / 1s audio)
- + add `@use_local_response` decorator to test methods that should pick up these responses (if `BasicDownloader` or `ProxiedDownloader` is used)
- - run all the tests and make sure they pass
- - Command: `neodb-manage python3 manage.py test [--keepdb]`.
- - See [this issue](https://github.com/neodb-social/neodb/issues/5) if `lxml.etree.ParserError` occurs on macOS.
- - add a site UI label style to `common/static/scss/_sitelabel.scss`
- - update documentation in [sites.md](../sites.md)
diff --git a/docs/internals/federation.md b/docs/internals/federation.md
deleted file mode 100644
index 496019cb..00000000
--- a/docs/internals/federation.md
+++ /dev/null
@@ -1,131 +0,0 @@
-# Federation
-
-## Supported federation protocols and standards
-
-- [ActivityPub](https://www.w3.org/TR/activitypub/) (Server-to-Server)
-- [WebFinger](https://webfinger.net/)
-- [Http Signatures](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures)
-- [NodeInfo](https://nodeinfo.diaspora.software/)
-
-## Supported FEPs
-
-- [FEP-f1d5: NodeInfo in Fediverse Software](https://codeberg.org/fediverse/fep/src/branch/main/fep/f1d5/fep-f1d5.md)
-
-## NodeInfo
-
-NeoDB instances can be identified from user agent string (`NeoDB/x.x (+https://example.org)`) and `protocols` in its nodeinfo, e.g. https://neodb.social/nodeinfo/2.0/ :
-```json
-{
- "version": "2.0",
- "software": {
- "name": "neodb",
- "version": "0.10.4.13",
- "repository": "https://github.com/neodb-social/neodb",
- "homepage": "https://neodb.net/"
- },
- "protocols": ["activitypub", "neodb"],
-}
-```
-
-
-## ActivityPub
-
-NeoDB's ActivityPub implementation is based on [Takahē](https://jointakahe.org), with some change to enable interchange of additional information between NeoDB instances.
-
-### Activity
-
-NeoDB add additional fields to `Note` activity:
-
- - `relatedWith` is a list of NeoDB specific activities which are associated with this `Note`. For each activity, `id` and `href` are both unique links to that activity, `withRegardTo` links to the catalog item, `attributedTo` links to the user, `type` is one of:
- - `Status`, its `status` can be one of: `complete`, `progress`, `wishlist` and `dropped`
- - `Rating`, its `value` is rating grade (int, 1-10), `worst` is always 1, `best` is always 10
- - `Comment`, its `content` is comment text
- - `Review`, its `name` is review title, `content` is its body, `mediaType` is always `text/markdown` for now
- - `Note`, its `content` is note text
- - `tag` is used to store list of NeoDB catalog items, which are related with this activity. `type` of NeoDB catalog item can be one of `Edition`, `Movie`, `TVShow`, `TVSeason`, `TVEpisode`, `Album`, `Game`, `Podcast`, `PodcastEpisode`, `Performance`, `PerformanceProduction`; href will be the link to that item.
-
-Example:
-```json
-{
- "@context": ["https://www.w3.org/ns/activitystreams", {
- "blurhash": "toot:blurhash",
- "Emoji": "toot:Emoji",
- "focalPoint": {
- "@container": "@list",
- "@id": "toot:focalPoint"
- },
- "Hashtag": "as:Hashtag",
- "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
- "sensitive": "as:sensitive",
- "toot": "http://joinmastodon.org/ns#",
- "votersCount": "toot:votersCount",
- "featured": {
- "@id": "toot:featured",
- "@type": "@id"
- }
- }, "https://w3id.org/security/v1"],
- "id": "https://neodb.social/@april_long_face@neodb.social/posts/380919151408919488/",
- "type": "Note",
- "relatedWith": [{
- "id": "https://neodb.social/p/5oyF0qRx96mKKmVpFzHtMM",
- "type": "Status",
- "status": "complete",
- "withRegardTo": "https://neodb.social/movie/7hfF7d0aFMaqHpFjUpq4zR",
- "attributedTo": "https://neodb.social/@april_long_face@neodb.social/",
- "href": "https://neodb.social/p/5oyF0qRx96mKKmVpFzHtMM",
- "published": "2024-11-17T10:16:42.745240+00:00",
- "updated": "2024-11-17T10:16:42.750917+00:00"
- }, {
- "id": "https://neodb.social/p/47cJnbQTkbSSN2izLwQMjo",
- "type": "Comment",
- "withRegardTo": "https://neodb.social/movie/7hfF7d0aFMaqHpFjUpq4zR",
- "attributedTo": "https://neodb.social/@april_long_face@neodb.social/",
- "content": "Broadway cin\u00e9math\u00e8que, at least I laughed hard.",
- "href": "https://neodb.social/p/47cJnbQTkbSSN2izLwQMjo",
- "published": "2024-11-17T10:16:42.745240+00:00",
- "updated": "2024-11-17T10:16:42.777276+00:00"
- }, {
- "id": "https://neodb.social/p/3AyYu974qo6OU09AAsPweQ",
- "type": "Rating",
- "best": 10,
- "value": 7,
- "withRegardTo": "https://neodb.social/movie/7hfF7d0aFMaqHpFjUpq4zR",
- "worst": 1,
- "attributedTo": "https://neodb.social/@april_long_face@neodb.social/",
- "href": "https://neodb.social/p/3AyYu974qo6OU09AAsPweQ",
- "published": "2024-11-17T10:16:42.784220+00:00",
- "updated": "2024-11-17T10:16:42.786458+00:00"
- }],
- "attributedTo": "https://neodb.social/@april_long_face@neodb.social/",
- "content": "
\u770b\u8fc7 \u963f\u8bfa\u62c9 \ud83c\udf15\ud83c\udf15\ud83c\udf15\ud83c\udf17\ud83c\udf11 Broadway cin\u00e9math\u00e8que, at least I laughed hard.
