switch ActivityPub upstream from Takahe to Incarnator

This commit is contained in:
Your Name 2024-06-01 01:52:22 -04:00 committed by Henri Dickson
parent c52ac69ae9
commit 9654e84568
9 changed files with 345 additions and 371 deletions

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "neodb-takahe"]
path = neodb-takahe
url = https://github.com/neodb-social/neodb-takahe.git
url = https://github.com/neodb-social/neodb-incarnator.git
branch = neodb

View file

@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
FROM python:3.11-slim as build
FROM python:3.12-slim as build
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
@ -21,10 +21,10 @@ RUN --mount=type=cache,sharing=locked,target=/root/.cache /neodb-venv/bin/python
WORKDIR /takahe
RUN python -m venv /takahe-venv
RUN --mount=type=cache,sharing=locked,target=/root/.cache /takahe-venv/bin/python3 -m pip install --upgrade -r requirements.txt
RUN --mount=type=cache,sharing=locked,target=/root/.cache /takahe-venv/bin/python3 -m pip install --upgrade -r requirements.lock
# runtime stage
FROM python:3.11-slim as runtime
FROM python:3.12-slim as runtime
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1

View file

@ -188,7 +188,7 @@ services:
neodb-worker-extra:
<<: *neodb-service
command: neodb-manage rqworker mastodon fetch crawl ap
command: neodb-manage rqworker-pool --num-workers ${NEODB_RQ_WORKER_NUM:-4} mastodon fetch crawl ap
depends_on:
migration:
condition: service_completed_successfully

View file

@ -8,7 +8,7 @@ NeoDB is a Django project, and it runs side by side with a [modified version](ht
Prerequisite
------------
- Python 3.11.x
- Python 3.12.x
- Docker Compose v2 or newer

View file

@ -142,6 +142,6 @@ It's possible to run multiple clusters in one host server, as long as `NEODB_SIT
## Scaling
For high-traffic instance, spin up `NEODB_WEB_WORKER_NUM`, `TAKAHE_WEB_WORKER_NUM`, `TAKAHE_STATOR_CONCURRENCY` and `TAKAHE_STATOR_CONCURRENCY_PER_MODEL` as long as the host server can handle them.
For high-traffic instance, spin up `NEODB_WEB_WORKER_NUM`, `NEODB_API_WORKER_NUM`, `NEODB_RQ_WORKER_NUM`, `TAKAHE_WEB_WORKER_NUM`, `TAKAHE_STATOR_CONCURRENCY` and `TAKAHE_STATOR_CONCURRENCY_PER_MODEL` as long as the host server can handle them.
Further scaling up with multiple nodes (e.g. via Kubernetes) is beyond the scope of this document, but consider run db/redis/typesense separately, and then duplicate web/worker/stator containers as long as connections and mounts are properly configured; `migration` only runs once when start or upgrade, it should be kept that way.

@ -1 +1 @@
Subproject commit bf2db828b2ae0a6847155ef25f283ebfab75cb23
Subproject commit f474e46fa1661460bc2443b7845395afe89b5503

View file

@ -2,6 +2,10 @@
# copy along with compose.yml, rename this file to .env
# Must uncomment these if you are doing development
# NEODB_DEBUG=True
# NEODB_IMAGE=neodb/neodb:edge
# Change these before start the instance for the first time!!
NEODB_SECRET_KEY=change_me
NEODB_SITE_NAME=Example Site
@ -13,9 +17,9 @@ NEODB_SITE_LOGO=/logo.png
NEODB_SITE_ICON=/icon.png
NEODB_SITE_LINKS=@NiceDB=https://donotban.com/@testie,@NeoDB=https://mastodon.social/@neodb
# Uncomment these if you are doing development
# NEODB_DEBUG=True
# NEODB_IMAGE=neodb/neodb:edge
# To enable push notification, generate a keypair from https://web-push-codelab.glitch.me
# TAKAHE_VAPID_PUBLIC_KEY=
# TAKAHE_VAPID_PRIVATE_KEY=tEPrD2JJkibcDC1G_sE8ZgZGTbd5w64spEJ7h2fGgxk
# HTTP port your reverse proxy should send request to
# NEODB_PORT=8000
@ -28,6 +32,8 @@ NEODB_SITE_LINKS=@NiceDB=https://donotban.com/@testie,@NeoDB=https://mastodon.so
# Scaling parameters
# NEODB_WEB_WORKER_NUM=32
# NEODB_API_WORKER_NUM=16
# NEODB_RQ_WORKER_NUM=8
# TAKAHE_WEB_WORKER_NUM=32
# TAKAHE_STATOR_CONCURRENCY=10
# TAKAHE_STATOR_CONCURRENCY_PER_MODEL=10

View file

@ -1,4 +1,4 @@
# Generated by Django 4.2.4 on 2023-08-12 16:48
# Generated by Django 4.2.13 on 2024-06-01 05:38
import functools
@ -10,33 +10,12 @@ import takahe.models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="TakaheSession",
fields=[
(
"session_key",
models.CharField(
max_length=40,
primary_key=True,
serialize=False,
verbose_name="session key",
),
),
("session_data", models.TextField(verbose_name="session data")),
(
"expire_date",
models.DateTimeField(db_index=True, verbose_name="expire date"),
),
],
options={
"db_table": "django_session",
},
),
migrations.CreateModel(
name="Domain",
fields=[
@ -148,22 +127,38 @@ class Migration(migrations.Migration):
("actor_uri", models.CharField(max_length=500, unique=True)),
("state", models.CharField(default="outdated", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("state_next_attempt", models.DateTimeField(blank=True, null=True)),
(
"state_locked_until",
models.DateTimeField(blank=True, db_index=True, null=True),
),
("local", models.BooleanField(db_index=True)),
("username", models.CharField(blank=True, max_length=500, null=True)),
(
"name",
models.CharField(
blank=True, max_length=500, null=True, verbose_name="昵称"
blank=True,
max_length=500,
null=True,
verbose_name="Display Name",
),
),
("summary", models.TextField(blank=True, null=True, verbose_name="简介")),
(
"summary",
models.TextField(blank=True, null=True, verbose_name="Bio"),
),
(
"manually_approves_followers",
models.BooleanField(default=False, verbose_name="手工审核关注者"),
models.BooleanField(
default=False, verbose_name="Manually approve new followers"
),
),
(
"discoverable",
models.BooleanField(default=True, verbose_name="允许被发现或推荐"),
models.BooleanField(
default=True,
verbose_name="Include profile and posts in search and discovery",
),
),
(
"profile_uri",
@ -190,6 +185,30 @@ class Migration(migrations.Migration):
models.CharField(blank=True, max_length=500, null=True),
),
("actor_type", models.CharField(default="person", max_length=100)),
(
"icon",
models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("profile_images",), **{}
),
verbose_name="Profile picture",
),
),
(
"image",
models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("background_images",), **{}
),
verbose_name="Header picture",
),
),
("metadata", models.JSONField(blank=True, null=True)),
("pinned", models.JSONField(blank=True, null=True)),
("sensitive", models.BooleanField(default=False)),
@ -225,40 +244,48 @@ class Migration(migrations.Migration):
"db_table": "users_identity",
},
),
migrations.AddField(
model_name="identity",
name="icon",
field=models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("profile_images",), **{}
migrations.CreateModel(
name="InboxMessage",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
verbose_name="头像",
),
("message", models.JSONField()),
("state", models.CharField(default="received", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
],
options={
"db_table": "users_inboxmessage",
},
),
migrations.AddField(
model_name="identity",
name="image",
field=models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("background_images",), **{}
migrations.CreateModel(
name="Invite",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
),
),
migrations.AddField(
model_name="identity",
name="state_locked_until",
field=models.DateTimeField(blank=True, db_index=True, null=True),
),
migrations.AddField(
model_name="identity",
name="state_next_attempt",
field=models.DateTimeField(blank=True, null=True),
("token", models.CharField(max_length=500, unique=True)),
("note", models.TextField(blank=True, null=True)),
("uses", models.IntegerField(blank=True, null=True)),
("expires", models.DateTimeField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
],
options={
"db_table": "users_invite",
},
),
migrations.CreateModel(
name="Post",
@ -294,6 +321,7 @@ class Migration(migrations.Migration):
),
),
("content", models.TextField()),
("language", models.CharField(blank=True, default="")),
(
"type",
models.CharField(
@ -360,38 +388,6 @@ class Migration(migrations.Migration):
"db_table": "activities_post",
},
),
migrations.CreateModel(
name="User",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("password", models.CharField(max_length=128, verbose_name="password")),
(
"last_login",
models.DateTimeField(
blank=True, null=True, verbose_name="last login"
),
),
("email", models.EmailField(max_length=254, unique=True)),
("admin", models.BooleanField(default=False)),
("moderator", models.BooleanField(default=False)),
("banned", models.BooleanField(default=False)),
("deleted", models.BooleanField(default=False)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("last_seen", models.DateTimeField(auto_now_add=True)),
],
options={
"db_table": "users_user",
},
),
migrations.CreateModel(
name="PostInteraction",
fields=[
@ -448,22 +444,67 @@ class Migration(migrations.Migration):
"db_table": "activities_postinteraction",
},
),
migrations.AddField(
model_name="identity",
name="users",
field=models.ManyToManyField(
blank=True, related_name="identities", to="takahe.user"
),
),
migrations.AddField(
model_name="domain",
name="users",
field=models.ManyToManyField(
blank=True, related_name="domains", to="takahe.user"
),
migrations.CreateModel(
name="Relay",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("inbox_uri", models.CharField(max_length=500, unique=True)),
("state", models.CharField(default="new", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("state_next_attempt", models.DateTimeField(blank=True, null=True)),
(
"state_locked_until",
models.DateTimeField(blank=True, db_index=True, null=True),
),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
],
options={
"db_table": "users_relay",
},
),
migrations.CreateModel(
name="Block",
name="User",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("password", models.CharField(max_length=128, verbose_name="password")),
(
"last_login",
models.DateTimeField(
blank=True, null=True, verbose_name="last login"
),
),
("email", models.EmailField(max_length=254, unique=True)),
("admin", models.BooleanField(default=False)),
("moderator", models.BooleanField(default=False)),
("banned", models.BooleanField(default=False)),
("deleted", models.BooleanField(default=False)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("last_seen", models.DateTimeField(auto_now_add=True)),
],
options={
"db_table": "users_user",
},
),
migrations.CreateModel(
name="PostAttachment",
fields=[
(
"id",
@ -476,176 +517,71 @@ class Migration(migrations.Migration):
),
("state", models.CharField(default="new", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("uri", models.CharField(blank=True, max_length=500, null=True)),
("mute", models.BooleanField()),
("include_notifications", models.BooleanField(default=False)),
("expires", models.DateTimeField(blank=True, null=True)),
("note", models.TextField(blank=True, null=True)),
("mimetype", models.CharField(max_length=200)),
(
"file",
models.FileField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("attachments",), **{}
),
),
),
(
"thumbnail",
models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer,
*("attachment_thumbnails",),
**{}
),
),
),
("remote_url", models.CharField(blank=True, max_length=500, null=True)),
("name", models.TextField(blank=True, null=True)),
("width", models.IntegerField(blank=True, null=True)),
("height", models.IntegerField(blank=True, null=True)),
("focal_x", models.FloatField(blank=True, null=True)),
("focal_y", models.FloatField(blank=True, null=True)),
("blurhash", models.TextField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
(
"source",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="outbound_blocks",
to="takahe.identity",
),
),
(
"target",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="inbound_blocks",
to="takahe.identity",
),
),
],
options={
"db_table": "users_block",
},
),
migrations.AlterUniqueTogether(
name="identity",
unique_together={("username", "domain")},
),
migrations.CreateModel(
name="Follow",
fields=[
(
"id",
models.BigIntegerField(
default=takahe.models.Snowflake.generate_follow,
primary_key=True,
serialize=False,
),
),
(
"boosts",
models.BooleanField(
default=True, help_text="Also follow boosts from this user"
),
),
("uri", models.CharField(blank=True, max_length=500, null=True)),
("note", models.TextField(blank=True, null=True)),
("state", models.CharField(default="unrequested", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
(
"source",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="outbound_follows",
to="takahe.identity",
),
),
(
"target",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="inbound_follows",
to="takahe.identity",
),
),
],
options={
"db_table": "users_follow",
"unique_together": {("source", "target")},
},
),
migrations.CreateModel(
name="InboxMessage",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("message", models.JSONField()),
("state", models.CharField(default="received", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
],
options={
"db_table": "users_inboxmessage",
},
),
migrations.CreateModel(
name="Config",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("key", models.CharField(max_length=500)),
("json", models.JSONField(blank=True, null=True)),
("image", models.ImageField(blank=True, null=True, upload_to="")),
(
"domain",
"author",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="configs",
to="takahe.domain",
),
),
(
"identity",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="configs",
related_name="attachments",
to="takahe.identity",
),
),
(
"user",
"post",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="configs",
to="takahe.user",
related_name="attachments",
to="takahe.post",
),
),
],
options={
"db_table": "core_config",
"unique_together": {("key", "user", "identity", "domain")},
"db_table": "activities_postattachment",
},
),
migrations.CreateModel(
name="Invite",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("token", models.CharField(max_length=500, unique=True)),
("note", models.TextField(blank=True, null=True)),
("uses", models.IntegerField(blank=True, null=True)),
("expires", models.DateTimeField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
],
options={
"db_table": "users_invite",
},
migrations.AddField(
model_name="identity",
name="users",
field=models.ManyToManyField(
blank=True, related_name="identities", to="takahe.user"
),
),
migrations.CreateModel(
name="FanOut",
@ -723,6 +659,80 @@ class Migration(migrations.Migration):
"db_table": "activities_fanout",
},
),
migrations.AddField(
model_name="domain",
name="users",
field=models.ManyToManyField(
blank=True, related_name="domains", to="takahe.user"
),
),
migrations.CreateModel(
name="Block",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("state", models.CharField(default="new", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("uri", models.CharField(blank=True, max_length=500, null=True)),
("mute", models.BooleanField()),
("include_notifications", models.BooleanField(default=False)),
("expires", models.DateTimeField(blank=True, null=True)),
("note", models.TextField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
(
"source",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="outbound_blocks",
to="takahe.identity",
),
),
(
"target",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="inbound_blocks",
to="takahe.identity",
),
),
],
options={
"db_table": "users_block",
},
),
migrations.CreateModel(
name="Announcement",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("text", models.TextField()),
("published", models.BooleanField(default=False)),
("start", models.DateTimeField(blank=True, null=True)),
("end", models.DateTimeField(blank=True, null=True)),
("include_unauthenticated", models.BooleanField(default=False)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("seen", models.ManyToManyField(blank=True, to="takahe.user")),
],
options={
"db_table": "users_announcement",
},
),
migrations.CreateModel(
name="TimelineEvent",
fields=[
@ -813,149 +823,104 @@ class Migration(migrations.Migration):
],
},
),
migrations.AlterUniqueTogether(
name="identity",
unique_together={("username", "domain")},
),
migrations.CreateModel(
name="PostAttachment",
name="Follow",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
models.BigIntegerField(
default=takahe.models.Snowflake.generate_follow,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("state", models.CharField(default="new", max_length=100)),
(
"boosts",
models.BooleanField(
default=True, help_text="Also follow boosts from this user"
),
),
("uri", models.CharField(blank=True, max_length=500, null=True)),
("note", models.TextField(blank=True, null=True)),
("state", models.CharField(default="unrequested", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("mimetype", models.CharField(max_length=200)),
(
"file",
models.FileField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer, *("attachments",), **{}
),
),
),
(
"thumbnail",
models.ImageField(
blank=True,
null=True,
storage=takahe.models.upload_store,
upload_to=functools.partial(
takahe.models.upload_namer,
*("attachment_thumbnails",),
**{}
),
),
),
("remote_url", models.CharField(blank=True, max_length=500, null=True)),
("name", models.TextField(blank=True, null=True)),
("width", models.IntegerField(blank=True, null=True)),
("height", models.IntegerField(blank=True, null=True)),
("focal_x", models.FloatField(blank=True, null=True)),
("focal_y", models.FloatField(blank=True, null=True)),
("blurhash", models.TextField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
(
"author",
"source",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="attachments",
related_name="outbound_follows",
to="takahe.identity",
),
),
(
"post",
"target",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="inbound_follows",
to="takahe.identity",
),
),
],
options={
"db_table": "users_follow",
"unique_together": {("source", "target")},
},
),
migrations.CreateModel(
name="Config",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("key", models.CharField(max_length=500)),
("json", models.JSONField(blank=True, null=True)),
("image", models.ImageField(blank=True, null=True, upload_to="")),
(
"domain",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="attachments",
to="takahe.post",
),
),
],
options={
"db_table": "activities_postattachment",
},
),
migrations.CreateModel(
name="Relay",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("inbox_uri", models.CharField(max_length=500, unique=True)),
("state", models.CharField(default="new", max_length=100)),
("state_changed", models.DateTimeField(auto_now_add=True)),
("state_next_attempt", models.DateTimeField(blank=True, null=True)),
(
"state_locked_until",
models.DateTimeField(blank=True, db_index=True, null=True),
),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
],
options={
"db_table": "users_relay",
},
),
migrations.CreateModel(
name="Announcement",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
related_name="configs",
to="takahe.domain",
),
),
(
"text",
models.TextField(),
),
(
"published",
models.BooleanField(
default=False,
),
),
(
"start",
models.DateTimeField(
"identity",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="configs",
to="takahe.identity",
),
),
(
"end",
models.DateTimeField(
"user",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="configs",
to="takahe.user",
),
),
("include_unauthenticated", models.BooleanField(default=False)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("seen", models.ManyToManyField(blank=True, to="takahe.user")),
],
options={
"db_table": "users_announcement",
"db_table": "core_config",
"unique_together": {("key", "user", "identity", "domain")},
},
),
]

View file

@ -919,6 +919,9 @@ class Post(models.Model):
# The main (HTML) content
content = models.TextField()
# The language of the content
language = models.CharField(default="", blank=True)
type = models.CharField(
max_length=20,
choices=Types.choices,