app, Infrastructure.Geo.IGeoLocator geo)
{
_options = options;
_app = app;
+ _geo = geo;
}
public bool CrawlerEnabled => _options.CurrentValue.Crawler.Enabled;
+ public string GeoSource => _geo.Source;
public string UserAgent => Federation.Crawler.Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress);
public void OnGet() => Harden();
diff --git a/PrivaPub/Web/Pages/Stargazing.cshtml b/PrivaPub/Web/Pages/Stargazing.cshtml
index 074b49f..110d761 100644
--- a/PrivaPub/Web/Pages/Stargazing.cshtml
+++ b/PrivaPub/Web/Pages/Stargazing.cshtml
@@ -15,6 +15,15 @@
A server we exchange activities with is described once a week, from its public NodeInfo and, when it has one, its
Mastodon instance API. Its location comes from the address we reached, looked up in an offline database: only the
country is shown publicly for small servers, and only the CDN for servers behind one.
+ @if (Model.GeoSource is { } geo)
+ {
+ Locations come from @geo, licensed under
+ CC BY 4.0.
+ }
+ else
+ {
+ No location database is loaded yet, so servers are not located.
+ }
The crawler
diff --git a/PrivaPub/appsettings.Production.json b/PrivaPub/appsettings.Production.json
index d257f77..81490ff 100644
--- a/PrivaPub/appsettings.Production.json
+++ b/PrivaPub/appsettings.Production.json
@@ -2,6 +2,26 @@
"Media": {
"Root": "/var/lib/privapub/media"
},
+ "Registrations": {
+ "Mode": "Invitations"
+ },
+ "Statistics": {
+ "Crawler": {
+ "Enabled": true,
+ "Seeds": [
+ "mastodon.social",
+ "fosstodon.org",
+ "mas.to",
+ "lemmy.world",
+ "lemmy.ml",
+ "misskey.io",
+ "pixelfed.social",
+ "framatube.org",
+ "piefed.social",
+ "bookwyrm.social"
+ ]
+ }
+ },
"MongoSettings": {
"Database": "PrivaPub",
"LogsDatabase": "logs",
diff --git a/deploy/max/geo-update.sh b/deploy/max/geo-update.sh
deleted file mode 100755
index c574d9e..0000000
--- a/deploy/max/geo-update.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# Fetches the month's DB-IP Lite city and ASN databases (CC BY 4.0, https://db-ip.com) into the directory PrivaPub reads
-# them from (Statistics:GeoDirectory). The app swaps to new files on its own; a failed download leaves the old ones.
-set -euo pipefail
-dir="${GEO_DIR:-/var/lib/privapub/geo}"
-tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
-this=$(date -u +%Y-%m)
-last=$(date -u -d "$(date -u +%Y-%m-15) -1 month" +%Y-%m)
-for kind in city asn; do
- got=""
- for month in "$this" "$last"; do
- if curl -fsS --max-time 900 -o "$tmp/$kind.gz" "https://download.db-ip.com/free/dbip-$kind-lite-$month.mmdb.gz"; then
- got=$month; break
- fi
- done
- [ -n "$got" ] || { echo "no $kind database for $this or $last" >&2; exit 1; }
- gunzip -t "$tmp/$kind.gz"
- gunzip -c "$tmp/$kind.gz" > "$tmp/dbip-$kind-lite.mmdb"
- [ "$(stat -c %s "$tmp/dbip-$kind-lite.mmdb")" -gt 1000000 ] || { echo "the $kind database is too small" >&2; exit 1; }
- install -m 640 "$tmp/dbip-$kind-lite.mmdb" "$dir/.dbip-$kind-lite.mmdb.new"
- mv -f "$dir/.dbip-$kind-lite.mmdb.new" "$dir/dbip-$kind-lite.mmdb"
- echo "$kind: DB-IP Lite $got"
-done
diff --git a/deploy/max/setup.sh b/deploy/max/setup.sh
index 832a516..a028224 100755
--- a/deploy/max/setup.sh
+++ b/deploy/max/setup.sh
@@ -12,7 +12,7 @@ ACME=/root/.acme.sh/acme.sh
echo "== directories"
install -d -o "$RUNNER" -g www-data -m 755 /var/www/$HOST
install -d -o "$RUNNER" -g "$RUNNER" -m 750 /var/backups/$HOST
-install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo /var/lib/privapub/geo
+install -d -o www-data -g www-data -m 750 /var/lib/privapub /var/lib/privapub/mongo
echo "== sudoers"
SUDOERS=/etc/sudoers.d/$RUNNER
@@ -23,15 +23,10 @@ visudo -cf "$SUDOERS"
echo "== units"
install -m 644 "$SRC/systemd/privapub-mongod.service" /etc/systemd/system/privapub-mongod.service
install -m 644 "$SRC/systemd/$UNIT.service" /etc/systemd/system/$UNIT.service
-install -m 755 "$SRC/max/geo-update.sh" /usr/local/bin/privapub-geo-update
-install -m 644 "$SRC/systemd/privapub-geo.service" /etc/systemd/system/privapub-geo.service
-install -m 644 "$SRC/systemd/privapub-geo.timer" /etc/systemd/system/privapub-geo.timer
systemctl daemon-reload
systemctl enable --now privapub-mongod >/dev/null
systemctl enable $UNIT >/dev/null
-systemctl enable --now privapub-geo.timer >/dev/null
systemctl is-active privapub-mongod
-[ -f /var/lib/privapub/geo/dbip-city-lite.mmdb ] || systemctl start privapub-geo.service || echo "geolocation databases not fetched yet; the timer retries"
echo "== nginx snippet and bootstrap vhost"
install -m 644 "$SRC/nginx/privapub-headers.conf" /etc/nginx/snippets/privapub-headers.conf
diff --git a/deploy/systemd/privapub-geo.service b/deploy/systemd/privapub-geo.service
deleted file mode 100644
index ac7dd72..0000000
--- a/deploy/systemd/privapub-geo.service
+++ /dev/null
@@ -1,15 +0,0 @@
-[Unit]
-Description=PrivaPub: fetch the DB-IP Lite geolocation databases
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=oneshot
-User=www-data
-Group=www-data
-ExecStart=/usr/local/bin/privapub-geo-update
-NoNewPrivileges=true
-ProtectSystem=strict
-ProtectHome=true
-PrivateTmp=true
-ReadWritePaths=/var/lib/privapub/geo
diff --git a/deploy/systemd/privapub-geo.timer b/deploy/systemd/privapub-geo.timer
deleted file mode 100644
index 997a57e..0000000
--- a/deploy/systemd/privapub-geo.timer
+++ /dev/null
@@ -1,10 +0,0 @@
-[Unit]
-Description=PrivaPub: refresh the geolocation databases monthly
-
-[Timer]
-OnCalendar=*-*-03 04:00:00
-RandomizedDelaySec=6h
-Persistent=true
-
-[Install]
-WantedBy=timers.target
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index 51ea8af..c650b2c 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -29,8 +29,11 @@ Written 2026-10-01 from the original 2023 code, the decePubClient UI, a federati
- followers-only posts arrived as DMs on Pleroma and Akkoma;
- Akkoma's open polls showed as ended and refused votes.
- Still open: installing the geolocation timer on Max (`deploy/max/setup.sh`, as root), and the smoke persona's
- `PRIVAPUB_SMOKE_TOKEN` secret, without which the deploy skips the signed-in half of its Mastodon API check.
+ Both open items were closed without a root step (owner decisions, 2026-10-04): the server fetches its geolocation
+ databases itself, and the deploy makes and signs in as @thepra.
+- [ ] Everything on in production (owner decisions 2026-10-04): v1.18.0 self-updating geolocation, @thepra, the crawler
+ on, sign-up by invitation, one registrations switch; then signed audiences with circles for everyone and SecureMode on,
+ account privacy, and one answer everywhere (the mismatch sweep).
- [ ] P7 Threads, communities, moderation, the social graph
- [ ] P8 Signatures, discovery, the long tail
@@ -159,10 +162,24 @@ and circles (see Owner decisions).
| Local side in statistics | **Only the kind of local actor** (person, group, application), **and only on public and unlisted traffic.** DMs, followers-only and circle traffic are one "private" class, never broken out per server in public. Circles are never named, whether as a kind or as a reason. Fetches of our own documents are counted per day, never per server. |
| Reading-driven traffic | **Counted per day, never logged per event:** the media proxy, lookups a client asks for, and the client API per endpoint group (admin only). Client app names are not recorded. |
| Describing servers | **Every server we exchange activities with is described weekly**, from its NodeInfo (including the user counts it publishes) and its Mastodon instance API, never its contact account. These requests are unsigned, because they are not ActivityPub documents. Never on read. |
-| Server locations | **City and network (ASN) from the offline DB-IP Lite databases** (CC BY 4.0, attributed), downloaded monthly outside the app. The location comes from the address we connected to; an inbound sender's address is never recorded, and no address is stored. In public: city and network only for servers reporting at least 10 users and not behind a CDN; the country otherwise; only the CDN's name for CDN-fronted servers. The admin sees everything. |
-| Crawler | **Off by default** (`Statistics:Crawler:Enabled`). When on, it identifies as `PrivaPub-Stargazer/ (+https://privapub.thepra.dev/stargazing)`, where `/stargazing` explains it and how to opt out. It honours robots.txt (an unreachable robots.txt means "keep out") and domain blocks. It visits one server a minute, each at most weekly, and at most 5000 servers. It reads only robots.txt, NodeInfo, the instance API and the peers list, never accounts, posts or directories. Crawled servers stay marked as crawled. |
+| Server locations | **City and network (ASN) from the offline DB-IP Lite databases** (CC BY 4.0, attributed), downloaded monthly (by the server itself since 2026-10-04). The location comes from the address we connected to; an inbound sender's address is never recorded, and no address is stored. In public: city and network only for servers reporting at least 10 users and not behind a CDN; the country otherwise; only the CDN's name for CDN-fronted servers. The admin sees everything. |
+| Crawler | **Off by default** (`Statistics:Crawler:Enabled`); **on in production since 2026-10-04**, see below. When on, it identifies as `PrivaPub-Stargazer/ (+https://privapub.thepra.dev/stargazing)`, where `/stargazing` explains it and how to opt out. It honours robots.txt (an unreachable robots.txt means "keep out") and domain blocks. It visits one server a minute, each at most weekly, and at most 5000 servers. It reads only robots.txt, NodeInfo, the instance API and the peers list, never accounts, posts or directories. Crawled servers stay marked as crawled. |
| A remote account deletes itself | **Its posts are kept but hidden everywhere** (`Post.AuthorGone`): from timelines, profiles, search and lookups by id. Its follows and timeline rows go, as before. |
-| Signed-in smoke check in production | **An undiscoverable persona**, whose read-only token is the Gitea secret `PRIVAPUB_SMOKE_TOKEN`; the deploy checks `verify_credentials`, home and notifications with it. The owner creates both. |
+| Signed-in smoke check in production | **An undiscoverable persona**, the deploy checks `verify_credentials`, home and notifications with it. *Superseded 2026-10-04: the deploy makes and keeps the persona itself, below.* |
+
+### Owner decisions on running everything in production (2026-10-04)
+
+| Question | Decision |
+|---|---|
+| What runs in production | **Everything that is built is on and checked by the deploy**, and nothing waits on a person running a command. |
+| Geolocation | **The server fetches DB-IP Lite itself** (`GeoUpdater`): it checks daily, installs a new month's databases once they are published, refuses a file that does not open as the right kind of database, and keeps the old one when anything fails. No timer and no root step. The deploy fails if the databases are missing or more than 40 days old. |
+| Crawler | **On in production**, seeded with a handful of large servers of different kinds (`appsettings.Production.json`); the deploy fails if `/stargazing` does not say it is on. |
+| Sign-up | **Closed: invitations only.** A group invitation creates an account; open sign-up answers 403. NodeInfo, `/api/v1/instance` and `/api/v2/instance` read the same switch (`Registrations:Mode`), and the deploy fails if they disagree. The first login on a server is made with `PrivaPub admin create-root`. |
+| Signed-in smoke check | **`@thepra`, undiscoverable, made and kept by the deploy**: `PrivaPub admin smoke thepra` creates or keeps the root `deploy-smoke` and the persona and gives the root a new password on every deploy; the deploy signs in through the real OAuth flow, checks the signed-in API and revokes its token. No secret is stored. |
+| Signed fetches (SecureMode) | **On in production** once the pasture passes with it on (Phase 2 of the 2026-10-04 plan). |
+| Circles on Mastodon and GoToSocial | **Each member's copy names that member** in `cc`; a member's refetch names the member, an instance actor's refetch the members on its server. Nothing new is revealed to anyone outside the circle. |
+| What circles and located posts reveal | **Unchanged**: circles still answer WebFinger, and circle and located posts still count in a persona's post count, "a good balance for the fediverse to work". |
+| Public `/stargazing` statistics | **Later**, as decided on 2026-10-03; the crawler and the admin API keep collecting meanwhile. |
## Libraries (researched; no maintained .NET ActivityPub library exists, so Letterbook and Iceshrimp.NET both wrote their own)
diff --git a/tools/pasture/appsettings.Pasture.json b/tools/pasture/appsettings.Pasture.json
index d5e7b24..51defee 100644
--- a/tools/pasture/appsettings.Pasture.json
+++ b/tools/pasture/appsettings.Pasture.json
@@ -26,6 +26,8 @@
},
"Media": { "Root": "/tmp/privapub-media" },
"RateLimits": { "AccountsPerMinute": 1000 },
+ "Registrations": { "Mode": "Open" },
+ "Statistics": { "Geo": { "AutoUpdate": false } },
"Kestrel": { "Endpoints": { "Http": { "Url": "http://0.0.0.0:80", "Protocols": "Http1AndHttp2" } } },
"Serilog": {
"MinimumLevel": { "Default": "Information", "Override": { "Microsoft": "Warning", "System": "Warning" } },
diff --git a/tools/pasture/lib/interop.sh b/tools/pasture/lib/interop.sh
index 6b44699..d416643 100644
--- a/tools/pasture/lib/interop.sh
+++ b/tools/pasture/lib/interop.sh
@@ -30,37 +30,14 @@ privapub_root() {
echo "$root" | j "print(d['token'])"
}
-# privapub_token : creates the persona under the pasture root if needed and returns a Mastodon token for it.
+# privapub_token : creates the persona under the pasture root if needed and returns a Mastodon token for it,
+# through the same OAuth flow the deploy's smoke check uses (tools/smoke/oauth.sh).
privapub_token() {
- local persona=$1 jwt cid cs q xt form code
+ local persona=$1 jwt
jwt=$(privapub_root)
curl -s -o /dev/null -X POST $P/clientapi/avatar/private/insert -H 'Content-Type: application/json' -H "Authorization: Bearer $jwt" \
-d "{\"userName\":\"$persona\",\"name\":\"$persona of PrivaPub\",\"biography\":\"testing federation\"}"
- local app; app=$(curl -s -X POST $P/api/v1/apps -d 'client_name=pasture&redirect_uris=urn:ietf:wg:oauth:2.0:oob&scopes=read+write+follow')
- cid=$(echo "$app" | j "print(d['client_id'])"); cs=$(echo "$app" | j "print(d['client_secret'])")
- q="client_id=$cid&redirect_uri=urn:ietf:wg:oauth:2.0:oob&response_type=code&scope=read+write+follow"
- local jar="$work/jar-$persona"
- xt=$(curl -s -c "$jar" -b "$jar" "$P/oauth/login?returnUrl=/oauth/authorize?$q" | grep -o 'name="__RequestVerificationToken" type="hidden" value="[^"]*"' | sed 's/.*value="//;s/"//')
- curl -s -o /dev/null -c "$jar" -b "$jar" -X POST $P/oauth/login --data-urlencode "returnUrl=/oauth/authorize?$q" --data-urlencode "__RequestVerificationToken=$xt" \
- --data-urlencode "userName=$ROOT_USER" --data-urlencode "password=$ROOT_PASS"
- curl -s -c "$jar" -b "$jar" "$P/oauth/authorize?$q&signed_in=1" > "$work/choose-$persona.html"
- form=$(python3 - "$work/choose-$persona.html" "$persona" <<'PY'
-import re,sys,urllib.parse,html
-s=open(sys.argv[1]).read()
-pairs=[(k,html.unescape(v)) for k,v in re.findall(r']*>(.*?)',s,re.S)
-avatar=None
-for b in blocks:
- if '@'+sys.argv[2]+'@' in b or '@'+sys.argv[2]+'<' in b or '>'+sys.argv[2]+'<' in b:
- m=re.search(r'name="avatarId" value="([^"]*)"',b)
- if m: avatar=m.group(1)
-if avatar is None:
- avatar=re.findall(r'name="avatarId" value="([^"]*)"',s)[0]
-print(urllib.parse.urlencode(pairs+[("avatarId",avatar),("decision","allow")]))
-PY
-)
- code=$(curl -s -c "$jar" -b "$jar" -X POST $P/oauth/authorize --data "$form" | grep -o '[^<]*' | sed 's/<[^>]*>//g')
- curl -s -X POST $P/oauth/token -d "grant_type=authorization_code&code=$code&client_id=$cid&client_secret=$cs&redirect_uri=urn:ietf:wg:oauth:2.0:oob" | j "print(d['access_token'])"
+ "$here/../smoke/oauth.sh" "$P" "$ROOT_USER" "$ROOT_PASS" "$persona" "read write follow" | cut -d' ' -f1
}
# stats_check : the admin statistics name the peer's software and count traffic both ways.
diff --git a/tools/smoke/oauth.sh b/tools/smoke/oauth.sh
new file mode 100755
index 0000000..a950cdd
--- /dev/null
+++ b/tools/smoke/oauth.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Gets a Mastodon API token for one persona through PrivaPub's real OAuth code flow, as a client would: register an app,
+# sign in with the root's password at /oauth/login, choose the persona at /oauth/authorize, read the out-of-band code and
+# exchange it. Prints " " so the caller can revoke the token afterwards.
+# usage: tools/smoke/oauth.sh [scopes]
+set -euo pipefail
+BASE="${1:?base url}"; LOGIN="${2:?root login}"; PASSWORD="${3:?root password}"; PERSONA="${4:?persona}"; SCOPES="${5:-read}"
+jar=$(mktemp -d); trap 'rm -rf "$jar"' EXIT
+json() { python3 -c "import sys,json; d=json.load(sys.stdin); print($1)"; }
+
+app=$(curl -fsS -X POST "$BASE/api/v1/apps" --data-urlencode client_name=privapub-oauth \
+ --data-urlencode redirect_uris=urn:ietf:wg:oauth:2.0:oob --data-urlencode "scopes=$SCOPES")
+cid=$(echo "$app" | json "d['client_id']"); cs=$(echo "$app" | json "d['client_secret']")
+scope_q=$(python3 -c "import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))" "$SCOPES")
+q="client_id=$cid&redirect_uri=urn:ietf:wg:oauth:2.0:oob&response_type=code&scope=$scope_q"
+
+xt=$(curl -fsS -c "$jar/c" -b "$jar/c" "$BASE/oauth/login?returnUrl=/oauth/authorize?$q" \
+ | grep -o 'name="__RequestVerificationToken" type="hidden" value="[^"]*"' | sed 's/.*value="//;s/"//')
+curl -sS -o /dev/null -c "$jar/c" -b "$jar/c" -X POST "$BASE/oauth/login" --data-urlencode "returnUrl=/oauth/authorize?$q" \
+ --data-urlencode "__RequestVerificationToken=$xt" --data-urlencode "userName=$LOGIN" --data-urlencode "password=$PASSWORD"
+curl -fsS -c "$jar/c" -b "$jar/c" "$BASE/oauth/authorize?$q&signed_in=1" > "$jar/choose.html"
+form=$(python3 - "$jar/choose.html" "$PERSONA" <<'PY'
+import re, sys, urllib.parse, html
+page = open(sys.argv[1]).read()
+pairs = [(k, html.unescape(v)) for k, v in re.findall(r']*>(.*?)', page, re.S):
+ if '@' + sys.argv[2] + '@' in block or '@' + sys.argv[2] + '<' in block or '>' + sys.argv[2] + '<' in block:
+ found = re.search(r'name="avatarId" value="([^"]*)"', block)
+ if found:
+ persona = found.group(1)
+if persona is None:
+ sys.exit("persona '%s' is not offered at /oauth/authorize" % sys.argv[2])
+print(urllib.parse.urlencode(pairs + [("avatarId", persona), ("decision", "allow")]))
+PY
+)
+code=$(curl -fsS -c "$jar/c" -b "$jar/c" -X POST "$BASE/oauth/authorize" --data "$form" | grep -o '[^<]*' | sed 's/<[^>]*>//g')
+[ -n "$code" ] || { echo "no authorization code" >&2; exit 1; }
+token=$(curl -fsS -X POST "$BASE/oauth/token" --data-urlencode grant_type=authorization_code --data-urlencode "code=$code" \
+ --data-urlencode "client_id=$cid" --data-urlencode "client_secret=$cs" --data-urlencode redirect_uri=urn:ietf:wg:oauth:2.0:oob \
+ | json "d['access_token']")
+echo "$token $cid $cs"