# The opt-in crawler against the pasture's own peers. Needs PrivaPub started with the crawler on:
#   PRIVAPUB_ENV="Statistics__Crawler__Enabled=true Statistics__Crawler__Seeds__0=mastodon.test" tools/pasture/run.sh up mastodon
# PrivaPub starts before its peers, so a first visit can find robots.txt answering 502 and rightly keep out for a week;
# the scenario then forgets that visit and restarts PrivaPub, which plans again.
echo "crawler"
privapub_root >/dev/null
podman exec -w /app pasture-privapub /app/PrivaPub admin promote "$ROOT_USER" >/dev/null 2>&1 || true
admin=$(curl -s -X POST $P/clientapi/user/login -H 'Content-Type: application/json' -d "{\"userName\":\"$ROOT_USER\",\"password\":\"$ROOT_PASS\"}" | j "print(d['token'])")
crawler() { curl -s -H "Authorization: Bearer $admin" "$P/clientapi/admin/statistics/crawler"; }
if [ "$(crawler | j "print(any(r['robotsDisallowed'] for r in d['recent']))")" = "True" ]; then
	podman exec pasture-mongo mongosh --quiet PrivaPub --eval 'db.Job.deleteMany({Kind:{$in:[8,9]}}); db.RemoteInstance.updateMany({}, {$unset:{LastCrawledAt:1, RobotsDisallowed:1, CrawlError:1}})' >/dev/null
	podman restart pasture-privapub >/dev/null
	until_true 60 'curl -fs -o /dev/null $P/build.json'
fi
[ "$(crawler | j "print(d['enabled'])")" = "True" ] && ok "the crawler is on" || { ko "the crawler is off: start PrivaPub with PRIVAPUB_ENV=Statistics__Crawler__Enabled=true"; return 0; }
until_true 60 '[ "$(crawler | j "print(any(r[\"host\"]==\"mastodon.test\" and r[\"lastCrawledAt\"] for r in d[\"recent\"]))")" = "True" ]' \
	&& ok "the crawler visited mastodon.test" || ko "the crawler never visited mastodon.test"
[ "$(crawler | j "print(next(r['peersCount'] is not None for r in d['recent'] if r['host']=='mastodon.test'))")" = "True" ] \
	&& ok "it read mastodon.test's peers list" || ko "no peers list read from mastodon.test"
[ "$(curl -s -H "Authorization: Bearer $admin" "$P/clientapi/admin/statistics/hosts/mastodon.test?days=1" | j "print((d['instance'] or {}).get('software'))")" = "mastodon" ] \
	&& ok "mastodon.test is described" || ko "mastodon.test is not described"