Files
SocialPub/PrivaPub/Web/Pages/Pages.cs
T
thepraandClaude Opus 5.5 4f5df743ae M11: the opt-in crawler, PrivaPub-Stargazer, and /stargazing
Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
  queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
  paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
  "PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
  - it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
    rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
  - it describes servers that only crawling ever found, through
    InstanceDescriber.Describe with robots.txt as the path filter;
  - it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
    read from the first 1 MB instead of refusing a large list;
  - it adds the names a server could ever be reached at as "crawled": DNS only,
    punycode, no addresses, ports or hidden services, and the reserved test names only on
    a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
    never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
  IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
  User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
  and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
2026-10-03 12:19:29 +02:00

149 lines
5.0 KiB
C#

using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.RazorPages;
using MongoDB.Entities;
using PrivaPub.Domain.Privacy;
using PrivaPub.Federation.Actors;
using PrivaPub.Federation.Rendering;
using PrivaPub.Models.Federation;
using PrivaPub.Models.Post;
using PrivaPub.StaticServices;
using PostEntity = PrivaPub.Models.Post.Post;
namespace PrivaPub.Web.Pages
{
public sealed record PostView(string Title, string Warning, string ContentHtml, string Url, DateTime Published, bool Edited)
{
public static PostView From(PostEntity post, LocalActor author) => new(
post.Title,
post.SpoilerText ?? (post.HasContentWarning ? ActivityPubRenderer.ContentWarning : default),
post.ContentHtml ?? ActivityPubRenderer.Html(post.Text),
author.PostHtmlUrl(post.ID),
DateTime.SpecifyKind(post.CreationDate, DateTimeKind.Utc),
post.EditedAt.HasValue);
}
public abstract class PublicPageModel : PageModel
{
protected bool WantsActivityJson()
{
var accept = Request.Headers.Accept.ToString();
return accept.Contains("activity+json", StringComparison.OrdinalIgnoreCase) || accept.Contains("ld+json", StringComparison.OrdinalIgnoreCase);
}
protected void Harden()
{
Response.Headers["Content-Security-Policy"] = "default-src 'none'; style-src 'unsafe-inline'; img-src https: data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'";
Response.Headers["Referrer-Policy"] = "no-referrer";
Response.Headers["X-Content-Type-Options"] = "nosniff";
}
}
public class ProfileModel : PublicPageModel
{
const int PageSize = 20;
readonly ILocalActorService _localActors;
readonly DbEntities _dbEntities;
public ProfileModel(ILocalActorService localActors, DbEntities dbEntities)
{
_localActors = localActors;
_dbEntities = dbEntities;
}
public LocalActor Actor { get; private set; }
public string BiographyHtml { get; private set; }
public IReadOnlyList<PostView> Posts { get; private set; } = Array.Empty<PostView>();
public async Task<IActionResult> OnGetAsync(string user, CancellationToken token)
{
Actor = await _localActors.FindByUserName(user, token);
if (Actor is not { IsFederated: true, IsCircle: false } || Actor.Kind == LocalActorKind.Application)
return NotFound();
if (WantsActivityJson())
return Redirect(Actor.Uri);
Harden();
BiographyHtml = ActivityPubRenderer.Html(Actor.Summary);
var posts = await (Actor.Kind == LocalActorKind.Group
? _dbEntities.Posts.Match(p => p.GroupId == Actor.Id)
: _dbEntities.Posts.Match(p => p.GroupUserId == Actor.Id && !p.IsFederatedCopy))
.Match(VisibilityPolicy.IsPublic)
.Match(p => p.ReblogOfPostId == null)
.Sort(p => p.ID, Order.Descending)
.Limit(PageSize)
.ExecuteAsync(token);
var authors = new Dictionary<string, LocalActor> { [Actor.Id] = Actor };
var views = new List<PostView>();
foreach (var post in posts)
{
if (post.IsFederatedCopy)
continue;
if (!authors.TryGetValue(post.GroupUserId, out var author))
authors[post.GroupUserId] = author = await _localActors.FindById(LocalActorKind.Person, post.GroupUserId, token);
if (author != default)
views.Add(PostView.From(post, author));
}
Posts = views;
return Page();
}
}
public class StatusModel : PublicPageModel
{
readonly ILocalActorService _localActors;
readonly DbEntities _dbEntities;
public StatusModel(ILocalActorService localActors, DbEntities dbEntities)
{
_localActors = localActors;
_dbEntities = dbEntities;
}
public LocalActor Actor { get; private set; }
public PostView Post { get; private set; }
public string ObjectUri { get; private set; }
public async Task<IActionResult> OnGetAsync(string user, string id, CancellationToken token)
{
Actor = await _localActors.FindByUserName(user, token);
if (Actor is not { IsFederated: true, Kind: LocalActorKind.Person })
return NotFound();
var post = await _dbEntities.Posts
.Match(p => p.ID == id && p.GroupUserId == Actor.Id && !p.IsFederatedCopy && p.ReblogOfPostId == null)
.Match(VisibilityPolicy.IsPublic)
.ExecuteFirstAsync(token);
if (post == default)
return NotFound();
ObjectUri = Actor.PostUri(post.ID);
if (WantsActivityJson())
return Redirect(ObjectUri);
Harden();
Post = PostView.From(post, Actor);
return Page();
}
}
public class StargazingModel : PublicPageModel
{
readonly Microsoft.Extensions.Options.IOptionsMonitor<Infrastructure.Statistics.StatisticsOptions> _options;
readonly Microsoft.Extensions.Options.IOptionsMonitor<Models.AppConfiguration> _app;
public StargazingModel(Microsoft.Extensions.Options.IOptionsMonitor<Infrastructure.Statistics.StatisticsOptions> options,
Microsoft.Extensions.Options.IOptionsMonitor<Models.AppConfiguration> app)
{
_options = options;
_app = app;
}
public bool CrawlerEnabled => _options.CurrentValue.Crawler.Enabled;
public string UserAgent => Federation.Crawler.Stargazer.UserAgent(_app.CurrentValue.BackendBaseAddress);
public void OnGet() => Harden();
}
}