Off by default (Statistics:Crawler:Enabled), as the owner decided. When it is on:
- CrawlPlan runs hourly, from StatisticsSchedule. It inserts the configured seeds and
queues up to HostsPerHour servers whose last visit is older than RevisitDays, are not
paused by the breaker and are not domain-blocked, spread across the hour.
- CrawlInstance visits one server at a time as
"PrivaPub-Stargazer/<ref> (+<base>/stargazing)":
- it reads robots.txt (RFC 9309: its own group first, then PrivaPub, then *; longest
rule wins; a 4xx allows everything; a 5xx or no answer keeps it out);
- it describes servers that only crawling ever found, through
InstanceDescriber.Describe with robots.txt as the path filter;
- it reads /api/v1/instance/peers through the new GetStringArray, which keeps what it
read from the first 1 MB instead of refusing a large list;
- it adds the names a server could ever be reached at as "crawled": DNS only,
punycode, no addresses, ports or hidden services, and the reserved test names only on
a test network. Never more than MaxNewHostsPerCrawl per visit or MaxHosts in all, and
never over a touched server.
- It reads nothing but robots.txt, NodeInfo, the instance API and the peers list.
IFederationHttp.GetText serves robots.txt, and HttpScope.Crawl carries the
User-Agent.
- /stargazing explains all this and how to keep the crawler out, says whether it is on,
and credits DB-IP. GET /clientapi/admin/statistics/crawler shows the frontier.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
240 lines
9.9 KiB
C#
240 lines
9.9 KiB
C#
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
|
using Microsoft.AspNetCore.ResponseCompression;
|
|
|
|
using OpenIddict.Validation.AspNetCore;
|
|
|
|
using PrivaPub.ClientModels;
|
|
using PrivaPub.Extensions;
|
|
using PrivaPub.Models;
|
|
using PrivaPub.Services;
|
|
using PrivaPub.StaticServices;
|
|
|
|
using System.Text.Json.Serialization;
|
|
using Microsoft.OpenApi;
|
|
using PrivaPub.Services.ClientToServer.Private;
|
|
using PrivaPub.Services.ClientToServer.Public;
|
|
using PrivaPub.Federation.Actors;
|
|
using PrivaPub.Federation.Outbox;
|
|
using PrivaPub.Federation.Signing;
|
|
using PrivaPub.Federation.Inbox;
|
|
using PrivaPub.Federation.Objects;
|
|
using PrivaPub.Federation.Moderation;
|
|
using PrivaPub.Federation.Inbox.Handlers;
|
|
using PrivaPub.Domain.Content;
|
|
using PrivaPub.Domain.Relationships;
|
|
using PrivaPub.Domain.Social;
|
|
using PrivaPub.Domain.Statuses;
|
|
using PrivaPub.Domain.Timelines;
|
|
using PrivaPub.Infrastructure.Http;
|
|
using PrivaPub.Infrastructure.Jobs;
|
|
using PrivaPub.Infrastructure.Geo;
|
|
using PrivaPub.Infrastructure.Statistics;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace PrivaPub.Middleware
|
|
{
|
|
public static class PrivaPubConfigurations
|
|
{
|
|
const string SchemeSelector = "PrivaPub";
|
|
|
|
public static IServiceCollection PrivaPubAppSettingsConfiguration(this IServiceCollection service, IConfiguration configuration)
|
|
{
|
|
return service
|
|
.Configure<MongoSettings>(configuration.GetSection(nameof(MongoSettings)))
|
|
.Configure<AppConfiguration>(configuration.GetSection(nameof(AppConfiguration)));
|
|
}
|
|
public static IServiceCollection PrivaPubWorkersConfiguration(this IServiceCollection service)
|
|
{
|
|
return service;
|
|
//.AddHostedService<DiscussionsWorker>()
|
|
//.AddHostedService<GroupsCleanerWorker>()
|
|
//.AddHostedService<PoliciesCleanerWorker>();
|
|
}
|
|
public static IServiceCollection PrivaPubFederationConfiguration(this IServiceCollection service, IConfiguration configuration)
|
|
{
|
|
service.Configure<FederationOptions>(configuration.GetSection("Federation"));
|
|
service.AddHttpClient(FederationHttp.ClientName, (provider, client) =>
|
|
{
|
|
var baseAddress = provider.GetRequiredService<IOptionsMonitor<AppConfiguration>>().CurrentValue.BackendBaseAddress?.TrimEnd('/');
|
|
client.Timeout = FederationHttp.RequestTimeout;
|
|
client.DefaultRequestHeaders.UserAgent.ParseAdd($"PrivaPub/{BuildInfo.Ref} (+{baseAddress}/)");
|
|
})
|
|
.ConfigurePrimaryHttpMessageHandler(provider =>
|
|
SafeHttpHandlerFactory.Create(provider.GetRequiredService<IOptions<FederationOptions>>().Value, provider.GetRequiredService<IConnectedAddresses>()));
|
|
return service
|
|
.AddSingleton<IConnectedAddresses, ConnectedAddresses>()
|
|
.AddSingleton<IFederationHttp, FederationHttp>()
|
|
.AddSingleton<IDomainBlocks, DomainBlocks>()
|
|
.AddSingleton<IContentRenderer, ContentRenderer>()
|
|
.AddSingleton<ILocalActorService, LocalActorService>()
|
|
.AddSingleton<IRemoteActorService, RemoteActorService>()
|
|
.AddSingleton<IDeliveryService, DeliveryService>()
|
|
.AddSingleton<IOutboxPublisher, OutboxPublisher>()
|
|
.AddSingleton<IGroupDistributor, GroupDistributor>()
|
|
.AddSingleton<ISignedFetchAuthorizer, SignedFetchAuthorizer>()
|
|
.AddSingleton<IFanout, Fanout>()
|
|
.AddSingleton<IInboxReceiver, InboxReceiver>()
|
|
.AddSingleton<IActivityHandler, FollowHandler>()
|
|
.AddSingleton<IActivityHandler, AcceptHandler>()
|
|
.AddSingleton<IActivityHandler, RejectHandler>()
|
|
.AddSingleton<IActivityHandler, UndoHandler>()
|
|
.AddSingleton<IActivityHandler, LikeHandler>()
|
|
.AddSingleton<IActivityHandler, DislikeHandler>()
|
|
.AddSingleton<IActivityHandler, EmojiReactHandler>()
|
|
.AddSingleton<IActivityHandler, QuoteRequestHandler>()
|
|
.AddSingleton<IReactions, Reactions>()
|
|
.AddSingleton<LinkPreviews>()
|
|
.AddSingleton<ILinkPreviews>(services => services.GetRequiredService<LinkPreviews>())
|
|
.AddSingleton<IJobHandler>(services => services.GetRequiredService<LinkPreviews>())
|
|
.AddSingleton<IActivityHandler, JoinHandler>()
|
|
.AddSingleton<IActivityHandler, AnnounceHandler>()
|
|
.AddSingleton<IActivityHandler, FlagHandler>()
|
|
.AddSingleton<IActivityHandler, CreateHandler>()
|
|
.AddSingleton<IActivityHandler, DeleteHandler>()
|
|
.AddSingleton<IActivityHandler, UpdateHandler>()
|
|
.AddSingleton<IJobHandler, InboxProcessor>()
|
|
.AddSingleton<IRemotePosts, RemotePosts>()
|
|
.AddSingleton<IObjectRecords, ObjectRecords>()
|
|
.AddSingleton<IPollService, PollService>()
|
|
.AddSingleton<IQuoteService, QuoteService>()
|
|
.AddSingleton<IJobHandler, PollRefreshJob>()
|
|
.AddSingleton<IJobHandler, PollCloseJob>()
|
|
.AddSingleton<InstanceDescriber>()
|
|
.AddSingleton<IJobHandler>(services => services.GetRequiredService<InstanceDescriber>())
|
|
.AddSingleton<IJobHandler, AncestorsJobHandler>()
|
|
.AddSingleton<IJobQueue, JobQueue>()
|
|
.AddSingleton<IHostCircuitBreaker, HostCircuitBreaker>()
|
|
.AddSingleton<IJobHandler, DeliveryJobHandler>()
|
|
.AddHostedService<JobWorker>();
|
|
}
|
|
public static IServiceCollection PrivaPubStatisticsConfiguration(this IServiceCollection service, IConfiguration configuration) =>
|
|
service
|
|
.Configure<StatisticsOptions>(configuration.GetSection("Statistics"))
|
|
.AddSingleton<InteractionSalts>()
|
|
.AddSingleton<InteractionLedger>()
|
|
.AddSingleton<IInteractionLedger>(services => services.GetRequiredService<InteractionLedger>())
|
|
.AddHostedService(services => services.GetRequiredService<InteractionLedger>())
|
|
.AddSingleton<IJobHandler, RollupJob>()
|
|
.AddSingleton<IGeoLocator, DbIpLocator>()
|
|
.AddSingleton<Domain.Statistics.StatisticsQueries>()
|
|
.AddSingleton<IJobHandler, Federation.Crawler.CrawlPlanner>()
|
|
.AddSingleton<IJobHandler, Federation.Crawler.InstanceCrawler>()
|
|
.AddHostedService<StatisticsSchedule>();
|
|
|
|
public static IServiceCollection PrivaPubAuthServicesConfiguration(this IServiceCollection service, IConfiguration configuration)
|
|
{
|
|
return service
|
|
.AddAuthorization(options =>
|
|
{
|
|
options.AddPolicy(Policies.IsUser, Extensions.Extensions.IsUserPolicy());
|
|
options.AddPolicy(Policies.IsAdmin, Extensions.Extensions.IsAdminPolicy());
|
|
options.AddPolicy(Policies.IsModerator, Extensions.Extensions.IsModeratorPolicy());
|
|
})
|
|
.AddAuthentication(options =>
|
|
{
|
|
options.DefaultAuthenticateScheme = SchemeSelector;
|
|
options.DefaultChallengeScheme = SchemeSelector;
|
|
options.DefaultScheme = SchemeSelector;
|
|
})
|
|
.AddPolicyScheme(SchemeSelector, SchemeSelector, options => options.ForwardDefaultSelector = context =>
|
|
context.Request.Path.StartsWithSegments("/api")
|
|
? OpenIddictValidationAspNetCoreDefaults.AuthenticationScheme
|
|
: JwtBearerDefaults.AuthenticationScheme)
|
|
.AddPrivaPubAuth(configuration)
|
|
.Services
|
|
.AddSingleton<AuthTokenManager>()
|
|
.AddSingleton<IPasswordHasher, PasswordHasher>();
|
|
}
|
|
public static IServiceCollection PrivaPubInternalizationConfiguration(this IServiceCollection service, IConfiguration configuration)
|
|
{
|
|
return service
|
|
.AddLocalization()
|
|
.AddSingleton<RequestLocalizationOptionsService>();
|
|
}
|
|
|
|
public static IServiceCollection PrivaPubOptimizationConfiguration(this IServiceCollection service)
|
|
{
|
|
return service.AddResponseCompression(opts =>
|
|
{
|
|
opts.Providers.Add<BrotliCompressionProvider>();
|
|
opts.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat(new[] { "application/octet-stream" });
|
|
});
|
|
}
|
|
|
|
public static IServiceCollection PrivaPubDataBaseConfiguration(this IServiceCollection service)
|
|
{
|
|
return service.AddSingleton<DbEntities>();
|
|
}
|
|
|
|
public static IServiceCollection PrivaPubServicesConfiguration(this IServiceCollection service)
|
|
{
|
|
return service
|
|
.AddTransient<IDataService, DataService>()
|
|
.AddTransient<IRootUsersService, RootUsersService>()
|
|
.AddTransient<IPublicAvatarUsersService, PublicAvatarUsersService>()
|
|
.AddTransient<IPrivateAvatarUsersService, PrivateAvatarUsersService>()
|
|
.AddTransient<IGroupUsersService, GroupUsersService>()
|
|
.AddTransient<IPostsService, PostsService>()
|
|
.AddTransient<IStatusService, StatusService>()
|
|
.AddTransient<IRelationshipService, RelationshipService>()
|
|
.AddTransient<IReportService, ReportService>()
|
|
.AddTransient<IFollowService, FollowService>()
|
|
.AddTransient<ITimelineService, TimelineService>()
|
|
.AddSingleton<AppConfigurationService>()
|
|
.AddHttpContextAccessor()
|
|
.AddMemoryCache()
|
|
.AddSingleton<IPasswordHasher, PasswordHasher>();
|
|
}
|
|
|
|
public static IServiceCollection PrivaPubMiddlewareConfiguration(this IServiceCollection service)
|
|
{
|
|
return service
|
|
.AddEndpointsApiExplorer()
|
|
.AddSwaggerGen(c =>
|
|
{
|
|
c.AddSecurityDefinition("Bearer", new OpenApiSecurityScheme
|
|
{
|
|
In = ParameterLocation.Header,
|
|
Description = "Please enter a valid token",
|
|
Name = "Authorization",
|
|
Type = SecuritySchemeType.Http,
|
|
BearerFormat = "JWT",
|
|
Scheme = "bearer"
|
|
});
|
|
c.AddSecurityRequirement(document => new OpenApiSecurityRequirement
|
|
{
|
|
[new OpenApiSecuritySchemeReference("Bearer", document)] = []
|
|
});
|
|
})
|
|
.AddRazorPages(options => options.RootDirectory = "/Web/Pages")
|
|
.Services
|
|
.AddControllers(options => { options.Filters.Add<OperationCancelledExceptionFilter>(); })
|
|
.AddJsonOptions(options =>
|
|
{
|
|
options.JsonSerializerOptions.IgnoreReadOnlyFields = false;
|
|
options.JsonSerializerOptions.IgnoreReadOnlyProperties = false;
|
|
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
|
|
options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
|
|
options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
|
|
}).Services;
|
|
}
|
|
|
|
public static IServiceCollection PrivaPubCORSConfiguration(this IServiceCollection service)
|
|
{
|
|
return service.AddCors(options =>
|
|
{
|
|
options.DefaultPolicyName = "DefaultCORS";
|
|
options.AddDefaultPolicy(configure =>
|
|
{
|
|
configure.AllowAnyMethod()
|
|
.AllowAnyHeader()
|
|
.AllowAnyOrigin()
|
|
.WithExposedHeaders("Link", "X-RateLimit-Remaining", "X-RateLimit-Reset")
|
|
.DisallowCredentials();
|
|
});
|
|
});
|
|
}
|
|
|
|
}
|
|
}
|