A backup is a directory <stamp>-<kind> under Backups:Root (/var/lib/privapub/backups, 2770, files 0640), written as .partial and renamed once whole: a manifest (host, build, newest migration, each collection's count, size, sha256 and indexes, what was left out and why, the media list), each collection as gzipped canonical Extended JSON read raw, and hard links to the files of untrashed media rows (copies where a link can't be made). On a replica set every collection is read in one snapshot session. Never in a backup: the statistics salt, jobs, recovery codes, sessions, the maintenance lock and the configuration's copy with its SMTP password. One backup or restore at a time (MaintenanceLock, a heartbeat document), and the janitor purges nothing meanwhile. BackupScheduler backs up nightly at 03:30 UTC (or at once after missing a night); rotation keeps 7 daily, 4 weekly, 3 pre-deploy and 3 pre-restore backups. CLI: admin backup [--kind] [--db-only], admin backups, admin backup verify; these run before migrations, so the deploy's own pre-deploy backup, which replaces mongodump, is of the database as the live build left it. EntityMaps.Warm runs once under a lock, since test hosts now boot side by side. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
245 lines
8.1 KiB
C#
245 lines
8.1 KiB
C#
using Microsoft.AspNetCore.HttpOverrides;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
using MongoDB.Bson;
|
|
using MongoDB.Bson.Serialization;
|
|
using MongoDB.Bson.Serialization.Serializers;
|
|
using MongoDB.Driver;
|
|
using MongoDB.Entities;
|
|
|
|
using Serilog;
|
|
|
|
using PrivaPub.Data;
|
|
using PrivaPub.Extensions;
|
|
using PrivaPub.Api.Mastodon.Auth;
|
|
using PrivaPub.Api.Mastodon.Infrastructure;
|
|
using PrivaPub.Infrastructure;
|
|
using PrivaPub.Infrastructure.Cli;
|
|
using PrivaPub.Infrastructure.Data;
|
|
using PrivaPub.Infrastructure.Http;
|
|
using PrivaPub.Infrastructure.Statistics;
|
|
using PrivaPub.Middleware;
|
|
using PrivaPub.Models;
|
|
using PrivaPub.Services;
|
|
using PrivaPub.StaticServices;
|
|
|
|
Log.Logger = new LoggerConfiguration().WriteTo.Console().CreateBootstrapLogger();
|
|
|
|
try
|
|
{
|
|
var builder = WebApplication.CreateBuilder(args);
|
|
builder.WebHost.ConfigureKestrel(serverOptions =>
|
|
{
|
|
if (builder.Environment.IsProduction())
|
|
{
|
|
serverOptions.ListenLocalhost(6970
|
|
//, options =>
|
|
//{
|
|
// options.Protocols = HttpProtocols.Http1AndHttp2AndHttp3;
|
|
//}
|
|
);
|
|
serverOptions.UseSystemd();
|
|
serverOptions.AddServerHeader = false;
|
|
}
|
|
});
|
|
builder.Host.UseSerilog((context, config) =>
|
|
{
|
|
config.ReadFrom.Configuration(context.Configuration);
|
|
});
|
|
|
|
try
|
|
{
|
|
builder.Services.PrivaPubAppSettingsConfiguration(builder.Configuration)
|
|
.PrivaPubWorkersConfiguration()
|
|
.PrivaPubAuthServicesConfiguration(builder.Configuration)
|
|
.PrivaPubInternalizationConfiguration(builder.Configuration)
|
|
.PrivaPubOptimizationConfiguration()
|
|
.PrivaPubDataBaseConfiguration()
|
|
.PrivaPubServicesConfiguration()
|
|
.PrivaPubFederationConfiguration(builder.Configuration)
|
|
.PrivaPubStatisticsConfiguration(builder.Configuration)
|
|
.PrivaPubCORSConfiguration()
|
|
.PrivaPubRateLimiting(builder.Configuration)
|
|
.PrivaPubOAuth(builder.Environment)
|
|
.AddScoped<PrivaPub.Api.Mastodon.Mappers.MastodonMapper>()
|
|
.AddScoped<PrivaPub.Api.Mastodon.Mappers.AccountSearch>()
|
|
.AddSingleton<PrivaPub.Domain.Social.Trends>()
|
|
.Configure<PrivaPub.Domain.Media.MediaOptions>(builder.Configuration.GetSection("Media"))
|
|
.AddSingleton<PrivaPub.Domain.Media.IMediaService, PrivaPub.Domain.Media.MediaService>()
|
|
.AddSingleton<PrivaPub.Domain.Media.IMediaProxy, PrivaPub.Domain.Media.MediaProxy>()
|
|
.AddHostedService<PrivaPub.Domain.Media.MediaJanitor>()
|
|
.Configure<PrivaPub.Infrastructure.Backup.BackupOptions>(builder.Configuration.GetSection("Backups"))
|
|
.AddSingleton<PrivaPub.Infrastructure.Backup.Backups>()
|
|
.AddHostedService<PrivaPub.Infrastructure.Backup.BackupScheduler>()
|
|
.PrivaPubMiddlewareConfiguration();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, "{0}.{1}()", nameof(Program), "ConfigureServices");
|
|
throw;
|
|
}
|
|
|
|
var federationOptions = builder.Configuration.GetSection("Federation").Get<FederationOptions>() ?? new();
|
|
if (builder.Environment.IsProduction() && (federationOptions.AllowPrivateNetworks || federationOptions.AllowPlainHttp || federationOptions.AcceptAnyCertificate))
|
|
throw new InvalidOperationException("Federation:AllowPrivateNetworks, AllowPlainHttp and AcceptAnyCertificate are for test networks and must stay off in Production.");
|
|
|
|
try
|
|
{
|
|
BsonSerializer.TryRegisterSerializer(new GuidSerializer(GuidRepresentation.Standard));
|
|
var mongoSettings = builder.Configuration.GetSection(nameof(MongoSettings)).Get<MongoSettings>();
|
|
await DB.InitAsync(mongoSettings.Database, MongoClientSettings.FromConnectionString(mongoSettings.ConnectionString));
|
|
EntityMaps.Warm();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}() DB Instantiation");
|
|
throw;
|
|
}
|
|
|
|
var app = default(WebApplication);
|
|
try
|
|
{
|
|
app = builder.Build();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, "{0}.{1}()", nameof(Program), "Build");
|
|
throw;
|
|
}
|
|
|
|
// Commands get every service but start nothing: no Kestrel, no hosted services, no media directory. The deploy runs
|
|
// them as its own user, which can read the configuration and reach the private mongod but owns no www-data directory.
|
|
// Backups are taken before migrations: the deploy's is of the database as the live build left it.
|
|
if (args is ["admin", .. var command] && AdminCommands.BeforeMigrations(command))
|
|
{
|
|
using var scope = app.Services.CreateScope();
|
|
Environment.ExitCode = await AdminCommands.Run(command, scope.ServiceProvider);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
await DB.Default.MigrateAsync<Program>();
|
|
await Indexes.Create();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}() DB Migrations");
|
|
throw;
|
|
}
|
|
|
|
if (args is ["admin", ..])
|
|
{
|
|
using var scope = app.Services.CreateScope();
|
|
Environment.ExitCode = await AdminCommands.Run(args[1..], scope.ServiceProvider);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var localizationService = app.Services.GetService<RequestLocalizationOptionsService>();
|
|
if (app.Environment.IsProduction())
|
|
{
|
|
app.UseResponseCompression();
|
|
app.UseForwardedHeaders(new()
|
|
{
|
|
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
|
|
});
|
|
}
|
|
|
|
if (app.Environment.IsDevelopment())
|
|
{
|
|
app.UseSwagger();
|
|
app.UseSwaggerUI();
|
|
}
|
|
|
|
app.UseHttpsRedirection();
|
|
app.UseCors("DefaultCORS");
|
|
app.UseMastodonErrorBodies();
|
|
|
|
app.UseStaticFiles();
|
|
var mediaRoot = app.Services.GetRequiredService<PrivaPub.Domain.Media.IMediaService>().Root;
|
|
Directory.CreateDirectory(mediaRoot);
|
|
app.UseStaticFiles(new StaticFileOptions
|
|
{
|
|
FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider(mediaRoot),
|
|
RequestPath = "/media/files",
|
|
ContentTypeProvider = PrivaPub.Domain.Media.MediaService.ContentTypes,
|
|
OnPrepareResponse = context =>
|
|
{
|
|
context.Context.Response.Headers["X-Content-Type-Options"] = "nosniff";
|
|
context.Context.Response.Headers["Content-Security-Policy"] = "default-src 'none'; sandbox";
|
|
context.Context.Response.Headers["Cache-Control"] = "public, max-age=31536000, immutable";
|
|
}
|
|
});
|
|
|
|
app.UseRequestLocalization(await localizationService.Get());
|
|
|
|
app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = TimeSpan.FromSeconds(30) });
|
|
app.UseRouting();
|
|
app.UseTrafficMeter();
|
|
app.UseRateLimiter();
|
|
|
|
// a stream's token may come as access_token or as the WebSocket's protocol, as Mastodon's clients send it
|
|
app.Use(async (context, next) =>
|
|
{
|
|
if (context.Request.Path.StartsWithSegments("/api/v1/streaming") && !context.Request.Headers.ContainsKey("Authorization"))
|
|
{
|
|
var token = context.Request.Query["access_token"].ToString();
|
|
if (string.IsNullOrEmpty(token))
|
|
token = context.Request.Headers["Sec-WebSocket-Protocol"].ToString();
|
|
if (!string.IsNullOrEmpty(token))
|
|
context.Request.Headers.Authorization = "Bearer " + token;
|
|
}
|
|
await next();
|
|
});
|
|
|
|
app.UseAuthentication();
|
|
app.UseAuthorization();
|
|
//app.UseWhen(context => context.Request.Path.StartsWithSegments("/peasants") ||
|
|
// context.Request.Path.StartsWithSegments("/users"),
|
|
// app => app.UseSignatureVerification().UseDigestVerification());
|
|
|
|
app.MapGet("/build.json", () => Results.Json(new
|
|
{
|
|
commit = BuildInfo.Commit,
|
|
buildRef = BuildInfo.Ref,
|
|
builtAt = BuildInfo.BuiltAt,
|
|
}));
|
|
app.MapControllers();
|
|
app.MapRazorPages();
|
|
//app.MapFallbackToFile("index.html");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, "{0}.{1}()", nameof(Program), "Use");
|
|
throw;
|
|
}
|
|
|
|
Log.ForContext<Program>().Information($"Starting collAnon at {nameof(Program)}()");
|
|
try
|
|
{
|
|
var dbClient = app.Services.GetService(typeof(DbEntities)) as DbEntities;
|
|
var passwordHasher = app.Services.GetService(typeof(IPasswordHasher)) as IPasswordHasher;
|
|
await dbClient.Init(passwordHasher);
|
|
await app.Services.GetRequiredService<PrivaPub.Federation.Moderation.IDomainBlocks>().Reload(CancellationToken.None);
|
|
await PrivaPub.Api.Mastodon.Auth.PersonaExchange.EnsureFirstPartyClient(app.Services, CancellationToken.None);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Warning(ex, $"{nameof(Program)}.{nameof(Program)}() DB Init");
|
|
}
|
|
|
|
await app.RunAsync();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext<Program>().Fatal(ex, $"{nameof(Program)}.{nameof(Program)}()");
|
|
Environment.ExitCode = 1;
|
|
}
|
|
finally
|
|
{
|
|
await Log.CloseAndFlushAsync();
|
|
}
|
|
|