M7: daily rollups
A RollupDay job folds each finished day's InteractionEvents into one InstanceDay per server:
- Counters for the admin, keyed channel:activity:object:outcome:reason, plus signature
schemes, audiences, local kinds and features;
- PublicCounters, everything a public page may ever read:
- inbound and outbound activities from an allowlist, on public, unlisted or unaddressed
traffic only, with the outcome collapsed (accepted or dropped, delivered or failed);
- no reasons, no Flag or Block;
- features of public objects;
- health:ok or health:failed from reachability (a 4xx means the server answered);
- latency, wait and byte histograms, and the number of distinct accounts from that day's
hashes.
Re-running a day replaces it and keeps the live Reads counters. The day's salt is then
deleted, so its hashes can never be recomputed, and the next day is queued.
StatisticsSchedule plans today's rollup every hour and catches up any of the last seven
days that have events but no rollup. Waits round up into their bucket.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ELjqpznMFMNrJoJUj6K5p2
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
bc8ce63707
commit
fc15f6356d
8 files changed
+441
-4
No files matched your search
@@ -0,0 +1,152 @@
|
||||
using MongoDB.Entities;
|
||||
|
||||
using PrivaPub.Infrastructure.Jobs;
|
||||
using PrivaPub.Infrastructure.Statistics;
|
||||
using PrivaPub.Models.Jobs;
|
||||
using PrivaPub.Models.Statistics;
|
||||
using PrivaPub.Tests.Support;
|
||||
|
||||
namespace PrivaPub.Tests.Statistics
|
||||
{
|
||||
public class RollupFoldTests
|
||||
{
|
||||
static IEnumerable<InteractionEvent> Day(string host) => new[]
|
||||
{
|
||||
new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 202, Reason = "queued", Signature = "cavage:rsa-sha256", LatencyMs = 40, Bytes = 900 },
|
||||
new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 401, Reason = "no-signature" },
|
||||
new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "public", LocalKind = "person", ActorHash = "a", Features = new() { "fep-044f-quote", "source:text/markdown" }, WaitMs = 1500 },
|
||||
new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "private", ActorHash = "b", Features = new() { "interaction-policy" } },
|
||||
new InteractionEvent { Host = host, Channel = "in", Activity = "Flag", Outcome = "accepted", Reason = "reported", Audience = "none", ActorHash = "a" },
|
||||
new InteractionEvent { Host = host, Channel = "in", Activity = "Like", Object = "Note", Outcome = "dropped", Reason = "unknown-object", Audience = "public" },
|
||||
new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Status = 202, Audience = "public", LocalKind = "person", LatencyMs = 120, Bytes = 2000 },
|
||||
new InteractionEvent { Host = host, Channel = "out", Activity = "Block", Outcome = "ok", Status = 202, Audience = "none" },
|
||||
new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "retry", Reason = "503", Audience = "private" },
|
||||
new InteractionEvent { Host = host, Channel = "http", Purpose = "actor", Trigger = "verify", Outcome = "ok", LatencyMs = 300, Bytes = 1400 },
|
||||
new InteractionEvent { Host = host, Channel = "http", Purpose = "object", Trigger = "fetchancestors", Outcome = "refused", Reason = "410" }
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Every_event_counts_for_the_admin_with_its_reason()
|
||||
{
|
||||
var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"];
|
||||
|
||||
Assert.Equal(1, fold.Counters["recv:Create:202:queued"]);
|
||||
Assert.Equal(1, fold.Counters["recv:Create:401:no-signature"]);
|
||||
Assert.Equal(1, fold.Counters["sig:cavage:rsa-sha256"]);
|
||||
Assert.Equal(2, fold.Counters["in:Create:Note:accepted:stored"]);
|
||||
Assert.Equal(1, fold.Counters["in:Like:Note:dropped:unknown-object"]);
|
||||
Assert.Equal(1, fold.Counters["out:Create:Note:retry:503"]);
|
||||
Assert.Equal(1, fold.Counters["out:Block:-:ok"]);
|
||||
Assert.Equal(1, fold.Counters["http:object:refused:410"]);
|
||||
Assert.Equal(1, fold.Counters["aud:in:private"]);
|
||||
Assert.Equal(1, fold.Counters["kind:out:person"]);
|
||||
Assert.Equal(1, fold.Counters["feat:source:text/markdown"]);
|
||||
Assert.Equal(1, fold.Latency["out:le250ms"]);
|
||||
Assert.Equal(1, fold.Latency["http:actor:le500ms"]);
|
||||
Assert.Equal(1, fold.Waits["in:le10s"]);
|
||||
Assert.Equal(2000, fold.Bytes["out"]);
|
||||
Assert.Equal(1400, fold.Bytes["http:actor"]);
|
||||
Assert.Equal(2, fold.Accounts.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_public_counters_keep_only_public_kinds_of_traffic_without_reasons()
|
||||
{
|
||||
var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"];
|
||||
|
||||
Assert.Equal(new Dictionary<string, long>
|
||||
{
|
||||
["in:Create:Note:accepted"] = 1,
|
||||
["in:Like:Note:dropped"] = 1,
|
||||
["out:Create:Note:delivered"] = 1,
|
||||
["feat:fep-044f-quote"] = 1,
|
||||
["feat:source:text/markdown"] = 1,
|
||||
["health:ok"] = 4,
|
||||
["health:failed"] = 1
|
||||
}, fold.PublicCounters);
|
||||
Assert.DoesNotContain(fold.PublicCounters.Keys, k => k.Contains("Flag") || k.Contains("Block") || k.Contains("recv") || k.Contains("interaction-policy"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Keys_are_safe_field_names_and_folding_is_repeatable()
|
||||
{
|
||||
Assert.Equal("in:-:Note:accepted", Rollup.Key("in", null, "Note", "accepted", null));
|
||||
Assert.Equal("feat:content:text/x_mfm", Rollup.Key("feat", "content:text/x.mfm"));
|
||||
var first = Rollup.Fold(Day("a.example").Concat(Day("b.example")));
|
||||
var second = Rollup.Fold(Day("a.example").Concat(Day("b.example")));
|
||||
|
||||
Assert.Equal(first.Hosts["a.example"].Counters, second.Hosts["a.example"].Counters);
|
||||
Assert.Equal(2, first.Accounts.Count);
|
||||
Assert.Equal(2, first.Hosts.Count);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class RollupJobTests : IAsyncLifetime
|
||||
{
|
||||
static readonly DateTime Day = new(2001, 2, 3, 0, 0, 0, DateTimeKind.Utc);
|
||||
readonly string _host = $"rollup{Guid.NewGuid():N}.example";
|
||||
|
||||
public ValueTask InitializeAsync()
|
||||
{
|
||||
Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task A_past_day_folds_into_server_rows_replaces_itself_keeps_reads_and_forgets_its_salt()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
await DB.Default.SaveAsync(new[]
|
||||
{
|
||||
new InteractionEvent { At = Day.AddHours(3), Host = _host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Audience = "public", ActorHash = "x" },
|
||||
new InteractionEvent { At = Day.AddHours(5), Host = _host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Audience = "public" },
|
||||
new InteractionEvent { At = Day.AddDays(1).AddMinutes(1), Host = _host, Channel = "in", Activity = "Like", Outcome = "accepted" }
|
||||
}, token);
|
||||
await DB.Default.SaveAsync(new InstanceDay { Day = Day, Host = _host, Reads = new() { ["media:hit"] = 7 } }, token);
|
||||
await DB.Default.SaveAsync(new InteractionSalt { Day = Day, Key = "AAAA", ExpiresAt = DateTime.UtcNow.AddHours(1) }, token);
|
||||
var job = new RollupJob(new JobQueue(j => j.DedupeKey == "never"));
|
||||
|
||||
Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result);
|
||||
var first = await DB.Default.Find<InstanceDay>().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token);
|
||||
Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result);
|
||||
var second = await DB.Default.Find<InstanceDay>().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token);
|
||||
|
||||
Assert.Equal(1, first.Counters["in:Create:Note:accepted"]);
|
||||
Assert.False(first.Counters.ContainsKey("in:Like:-:accepted"));
|
||||
Assert.Equal(1, first.PublicCounters["out:Create:Note:delivered"]);
|
||||
Assert.Equal(1, first.Accounts);
|
||||
Assert.Equal(7, first.Reads["media:hit"]);
|
||||
Assert.Equal(first.Counters, second.Counters);
|
||||
Assert.Equal(7, second.Reads["media:hit"]);
|
||||
Assert.NotNull(second.RolledUpAt);
|
||||
Assert.False(await DB.Default.Find<InteractionSalt>().Match(s => s.Day == Day).ExecuteAnyAsync(token));
|
||||
Assert.True(await DB.Default.Find<Job>().Match(j => j.DedupeKey == "rollup|2001-02-04" && j.Kind == JobKind.RollupDay).ExecuteAnyAsync(token));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Today_is_not_folded_before_it_ends()
|
||||
{
|
||||
var outcome = await new RollupJob(new JobQueue(j => j.DedupeKey == "never")).Handle(RollupJob.For(DateTime.UtcNow.Date, "x"), TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Equal(JobResult.Defer, outcome.Result);
|
||||
Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, outcome.RetryAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_schedule_plans_today_and_catches_up_a_missed_day()
|
||||
{
|
||||
var token = TestContext.Current.CancellationToken;
|
||||
var missed = DateTime.UtcNow.Date.AddDays(-3);
|
||||
await DB.Default.SaveAsync(new InteractionEvent { At = missed.AddHours(12), Host = _host, Channel = "in", Activity = "Create", Outcome = "accepted" }, token);
|
||||
|
||||
await new StatisticsSchedule(new JobQueue(j => j.DedupeKey == "never"), Microsoft.Extensions.Logging.Abstractions.NullLogger<StatisticsSchedule>.Instance).Plan(token);
|
||||
|
||||
var today = await DB.Default.Find<Job>().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(DateTime.UtcNow.Date)}").ExecuteFirstAsync(token);
|
||||
Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, today.RunAt);
|
||||
Assert.True(await DB.Default.Find<Job>().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(missed)}").ExecuteAnyAsync(token));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user