using MongoDB.Entities; using PrivaPub.Infrastructure.Jobs; using PrivaPub.Infrastructure.Statistics; using PrivaPub.Models.Jobs; using PrivaPub.Models.Statistics; using PrivaPub.Tests.Support; namespace PrivaPub.Tests.Statistics { public class RollupFoldTests { static IEnumerable Day(string host) => new[] { new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 202, Reason = "queued", Signature = "cavage:rsa-sha256", LatencyMs = 40, Bytes = 900 }, new InteractionEvent { Host = host, Channel = "recv", Activity = "Create", Status = 401, Reason = "no-signature" }, new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "public", LocalKind = "person", ActorHash = "a", Features = new() { "fep-044f-quote", "source:text/markdown" }, WaitMs = 1500 }, new InteractionEvent { Host = host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Reason = "stored", Audience = "private", ActorHash = "b", Features = new() { "interaction-policy" } }, new InteractionEvent { Host = host, Channel = "in", Activity = "Flag", Outcome = "accepted", Reason = "reported", Audience = "none", ActorHash = "a" }, new InteractionEvent { Host = host, Channel = "in", Activity = "Like", Object = "Note", Outcome = "dropped", Reason = "unknown-object", Audience = "public" }, new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Status = 202, Audience = "public", LocalKind = "person", LatencyMs = 120, Bytes = 2000 }, new InteractionEvent { Host = host, Channel = "out", Activity = "Block", Outcome = "ok", Status = 202, Audience = "none" }, new InteractionEvent { Host = host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "retry", Reason = "503", Audience = "private" }, new InteractionEvent { Host = host, Channel = "http", Purpose = "actor", Trigger = "verify", Outcome = "ok", LatencyMs = 300, Bytes = 1400 }, new InteractionEvent { Host = host, Channel = "http", Purpose = "object", Trigger = "fetchancestors", Outcome = "refused", Reason = "410" } }; [Fact] public void Every_event_counts_for_the_admin_with_its_reason() { var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"]; Assert.Equal(1, fold.Counters["recv:Create:202:queued"]); Assert.Equal(1, fold.Counters["recv:Create:401:no-signature"]); Assert.Equal(1, fold.Counters["sig:cavage:rsa-sha256"]); Assert.Equal(2, fold.Counters["in:Create:Note:accepted:stored"]); Assert.Equal(1, fold.Counters["in:Like:Note:dropped:unknown-object"]); Assert.Equal(1, fold.Counters["out:Create:Note:retry:503"]); Assert.Equal(1, fold.Counters["out:Block:-:ok"]); Assert.Equal(1, fold.Counters["http:object:refused:410"]); Assert.Equal(1, fold.Counters["aud:in:private"]); Assert.Equal(1, fold.Counters["kind:out:person"]); Assert.Equal(1, fold.Counters["feat:source:text/markdown"]); Assert.Equal(1, fold.Latency["out:le250ms"]); Assert.Equal(1, fold.Latency["http:actor:le500ms"]); Assert.Equal(1, fold.Waits["in:le10s"]); Assert.Equal(2000, fold.Bytes["out"]); Assert.Equal(1400, fold.Bytes["http:actor"]); Assert.Equal(2, fold.Accounts.Count); } [Fact] public void The_public_counters_keep_only_public_kinds_of_traffic_without_reasons() { var fold = Rollup.Fold(Day("social.example")).Hosts["social.example"]; Assert.Equal(new Dictionary { ["in:Create:Note:accepted"] = 1, ["in:Like:Note:dropped"] = 1, ["out:Create:Note:delivered"] = 1, ["feat:fep-044f-quote"] = 1, ["feat:source:text/markdown"] = 1, ["health:ok"] = 4, ["health:failed"] = 1 }, fold.PublicCounters); Assert.DoesNotContain(fold.PublicCounters.Keys, k => k.Contains("Flag") || k.Contains("Block") || k.Contains("recv") || k.Contains("interaction-policy")); } [Fact] public void Keys_are_safe_field_names_and_folding_is_repeatable() { Assert.Equal("in:-:Note:accepted", Rollup.Key("in", null, "Note", "accepted", null)); Assert.Equal("feat:content:text/x_mfm", Rollup.Key("feat", "content:text/x.mfm")); var first = Rollup.Fold(Day("a.example").Concat(Day("b.example"))); var second = Rollup.Fold(Day("a.example").Concat(Day("b.example"))); Assert.Equal(first.Hosts["a.example"].Counters, second.Hosts["a.example"].Counters); Assert.Equal(2, first.Accounts.Count); Assert.Equal(2, first.Hosts.Count); } } [Trait("Category", "Integration")] public sealed class RollupJobTests : IAsyncLifetime { static readonly DateTime Day = new(2001, 2, 3, 0, 0, 0, DateTimeKind.Utc); readonly string _host = $"rollup{Guid.NewGuid():N}.example"; public ValueTask InitializeAsync() { Assert.SkipUnless(MongoFixture.Enabled, MongoFixture.Skip); return ValueTask.CompletedTask; } public ValueTask DisposeAsync() => ValueTask.CompletedTask; [Fact] public async Task A_past_day_folds_into_server_rows_replaces_itself_keeps_reads_and_forgets_its_salt() { var token = TestContext.Current.CancellationToken; await DB.Default.SaveAsync(new[] { new InteractionEvent { At = Day.AddHours(3), Host = _host, Channel = "in", Activity = "Create", Object = "Note", Outcome = "accepted", Audience = "public", ActorHash = "x" }, new InteractionEvent { At = Day.AddHours(5), Host = _host, Channel = "out", Activity = "Create", Object = "Note", Outcome = "ok", Audience = "public" }, new InteractionEvent { At = Day.AddDays(1).AddMinutes(1), Host = _host, Channel = "in", Activity = "Like", Outcome = "accepted" } }, token); await DB.Default.SaveAsync(new InstanceDay { Day = Day, Host = _host, Reads = new() { ["media:hit"] = 7 } }, token); await DB.Default.SaveAsync(new InteractionSalt { Day = Day, Key = "AAAA", ExpiresAt = DateTime.UtcNow.AddHours(1) }, token); var job = new RollupJob(new JobQueue(j => j.DedupeKey == "never")); Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result); var first = await DB.Default.Find().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token); Assert.Equal(JobResult.Done, (await job.Handle(RollupJob.For(Day, $"test|{Guid.NewGuid():N}"), token)).Result); var second = await DB.Default.Find().Match(d => d.Day == Day && d.Host == _host).ExecuteSingleAsync(token); Assert.Equal(1, first.Counters["in:Create:Note:accepted"]); Assert.False(first.Counters.ContainsKey("in:Like:-:accepted")); Assert.Equal(1, first.PublicCounters["out:Create:Note:delivered"]); Assert.Equal(1, first.Accounts); Assert.Equal(7, first.Reads["media:hit"]); Assert.Equal(first.Counters, second.Counters); Assert.Equal(7, second.Reads["media:hit"]); Assert.NotNull(second.RolledUpAt); Assert.False(await DB.Default.Find().Match(s => s.Day == Day).ExecuteAnyAsync(token)); Assert.True(await DB.Default.Find().Match(j => j.DedupeKey == "rollup|2001-02-04" && j.Kind == JobKind.RollupDay).ExecuteAnyAsync(token)); } [Fact] public async Task Today_is_not_folded_before_it_ends() { var outcome = await new RollupJob(new JobQueue(j => j.DedupeKey == "never")).Handle(RollupJob.For(DateTime.UtcNow.Date, "x"), TestContext.Current.CancellationToken); Assert.Equal(JobResult.Defer, outcome.Result); Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, outcome.RetryAt); } [Fact] public async Task The_schedule_plans_today_and_catches_up_a_missed_day() { var token = TestContext.Current.CancellationToken; var missed = DateTime.UtcNow.Date.AddDays(-3); await DB.Default.SaveAsync(new InteractionEvent { At = missed.AddHours(12), Host = _host, Channel = "in", Activity = "Create", Outcome = "accepted" }, token); await new StatisticsSchedule(new JobQueue(j => j.DedupeKey == "never"), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance).Plan(token); var today = await DB.Default.Find().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(DateTime.UtcNow.Date)}").ExecuteFirstAsync(token); Assert.Equal(DateTime.UtcNow.Date.AddDays(1) + RollupJob.After, today.RunAt); Assert.True(await DB.Default.Find().Match(j => j.DedupeKey == $"rollup|{RollupJob.DayKey(missed)}").ExecuteAnyAsync(token)); } } }