From 50c40195d3a33caa671d452c85d446e3ed328b82 Mon Sep 17 00:00:00 2001 From: thepra Date: Fri, 16 Oct 2026 19:41:56 +0200 Subject: [PATCH] "me-web?" 0.12.0: every address block travels with the app - The app now carries the whole RIPE Database list of address blocks, all five million, small single-customer blocks included, so the provider card and the path can name the block your connection belongs to without a connection to the publisher's server. The downloads are about 25 MB larger for it. - When the publisher's server can't be reached, the provider card still names your network and its block, from the public address your router reports or the one the STUN test sees. - The table is read a small piece at a time, so it takes almost no memory on a phone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB --- .gitea/scripts/wait-knowledge.sh | 1 + CLAUDE.md | 24 ++- Directory.Build.props | 2 +- README.md | 9 +- src/MeWeb.Android/AndroidPlatform.cs | 17 ++ src/MeWeb.Android/MeWeb.Android.csproj | 5 + src/MeWeb.Core/Checks/CheckRunner.cs | 12 +- src/MeWeb.Core/Knowledge/Providers.cs | 21 +- src/MeWeb.Core/Knowledge/RegistryTable.cs | 238 ++++++++++++++-------- src/MeWeb.Core/Knowledge/Sources.cs | 23 --- src/MeWeb.Core/MeWeb.Core.csproj | 1 - src/MeWeb.Core/Net/LocalNetwork.cs | 5 + src/MeWeb.Desktop/MeWeb.Desktop.csproj | 4 + src/MeWeb.Server/KnowledgeFeeder.cs | 37 ++-- src/MeWeb.Server/RegistryBuilder.cs | 52 +++++ src/MeWeb.Server/RegistryIndex.cs | 112 ---------- src/MeWeb.Shared/ServerContracts.cs | 1 + tools/Screenshots/Screenshots.csproj | 4 + tools/knowledge.cs | 6 +- 19 files changed, 318 insertions(+), 256 deletions(-) create mode 100644 src/MeWeb.Server/RegistryBuilder.cs delete mode 100644 src/MeWeb.Server/RegistryIndex.cs diff --git a/.gitea/scripts/wait-knowledge.sh b/.gitea/scripts/wait-knowledge.sh index f714b38..3fc3235 100755 --- a/.gitea/scripts/wait-knowledge.sh +++ b/.gitea/scripts/wait-knowledge.sh @@ -7,6 +7,7 @@ for attempt in $(seq 1 240); do for file in ip2asn.bin networks.bin carriers.json registry.bin; do case "$body" in *"\"$file\""*) ;; *) missing="$missing $file" ;; esac done + case "$body" in *'"format":2,'*) ;; *) missing="$missing format-2" ;; esac if [ -n "$body" ] && [ -z "$missing" ]; then echo "network data ready: $(printf '%s' "$body" | head -c 200)" exit 0 diff --git a/CLAUDE.md b/CLAUDE.md index ceed9f7..baebc90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ The Android release key: alias `meweb`, certificate SHA-256 | `data/observed.json` | notes counted from shared findings and accepted by a person: per AS number and line (`home`, `mobile`), IPv4 shared or public, IPv6 offered or not, port 25, each with how many networks agree; embedded in every release | | `src/MeWeb.Server/ObservedNotes.cs` | every six hours (with the share sweep) counts the shared findings into `/var/lib/meweb/observed.json` on the box: one vote per /24 or /48, the latest check of each network, the publisher's own network left out; nothing is served | | `tools/observed.cs` | `show` prints that count, `accept AS/ ...` copies the ready notes (at least 5 networks, 4 in 5 agreeing) into `data/observed.json` | -| `src/MeWeb.Server/KnowledgeFeeder.cs` | refreshes the open network data under `/var/lib/meweb/knowledge` every six hours (iptoasn daily, DB-IP monthly, eight RouteViews RIBs weekly through `bzcat` (one or two collectors per continent, `Collectors`; each counts once per network in `UpstreamTally`, and a path's first AS, the collector's own neighbour, is never taken for an upstream), the AOSP carrier list weekly, RIPE's `inetnum` and `inet6num` split files weekly), builds `ip2asn.bin`, `networks.bin`, `carriers.json`, `registry.bin` and `SOURCES.txt` with `MeWeb.Core/Knowledge/Sources.cs`, and serves them at `/api/knowledge`; it also writes `registry-full.bin` (`RegistryIndex`), every RIPE block, which never leaves the box | +| `src/MeWeb.Server/KnowledgeFeeder.cs` | refreshes the open network data under `/var/lib/meweb/knowledge` every six hours (iptoasn daily, DB-IP monthly, eight RouteViews RIBs weekly through `bzcat` (one or two collectors per continent, `Collectors`; each counts once per network in `UpstreamTally`, and a path's first AS, the collector's own neighbour, is never taken for an upstream), the AOSP carrier list weekly, RIPE's `inetnum` and `inet6num` split files weekly), builds `ip2asn.bin`, `networks.bin`, `carriers.json`, `registry.bin` (every RIPE block, `RegistryBuilder`) and `SOURCES.txt` with `MeWeb.Core/Knowledge/Sources.cs`, and serves them at `/api/knowledge` | | `tools/knowledge.cs` | takes the server's knowledge files (`MEWEB_KNOWLEDGE_URL`, default the public server; the release sets `http://127.0.0.1:5480` after deploying the server first) into `artifacts/knowledge/`, checking each SHA-256, and falls back to building `ip2asn.bin` alone from iptoasn (`--local` forces that); `MeWeb.Core` embeds whatever is there; `tools/release.cs` runs it before every app build | | `tools/setup.cs` + `setup.json` | pinned build tools | | `tools/Screenshots` | headless renders of every view in every language | @@ -129,14 +129,20 @@ The Android release key: alias `meweb`, certificate SHA-256 persons, contacts, `descr` or `remarks`), RIPE credited with a link, the extract's date and a note that it may lag; `SOURCES.txt` (the release notes) and `RegistryCredit` carry that. The RIR's own allocations (`ALLOCATED PA`, `ALLOCATED UNSPECIFIED`, `ALLOCATED-BY-RIR`) are skipped: their names are registry-made (`IT-FASTWEB-20101230`). -- **No block is left out, but the apps cannot carry all of them.** Every RIPE block compresses to about 24 MB and took - 2.2 GB to build, so `registry.bin` (about 0.7 MB) holds the blocks of 2,048+ IPv4 addresses or a /40, plus those of - 256+ or a /48 whose name recurs (a provider's pool label, `FASTWEB-L3-PAT_NAT`); `RegistryIndex` keeps all five - million on disk (about 160 MB, binary search with `RandomAccess`, nothing resident) and `/api/ip` answers each - caller with the block of the address it sees, never another. The app prefers that answer and falls back to the - embedded table. `RegistryIndex.Build` reads IPv4 and then IPv6 and streams the flattened ranges out - (`RegistryTable.Narrowest`, the narrowest block wins): about 280 MB peak, under the unit's 512 MB. The first build - after a deploy takes some fifteen minutes, so `wait-knowledge.sh` waits for `registry.bin` in the manifest. +- **Every RIPE block travels with the app, because the app works offline.** The owner, 16 October 2026: "this is an + offline first capable app it needs to hold value even in absence of connection". `registry.bin` holds all five + million blocks, flattened so the narrowest one wins (`RegistryTable.Narrowest`), in Brotli chunks of 16,384 rows + behind an index: about 26 MB, opened in a millisecond, one chunk decompressed per lookup, nothing else resident. It + is not inside `MeWeb.Core`, because the APK stores every assembly once per processor type: the desktop head and the + screenshot tool embed it in their entry assembly, the Android head ships it as an uncompressed asset + (`AndroidStoreUncompressedFileExtensions`) that `AndroidPlatform.OpenRegistry` reads in place through `OpenFd`. + `RegistryBuilder` reads IPv4, then IPv6, with the names in a temporary file: about 200 MB peak, under the unit's + 512 MB, and some three minutes on top of the RouteViews tally. `/api/ip` still answers each caller with its own + block from the server's newer copy; without that answer the app looks the address up itself, and when the server + cannot be reached at all it uses the router's public outside address or the STUN answer, so the provider card names + the network and the block without the publisher's server. A change to any knowledge file's format bumps + `KnowledgeFeeder.Format` and the `"format":N` that `wait-knowledge.sh` waits for: the feeder rebuilds only when a + source changes or the format differs, and a release must not embed the previous build. - **The server cannot ping**: `NoNewPrivileges` drops `ping`'s file capability and `ping_group_range` is closed, so the trace back to a caller uses `tracepath` (UDP, unprivileged) and keeps only public hops outside `Probe:Refuse`, so the box's own network never reaches an app. diff --git a/Directory.Build.props b/Directory.Build.props index 2b186c1..4d1066f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - 0.11.0 + 0.12.0 “me-web?” Arasaka di Hvozdikov Yevheniy © 2026 Arasaka di Hvozdikov Yevheniy diff --git a/README.md b/README.md index bf3cfa4..effa846 100644 --- a/README.md +++ b/README.md @@ -74,11 +74,10 @@ The APK is signed with the key whose certificate SHA-256 is - mobile operators by network code: the Android Open Source Project's carrier list, Apache License 2.0; - the name of the block of addresses a connection belongs to: the [RIPE Database](https://www.ripe.net/manage-ips-and-asns/db/) of the RIPE NCC, used with the RIPE NCC's permission (16 October 2026) on condition that it is never put behind - a paywall or used to make money. Only each block's range, name and country are kept, nothing about people. The - apps carry every block of at least 2,048 IPv4 addresses or a /40 in IPv6, and the smaller ones, down to 256 - addresses or a /48, whose name recurs; the server keeps every block and tells each app only the block of the - address it sees. The - data may lag behind the live database; the release notes give its date. + a paywall or used to make money. Only each block's range, name and country are kept, nothing about people. + Every block travels with the apps (about 26 MB), so the name is there without a connection; the server also + tells each app the block of the address it sees, from its own weekly copy. The data may lag behind the live + database; the release notes give its date. - Provider notes: each operator's own technical sheet (AGCOM 156/23 in Italy) and users' reports on [FibraClick](https://forum.fibra.click), each cited in `data/isps/`; and, for any country, totals from findings people chose to share (`data/observed.json`), published only after a person has reviewed them and only where at diff --git a/src/MeWeb.Android/AndroidPlatform.cs b/src/MeWeb.Android/AndroidPlatform.cs index c34a3bd..a38ba29 100644 --- a/src/MeWeb.Android/AndroidPlatform.cs +++ b/src/MeWeb.Android/AndroidPlatform.cs @@ -7,6 +7,8 @@ using Android.Telephony; using MeWeb.Core.Net; +using Microsoft.Win32.SafeHandles; + namespace MeWeb.Android; public class AndroidPlatform : SystemPlatform @@ -144,6 +146,21 @@ public class AndroidPlatform : SystemPlatform } } + public override Stream OpenRegistry() + { + try + { + using var asset = Context.Assets?.OpenFd("registry.bin"); + if (asset?.ParcelFileDescriptor is null) return null; + using var copy = asset.ParcelFileDescriptor.Dup(); + return new FileStream(new SafeFileHandle(copy.DetachFd(), true), FileAccess.Read) { Position = asset.StartOffset }; + } + catch (Exception) + { + return null; + } + } + public override void Quit() { MainActivity.Current?.FinishAndRemoveTask(); diff --git a/src/MeWeb.Android/MeWeb.Android.csproj b/src/MeWeb.Android/MeWeb.Android.csproj index e3892be..ff470d9 100644 --- a/src/MeWeb.Android/MeWeb.Android.csproj +++ b/src/MeWeb.Android/MeWeb.Android.csproj @@ -13,6 +13,7 @@ $([MSBuild]::Add($([MSBuild]::Multiply($(VersionMajor), 10000)), $([MSBuild]::Add($([MSBuild]::Multiply($(VersionMinor), 100)), $(VersionPatch))))) apk false + .bin false @@ -24,4 +25,8 @@ + + + + diff --git a/src/MeWeb.Core/Checks/CheckRunner.cs b/src/MeWeb.Core/Checks/CheckRunner.cs index ea4af36..1fa970b 100644 --- a/src/MeWeb.Core/Checks/CheckRunner.cs +++ b/src/MeWeb.Core/Checks/CheckRunner.cs @@ -1,3 +1,5 @@ +using System.Net; + using MeWeb.Core.History; using MeWeb.Core.Knowledge; using MeWeb.Core.Net; @@ -190,6 +192,14 @@ public class CheckRunner(IPlatform platform, ServerProbe server, KnowledgeBase k await Step(CheckId.History, async () => { await Task.WhenAll(onlineDone, provider, nat, behaviour, mail, names, device, speed); + if (run.PublicV4 is null && run.PublicV6 is null) + { + var wan = Addresses.Parse(run.Router?.WanAddress); + var seen = Addresses.IsPublic(wan) ? wan : IPEndPoint.TryParse(run.Stun?.FirstMapped ?? run.Stun?.SecondMapped ?? "", out var mapped) && Addresses.IsPublic(mapped.Address) ? mapped.Address : null; + run.NetworkV4 = knowledge.Asn?.Find(seen); + run.BlockV4 = knowledge.Registry?.Find(seen); + run.Provider = knowledge.ProviderFor(run.NetworkV4?.Number ?? 0); + } var number = (run.NetworkV4 ?? run.NetworkV6)?.Number ?? 0; var network = $"{number}/{run.Local?.GatewayV4 ?? run.Local?.Active?.Kind.ToString() ?? "none"}"; run.History = history.Append(new HistoryEntry @@ -203,7 +213,7 @@ public class CheckRunner(IPlatform platform, ServerProbe server, KnowledgeBase k PortLow = run.Stun?.Ports?.Pattern is PortPattern.InOrder or PortPattern.Block ? run.Stun.Ports.Low : 0, PortHigh = run.Stun?.Ports?.Pattern is PortPattern.InOrder or PortPattern.Block ? run.Stun.Ports.High : 0, }); - Settle(Verdicts.Provider(knowledge, run.NetworkV4, run.NetworkV6, run.BlockV4, run.BlockV6, run.PublicV4 is not null || run.PublicV6 is not null, run.Local?.Active?.Kind == LinkKind.Mobile, run)); + Settle(Verdicts.Provider(knowledge, run.NetworkV4, run.NetworkV6, run.BlockV4, run.BlockV6, run.PublicV4 is not null || run.PublicV6 is not null || run.NetworkV4 is not null, run.Local?.Active?.Kind == LinkKind.Mobile, run)); Settle(Verdicts.History(run.History)); run.Uses = Uses.Decide(run); }); diff --git a/src/MeWeb.Core/Knowledge/Providers.cs b/src/MeWeb.Core/Knowledge/Providers.cs index dbabe88..1fdbc24 100644 --- a/src/MeWeb.Core/Knowledge/Providers.cs +++ b/src/MeWeb.Core/Knowledge/Providers.cs @@ -1,6 +1,8 @@ using System.Text.Json; using System.Text.Json.Serialization; +using MeWeb.Core.Net; + namespace MeWeb.Core.Knowledge; public class ProviderFact @@ -33,12 +35,12 @@ public partial class KnowledgeJsonContext : JsonSerializerContext { } -public sealed class KnowledgeBase +public sealed class KnowledgeBase(IPlatform platform) { readonly Lazy asn = new(LoadAsn); readonly Lazy> providers = new(LoadProviders); readonly Lazy networks = new(LoadNetworks); - readonly Lazy registry = new(LoadRegistry); + readonly Lazy registry = new(() => LoadRegistry(platform)); readonly Lazy> carriers = new(LoadCarriers); readonly Lazy> observed = new(LoadObserved); @@ -67,10 +69,19 @@ public sealed class KnowledgeBase return stream is null ? null : NetworkTable.Read(stream); } - static RegistryTable LoadRegistry() + static RegistryTable LoadRegistry(IPlatform platform) { - using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("registry.bin"); - return stream is null ? null : RegistryTable.Read(stream); + var stream = platform.OpenRegistry(); + if (stream is null) return null; + try + { + return RegistryTable.Open(stream); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or EndOfStreamException) + { + stream.Dispose(); + return null; + } } static Dictionary LoadCarriers() diff --git a/src/MeWeb.Core/Knowledge/RegistryTable.cs b/src/MeWeb.Core/Knowledge/RegistryTable.cs index a02912a..451b5e4 100644 --- a/src/MeWeb.Core/Knowledge/RegistryTable.cs +++ b/src/MeWeb.Core/Knowledge/RegistryTable.cs @@ -22,114 +22,109 @@ public class RegistryRange public string Country { get; set; } } -public sealed class RegistryTable +public sealed class RegistryTable : IDisposable { - const uint Magic = 0x334B574D; + internal const uint Magic = 0x354B574D; + internal const int HeaderBytes = 32; - RegistryBlock[] blocks; - uint[] v4Start; - int[] v4Block; - UInt128[] v6Start; - int[] v6Block; + readonly Lock gate = new(); + Stream stream; + long origin; + UInt128[][] starts; + long[][] offsets; + int cachedFamily = -1; + int cachedChunk = -1; + List<(UInt128 Start, RegistryBlock Block)> cached = []; public DateTimeOffset SourceDate { get; private set; } - public int Count => blocks.Length; + public int Count { get; private set; } + + public static RegistryTable Open(Stream seekable) + { + var table = new RegistryTable { stream = seekable, origin = seekable.Position }; + using var reader = new BinaryReader(seekable, Encoding.UTF8, leaveOpen: true); + if (reader.ReadUInt32() != Magic) throw new InvalidDataException("not a “me-web?” registry table"); + table.SourceDate = DateTimeOffset.FromUnixTimeSeconds(reader.ReadInt64()); + table.Count = reader.ReadInt32(); + var index = reader.ReadInt64(); + seekable.Position = table.origin + index; + table.starts = new UInt128[2][]; + table.offsets = new long[2][]; + var at = (long)HeaderBytes; + for (var family = 0; family < 2; family++) + { + var count = reader.Read7BitEncodedInt(); + table.starts[family] = new UInt128[count]; + table.offsets[family] = new long[count + 1]; + var previous = UInt128.Zero; + for (var i = 0; i < count; i++) + { + previous += AsnTable.ReadBig(reader); + table.starts[family][i] = previous; + table.offsets[family][i] = at; + at += reader.Read7BitEncodedInt64(); + } + table.offsets[family][count] = at; + } + return table; + } public RegistryBlock Find(IPAddress address) { if (address is null) return null; if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); - var index = address.AddressFamily == AddressFamily.InterNetwork - ? AsnTable.Search(v4Start, BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()), v4Block) - : AsnTable.Search(v6Start, BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()), v6Block); - return index < 0 ? null : blocks[index]; + var family = address.AddressFamily == AddressFamily.InterNetwork ? 0 : 1; + var value = family == 0 ? BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()) : BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()); + var chunk = Last(starts[family], value, s => s); + if (chunk < 0) return null; + lock (gate) + { + if (cachedFamily != family || cachedChunk != chunk) Load(family, chunk); + var entry = Last(cached, value, e => e.Start); + return entry < 0 ? null : cached[entry].Block; + } } - public static RegistryTable Read(Stream compressed) + public void Dispose() => stream?.Dispose(); + + void Load(int family, int chunk) { - using var brotli = new BrotliStream(compressed, CompressionMode.Decompress); + var compressed = new byte[offsets[family][chunk + 1] - offsets[family][chunk]]; + stream.Position = origin + offsets[family][chunk]; + stream.ReadExactly(compressed); + using var brotli = new BrotliStream(new MemoryStream(compressed), CompressionMode.Decompress); using var reader = new BinaryReader(brotli, Encoding.UTF8); - if (reader.ReadUInt32() != Magic) throw new InvalidDataException("not a “me-web?” registry table"); - var table = new RegistryTable { SourceDate = DateTimeOffset.FromUnixTimeSeconds(reader.ReadInt64()) }; - table.blocks = new RegistryBlock[reader.Read7BitEncodedInt()]; - for (var i = 0; i < table.blocks.Length; i++) - table.blocks[i] = new RegistryBlock { Name = reader.ReadString(), Country = reader.ReadString() }; - var count = reader.Read7BitEncodedInt(); - table.v4Start = new uint[count]; - table.v4Block = new int[count]; - var previous4 = 0u; + var entries = new List<(UInt128 Start, RegistryBlock Block)>(count); + var start = starts[family][chunk]; for (var i = 0; i < count; i++) { - previous4 += (uint)reader.Read7BitEncodedInt64(); - table.v4Start[i] = previous4; - table.v4Block[i] = reader.Read7BitEncodedInt() - 1; + start += AsnTable.ReadBig(reader); + var name = reader.ReadString(); + entries.Add((start, name.Length == 0 ? null : new RegistryBlock { Name = name, Country = reader.ReadString() })); } - - count = reader.Read7BitEncodedInt(); - table.v6Start = new UInt128[count]; - table.v6Block = new int[count]; - var previous6 = UInt128.Zero; - for (var i = 0; i < count; i++) - { - previous6 += AsnTable.ReadBig(reader); - table.v6Start[i] = previous6; - table.v6Block[i] = reader.Read7BitEncodedInt() - 1; - } - return table; + cached = entries; + cachedFamily = family; + cachedChunk = chunk; } - public static void Write(Stream output, DateTimeOffset sourceDate, List ranges) + static int Last(IReadOnlyList list, UInt128 value, Func start) { - var index = new Dictionary<(string, string), int>(); - var list = new List(); - var v4 = new List<(uint Start, uint End, int Block)>(); - var v6 = new List<(UInt128 Start, UInt128 End, int Block)>(); - foreach (var range in ranges.OrderBy(r => r.Name, StringComparer.Ordinal).ThenBy(r => r.Country, StringComparer.Ordinal)) + var low = 0; + var high = list.Count - 1; + var found = -1; + while (low <= high) { - var key = (range.Name ?? "", range.Country ?? ""); - if (!index.TryGetValue(key, out var block)) + var middle = low + (high - low) / 2; + if (start(list[middle]) <= value) { - index[key] = block = list.Count; - list.Add(new RegistryBlock { Name = key.Item1, Country = key.Item2 }); + found = middle; + low = middle + 1; } - if (range.V6) v6.Add((range.Start, range.End, block)); - else v4.Add(((uint)range.Start, (uint)range.End, block)); - } - - using var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true); - using var writer = new BinaryWriter(brotli, Encoding.UTF8); - writer.Write(Magic); - writer.Write(sourceDate.ToUnixTimeSeconds()); - writer.Write7BitEncodedInt(list.Count); - foreach (var block in list) - { - writer.Write(block.Name); - writer.Write(block.Country); - } - - var flat4 = new List<(uint Start, int Block)>(); - Narrowest(v4, uint.MaxValue, (start, block) => flat4.Add((start, block))); - writer.Write7BitEncodedInt(flat4.Count); - var previous4 = 0u; - foreach (var (start, block) in flat4) - { - writer.Write7BitEncodedInt64(start - previous4); - writer.Write7BitEncodedInt(block + 1); - previous4 = start; - } - - var flat6 = new List<(UInt128 Start, int Block)>(); - Narrowest(v6, UInt128.MaxValue, (start, block) => flat6.Add((start, block))); - writer.Write7BitEncodedInt(flat6.Count); - var previous6 = UInt128.Zero; - foreach (var (start, block) in flat6) - { - AsnTable.WriteBig(writer, start - previous6); - writer.Write7BitEncodedInt(block + 1); - previous6 = start; + else high = middle - 1; } + return found; } public static void Narrowest(List<(T Start, T End, int Block)> ranges, T max, Action write) where T : struct, IBinaryInteger @@ -172,3 +167,80 @@ public sealed class RegistryTable if (pending is { } last) write(last.Start, last.Block); } } + +public sealed class RegistryWriter : IDisposable +{ + const int ChunkEntries = 16384; + + readonly Stream output; + readonly DateTimeOffset date; + readonly List<(UInt128 Start, string Name, string Country)> pending = []; + readonly List<(UInt128 Start, long Length)>[] index = [[], []]; + int family; + int count; + + public RegistryWriter(Stream output, DateTimeOffset date) + { + this.output = output; + this.date = date; + output.Write(new byte[RegistryTable.HeaderBytes]); + } + + public void Add(bool v6, UInt128 start, string name, string country) + { + var next = v6 ? 1 : 0; + if (next < family) throw new InvalidOperationException("IPv4 rows come before IPv6 rows"); + if (next != family || pending.Count == ChunkEntries) Flush(); + family = next; + pending.Add((start, name ?? "", country ?? "")); + if (!string.IsNullOrEmpty(name)) count++; + } + + public void Dispose() + { + Flush(); + var at = output.Position; + using var writer = new BinaryWriter(output, Encoding.UTF8, leaveOpen: true); + foreach (var chunks in index) + { + writer.Write7BitEncodedInt(chunks.Count); + var previous = UInt128.Zero; + foreach (var (start, length) in chunks) + { + AsnTable.WriteBig(writer, start - previous); + writer.Write7BitEncodedInt64(length); + previous = start; + } + } + writer.Flush(); + var end = output.Position; + output.Position = 0; + writer.Write(RegistryTable.Magic); + writer.Write(date.ToUnixTimeSeconds()); + writer.Write(count); + writer.Write(at); + writer.Flush(); + output.Position = end; + } + + void Flush() + { + if (pending.Count == 0) return; + var before = output.Position; + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + using (var writer = new BinaryWriter(brotli, Encoding.UTF8)) + { + writer.Write7BitEncodedInt(pending.Count); + var previous = pending[0].Start; + foreach (var (start, name, country) in pending) + { + AsnTable.WriteBig(writer, start - previous); + writer.Write(name); + if (name.Length > 0) writer.Write(country); + previous = start; + } + } + index[family].Add((pending[0].Start, output.Position - before)); + pending.Clear(); + } +} diff --git a/src/MeWeb.Core/Knowledge/Sources.cs b/src/MeWeb.Core/Knowledge/Sources.cs index 6075146..e5a5cca 100644 --- a/src/MeWeb.Core/Knowledge/Sources.cs +++ b/src/MeWeb.Core/Knowledge/Sources.cs @@ -15,11 +15,6 @@ public class Carrier public static class Sources { static readonly HashSet Allocations = ["ALLOCATED PA", "ALLOCATED UNSPECIFIED", "ALLOCATED-BY-RIR"]; - static readonly UInt128 WideV4 = 2048; - static readonly UInt128 NarrowV4 = 256; - static readonly UInt128 WideV6 = UInt128.One << 88; - static readonly UInt128 NarrowV6 = UInt128.One << 80; - const int Repeated = 3; public static IEnumerable IpToAsn(Stream gzip) { @@ -82,24 +77,6 @@ public static class Sources if (range?.Name is not null && !Allocations.Contains(status ?? "")) yield return range; } - public static List RipeSelection(IEnumerable blocks) - { - var counts = new Dictionary(StringComparer.Ordinal); - foreach (var block in blocks) - if (block.End - block.Start >= (block.V6 ? NarrowV6 : NarrowV4) - 1) counts[block.Name] = counts.GetValueOrDefault(block.Name) + 1; - var names = new Dictionary(StringComparer.Ordinal); - var kept = new List(); - foreach (var block in blocks) - { - var size = block.End - block.Start; - if (size < (block.V6 ? NarrowV6 : NarrowV4) - 1 || (size < (block.V6 ? WideV6 : WideV4) - 1 && counts[block.Name] < Repeated)) continue; - block.Name = Intern(names, block.Name); - block.Country = Intern(names, block.Country ?? ""); - kept.Add(block); - } - return kept; - } - static RegistryRange RipeRange(string line) { if (line.StartsWith("inetnum:", StringComparison.Ordinal)) diff --git a/src/MeWeb.Core/MeWeb.Core.csproj b/src/MeWeb.Core/MeWeb.Core.csproj index fe128ae..a92503e 100644 --- a/src/MeWeb.Core/MeWeb.Core.csproj +++ b/src/MeWeb.Core/MeWeb.Core.csproj @@ -12,7 +12,6 @@ - diff --git a/src/MeWeb.Core/Net/LocalNetwork.cs b/src/MeWeb.Core/Net/LocalNetwork.cs index a3d26f6..86241f9 100644 --- a/src/MeWeb.Core/Net/LocalNetwork.cs +++ b/src/MeWeb.Core/Net/LocalNetwork.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Reflection; using System.Net.NetworkInformation; using System.Net.Sockets; using System.Runtime.InteropServices; @@ -82,6 +83,8 @@ public interface IPlatform Task ReadCellular(CancellationToken ct); IDisposable AllowMulticast(); + + Stream OpenRegistry(); } public class SystemPlatform : IPlatform @@ -148,6 +151,8 @@ public class SystemPlatform : IPlatform public virtual IDisposable AllowMulticast() => new Nothing(); + public virtual Stream OpenRegistry() => Assembly.GetEntryAssembly()?.GetManifestResourceStream("registry.bin"); + public virtual Task ReadCellular(CancellationToken ct) => Task.FromResult(null); static readonly string[] Placeholders = ["o.e.m", "system product", "system manufacturer", "system version", "default string", "not applicable", "not specified", "to be filled", "none"]; diff --git a/src/MeWeb.Desktop/MeWeb.Desktop.csproj b/src/MeWeb.Desktop/MeWeb.Desktop.csproj index ab47e91..6c97f30 100644 --- a/src/MeWeb.Desktop/MeWeb.Desktop.csproj +++ b/src/MeWeb.Desktop/MeWeb.Desktop.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/MeWeb.Server/KnowledgeFeeder.cs b/src/MeWeb.Server/KnowledgeFeeder.cs index 47b3416..f335840 100644 --- a/src/MeWeb.Server/KnowledgeFeeder.cs +++ b/src/MeWeb.Server/KnowledgeFeeder.cs @@ -23,6 +23,7 @@ public class KnowledgeOptions public partial class KnowledgeFeeder(IOptionsMonitor options, IHttpClientFactory factory, ILogger log) : BackgroundService { public const string Client = "knowledge"; + public const int Format = 2; public static readonly string[] Published = ["ip2asn.bin", "networks.bin", "carriers.json", "registry.bin", "SOURCES.txt"]; const string IpToAsn = "iptoasn"; @@ -57,7 +58,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, [RipeV6] = RipeLicence, }; - const string RipeLicence = "RIPE Database extract, RIPE NCC (https://www.ripe.net), used with the RIPE NCC's permission (16 October 2026, ticket #1122623) for a free app with no commercial use; only each block's range, name and country are kept, from blocks of at least 2048 IPv4 addresses or a /40, or of at least 256 addresses or a /48 whose name recurs; it may lag behind the live database"; + const string RipeLicence = "RIPE Database extract, RIPE NCC (https://www.ripe.net), used with the RIPE NCC's permission (16 October 2026, ticket #1122623) for a free app with no commercial use; only each block's range, name and country are kept, of every block but the RIPE NCC's own allocations; it may lag behind the live database"; public string Folder => options.CurrentValue.Folder; @@ -79,7 +80,20 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, while (await timer.WaitForNextTickAsync(stoppingToken)); } - public RegistryBlock BlockFor(IPAddress address) => RegistryIndex.Find(Path.Combine(Folder, "current", RegistryIndex.FileName), address); + public RegistryBlock BlockFor(IPAddress address) + { + var path = Path.Combine(Folder, "current", "registry.bin"); + if (!File.Exists(path)) return null; + try + { + using var table = RegistryTable.Open(File.OpenRead(path)); + return table.Find(address); + } + catch (Exception ex) when (ex is IOException or InvalidDataException) + { + return null; + } + } public ViewKnowledge Manifest() { @@ -142,7 +156,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, } } - if (!changed && Manifest() is not null) return; + if (!changed && Manifest()?.Format == Format) return; Build(sources, known.Values.ToList(), ct); } @@ -258,6 +272,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, var manifest = new ViewKnowledge { + Format = Format, BuiltAt = built, Sources = known.OrderBy(s => s.Name, StringComparer.Ordinal).ToList(), Files = Published.Where(f => File.Exists(Path.Combine(staging, f))).Select(f => new ViewKnowledgeFile @@ -274,19 +289,17 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, if (Directory.Exists(previous)) Directory.Delete(previous, true); if (Directory.Exists(current)) Directory.Move(current, previous); Directory.Move(staging, current); - log.LogInformation("{Service} built {Files} in {Seconds:0.0} s: {Networks} networks, upstreams from {Collectors} routing tables, {Carriers} carriers, {Kept} of {Blocks} registry blocks embedded", + log.LogInformation("{Service} built {Files} in {Seconds:0.0} s: {Networks} networks, upstreams from {Collectors} routing tables, {Carriers} carriers, {Blocks} registry blocks", nameof(KnowledgeFeeder), string.Join(", ", manifest.Files.Select(f => $"{f.Name} {f.Bytes / 1024} KiB")), clock.Elapsed.TotalSeconds, - organisations.Keys.Union(upstreams.Keys).Count(), tally.Collectors, list.Count, registry.Kept, registry.All); + organisations.Keys.Union(upstreams.Keys).Count(), tally.Collectors, list.Count, registry); } - static (int Kept, int All) Registry(string sources, string staging, DateTimeOffset date) + static int Registry(string sources, string staging, DateTimeOffset date) { - var v4 = Read(Path.Combine(sources, "ripe-inetnum.gz")); - var v6 = Read(Path.Combine(sources, "ripe-inet6num.gz")); - var kept = Sources.RipeSelection(v4.Concat(v6)); - if (kept.Count == 0) return (0, 0); - using (var output = File.Create(Path.Combine(staging, "registry.bin"))) RegistryTable.Write(output, date, kept); - return (kept.Count, RegistryIndex.Build(v4, v6, Path.Combine(staging, RegistryIndex.FileName), date)); + var v4 = Path.Combine(sources, "ripe-inetnum.gz"); + var v6 = Path.Combine(sources, "ripe-inet6num.gz"); + if (!File.Exists(v4) && !File.Exists(v6)) return 0; + return RegistryBuilder.Build(Read(v4), Read(v6), Path.Combine(staging, "registry.bin"), date); static IEnumerable Read(string path) { diff --git a/src/MeWeb.Server/RegistryBuilder.cs b/src/MeWeb.Server/RegistryBuilder.cs new file mode 100644 index 0000000..d6f2226 --- /dev/null +++ b/src/MeWeb.Server/RegistryBuilder.cs @@ -0,0 +1,52 @@ +using System.Numerics; +using System.Text; + +using MeWeb.Core.Knowledge; + +namespace MeWeb.Server; + +public static class RegistryBuilder +{ + public static int Build(IEnumerable v4Blocks, IEnumerable v6Blocks, string path, DateTimeOffset date) + { + var namesPath = path + ".names"; + var count = 0; + using (var output = File.Create(path)) + using (var writer = new RegistryWriter(output, date)) + { + Family(false, v4Blocks, uint.MaxValue, 5_000_000); + Family(true, v6Blocks, UInt128.MaxValue, 1_200_000); + + void Family(bool v6, IEnumerable blocks, T max, int capacity) where T : struct, IBinaryInteger + { + var ranges = new List<(T Start, T End, int Block)>(capacity); + using (var names = File.Create(namesPath)) + using (var labels = new BinaryWriter(names, Encoding.UTF8)) + { + foreach (var block in blocks.Where(b => b.V6 == v6)) + { + ranges.Add((T.CreateTruncating(block.Start), T.CreateTruncating(block.End), checked((int)names.Position))); + labels.Write(block.Name); + labels.Write(block.Country ?? ""); + count++; + } + } + using var handle = File.OpenHandle(namesPath); + var buffer = new byte[640]; + RegistryTable.Narrowest(ranges, max, (start, at) => + { + if (at < 0) + { + writer.Add(v6, UInt128.CreateTruncating(start), null, null); + return; + } + var read = RandomAccess.Read(handle, buffer, at); + using var reader = new BinaryReader(new MemoryStream(buffer, 0, read), Encoding.UTF8); + writer.Add(v6, UInt128.CreateTruncating(start), reader.ReadString(), reader.ReadString()); + }); + } + } + File.Delete(namesPath); + return count; + } +} diff --git a/src/MeWeb.Server/RegistryIndex.cs b/src/MeWeb.Server/RegistryIndex.cs deleted file mode 100644 index 99e6dc6..0000000 --- a/src/MeWeb.Server/RegistryIndex.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System.Buffers.Binary; -using System.Net; -using System.Net.Sockets; -using System.Text; - -using MeWeb.Core.Knowledge; - -namespace MeWeb.Server; - -public static class RegistryIndex -{ - public const string FileName = "registry-full.bin"; - - const uint Magic = 0x344B574D; - const int HeaderBytes = 32; - const int V4Bytes = 8; - const int V6Bytes = 20; - - public static int Build(IEnumerable v4Blocks, IEnumerable v6Blocks, string path, DateTimeOffset date) - { - var namesPath = path + ".names"; - var count = 0; - using (var names = File.Create(namesPath)) - using (var labels = new BinaryWriter(names, Encoding.UTF8)) - using (var output = File.Create(path)) - using (var writer = new BinaryWriter(output)) - { - int Label(RegistryRange block) - { - var at = checked((int)names.Position); - labels.Write(block.Name); - labels.Write(block.Country ?? ""); - count++; - return at; - } - writer.Write(new byte[HeaderBytes]); - var v4 = new List<(uint Start, uint End, int Block)>(5_000_000); - foreach (var block in v4Blocks.Where(b => !b.V6)) v4.Add(((uint)block.Start, (uint)block.End, Label(block))); - var count4 = 0; - RegistryTable.Narrowest(v4, uint.MaxValue, (start, block) => - { - writer.Write(start); - writer.Write(block); - count4++; - }); - v4 = null; - var v6 = new List<(UInt128 Start, UInt128 End, int Block)>(1_200_000); - foreach (var block in v6Blocks.Where(b => b.V6)) v6.Add((block.Start, block.End, Label(block))); - var count6 = 0; - RegistryTable.Narrowest(v6, UInt128.MaxValue, (start, block) => - { - writer.Write((ulong)(start >> 64)); - writer.Write((ulong)start); - writer.Write(block); - count6++; - }); - v6 = null; - labels.Flush(); - writer.Flush(); - var namesAt = output.Position; - names.Position = 0; - names.CopyTo(output); - output.Position = 0; - writer.Write(Magic); - writer.Write(date.ToUnixTimeSeconds()); - writer.Write(count4); - writer.Write(count6); - writer.Write(namesAt); - } - File.Delete(namesPath); - return count; - } - - public static RegistryBlock Find(string path, IPAddress address) - { - if (address is null || !File.Exists(path)) return null; - if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); - using var handle = File.OpenHandle(path); - Span header = stackalloc byte[HeaderBytes]; - if (RandomAccess.Read(handle, header, 0) < HeaderBytes || BinaryPrimitives.ReadUInt32LittleEndian(header) != Magic) return null; - var count4 = BinaryPrimitives.ReadInt32LittleEndian(header[12..]); - var count6 = BinaryPrimitives.ReadInt32LittleEndian(header[16..]); - var namesAt = BinaryPrimitives.ReadInt64LittleEndian(header[20..]); - var v4 = address.AddressFamily == AddressFamily.InterNetwork; - var value = v4 ? BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()) : BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()); - var first = v4 ? HeaderBytes : HeaderBytes + (long)count4 * V4Bytes; - var size = v4 ? V4Bytes : V6Bytes; - Span record = stackalloc byte[V6Bytes]; - var low = 0; - var high = (v4 ? count4 : count6) - 1; - var block = -1; - while (low <= high) - { - var middle = low + (high - low) / 2; - RandomAccess.Read(handle, record[..size], first + (long)middle * size); - var start = v4 - ? BinaryPrimitives.ReadUInt32LittleEndian(record) - : ((UInt128)BinaryPrimitives.ReadUInt64LittleEndian(record) << 64) | BinaryPrimitives.ReadUInt64LittleEndian(record[8..]); - if (start <= value) - { - block = BinaryPrimitives.ReadInt32LittleEndian(record[(size - 4)..]); - low = middle + 1; - } - else high = middle - 1; - } - if (block < 0) return null; - var text = new byte[640]; - var read = RandomAccess.Read(handle, text, namesAt + block); - using var reader = new BinaryReader(new MemoryStream(text, 0, read), Encoding.UTF8); - return new RegistryBlock { Name = reader.ReadString(), Country = reader.ReadString() }; - } -} diff --git a/src/MeWeb.Shared/ServerContracts.cs b/src/MeWeb.Shared/ServerContracts.cs index 831d9b0..4928d07 100644 --- a/src/MeWeb.Shared/ServerContracts.cs +++ b/src/MeWeb.Shared/ServerContracts.cs @@ -17,6 +17,7 @@ public class ViewIp public class ViewKnowledge { + public int Format { get; set; } public DateTimeOffset BuiltAt { get; set; } public List Files { get; set; } = []; public List Sources { get; set; } = []; diff --git a/tools/Screenshots/Screenshots.csproj b/tools/Screenshots/Screenshots.csproj index 4e4e171..37b0044 100644 --- a/tools/Screenshots/Screenshots.csproj +++ b/tools/Screenshots/Screenshots.csproj @@ -15,4 +15,8 @@ + + + + diff --git a/tools/knowledge.cs b/tools/knowledge.cs index bdecb83..934a3ff 100644 --- a/tools/knowledge.cs +++ b/tools/knowledge.cs @@ -73,11 +73,9 @@ void Report() NetworkTable networks = null; if (File.Exists(path)) using (var input = File.OpenRead(path)) networks = NetworkTable.Read(input); - RegistryTable registry = null; path = Path.Combine(output, "registry.bin"); - if (File.Exists(path)) - using (var input = File.OpenRead(path)) registry = RegistryTable.Read(input); - Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} address block names"); + using var registry = File.Exists(path) ? RegistryTable.Open(File.OpenRead(path)) : null; + Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} named address blocks"); } static string FindRoot()