diff --git a/.gitea/scripts/wait-knowledge.sh b/.gitea/scripts/wait-knowledge.sh index f714b38..3fc3235 100755 --- a/.gitea/scripts/wait-knowledge.sh +++ b/.gitea/scripts/wait-knowledge.sh @@ -7,6 +7,7 @@ for attempt in $(seq 1 240); do for file in ip2asn.bin networks.bin carriers.json registry.bin; do case "$body" in *"\"$file\""*) ;; *) missing="$missing $file" ;; esac done + case "$body" in *'"format":2,'*) ;; *) missing="$missing format-2" ;; esac if [ -n "$body" ] && [ -z "$missing" ]; then echo "network data ready: $(printf '%s' "$body" | head -c 200)" exit 0 diff --git a/CLAUDE.md b/CLAUDE.md index ceed9f7..baebc90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ The Android release key: alias `meweb`, certificate SHA-256 | `data/observed.json` | notes counted from shared findings and accepted by a person: per AS number and line (`home`, `mobile`), IPv4 shared or public, IPv6 offered or not, port 25, each with how many networks agree; embedded in every release | | `src/MeWeb.Server/ObservedNotes.cs` | every six hours (with the share sweep) counts the shared findings into `/var/lib/meweb/observed.json` on the box: one vote per /24 or /48, the latest check of each network, the publisher's own network left out; nothing is served | | `tools/observed.cs` | `show` prints that count, `accept AS/ ...` copies the ready notes (at least 5 networks, 4 in 5 agreeing) into `data/observed.json` | -| `src/MeWeb.Server/KnowledgeFeeder.cs` | refreshes the open network data under `/var/lib/meweb/knowledge` every six hours (iptoasn daily, DB-IP monthly, eight RouteViews RIBs weekly through `bzcat` (one or two collectors per continent, `Collectors`; each counts once per network in `UpstreamTally`, and a path's first AS, the collector's own neighbour, is never taken for an upstream), the AOSP carrier list weekly, RIPE's `inetnum` and `inet6num` split files weekly), builds `ip2asn.bin`, `networks.bin`, `carriers.json`, `registry.bin` and `SOURCES.txt` with `MeWeb.Core/Knowledge/Sources.cs`, and serves them at `/api/knowledge`; it also writes `registry-full.bin` (`RegistryIndex`), every RIPE block, which never leaves the box | +| `src/MeWeb.Server/KnowledgeFeeder.cs` | refreshes the open network data under `/var/lib/meweb/knowledge` every six hours (iptoasn daily, DB-IP monthly, eight RouteViews RIBs weekly through `bzcat` (one or two collectors per continent, `Collectors`; each counts once per network in `UpstreamTally`, and a path's first AS, the collector's own neighbour, is never taken for an upstream), the AOSP carrier list weekly, RIPE's `inetnum` and `inet6num` split files weekly), builds `ip2asn.bin`, `networks.bin`, `carriers.json`, `registry.bin` (every RIPE block, `RegistryBuilder`) and `SOURCES.txt` with `MeWeb.Core/Knowledge/Sources.cs`, and serves them at `/api/knowledge` | | `tools/knowledge.cs` | takes the server's knowledge files (`MEWEB_KNOWLEDGE_URL`, default the public server; the release sets `http://127.0.0.1:5480` after deploying the server first) into `artifacts/knowledge/`, checking each SHA-256, and falls back to building `ip2asn.bin` alone from iptoasn (`--local` forces that); `MeWeb.Core` embeds whatever is there; `tools/release.cs` runs it before every app build | | `tools/setup.cs` + `setup.json` | pinned build tools | | `tools/Screenshots` | headless renders of every view in every language | @@ -129,14 +129,20 @@ The Android release key: alias `meweb`, certificate SHA-256 persons, contacts, `descr` or `remarks`), RIPE credited with a link, the extract's date and a note that it may lag; `SOURCES.txt` (the release notes) and `RegistryCredit` carry that. The RIR's own allocations (`ALLOCATED PA`, `ALLOCATED UNSPECIFIED`, `ALLOCATED-BY-RIR`) are skipped: their names are registry-made (`IT-FASTWEB-20101230`). -- **No block is left out, but the apps cannot carry all of them.** Every RIPE block compresses to about 24 MB and took - 2.2 GB to build, so `registry.bin` (about 0.7 MB) holds the blocks of 2,048+ IPv4 addresses or a /40, plus those of - 256+ or a /48 whose name recurs (a provider's pool label, `FASTWEB-L3-PAT_NAT`); `RegistryIndex` keeps all five - million on disk (about 160 MB, binary search with `RandomAccess`, nothing resident) and `/api/ip` answers each - caller with the block of the address it sees, never another. The app prefers that answer and falls back to the - embedded table. `RegistryIndex.Build` reads IPv4 and then IPv6 and streams the flattened ranges out - (`RegistryTable.Narrowest`, the narrowest block wins): about 280 MB peak, under the unit's 512 MB. The first build - after a deploy takes some fifteen minutes, so `wait-knowledge.sh` waits for `registry.bin` in the manifest. +- **Every RIPE block travels with the app, because the app works offline.** The owner, 16 October 2026: "this is an + offline first capable app it needs to hold value even in absence of connection". `registry.bin` holds all five + million blocks, flattened so the narrowest one wins (`RegistryTable.Narrowest`), in Brotli chunks of 16,384 rows + behind an index: about 26 MB, opened in a millisecond, one chunk decompressed per lookup, nothing else resident. It + is not inside `MeWeb.Core`, because the APK stores every assembly once per processor type: the desktop head and the + screenshot tool embed it in their entry assembly, the Android head ships it as an uncompressed asset + (`AndroidStoreUncompressedFileExtensions`) that `AndroidPlatform.OpenRegistry` reads in place through `OpenFd`. + `RegistryBuilder` reads IPv4, then IPv6, with the names in a temporary file: about 200 MB peak, under the unit's + 512 MB, and some three minutes on top of the RouteViews tally. `/api/ip` still answers each caller with its own + block from the server's newer copy; without that answer the app looks the address up itself, and when the server + cannot be reached at all it uses the router's public outside address or the STUN answer, so the provider card names + the network and the block without the publisher's server. A change to any knowledge file's format bumps + `KnowledgeFeeder.Format` and the `"format":N` that `wait-knowledge.sh` waits for: the feeder rebuilds only when a + source changes or the format differs, and a release must not embed the previous build. - **The server cannot ping**: `NoNewPrivileges` drops `ping`'s file capability and `ping_group_range` is closed, so the trace back to a caller uses `tracepath` (UDP, unprivileged) and keeps only public hops outside `Probe:Refuse`, so the box's own network never reaches an app. diff --git a/Directory.Build.props b/Directory.Build.props index 2b186c1..4d1066f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - 0.11.0 + 0.12.0 “me-web?” Arasaka di Hvozdikov Yevheniy © 2026 Arasaka di Hvozdikov Yevheniy diff --git a/README.md b/README.md index bf3cfa4..effa846 100644 --- a/README.md +++ b/README.md @@ -74,11 +74,10 @@ The APK is signed with the key whose certificate SHA-256 is - mobile operators by network code: the Android Open Source Project's carrier list, Apache License 2.0; - the name of the block of addresses a connection belongs to: the [RIPE Database](https://www.ripe.net/manage-ips-and-asns/db/) of the RIPE NCC, used with the RIPE NCC's permission (16 October 2026) on condition that it is never put behind - a paywall or used to make money. Only each block's range, name and country are kept, nothing about people. The - apps carry every block of at least 2,048 IPv4 addresses or a /40 in IPv6, and the smaller ones, down to 256 - addresses or a /48, whose name recurs; the server keeps every block and tells each app only the block of the - address it sees. The - data may lag behind the live database; the release notes give its date. + a paywall or used to make money. Only each block's range, name and country are kept, nothing about people. + Every block travels with the apps (about 26 MB), so the name is there without a connection; the server also + tells each app the block of the address it sees, from its own weekly copy. The data may lag behind the live + database; the release notes give its date. - Provider notes: each operator's own technical sheet (AGCOM 156/23 in Italy) and users' reports on [FibraClick](https://forum.fibra.click), each cited in `data/isps/`; and, for any country, totals from findings people chose to share (`data/observed.json`), published only after a person has reviewed them and only where at diff --git a/src/MeWeb.Android/AndroidPlatform.cs b/src/MeWeb.Android/AndroidPlatform.cs index c34a3bd..a38ba29 100644 --- a/src/MeWeb.Android/AndroidPlatform.cs +++ b/src/MeWeb.Android/AndroidPlatform.cs @@ -7,6 +7,8 @@ using Android.Telephony; using MeWeb.Core.Net; +using Microsoft.Win32.SafeHandles; + namespace MeWeb.Android; public class AndroidPlatform : SystemPlatform @@ -144,6 +146,21 @@ public class AndroidPlatform : SystemPlatform } } + public override Stream OpenRegistry() + { + try + { + using var asset = Context.Assets?.OpenFd("registry.bin"); + if (asset?.ParcelFileDescriptor is null) return null; + using var copy = asset.ParcelFileDescriptor.Dup(); + return new FileStream(new SafeFileHandle(copy.DetachFd(), true), FileAccess.Read) { Position = asset.StartOffset }; + } + catch (Exception) + { + return null; + } + } + public override void Quit() { MainActivity.Current?.FinishAndRemoveTask(); diff --git a/src/MeWeb.Android/MeWeb.Android.csproj b/src/MeWeb.Android/MeWeb.Android.csproj index e3892be..ff470d9 100644 --- a/src/MeWeb.Android/MeWeb.Android.csproj +++ b/src/MeWeb.Android/MeWeb.Android.csproj @@ -13,6 +13,7 @@ $([MSBuild]::Add($([MSBuild]::Multiply($(VersionMajor), 10000)), $([MSBuild]::Add($([MSBuild]::Multiply($(VersionMinor), 100)), $(VersionPatch))))) apk false + .bin false @@ -24,4 +25,8 @@ + + + + diff --git a/src/MeWeb.Core/Checks/CheckRunner.cs b/src/MeWeb.Core/Checks/CheckRunner.cs index ea4af36..1fa970b 100644 --- a/src/MeWeb.Core/Checks/CheckRunner.cs +++ b/src/MeWeb.Core/Checks/CheckRunner.cs @@ -1,3 +1,5 @@ +using System.Net; + using MeWeb.Core.History; using MeWeb.Core.Knowledge; using MeWeb.Core.Net; @@ -190,6 +192,14 @@ public class CheckRunner(IPlatform platform, ServerProbe server, KnowledgeBase k await Step(CheckId.History, async () => { await Task.WhenAll(onlineDone, provider, nat, behaviour, mail, names, device, speed); + if (run.PublicV4 is null && run.PublicV6 is null) + { + var wan = Addresses.Parse(run.Router?.WanAddress); + var seen = Addresses.IsPublic(wan) ? wan : IPEndPoint.TryParse(run.Stun?.FirstMapped ?? run.Stun?.SecondMapped ?? "", out var mapped) && Addresses.IsPublic(mapped.Address) ? mapped.Address : null; + run.NetworkV4 = knowledge.Asn?.Find(seen); + run.BlockV4 = knowledge.Registry?.Find(seen); + run.Provider = knowledge.ProviderFor(run.NetworkV4?.Number ?? 0); + } var number = (run.NetworkV4 ?? run.NetworkV6)?.Number ?? 0; var network = $"{number}/{run.Local?.GatewayV4 ?? run.Local?.Active?.Kind.ToString() ?? "none"}"; run.History = history.Append(new HistoryEntry @@ -203,7 +213,7 @@ public class CheckRunner(IPlatform platform, ServerProbe server, KnowledgeBase k PortLow = run.Stun?.Ports?.Pattern is PortPattern.InOrder or PortPattern.Block ? run.Stun.Ports.Low : 0, PortHigh = run.Stun?.Ports?.Pattern is PortPattern.InOrder or PortPattern.Block ? run.Stun.Ports.High : 0, }); - Settle(Verdicts.Provider(knowledge, run.NetworkV4, run.NetworkV6, run.BlockV4, run.BlockV6, run.PublicV4 is not null || run.PublicV6 is not null, run.Local?.Active?.Kind == LinkKind.Mobile, run)); + Settle(Verdicts.Provider(knowledge, run.NetworkV4, run.NetworkV6, run.BlockV4, run.BlockV6, run.PublicV4 is not null || run.PublicV6 is not null || run.NetworkV4 is not null, run.Local?.Active?.Kind == LinkKind.Mobile, run)); Settle(Verdicts.History(run.History)); run.Uses = Uses.Decide(run); }); diff --git a/src/MeWeb.Core/Knowledge/Providers.cs b/src/MeWeb.Core/Knowledge/Providers.cs index dbabe88..1fdbc24 100644 --- a/src/MeWeb.Core/Knowledge/Providers.cs +++ b/src/MeWeb.Core/Knowledge/Providers.cs @@ -1,6 +1,8 @@ using System.Text.Json; using System.Text.Json.Serialization; +using MeWeb.Core.Net; + namespace MeWeb.Core.Knowledge; public class ProviderFact @@ -33,12 +35,12 @@ public partial class KnowledgeJsonContext : JsonSerializerContext { } -public sealed class KnowledgeBase +public sealed class KnowledgeBase(IPlatform platform) { readonly Lazy asn = new(LoadAsn); readonly Lazy> providers = new(LoadProviders); readonly Lazy networks = new(LoadNetworks); - readonly Lazy registry = new(LoadRegistry); + readonly Lazy registry = new(() => LoadRegistry(platform)); readonly Lazy> carriers = new(LoadCarriers); readonly Lazy> observed = new(LoadObserved); @@ -67,10 +69,19 @@ public sealed class KnowledgeBase return stream is null ? null : NetworkTable.Read(stream); } - static RegistryTable LoadRegistry() + static RegistryTable LoadRegistry(IPlatform platform) { - using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("registry.bin"); - return stream is null ? null : RegistryTable.Read(stream); + var stream = platform.OpenRegistry(); + if (stream is null) return null; + try + { + return RegistryTable.Open(stream); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or EndOfStreamException) + { + stream.Dispose(); + return null; + } } static Dictionary LoadCarriers() diff --git a/src/MeWeb.Core/Knowledge/RegistryTable.cs b/src/MeWeb.Core/Knowledge/RegistryTable.cs index a02912a..451b5e4 100644 --- a/src/MeWeb.Core/Knowledge/RegistryTable.cs +++ b/src/MeWeb.Core/Knowledge/RegistryTable.cs @@ -22,114 +22,109 @@ public class RegistryRange public string Country { get; set; } } -public sealed class RegistryTable +public sealed class RegistryTable : IDisposable { - const uint Magic = 0x334B574D; + internal const uint Magic = 0x354B574D; + internal const int HeaderBytes = 32; - RegistryBlock[] blocks; - uint[] v4Start; - int[] v4Block; - UInt128[] v6Start; - int[] v6Block; + readonly Lock gate = new(); + Stream stream; + long origin; + UInt128[][] starts; + long[][] offsets; + int cachedFamily = -1; + int cachedChunk = -1; + List<(UInt128 Start, RegistryBlock Block)> cached = []; public DateTimeOffset SourceDate { get; private set; } - public int Count => blocks.Length; + public int Count { get; private set; } + + public static RegistryTable Open(Stream seekable) + { + var table = new RegistryTable { stream = seekable, origin = seekable.Position }; + using var reader = new BinaryReader(seekable, Encoding.UTF8, leaveOpen: true); + if (reader.ReadUInt32() != Magic) throw new InvalidDataException("not a “me-web?” registry table"); + table.SourceDate = DateTimeOffset.FromUnixTimeSeconds(reader.ReadInt64()); + table.Count = reader.ReadInt32(); + var index = reader.ReadInt64(); + seekable.Position = table.origin + index; + table.starts = new UInt128[2][]; + table.offsets = new long[2][]; + var at = (long)HeaderBytes; + for (var family = 0; family < 2; family++) + { + var count = reader.Read7BitEncodedInt(); + table.starts[family] = new UInt128[count]; + table.offsets[family] = new long[count + 1]; + var previous = UInt128.Zero; + for (var i = 0; i < count; i++) + { + previous += AsnTable.ReadBig(reader); + table.starts[family][i] = previous; + table.offsets[family][i] = at; + at += reader.Read7BitEncodedInt64(); + } + table.offsets[family][count] = at; + } + return table; + } public RegistryBlock Find(IPAddress address) { if (address is null) return null; if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); - var index = address.AddressFamily == AddressFamily.InterNetwork - ? AsnTable.Search(v4Start, BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()), v4Block) - : AsnTable.Search(v6Start, BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()), v6Block); - return index < 0 ? null : blocks[index]; + var family = address.AddressFamily == AddressFamily.InterNetwork ? 0 : 1; + var value = family == 0 ? BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()) : BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()); + var chunk = Last(starts[family], value, s => s); + if (chunk < 0) return null; + lock (gate) + { + if (cachedFamily != family || cachedChunk != chunk) Load(family, chunk); + var entry = Last(cached, value, e => e.Start); + return entry < 0 ? null : cached[entry].Block; + } } - public static RegistryTable Read(Stream compressed) + public void Dispose() => stream?.Dispose(); + + void Load(int family, int chunk) { - using var brotli = new BrotliStream(compressed, CompressionMode.Decompress); + var compressed = new byte[offsets[family][chunk + 1] - offsets[family][chunk]]; + stream.Position = origin + offsets[family][chunk]; + stream.ReadExactly(compressed); + using var brotli = new BrotliStream(new MemoryStream(compressed), CompressionMode.Decompress); using var reader = new BinaryReader(brotli, Encoding.UTF8); - if (reader.ReadUInt32() != Magic) throw new InvalidDataException("not a “me-web?” registry table"); - var table = new RegistryTable { SourceDate = DateTimeOffset.FromUnixTimeSeconds(reader.ReadInt64()) }; - table.blocks = new RegistryBlock[reader.Read7BitEncodedInt()]; - for (var i = 0; i < table.blocks.Length; i++) - table.blocks[i] = new RegistryBlock { Name = reader.ReadString(), Country = reader.ReadString() }; - var count = reader.Read7BitEncodedInt(); - table.v4Start = new uint[count]; - table.v4Block = new int[count]; - var previous4 = 0u; + var entries = new List<(UInt128 Start, RegistryBlock Block)>(count); + var start = starts[family][chunk]; for (var i = 0; i < count; i++) { - previous4 += (uint)reader.Read7BitEncodedInt64(); - table.v4Start[i] = previous4; - table.v4Block[i] = reader.Read7BitEncodedInt() - 1; + start += AsnTable.ReadBig(reader); + var name = reader.ReadString(); + entries.Add((start, name.Length == 0 ? null : new RegistryBlock { Name = name, Country = reader.ReadString() })); } - - count = reader.Read7BitEncodedInt(); - table.v6Start = new UInt128[count]; - table.v6Block = new int[count]; - var previous6 = UInt128.Zero; - for (var i = 0; i < count; i++) - { - previous6 += AsnTable.ReadBig(reader); - table.v6Start[i] = previous6; - table.v6Block[i] = reader.Read7BitEncodedInt() - 1; - } - return table; + cached = entries; + cachedFamily = family; + cachedChunk = chunk; } - public static void Write(Stream output, DateTimeOffset sourceDate, List ranges) + static int Last(IReadOnlyList list, UInt128 value, Func start) { - var index = new Dictionary<(string, string), int>(); - var list = new List(); - var v4 = new List<(uint Start, uint End, int Block)>(); - var v6 = new List<(UInt128 Start, UInt128 End, int Block)>(); - foreach (var range in ranges.OrderBy(r => r.Name, StringComparer.Ordinal).ThenBy(r => r.Country, StringComparer.Ordinal)) + var low = 0; + var high = list.Count - 1; + var found = -1; + while (low <= high) { - var key = (range.Name ?? "", range.Country ?? ""); - if (!index.TryGetValue(key, out var block)) + var middle = low + (high - low) / 2; + if (start(list[middle]) <= value) { - index[key] = block = list.Count; - list.Add(new RegistryBlock { Name = key.Item1, Country = key.Item2 }); + found = middle; + low = middle + 1; } - if (range.V6) v6.Add((range.Start, range.End, block)); - else v4.Add(((uint)range.Start, (uint)range.End, block)); - } - - using var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true); - using var writer = new BinaryWriter(brotli, Encoding.UTF8); - writer.Write(Magic); - writer.Write(sourceDate.ToUnixTimeSeconds()); - writer.Write7BitEncodedInt(list.Count); - foreach (var block in list) - { - writer.Write(block.Name); - writer.Write(block.Country); - } - - var flat4 = new List<(uint Start, int Block)>(); - Narrowest(v4, uint.MaxValue, (start, block) => flat4.Add((start, block))); - writer.Write7BitEncodedInt(flat4.Count); - var previous4 = 0u; - foreach (var (start, block) in flat4) - { - writer.Write7BitEncodedInt64(start - previous4); - writer.Write7BitEncodedInt(block + 1); - previous4 = start; - } - - var flat6 = new List<(UInt128 Start, int Block)>(); - Narrowest(v6, UInt128.MaxValue, (start, block) => flat6.Add((start, block))); - writer.Write7BitEncodedInt(flat6.Count); - var previous6 = UInt128.Zero; - foreach (var (start, block) in flat6) - { - AsnTable.WriteBig(writer, start - previous6); - writer.Write7BitEncodedInt(block + 1); - previous6 = start; + else high = middle - 1; } + return found; } public static void Narrowest(List<(T Start, T End, int Block)> ranges, T max, Action write) where T : struct, IBinaryInteger @@ -172,3 +167,80 @@ public sealed class RegistryTable if (pending is { } last) write(last.Start, last.Block); } } + +public sealed class RegistryWriter : IDisposable +{ + const int ChunkEntries = 16384; + + readonly Stream output; + readonly DateTimeOffset date; + readonly List<(UInt128 Start, string Name, string Country)> pending = []; + readonly List<(UInt128 Start, long Length)>[] index = [[], []]; + int family; + int count; + + public RegistryWriter(Stream output, DateTimeOffset date) + { + this.output = output; + this.date = date; + output.Write(new byte[RegistryTable.HeaderBytes]); + } + + public void Add(bool v6, UInt128 start, string name, string country) + { + var next = v6 ? 1 : 0; + if (next < family) throw new InvalidOperationException("IPv4 rows come before IPv6 rows"); + if (next != family || pending.Count == ChunkEntries) Flush(); + family = next; + pending.Add((start, name ?? "", country ?? "")); + if (!string.IsNullOrEmpty(name)) count++; + } + + public void Dispose() + { + Flush(); + var at = output.Position; + using var writer = new BinaryWriter(output, Encoding.UTF8, leaveOpen: true); + foreach (var chunks in index) + { + writer.Write7BitEncodedInt(chunks.Count); + var previous = UInt128.Zero; + foreach (var (start, length) in chunks) + { + AsnTable.WriteBig(writer, start - previous); + writer.Write7BitEncodedInt64(length); + previous = start; + } + } + writer.Flush(); + var end = output.Position; + output.Position = 0; + writer.Write(RegistryTable.Magic); + writer.Write(date.ToUnixTimeSeconds()); + writer.Write(count); + writer.Write(at); + writer.Flush(); + output.Position = end; + } + + void Flush() + { + if (pending.Count == 0) return; + var before = output.Position; + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + using (var writer = new BinaryWriter(brotli, Encoding.UTF8)) + { + writer.Write7BitEncodedInt(pending.Count); + var previous = pending[0].Start; + foreach (var (start, name, country) in pending) + { + AsnTable.WriteBig(writer, start - previous); + writer.Write(name); + if (name.Length > 0) writer.Write(country); + previous = start; + } + } + index[family].Add((pending[0].Start, output.Position - before)); + pending.Clear(); + } +} diff --git a/src/MeWeb.Core/Knowledge/Sources.cs b/src/MeWeb.Core/Knowledge/Sources.cs index 6075146..e5a5cca 100644 --- a/src/MeWeb.Core/Knowledge/Sources.cs +++ b/src/MeWeb.Core/Knowledge/Sources.cs @@ -15,11 +15,6 @@ public class Carrier public static class Sources { static readonly HashSet Allocations = ["ALLOCATED PA", "ALLOCATED UNSPECIFIED", "ALLOCATED-BY-RIR"]; - static readonly UInt128 WideV4 = 2048; - static readonly UInt128 NarrowV4 = 256; - static readonly UInt128 WideV6 = UInt128.One << 88; - static readonly UInt128 NarrowV6 = UInt128.One << 80; - const int Repeated = 3; public static IEnumerable IpToAsn(Stream gzip) { @@ -82,24 +77,6 @@ public static class Sources if (range?.Name is not null && !Allocations.Contains(status ?? "")) yield return range; } - public static List RipeSelection(IEnumerable blocks) - { - var counts = new Dictionary(StringComparer.Ordinal); - foreach (var block in blocks) - if (block.End - block.Start >= (block.V6 ? NarrowV6 : NarrowV4) - 1) counts[block.Name] = counts.GetValueOrDefault(block.Name) + 1; - var names = new Dictionary(StringComparer.Ordinal); - var kept = new List(); - foreach (var block in blocks) - { - var size = block.End - block.Start; - if (size < (block.V6 ? NarrowV6 : NarrowV4) - 1 || (size < (block.V6 ? WideV6 : WideV4) - 1 && counts[block.Name] < Repeated)) continue; - block.Name = Intern(names, block.Name); - block.Country = Intern(names, block.Country ?? ""); - kept.Add(block); - } - return kept; - } - static RegistryRange RipeRange(string line) { if (line.StartsWith("inetnum:", StringComparison.Ordinal)) diff --git a/src/MeWeb.Core/MeWeb.Core.csproj b/src/MeWeb.Core/MeWeb.Core.csproj index fe128ae..a92503e 100644 --- a/src/MeWeb.Core/MeWeb.Core.csproj +++ b/src/MeWeb.Core/MeWeb.Core.csproj @@ -12,7 +12,6 @@ - diff --git a/src/MeWeb.Core/Net/LocalNetwork.cs b/src/MeWeb.Core/Net/LocalNetwork.cs index a3d26f6..86241f9 100644 --- a/src/MeWeb.Core/Net/LocalNetwork.cs +++ b/src/MeWeb.Core/Net/LocalNetwork.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Reflection; using System.Net.NetworkInformation; using System.Net.Sockets; using System.Runtime.InteropServices; @@ -82,6 +83,8 @@ public interface IPlatform Task ReadCellular(CancellationToken ct); IDisposable AllowMulticast(); + + Stream OpenRegistry(); } public class SystemPlatform : IPlatform @@ -148,6 +151,8 @@ public class SystemPlatform : IPlatform public virtual IDisposable AllowMulticast() => new Nothing(); + public virtual Stream OpenRegistry() => Assembly.GetEntryAssembly()?.GetManifestResourceStream("registry.bin"); + public virtual Task ReadCellular(CancellationToken ct) => Task.FromResult(null); static readonly string[] Placeholders = ["o.e.m", "system product", "system manufacturer", "system version", "default string", "not applicable", "not specified", "to be filled", "none"]; diff --git a/src/MeWeb.Desktop/MeWeb.Desktop.csproj b/src/MeWeb.Desktop/MeWeb.Desktop.csproj index ab47e91..6c97f30 100644 --- a/src/MeWeb.Desktop/MeWeb.Desktop.csproj +++ b/src/MeWeb.Desktop/MeWeb.Desktop.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/MeWeb.Server/KnowledgeFeeder.cs b/src/MeWeb.Server/KnowledgeFeeder.cs index 47b3416..f335840 100644 --- a/src/MeWeb.Server/KnowledgeFeeder.cs +++ b/src/MeWeb.Server/KnowledgeFeeder.cs @@ -23,6 +23,7 @@ public class KnowledgeOptions public partial class KnowledgeFeeder(IOptionsMonitor options, IHttpClientFactory factory, ILogger log) : BackgroundService { public const string Client = "knowledge"; + public const int Format = 2; public static readonly string[] Published = ["ip2asn.bin", "networks.bin", "carriers.json", "registry.bin", "SOURCES.txt"]; const string IpToAsn = "iptoasn"; @@ -57,7 +58,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, [RipeV6] = RipeLicence, }; - const string RipeLicence = "RIPE Database extract, RIPE NCC (https://www.ripe.net), used with the RIPE NCC's permission (16 October 2026, ticket #1122623) for a free app with no commercial use; only each block's range, name and country are kept, from blocks of at least 2048 IPv4 addresses or a /40, or of at least 256 addresses or a /48 whose name recurs; it may lag behind the live database"; + const string RipeLicence = "RIPE Database extract, RIPE NCC (https://www.ripe.net), used with the RIPE NCC's permission (16 October 2026, ticket #1122623) for a free app with no commercial use; only each block's range, name and country are kept, of every block but the RIPE NCC's own allocations; it may lag behind the live database"; public string Folder => options.CurrentValue.Folder; @@ -79,7 +80,20 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, while (await timer.WaitForNextTickAsync(stoppingToken)); } - public RegistryBlock BlockFor(IPAddress address) => RegistryIndex.Find(Path.Combine(Folder, "current", RegistryIndex.FileName), address); + public RegistryBlock BlockFor(IPAddress address) + { + var path = Path.Combine(Folder, "current", "registry.bin"); + if (!File.Exists(path)) return null; + try + { + using var table = RegistryTable.Open(File.OpenRead(path)); + return table.Find(address); + } + catch (Exception ex) when (ex is IOException or InvalidDataException) + { + return null; + } + } public ViewKnowledge Manifest() { @@ -142,7 +156,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, } } - if (!changed && Manifest() is not null) return; + if (!changed && Manifest()?.Format == Format) return; Build(sources, known.Values.ToList(), ct); } @@ -258,6 +272,7 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, var manifest = new ViewKnowledge { + Format = Format, BuiltAt = built, Sources = known.OrderBy(s => s.Name, StringComparer.Ordinal).ToList(), Files = Published.Where(f => File.Exists(Path.Combine(staging, f))).Select(f => new ViewKnowledgeFile @@ -274,19 +289,17 @@ public partial class KnowledgeFeeder(IOptionsMonitor options, if (Directory.Exists(previous)) Directory.Delete(previous, true); if (Directory.Exists(current)) Directory.Move(current, previous); Directory.Move(staging, current); - log.LogInformation("{Service} built {Files} in {Seconds:0.0} s: {Networks} networks, upstreams from {Collectors} routing tables, {Carriers} carriers, {Kept} of {Blocks} registry blocks embedded", + log.LogInformation("{Service} built {Files} in {Seconds:0.0} s: {Networks} networks, upstreams from {Collectors} routing tables, {Carriers} carriers, {Blocks} registry blocks", nameof(KnowledgeFeeder), string.Join(", ", manifest.Files.Select(f => $"{f.Name} {f.Bytes / 1024} KiB")), clock.Elapsed.TotalSeconds, - organisations.Keys.Union(upstreams.Keys).Count(), tally.Collectors, list.Count, registry.Kept, registry.All); + organisations.Keys.Union(upstreams.Keys).Count(), tally.Collectors, list.Count, registry); } - static (int Kept, int All) Registry(string sources, string staging, DateTimeOffset date) + static int Registry(string sources, string staging, DateTimeOffset date) { - var v4 = Read(Path.Combine(sources, "ripe-inetnum.gz")); - var v6 = Read(Path.Combine(sources, "ripe-inet6num.gz")); - var kept = Sources.RipeSelection(v4.Concat(v6)); - if (kept.Count == 0) return (0, 0); - using (var output = File.Create(Path.Combine(staging, "registry.bin"))) RegistryTable.Write(output, date, kept); - return (kept.Count, RegistryIndex.Build(v4, v6, Path.Combine(staging, RegistryIndex.FileName), date)); + var v4 = Path.Combine(sources, "ripe-inetnum.gz"); + var v6 = Path.Combine(sources, "ripe-inet6num.gz"); + if (!File.Exists(v4) && !File.Exists(v6)) return 0; + return RegistryBuilder.Build(Read(v4), Read(v6), Path.Combine(staging, "registry.bin"), date); static IEnumerable Read(string path) { diff --git a/src/MeWeb.Server/RegistryBuilder.cs b/src/MeWeb.Server/RegistryBuilder.cs new file mode 100644 index 0000000..d6f2226 --- /dev/null +++ b/src/MeWeb.Server/RegistryBuilder.cs @@ -0,0 +1,52 @@ +using System.Numerics; +using System.Text; + +using MeWeb.Core.Knowledge; + +namespace MeWeb.Server; + +public static class RegistryBuilder +{ + public static int Build(IEnumerable v4Blocks, IEnumerable v6Blocks, string path, DateTimeOffset date) + { + var namesPath = path + ".names"; + var count = 0; + using (var output = File.Create(path)) + using (var writer = new RegistryWriter(output, date)) + { + Family(false, v4Blocks, uint.MaxValue, 5_000_000); + Family(true, v6Blocks, UInt128.MaxValue, 1_200_000); + + void Family(bool v6, IEnumerable blocks, T max, int capacity) where T : struct, IBinaryInteger + { + var ranges = new List<(T Start, T End, int Block)>(capacity); + using (var names = File.Create(namesPath)) + using (var labels = new BinaryWriter(names, Encoding.UTF8)) + { + foreach (var block in blocks.Where(b => b.V6 == v6)) + { + ranges.Add((T.CreateTruncating(block.Start), T.CreateTruncating(block.End), checked((int)names.Position))); + labels.Write(block.Name); + labels.Write(block.Country ?? ""); + count++; + } + } + using var handle = File.OpenHandle(namesPath); + var buffer = new byte[640]; + RegistryTable.Narrowest(ranges, max, (start, at) => + { + if (at < 0) + { + writer.Add(v6, UInt128.CreateTruncating(start), null, null); + return; + } + var read = RandomAccess.Read(handle, buffer, at); + using var reader = new BinaryReader(new MemoryStream(buffer, 0, read), Encoding.UTF8); + writer.Add(v6, UInt128.CreateTruncating(start), reader.ReadString(), reader.ReadString()); + }); + } + } + File.Delete(namesPath); + return count; + } +} diff --git a/src/MeWeb.Server/RegistryIndex.cs b/src/MeWeb.Server/RegistryIndex.cs deleted file mode 100644 index 99e6dc6..0000000 --- a/src/MeWeb.Server/RegistryIndex.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System.Buffers.Binary; -using System.Net; -using System.Net.Sockets; -using System.Text; - -using MeWeb.Core.Knowledge; - -namespace MeWeb.Server; - -public static class RegistryIndex -{ - public const string FileName = "registry-full.bin"; - - const uint Magic = 0x344B574D; - const int HeaderBytes = 32; - const int V4Bytes = 8; - const int V6Bytes = 20; - - public static int Build(IEnumerable v4Blocks, IEnumerable v6Blocks, string path, DateTimeOffset date) - { - var namesPath = path + ".names"; - var count = 0; - using (var names = File.Create(namesPath)) - using (var labels = new BinaryWriter(names, Encoding.UTF8)) - using (var output = File.Create(path)) - using (var writer = new BinaryWriter(output)) - { - int Label(RegistryRange block) - { - var at = checked((int)names.Position); - labels.Write(block.Name); - labels.Write(block.Country ?? ""); - count++; - return at; - } - writer.Write(new byte[HeaderBytes]); - var v4 = new List<(uint Start, uint End, int Block)>(5_000_000); - foreach (var block in v4Blocks.Where(b => !b.V6)) v4.Add(((uint)block.Start, (uint)block.End, Label(block))); - var count4 = 0; - RegistryTable.Narrowest(v4, uint.MaxValue, (start, block) => - { - writer.Write(start); - writer.Write(block); - count4++; - }); - v4 = null; - var v6 = new List<(UInt128 Start, UInt128 End, int Block)>(1_200_000); - foreach (var block in v6Blocks.Where(b => b.V6)) v6.Add((block.Start, block.End, Label(block))); - var count6 = 0; - RegistryTable.Narrowest(v6, UInt128.MaxValue, (start, block) => - { - writer.Write((ulong)(start >> 64)); - writer.Write((ulong)start); - writer.Write(block); - count6++; - }); - v6 = null; - labels.Flush(); - writer.Flush(); - var namesAt = output.Position; - names.Position = 0; - names.CopyTo(output); - output.Position = 0; - writer.Write(Magic); - writer.Write(date.ToUnixTimeSeconds()); - writer.Write(count4); - writer.Write(count6); - writer.Write(namesAt); - } - File.Delete(namesPath); - return count; - } - - public static RegistryBlock Find(string path, IPAddress address) - { - if (address is null || !File.Exists(path)) return null; - if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); - using var handle = File.OpenHandle(path); - Span header = stackalloc byte[HeaderBytes]; - if (RandomAccess.Read(handle, header, 0) < HeaderBytes || BinaryPrimitives.ReadUInt32LittleEndian(header) != Magic) return null; - var count4 = BinaryPrimitives.ReadInt32LittleEndian(header[12..]); - var count6 = BinaryPrimitives.ReadInt32LittleEndian(header[16..]); - var namesAt = BinaryPrimitives.ReadInt64LittleEndian(header[20..]); - var v4 = address.AddressFamily == AddressFamily.InterNetwork; - var value = v4 ? BinaryPrimitives.ReadUInt32BigEndian(address.GetAddressBytes()) : BinaryPrimitives.ReadUInt128BigEndian(address.GetAddressBytes()); - var first = v4 ? HeaderBytes : HeaderBytes + (long)count4 * V4Bytes; - var size = v4 ? V4Bytes : V6Bytes; - Span record = stackalloc byte[V6Bytes]; - var low = 0; - var high = (v4 ? count4 : count6) - 1; - var block = -1; - while (low <= high) - { - var middle = low + (high - low) / 2; - RandomAccess.Read(handle, record[..size], first + (long)middle * size); - var start = v4 - ? BinaryPrimitives.ReadUInt32LittleEndian(record) - : ((UInt128)BinaryPrimitives.ReadUInt64LittleEndian(record) << 64) | BinaryPrimitives.ReadUInt64LittleEndian(record[8..]); - if (start <= value) - { - block = BinaryPrimitives.ReadInt32LittleEndian(record[(size - 4)..]); - low = middle + 1; - } - else high = middle - 1; - } - if (block < 0) return null; - var text = new byte[640]; - var read = RandomAccess.Read(handle, text, namesAt + block); - using var reader = new BinaryReader(new MemoryStream(text, 0, read), Encoding.UTF8); - return new RegistryBlock { Name = reader.ReadString(), Country = reader.ReadString() }; - } -} diff --git a/src/MeWeb.Shared/ServerContracts.cs b/src/MeWeb.Shared/ServerContracts.cs index 831d9b0..4928d07 100644 --- a/src/MeWeb.Shared/ServerContracts.cs +++ b/src/MeWeb.Shared/ServerContracts.cs @@ -17,6 +17,7 @@ public class ViewIp public class ViewKnowledge { + public int Format { get; set; } public DateTimeOffset BuiltAt { get; set; } public List Files { get; set; } = []; public List Sources { get; set; } = []; diff --git a/tools/Screenshots/Screenshots.csproj b/tools/Screenshots/Screenshots.csproj index 4e4e171..37b0044 100644 --- a/tools/Screenshots/Screenshots.csproj +++ b/tools/Screenshots/Screenshots.csproj @@ -15,4 +15,8 @@ + + + + diff --git a/tools/knowledge.cs b/tools/knowledge.cs index bdecb83..934a3ff 100644 --- a/tools/knowledge.cs +++ b/tools/knowledge.cs @@ -73,11 +73,9 @@ void Report() NetworkTable networks = null; if (File.Exists(path)) using (var input = File.OpenRead(path)) networks = NetworkTable.Read(input); - RegistryTable registry = null; path = Path.Combine(output, "registry.bin"); - if (File.Exists(path)) - using (var input = File.OpenRead(path)) registry = RegistryTable.Read(input); - Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} address block names"); + using var registry = File.Exists(path) ? RegistryTable.Open(File.OpenRead(path)) : null; + Console.WriteLine($"knowledge: {table.Count} networks in the provider map (source {table.SourceDate:yyyy-MM-dd}), {networks?.Count ?? 0} with names or upstreams, {registry?.Count ?? 0} named address blocks"); } static string FindRoot()