me-web? 0.5: the shared exit measured, the route from the server, the network's upstreams

- Behind your provider's shared address, me-web? now shows how its exit hands out ports
  (kept, one after another, from one block, or at random), the range it gave this device,
  and how many customers may share the address at most; the estimate sharpens with each
  check on the same address.
- It tests whether two devices behind the same address can reach each other (hairpin), and
  how long an unused connection stays open: the card says "measuring" for about two minutes
  after the check, then the answer.
- With the test from outside, the publisher's server also traces the route back to your
  address: your provider's routers and the networks in between appear on the path, with
  their names.
- The provider card says which networks your provider reaches the internet through.
- On mobile data, the path shows the operator and the type of network (5G, 4G) read from the
  phone; on Android 13 and later the phone asks once for permission to read the network type.
- The speed card adds the round trip to the publisher's server.
- The network data now comes from the publisher's server, refreshed there from open sources:
  iptoasn.com, DB-IP Lite (CC BY 4.0), the University of Oregon Route Views Project (CC BY 4.0)
  and the Android Open Source Project's list of mobile operators (Apache 2.0); the release
  notes list each source's date and checksum.
- Source links in the cards wrap instead of being cut off.
- New strings, drafted in every language.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012CzABvBkbcFqoHdmi8b9WB

src/MeWeb.Core/Knowledge/AsnTable.cs

@@ -46,6 +46,21 @@ public sealed class AsnTable
     return index < 0 ? null : systems[index];
   }
 
+  public AsnEntry Find(uint number)
+  {
+    var low = 0;
+    var high = systems.Length - 1;
+    while (low <= high)
+    {
+      var middle = (low + high) / 2;
+      var found = systems[middle].Number;
+      if (found == number) return systems[middle];
+      if (found < number) low = middle + 1;
+      else high = middle - 1;
+    }
+    return null;
+  }
+
   public static AsnTable Read(Stream compressed)
   {
     using var brotli = new BrotliStream(compressed, CompressionMode.Decompress);
@@ -82,9 +97,19 @@ public sealed class AsnTable
 
   public static void Write(Stream output, DateTimeOffset sourceDate, IEnumerable<AsnRange> ranges)
   {
-    var list = ranges.Where(r => r.Number != 0).ToList();
-    var systems = list.GroupBy(r => r.Number).OrderBy(g => g.Key)
-      .Select(g => new AsnEntry { Number = g.Key, Country = g.First().Country ?? "", Name = g.First().Name ?? "" }).ToList();
+    var known = new Dictionary<uint, AsnEntry>();
+    var v4Ranges = new List<(UInt128 Start, UInt128 End, uint Number)>();
+    var v6Ranges = new List<(UInt128 Start, UInt128 End, uint Number)>();
+    foreach (var range in ranges)
+    {
+      if (range.Number == 0) continue;
+      if (!known.ContainsKey(range.Number)) known[range.Number] = new AsnEntry { Number = range.Number, Country = range.Country ?? "", Name = range.Name ?? "" };
+      if (range.Start.AddressFamily == AddressFamily.InterNetwork)
+        v4Ranges.Add((BinaryPrimitives.ReadUInt32BigEndian(range.Start.GetAddressBytes()), BinaryPrimitives.ReadUInt32BigEndian(range.End.GetAddressBytes()), range.Number));
+      else
+        v6Ranges.Add((BinaryPrimitives.ReadUInt128BigEndian(range.Start.GetAddressBytes()), BinaryPrimitives.ReadUInt128BigEndian(range.End.GetAddressBytes()), range.Number));
+    }
+    var systems = known.Values.OrderBy(s => s.Number).ToList();
     var index = systems.Select((s, i) => (s.Number, i)).ToDictionary(p => p.Number, p => p.i);
 
     using var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true);
@@ -99,7 +124,7 @@ public sealed class AsnTable
       writer.Write(system.Name);
     }
 
-    var v4 = Flatten(list.Where(r => r.Start.AddressFamily == AddressFamily.InterNetwork), a => BinaryPrimitives.ReadUInt32BigEndian(a.GetAddressBytes()), uint.MaxValue, index);
+    var v4 = Flatten(v4Ranges, uint.MaxValue, index);
     writer.Write7BitEncodedInt(v4.Count);
     var previous4 = 0u;
     foreach (var (start, system) in v4)
@@ -109,7 +134,7 @@ public sealed class AsnTable
       previous4 = (uint)start;
     }
 
-    var v6 = Flatten(list.Where(r => r.Start.AddressFamily == AddressFamily.InterNetworkV6), a => BinaryPrimitives.ReadUInt128BigEndian(a.GetAddressBytes()), UInt128.MaxValue, index);
+    var v6 = Flatten(v6Ranges, UInt128.MaxValue, index);
     writer.Write7BitEncodedInt(v6.Count);
     var previous6 = UInt128.Zero;
     foreach (var (start, system) in v6)
@@ -120,14 +145,16 @@ public sealed class AsnTable
     }
   }
 
-  static List<(UInt128 Start, int System)> Flatten(IEnumerable<AsnRange> ranges, Func<IPAddress, UInt128> number, UInt128 max, Dictionary<uint, int> index)
+  static List<(UInt128 Start, int System)> Flatten(List<(UInt128 Start, UInt128 End, uint Number)> ranges, UInt128 max, Dictionary<uint, int> index)
   {
+    ranges.Sort((a, b) => a.Start.CompareTo(b.Start));
     var result = new List<(UInt128 Start, int System)>();
     var nextFree = UInt128.Zero;
     var lastSystem = -1;
     var full = false;
-    foreach (var range in ranges.Select(r => (Start: number(r.Start), End: number(r.End), System: index[r.Number])).OrderBy(r => r.Start))
+    foreach (var (rangeStart, rangeEnd, number) in ranges)
     {
+      var range = (Start: rangeStart, End: rangeEnd, System: index[number]);
       if (full || range.End < nextFree) continue;
       var start = range.Start < nextFree ? nextFree : range.Start;
       if (start > nextFree && lastSystem != -1)

src/MeWeb.Core/Knowledge/NetworkTable.cs

@@ -0,0 +1,63 @@
+using System.IO.Compression;
+using System.Text;
+
+namespace MeWeb.Core.Knowledge;
+
+public class NetworkInfo
+{
+  public uint Number { get; set; }
+  public string Organisation { get; set; }
+  public List<uint> Upstreams { get; set; } = [];
+}
+
+public sealed class NetworkTable
+{
+  const uint Magic = 0x324B574D;
+
+  readonly Dictionary<uint, NetworkInfo> networks = [];
+
+  public DateTimeOffset BuiltAt { get; private set; }
+
+  public int Count => networks.Count;
+
+  public NetworkInfo Find(uint number) => networks.GetValueOrDefault(number);
+
+  public static NetworkTable Read(Stream compressed)
+  {
+    using var brotli = new BrotliStream(compressed, CompressionMode.Decompress);
+    using var reader = new BinaryReader(brotli, Encoding.UTF8);
+    if (reader.ReadUInt32() != Magic) throw new InvalidDataException("not a me-web? network table");
+    var table = new NetworkTable { BuiltAt = DateTimeOffset.FromUnixTimeSeconds(reader.ReadInt64()) };
+    var count = reader.Read7BitEncodedInt();
+    var previous = 0u;
+    for (var i = 0; i < count; i++)
+    {
+      previous += (uint)reader.Read7BitEncodedInt64();
+      var info = new NetworkInfo { Number = previous, Organisation = reader.ReadString() };
+      var upstreams = reader.Read7BitEncodedInt();
+      for (var u = 0; u < upstreams; u++) info.Upstreams.Add((uint)reader.Read7BitEncodedInt64());
+      if (info.Organisation.Length == 0) info.Organisation = null;
+      table.networks[info.Number] = info;
+    }
+    return table;
+  }
+
+  public static void Write(Stream output, DateTimeOffset builtAt, IEnumerable<NetworkInfo> networks)
+  {
+    var list = networks.Where(n => n.Number != 0).OrderBy(n => n.Number).ToList();
+    using var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true);
+    using var writer = new BinaryWriter(brotli, Encoding.UTF8);
+    writer.Write(Magic);
+    writer.Write(builtAt.ToUnixTimeSeconds());
+    writer.Write7BitEncodedInt(list.Count);
+    var previous = 0u;
+    foreach (var info in list)
+    {
+      writer.Write7BitEncodedInt64(info.Number - previous);
+      writer.Write(info.Organisation ?? "");
+      writer.Write7BitEncodedInt(info.Upstreams.Count);
+      foreach (var upstream in info.Upstreams) writer.Write7BitEncodedInt64(upstream);
+      previous = info.Number;
+    }
+  }
+}

src/MeWeb.Core/Knowledge/Providers.cs

@@ -27,6 +27,7 @@ public class ProviderProfile
 
 [JsonSourceGenerationOptions(JsonSerializerDefaults.Web)]
 [JsonSerializable(typeof(ProviderProfile))]
+[JsonSerializable(typeof(List<Carrier>))]
 public partial class KnowledgeJsonContext : JsonSerializerContext
 {
 }
@@ -35,6 +36,8 @@ public sealed class KnowledgeBase
 {
   readonly Lazy<AsnTable> asn = new(LoadAsn);
   readonly Lazy<List<ProviderProfile>> providers = new(LoadProviders);
+  readonly Lazy<NetworkTable> networks = new(LoadNetworks);
+  readonly Lazy<Dictionary<string, string>> carriers = new(LoadCarriers);
 
   public bool HasAsnTable => typeof(KnowledgeBase).Assembly.GetManifestResourceInfo("ip2asn.bin") is not null;
 
@@ -44,6 +47,26 @@ public sealed class KnowledgeBase
 
   public ProviderProfile ProviderFor(uint number) => number == 0 ? null : Providers.FirstOrDefault(p => p.Asns.Contains(number));
 
+  public NetworkInfo NetworkFor(uint number) => number == 0 ? null : networks.Value?.Find(number);
+
+  public string CarrierFor(string mccMnc) => string.IsNullOrEmpty(mccMnc) ? null : carriers.Value.GetValueOrDefault(mccMnc);
+
+  public string NameOf(uint number) => NetworkFor(number)?.Organisation ?? Asn?.Find(number)?.Name;
+
+  static NetworkTable LoadNetworks()
+  {
+    using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("networks.bin");
+    return stream is null ? null : NetworkTable.Read(stream);
+  }
+
+  static Dictionary<string, string> LoadCarriers()
+  {
+    using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("carriers.json");
+    if (stream is null) return [];
+    var list = JsonSerializer.Deserialize(stream, KnowledgeJsonContext.Default.ListCarrier) ?? [];
+    return list.Where(c => c.MccMnc is not null && c.Name is not null).GroupBy(c => c.MccMnc).ToDictionary(g => g.Key, g => g.First().Name);
+  }
+
   static AsnTable LoadAsn()
   {
     using var stream = typeof(KnowledgeBase).Assembly.GetManifestResourceStream("ip2asn.bin");

src/MeWeb.Core/Knowledge/Sources.cs

@@ -0,0 +1,202 @@
+using System.Buffers.Binary;
+using System.IO.Compression;
+using System.Net;
+
+namespace MeWeb.Core.Knowledge;
+
+public class Carrier
+{
+  public string MccMnc { get; set; }
+  public string Name { get; set; }
+}
+
+public static class Sources
+{
+  public static IEnumerable<AsnRange> IpToAsn(Stream gzip)
+  {
+    var names = new Dictionary<string, string>(StringComparer.Ordinal);
+    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
+    string line;
+    while ((line = reader.ReadLine()) is not null)
+    {
+      var parts = line.Split('\t');
+      if (parts.Length < 5 || !uint.TryParse(parts[2], out var number) || number == 0) continue;
+      if (!IPAddress.TryParse(parts[0], out var start) || !IPAddress.TryParse(parts[1], out var end)) continue;
+      yield return new AsnRange { Start = start, End = end, Number = number, Country = Intern(names, parts[3] == "None" ? "" : parts[3]), Name = Intern(names, parts[4]) };
+    }
+  }
+
+  public static Dictionary<uint, string> DbIpOrganisations(Stream gzip)
+  {
+    var seen = new Dictionary<uint, Dictionary<string, int>>();
+    using var reader = new StreamReader(new GZipStream(gzip, CompressionMode.Decompress));
+    string line;
+    while ((line = reader.ReadLine()) is not null)
+    {
+      var first = line.IndexOf(',');
+      var second = first < 0 ? -1 : line.IndexOf(',', first + 1);
+      var third = second < 0 ? -1 : line.IndexOf(',', second + 1);
+      if (third < 0 || !uint.TryParse(line.AsSpan(second + 1, third - second - 1), out var number) || number == 0) continue;
+      var name = line[(third + 1)..].Trim();
+      if (name.Length >= 2 && name[0] == '"' && name[^1] == '"') name = name[1..^1].Replace("\"\"", "\"");
+      if (name.Length == 0) continue;
+      if (!seen.TryGetValue(number, out var names)) seen[number] = names = [];
+      names[name] = names.GetValueOrDefault(name) + 1;
+    }
+    return seen.ToDictionary(p => p.Key, p => p.Value.MaxBy(n => n.Value).Key);
+  }
+
+  public static Dictionary<uint, List<uint>> Upstreams(Stream mrt, int keep = 4, double share = 0.05)
+  {
+    var counts = new Dictionary<uint, Dictionary<uint, int>>();
+    var header = new byte[12];
+    var body = new byte[1 << 16];
+    while (Fill(mrt, header))
+    {
+      var type = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(4));
+      var subtype = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(6));
+      var length = (int)BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(8));
+      if (body.Length < length) body = new byte[length];
+      if (!Fill(mrt, body.AsSpan(0, length))) break;
+      if (type != 13 || subtype is not (2 or 4)) continue;
+      Rib(body.AsSpan(0, length), counts);
+    }
+    var result = new Dictionary<uint, List<uint>>();
+    foreach (var (origin, neighbours) in counts)
+    {
+      var total = (double)neighbours.Values.Sum();
+      var chosen = neighbours.Where(n => n.Value / total >= share).OrderByDescending(n => n.Value).ThenBy(n => n.Key).Take(keep).Select(n => n.Key).ToList();
+      if (chosen.Count > 0) result[origin] = chosen;
+    }
+    return result;
+  }
+
+  public static List<Carrier> Carriers(TextReader textpb)
+  {
+    var list = new List<Carrier>();
+    string name = null;
+    string tuple = null;
+    var plain = false;
+    string line;
+    while ((line = textpb.ReadLine()) is not null)
+    {
+      var text = line.Trim();
+      if (text.StartsWith("carrier_id {", StringComparison.Ordinal)) name = null;
+      else if (text.StartsWith("carrier_name:", StringComparison.Ordinal)) name = Quoted(text);
+      else if (text.StartsWith("carrier_attribute {", StringComparison.Ordinal))
+      {
+        tuple = null;
+        plain = true;
+      }
+      else if (text.StartsWith("mccmnc_tuple:", StringComparison.Ordinal)) tuple = tuple is null ? Quoted(text) : null;
+      else if (text == "}" && tuple is not null)
+      {
+        if (plain && name is not null && list.All(c => c.MccMnc != tuple)) list.Add(new Carrier { MccMnc = tuple, Name = name });
+        tuple = null;
+      }
+      else if (text.Length > 0 && text != "}") plain = false;
+    }
+    return list.OrderBy(c => c.MccMnc, StringComparer.Ordinal).ToList();
+  }
+
+  static string Intern(Dictionary<string, string> names, string text)
+  {
+    if (names.TryGetValue(text, out var known)) return known;
+    names[text] = text;
+    return text;
+  }
+
+  static string Quoted(string text)
+  {
+    var start = text.IndexOf('"');
+    var end = text.LastIndexOf('"');
+    return start >= 0 && end > start ? text[(start + 1)..end] : null;
+  }
+
+  static void Rib(ReadOnlySpan<byte> rib, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    if (rib.Length < 5) return;
+    var at = 4;
+    var bits = rib[at++];
+    at += (bits + 7) / 8;
+    if (at + 2 > rib.Length) return;
+    var entries = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
+    at += 2;
+    for (var e = 0; e < entries && at + 8 <= rib.Length; e++)
+    {
+      at += 6;
+      var size = BinaryPrimitives.ReadUInt16BigEndian(rib[at..]);
+      at += 2;
+      if (at + size > rib.Length) return;
+      Attributes(rib.Slice(at, size), counts);
+      at += size;
+    }
+  }
+
+  static void Attributes(ReadOnlySpan<byte> attributes, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    var at = 0;
+    while (at + 3 <= attributes.Length)
+    {
+      var flags = attributes[at];
+      var type = attributes[at + 1];
+      var extended = (flags & 0x10) != 0;
+      if (extended && at + 4 > attributes.Length) return;
+      var length = extended ? BinaryPrimitives.ReadUInt16BigEndian(attributes[(at + 2)..]) : attributes[at + 2];
+      var start = at + (extended ? 4 : 3);
+      if (start + length > attributes.Length) return;
+      if (type == 2)
+      {
+        Path(attributes.Slice(start, length), counts);
+        return;
+      }
+      at = start + length;
+    }
+  }
+
+  static void Path(ReadOnlySpan<byte> path, Dictionary<uint, Dictionary<uint, int>> counts)
+  {
+    uint last = 0;
+    uint before = 0;
+    var endsInSet = false;
+    var at = 0;
+    while (at + 2 <= path.Length)
+    {
+      var segment = path[at];
+      var count = path[at + 1];
+      at += 2;
+      if (at + count * 4 > path.Length) return;
+      for (var i = 0; i < count; i++)
+      {
+        var number = BinaryPrimitives.ReadUInt32BigEndian(path[(at + i * 4)..]);
+        if (segment != 2)
+        {
+          endsInSet = true;
+          continue;
+        }
+        endsInSet = false;
+        if (number == last) continue;
+        before = last;
+        last = number;
+      }
+      at += count * 4;
+    }
+    if (endsInSet || last == 0 || before == 0 || Reserved(before) || Reserved(last)) return;
+    if (!counts.TryGetValue(last, out var neighbours)) counts[last] = neighbours = [];
+    neighbours[before] = neighbours.GetValueOrDefault(before) + 1;
+  }
+
+  static bool Reserved(uint number) => number is 23456 or (>= 64496 and <= 131071) or >= 4200000000;
+
+  static bool Fill(Stream stream, Span<byte> buffer)
+  {
+    var read = 0;
+    while (read < buffer.Length)
+    {
+      var n = stream.Read(buffer[read..]);
+      if (n == 0) return false;
+      read += n;
+    }
+    return true;
+  }
+}