using System.Globalization; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Nodes; const string Cldr = "47.0.0"; string[] unLanguages = ["ar", "zh", "en", "fr", "ru", "es"]; string[] reviewed = ["en", "it"]; string[] officialStatuses = ["official", "de_facto_official"]; string[] caselessScripts = ["ka"]; var supplied = new Dictionary { ["ay"] = ("Aymar aru", "Aymara", false), ["bi"] = ("Bislama", "Bislama", false), ["dv"] = ("ދިވެހި", "Dhivehi", true), ["fj"] = ("Na Vosa Vakaviti", "Fijian", false), ["gil"] = ("Taetae ni Kiribati", "Gilbertese", false), ["gn"] = ("Avañe'ẽ", "Guarani", false), ["hif"] = ("Fiji Baat", "Fiji Hindi", false), ["ho"] = ("Hiri Motu", "Hiri Motu", false), ["ht"] = ("Kreyòl ayisyen", "Haitian Creole", false), ["mh"] = ("Kajin M̧ajeļ", "Marshallese", false), ["ms-Arab"] = ("بهاس ملايو", "Malay (Arabic script)", true), ["na"] = ("Dorerin Naoero", "Nauruan", false), ["ny"] = ("Chichewa", "Chichewa", false), ["pau"] = ("Tekoi ra Belau", "Palauan", false), ["sm"] = ("Gagana Samoa", "Samoan", false), ["ss"] = ("SiSwati", "Swati", false), ["tet"] = ("Tetun", "Tetum", false), ["tpi"] = ("Tok Pisin", "Tok Pisin", false), ["tvl"] = ("Te Ggana Tuuvalu", "Tuvaluan", false), ["wni"] = ("Shindzuani", "Ndzwani Comorian", false), ["zdj"] = ("Shingazidja", "Ngazidja Comorian", false), }; var root = FindRoot(); var target = Path.Combine(root, "data", "languages.json"); using var http = new HttpClient(); http.DefaultRequestHeaders.UserAgent.ParseAdd("meweb-languages/1.0"); var baseUrl = $"https://raw.githubusercontent.com/unicode-org/cldr-json/{Cldr}/cldr-json/cldr-core/supplemental"; var territories = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryInfo.json"))["supplemental"]["territoryInfo"].AsObject(); var containment = JsonNode.Parse(await http.GetStringAsync($"{baseUrl}/territoryContainment.json"))["supplemental"]["territoryContainment"]; var members = containment["UN"]["_contains"].AsArray().Select(n => (string)n).ToHashSet(); var known = CultureInfo.GetCultures(CultureTypes.NeutralCultures | CultureTypes.SpecificCultures).Select(c => c.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); var codes = new SortedSet(unLanguages, StringComparer.Ordinal); var missing = new SortedSet(StringComparer.Ordinal); foreach (var (territory, info) in territories) { if (!members.Contains(territory)) continue; var population = info["languagePopulation"]?.AsObject(); if (population is null) continue; foreach (var (language, details) in population) { var status = (string)details["_officialStatus"]; if (status is null || !officialStatuses.Contains(status)) continue; var code = language.Replace('_', '-'); if (known.Contains(code) || supplied.ContainsKey(code)) codes.Add(code); else missing.Add(code); } } var previous = File.Exists(target) ? JsonNode.Parse(File.ReadAllText(target)).AsArray().ToDictionary(n => (string)n["code"], n => (string)n["status"]) : []; var list = new JsonArray(); foreach (var code in codes) { var (name, english, rtl) = supplied.TryGetValue(code, out var given) ? given : FromCulture(code); var status = reviewed.Contains(code) ? "reviewed" : previous.GetValueOrDefault(code) ?? "draft"; list.Add((JsonNode)new JsonObject { ["code"] = code, ["name"] = name, ["english"] = english, ["rtl"] = rtl, ["status"] = status, }); } File.WriteAllText(target, list.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }) + "\n"); Console.WriteLine($"{list.Count} languages written to data/languages.json (CLDR {Cldr}, {members.Count} UN member states)"); if (missing.Count > 0) Console.WriteLine($"official but neither in ICU nor supplied, left out: {string.Join(", ", missing)}"); return 0; (string Name, string English, bool Rtl) FromCulture(string code) { var culture = CultureInfo.GetCultureInfo(code); var native = culture.NativeName; var name = caselessScripts.Contains(code.Split('-')[0]) || native.Length == 0 ? native : culture.TextInfo.ToUpper(native[0]) + native[1..]; return (name, culture.EnglishName, culture.TextInfo.IsRightToLeft); } static string FindRoot() { var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeWeb.slnx"))) dir = dir.Parent; return dir?.FullName ?? throw new InvalidOperationException("run from inside the repository (MeWeb.slnx not found)"); }