using System.Formats.Tar; using System.Text; using System.Text.RegularExpressions; namespace PrivaPub.Infrastructure.Backup { // A backup as one tar file, for the administrator to download and to upload again: every file under /, its length // known before the first byte (so a download has a Content-Length and resumes with Range), written straight from the // backup's files, never held in memory. public sealed partial class BackupTar { const int Block = 512; readonly List _entries = []; public long Length { get; private set; } sealed record Entry(string Name, string Path, long Size, DateTime ModifiedAt, long Offset); public static BackupTar Of(string backupsRoot, string id) { var directory = Path.Combine(backupsRoot, id); var tar = new BackupTar(); var offset = 0L; foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)) { var info = new FileInfo(file); var name = id + "/" + Path.GetRelativePath(directory, file).Replace('\\', '/'); tar._entries.Add(new Entry(name, file, info.Length, info.LastWriteTimeUtc, offset)); offset += Block + Padded(info.Length); } tar.Length = offset + 2 * Block; return tar; } static long Padded(long size) => (size + Block - 1) / Block * Block; /// Writes bytes from to to (inclusive) of the tar. public async Task Write(Stream output, long from, long to, CancellationToken token) { var buffer = new byte[81920]; foreach (var entry in _entries) { var headerEnd = entry.Offset + Block; var dataEnd = headerEnd + entry.Size; var entryEnd = headerEnd + Padded(entry.Size); if (entryEnd <= from) continue; if (entry.Offset > to) return; if (from < headerEnd) { var header = Header(entry); var start = (int)Math.Max(0, from - entry.Offset); var end = (int)Math.Min(Block, to - entry.Offset + 1); await output.WriteAsync(header.AsMemory(start, end - start), token); } if (from < dataEnd && to >= headerEnd && entry.Size > 0) { var start = Math.Max(0, from - headerEnd); var end = Math.Min(entry.Size, to - headerEnd + 1); await using var file = new FileStream(entry.Path, FileMode.Open, FileAccess.Read, FileShare.Read, 1, FileOptions.SequentialScan); file.Seek(start, SeekOrigin.Begin); var left = end - start; while (left > 0) { var read = await file.ReadAsync(buffer.AsMemory(0, (int)Math.Min(buffer.Length, left)), token); if (read == 0) throw new IOException($"{entry.Name} shrank while it was sent"); await output.WriteAsync(buffer.AsMemory(0, read), token); left -= read; } } if (to >= dataEnd && from < entryEnd) { var start = Math.Max(from, dataEnd); var end = Math.Min(entryEnd - 1, to); if (end >= start) await output.WriteAsync(new byte[end - start + 1], token); } } var trailer = Length - 2 * Block; if (to >= trailer) await output.WriteAsync(new byte[to - Math.Max(from, trailer) + 1], token); } // a ustar header (name split into prefix and name past 100 bytes; sizes past 8 GiB in base 256) static byte[] Header(Entry entry) { var header = new byte[Block]; var name = entry.Name; var prefix = string.Empty; if (Encoding.UTF8.GetByteCount(name) > 100) { var cut = name.LastIndexOf('/', Math.Min(name.Length - 1, 155)); while (cut > 0 && Encoding.UTF8.GetByteCount(name[(cut + 1)..]) > 100) cut = name.IndexOf('/', cut + 1); if (cut <= 0 || Encoding.UTF8.GetByteCount(name[..cut]) > 155) throw new InvalidOperationException($"{name} is too long for a tar header"); prefix = name[..cut]; name = name[(cut + 1)..]; } Text(header, 0, 100, name); Octal(header, 100, 8, Convert.ToInt32("640", 8)); Octal(header, 108, 8, 0); Octal(header, 116, 8, 0); if (entry.Size < 1L << 33) Octal(header, 124, 12, entry.Size); else { header[124] = 0x80; for (var i = 0; i < 8; i++) header[135 - i] = (byte)(entry.Size >> (8 * i)); } Octal(header, 136, 12, new DateTimeOffset(entry.ModifiedAt).ToUnixTimeSeconds()); header[156] = (byte)'0'; Text(header, 257, 6, "ustar"); Text(header, 263, 2, "00"); Text(header, 265, 32, "privapub"); Text(header, 297, 32, "privapub"); Text(header, 345, 155, prefix); for (var i = 148; i < 156; i++) header[i] = (byte)' '; var sum = header.Sum(b => (long)b); Text(header, 148, 7, Convert.ToString(sum, 8).PadLeft(6, '0')); header[155] = (byte)' '; return header; } static void Text(byte[] header, int at, int length, string value) { var bytes = Encoding.UTF8.GetBytes(value); Array.Copy(bytes, 0, header, at, Math.Min(bytes.Length, length)); } static void Octal(byte[] header, int at, int length, long value) => Text(header, at, length - 1, Convert.ToString(value, 8).PadLeft(length - 1, '0')); [GeneratedRegex(@"^\d{8}-\d{6}-[a-z-]{1,20}$")] public static partial Regex BackupId(); /// /// A backup read from an uploaded tar into the backups' root: refused when it holds anything but plain files and /// folders under one backup's folder, a path leaving it, or more than the disk can take. The backup, or why not. /// public static async Task<(BackupInfo Backup, string Error)> Import(Stream tar, string backupsRoot, CancellationToken token) { const int MaxEntries = 2_000_000; var id = default(string); var partial = default(string); var entries = 0; var free = new DriveInfo(Path.GetFullPath(backupsRoot)).AvailableFreeSpace; var written = 0L; try { await using var reader = new TarReader(tar, leaveOpen: true); while (await reader.GetNextEntryAsync(copyData: false, token) is { } entry) { if (++entries > MaxEntries) return (default, "too many files"); var name = entry.Name.TrimEnd('/'); var slash = name.IndexOf('/'); var top = slash < 0 ? name : name[..slash]; if (id == default) { if (!BackupId().IsMatch(top)) return (default, $"{top} is not a backup's folder"); id = top; if (Directory.Exists(Path.Combine(backupsRoot, id))) return (default, $"{id} is already here"); partial = Path.Combine(backupsRoot, id + ServerBackup.PartialSuffix); if (Directory.Exists(partial)) Directory.Delete(partial, recursive: true); ServerBackup.MakeDirectory(partial); } if (top != id) return (default, $"{name} is outside {id}"); if (entry.EntryType is TarEntryType.Directory) continue; if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile)) return (default, $"{name} is not a plain file"); if (slash < 0) return (default, $"{name} is not inside a folder"); written += entry.Length; if (written > free - 512L * 1024 * 1024) return (default, "not enough free disk"); var path = ServerBackup.Inside(partial, name[(slash + 1)..]); Directory.CreateDirectory(Path.GetDirectoryName(path)!); await using var file = OperatingSystem.IsWindows() ? new FileStream(path, FileMode.CreateNew, FileAccess.Write) : new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write, UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead }); if (entry.DataStream != default) await entry.DataStream.CopyToAsync(file, token); } if (id == default) return (default, "the file holds nothing"); var manifest = ArchiveManifest.Read(partial); if (manifest == default) return (default, "it has no manifest"); manifest.Kind = "uploaded"; manifest.Write(partial); Directory.Move(partial, Path.Combine(backupsRoot, id)); partial = default; var problems = await ServerBackup.Verify(backupsRoot, id, token); if (problems.Count > 0) { ServerBackup.Delete(backupsRoot, id); return (default, string.Join("; ", problems.Take(5))); } return (ServerBackup.Find(backupsRoot, id), default); } catch (Exception ex) when (ex is FormatException or InvalidDataException or InvalidOperationException or EndOfStreamException or System.Text.Json.JsonException) { return (default, $"not a backup: {ex.Message}"); } finally { if (partial != default && Directory.Exists(partial)) Directory.Delete(partial, recursive: true); } } } }