SocialPub/PrivaPub/Infrastructure/Backup/BackupTar.cs

using System.Formats.Tar;
using System.Text;
using System.Text.RegularExpressions;

namespace PrivaPub.Infrastructure.Backup
{
	// A backup as one tar file, for the administrator to download and to upload again: every file under <id>/, its length
	// known before the first byte (so a download has a Content-Length and resumes with Range), written straight from the
	// backup's files, never held in memory.
	public sealed partial class BackupTar
	{
		const int Block = 512;

		readonly List<Entry> _entries = [];

		public long Length { get; private set; }

		sealed record Entry(string Name, string Path, long Size, DateTime ModifiedAt, long Offset);

		public static BackupTar Of(string backupsRoot, string id)
		{
			var directory = Path.Combine(backupsRoot, id);
			var tar = new BackupTar();
			var offset = 0L;
			foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal))
			{
				var info = new FileInfo(file);
				var name = id + "/" + Path.GetRelativePath(directory, file).Replace('\\', '/');
				tar._entries.Add(new Entry(name, file, info.Length, info.LastWriteTimeUtc, offset));
				offset += Block + Padded(info.Length);
			}
			tar.Length = offset + 2 * Block;
			return tar;
		}

		static long Padded(long size) => (size + Block - 1) / Block * Block;

		/// <summary>Writes bytes from to to (inclusive) of the tar.</summary>
		public async Task Write(Stream output, long from, long to, CancellationToken token)
		{
			var buffer = new byte[81920];
			foreach (var entry in _entries)
			{
				var headerEnd = entry.Offset + Block;
				var dataEnd = headerEnd + entry.Size;
				var entryEnd = headerEnd + Padded(entry.Size);
				if (entryEnd <= from)
					continue;
				if (entry.Offset > to)
					return;
				if (from < headerEnd)
				{
					var header = Header(entry);
					var start = (int)Math.Max(0, from - entry.Offset);
					var end = (int)Math.Min(Block, to - entry.Offset + 1);
					await output.WriteAsync(header.AsMemory(start, end - start), token);
				}
				if (from < dataEnd && to >= headerEnd && entry.Size > 0)
				{
					var start = Math.Max(0, from - headerEnd);
					var end = Math.Min(entry.Size, to - headerEnd + 1);
					await using var file = new FileStream(entry.Path, FileMode.Open, FileAccess.Read, FileShare.Read, 1, FileOptions.SequentialScan);
					file.Seek(start, SeekOrigin.Begin);
					var left = end - start;
					while (left > 0)
					{
						var read = await file.ReadAsync(buffer.AsMemory(0, (int)Math.Min(buffer.Length, left)), token);
						if (read == 0)
							throw new IOException($"{entry.Name} shrank while it was sent");
						await output.WriteAsync(buffer.AsMemory(0, read), token);
						left -= read;
					}
				}
				if (to >= dataEnd && from < entryEnd)
				{
					var start = Math.Max(from, dataEnd);
					var end = Math.Min(entryEnd - 1, to);
					if (end >= start)
						await output.WriteAsync(new byte[end - start + 1], token);
				}
			}
			var trailer = Length - 2 * Block;
			if (to >= trailer)
				await output.WriteAsync(new byte[to - Math.Max(from, trailer) + 1], token);
		}

		// a ustar header (name split into prefix and name past 100 bytes; sizes past 8 GiB in base 256)
		static byte[] Header(Entry entry)
		{
			var header = new byte[Block];
			var name = entry.Name;
			var prefix = string.Empty;
			if (Encoding.UTF8.GetByteCount(name) > 100)
			{
				var cut = name.LastIndexOf('/', Math.Min(name.Length - 1, 155));
				while (cut > 0 && Encoding.UTF8.GetByteCount(name[(cut + 1)..]) > 100)
					cut = name.IndexOf('/', cut + 1);
				if (cut <= 0 || Encoding.UTF8.GetByteCount(name[..cut]) > 155)
					throw new InvalidOperationException($"{name} is too long for a tar header");
				prefix = name[..cut];
				name = name[(cut + 1)..];
			}
			Text(header, 0, 100, name);
			Octal(header, 100, 8, Convert.ToInt32("640", 8));
			Octal(header, 108, 8, 0);
			Octal(header, 116, 8, 0);
			if (entry.Size < 1L << 33)
				Octal(header, 124, 12, entry.Size);
			else
			{
				header[124] = 0x80;
				for (var i = 0; i < 8; i++)
					header[135 - i] = (byte)(entry.Size >> (8 * i));
			}
			Octal(header, 136, 12, new DateTimeOffset(entry.ModifiedAt).ToUnixTimeSeconds());
			header[156] = (byte)'0';
			Text(header, 257, 6, "ustar");
			Text(header, 263, 2, "00");
			Text(header, 265, 32, "privapub");
			Text(header, 297, 32, "privapub");
			Text(header, 345, 155, prefix);
			for (var i = 148; i < 156; i++)
				header[i] = (byte)' ';
			var sum = header.Sum(b => (long)b);
			Text(header, 148, 7, Convert.ToString(sum, 8).PadLeft(6, '0'));
			header[155] = (byte)' ';
			return header;
		}

		static void Text(byte[] header, int at, int length, string value)
		{
			var bytes = Encoding.UTF8.GetBytes(value);
			Array.Copy(bytes, 0, header, at, Math.Min(bytes.Length, length));
		}

		static void Octal(byte[] header, int at, int length, long value) =>
			Text(header, at, length - 1, Convert.ToString(value, 8).PadLeft(length - 1, '0'));

		[GeneratedRegex(@"^\d{8}-\d{6}-[a-z-]{1,20}$")]
		public static partial Regex BackupId();

		/// <summary>
		/// A backup read from an uploaded tar into the backups' root: refused when it holds anything but plain files and
		/// folders under one backup's folder, a path leaving it, or more than the disk can take. The backup, or why not.
		/// </summary>
		public static async Task<(BackupInfo Backup, string Error)> Import(Stream tar, string backupsRoot, CancellationToken token)
		{
			const int MaxEntries = 2_000_000;
			var id = default(string);
			var partial = default(string);
			var entries = 0;
			var free = new DriveInfo(Path.GetFullPath(backupsRoot)).AvailableFreeSpace;
			var written = 0L;
			try
			{
				await using var reader = new TarReader(tar, leaveOpen: true);
				while (await reader.GetNextEntryAsync(copyData: false, token) is { } entry)
				{
					if (++entries > MaxEntries)
						return (default, "too many files");
					var name = entry.Name.TrimEnd('/');
					var slash = name.IndexOf('/');
					var top = slash < 0 ? name : name[..slash];
					if (id == default)
					{
						if (!BackupId().IsMatch(top))
							return (default, $"{top} is not a backup's folder");
						id = top;
						if (Directory.Exists(Path.Combine(backupsRoot, id)))
							return (default, $"{id} is already here");
						partial = Path.Combine(backupsRoot, id + ServerBackup.PartialSuffix);
						if (Directory.Exists(partial))
							Directory.Delete(partial, recursive: true);
						ServerBackup.MakeDirectory(partial);
					}
					if (top != id)
						return (default, $"{name} is outside {id}");
					if (entry.EntryType is TarEntryType.Directory)
						continue;
					if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile))
						return (default, $"{name} is not a plain file");
					if (slash < 0)
						return (default, $"{name} is not inside a folder");
					written += entry.Length;
					if (written > free - 512L * 1024 * 1024)
						return (default, "not enough free disk");
					var path = ServerBackup.Inside(partial, name[(slash + 1)..]);
					Directory.CreateDirectory(Path.GetDirectoryName(path)!);
					await using var file = OperatingSystem.IsWindows()
						? new FileStream(path, FileMode.CreateNew, FileAccess.Write)
						: new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write,
							UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead });
					if (entry.DataStream != default)
						await entry.DataStream.CopyToAsync(file, token);
				}
				if (id == default)
					return (default, "the file holds nothing");
				var manifest = ArchiveManifest.Read(partial);
				if (manifest == default)
					return (default, "it has no manifest");
				manifest.Kind = "uploaded";
				manifest.Write(partial);
				Directory.Move(partial, Path.Combine(backupsRoot, id));
				partial = default;
				var problems = await ServerBackup.Verify(backupsRoot, id, token);
				if (problems.Count > 0)
				{
					ServerBackup.Delete(backupsRoot, id);
					return (default, string.Join("; ", problems.Take(5)));
				}
				return (ServerBackup.Find(backupsRoot, id), default);
			}
			catch (Exception ex) when (ex is FormatException or InvalidDataException or InvalidOperationException or EndOfStreamException or System.Text.Json.JsonException)
			{
				return (default, $"not a backup: {ex.Message}");
			}
			finally
			{
				if (partial != default && Directory.Exists(partial))
					Directory.Delete(partial, recursive: true);
			}
		}
	}
}