The administrator's page backs up, downloads, uploads and restores
/clientapi/admin/backups (owner decision 2026-10-07, approving these endpoints in production): the list with what runs, the restore waiting and the last restore's report; back up now; delete; a download for the password, through a ticket good for ten minutes in the link's path, as one tar whose length is known first and which honours Range (BackupTar); an upload in pieces of at most 32 MB, each at the offset already received or refused with it, so a broken upload resumes, read into a backup only when it holds nothing but plain files under one backup's folder; and a restore for the password and the host typed out. Every call checks the administrator against the database. nginx streams downloads for an hour and takes upload pieces unbuffered, rate-limited. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LsXgEaXee4GCU1hwYgPJXw
This commit is contained in:
thepraandClaude Opus 5.5 committed
1 parent
fba57318fa
commit
bc5f2beaf7
8 files changed
+987
No files matched your search
@@ -0,0 +1,224 @@
|
||||
using System.Formats.Tar;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
// A backup as one tar file, for the administrator to download and to upload again: every file under <id>/, its length
|
||||
// known before the first byte (so a download has a Content-Length and resumes with Range), written straight from the
|
||||
// backup's files, never held in memory.
|
||||
public sealed partial class BackupTar
|
||||
{
|
||||
const int Block = 512;
|
||||
|
||||
readonly List<Entry> _entries = [];
|
||||
|
||||
public long Length { get; private set; }
|
||||
|
||||
sealed record Entry(string Name, string Path, long Size, DateTime ModifiedAt, long Offset);
|
||||
|
||||
public static BackupTar Of(string backupsRoot, string id)
|
||||
{
|
||||
var directory = Path.Combine(backupsRoot, id);
|
||||
var tar = new BackupTar();
|
||||
var offset = 0L;
|
||||
foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal))
|
||||
{
|
||||
var info = new FileInfo(file);
|
||||
var name = id + "/" + Path.GetRelativePath(directory, file).Replace('\\', '/');
|
||||
tar._entries.Add(new Entry(name, file, info.Length, info.LastWriteTimeUtc, offset));
|
||||
offset += Block + Padded(info.Length);
|
||||
}
|
||||
tar.Length = offset + 2 * Block;
|
||||
return tar;
|
||||
}
|
||||
|
||||
static long Padded(long size) => (size + Block - 1) / Block * Block;
|
||||
|
||||
/// <summary>Writes bytes from to to (inclusive) of the tar.</summary>
|
||||
public async Task Write(Stream output, long from, long to, CancellationToken token)
|
||||
{
|
||||
var buffer = new byte[81920];
|
||||
foreach (var entry in _entries)
|
||||
{
|
||||
var headerEnd = entry.Offset + Block;
|
||||
var dataEnd = headerEnd + entry.Size;
|
||||
var entryEnd = headerEnd + Padded(entry.Size);
|
||||
if (entryEnd <= from)
|
||||
continue;
|
||||
if (entry.Offset > to)
|
||||
return;
|
||||
if (from < headerEnd)
|
||||
{
|
||||
var header = Header(entry);
|
||||
var start = (int)Math.Max(0, from - entry.Offset);
|
||||
var end = (int)Math.Min(Block, to - entry.Offset + 1);
|
||||
await output.WriteAsync(header.AsMemory(start, end - start), token);
|
||||
}
|
||||
if (from < dataEnd && to >= headerEnd && entry.Size > 0)
|
||||
{
|
||||
var start = Math.Max(0, from - headerEnd);
|
||||
var end = Math.Min(entry.Size, to - headerEnd + 1);
|
||||
await using var file = new FileStream(entry.Path, FileMode.Open, FileAccess.Read, FileShare.Read, 1, FileOptions.SequentialScan);
|
||||
file.Seek(start, SeekOrigin.Begin);
|
||||
var left = end - start;
|
||||
while (left > 0)
|
||||
{
|
||||
var read = await file.ReadAsync(buffer.AsMemory(0, (int)Math.Min(buffer.Length, left)), token);
|
||||
if (read == 0)
|
||||
throw new IOException($"{entry.Name} shrank while it was sent");
|
||||
await output.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
left -= read;
|
||||
}
|
||||
}
|
||||
if (to >= dataEnd && from < entryEnd)
|
||||
{
|
||||
var start = Math.Max(from, dataEnd);
|
||||
var end = Math.Min(entryEnd - 1, to);
|
||||
if (end >= start)
|
||||
await output.WriteAsync(new byte[end - start + 1], token);
|
||||
}
|
||||
}
|
||||
var trailer = Length - 2 * Block;
|
||||
if (to >= trailer)
|
||||
await output.WriteAsync(new byte[to - Math.Max(from, trailer) + 1], token);
|
||||
}
|
||||
|
||||
// a ustar header (name split into prefix and name past 100 bytes; sizes past 8 GiB in base 256)
|
||||
static byte[] Header(Entry entry)
|
||||
{
|
||||
var header = new byte[Block];
|
||||
var name = entry.Name;
|
||||
var prefix = string.Empty;
|
||||
if (Encoding.UTF8.GetByteCount(name) > 100)
|
||||
{
|
||||
var cut = name.LastIndexOf('/', Math.Min(name.Length - 1, 155));
|
||||
while (cut > 0 && Encoding.UTF8.GetByteCount(name[(cut + 1)..]) > 100)
|
||||
cut = name.IndexOf('/', cut + 1);
|
||||
if (cut <= 0 || Encoding.UTF8.GetByteCount(name[..cut]) > 155)
|
||||
throw new InvalidOperationException($"{name} is too long for a tar header");
|
||||
prefix = name[..cut];
|
||||
name = name[(cut + 1)..];
|
||||
}
|
||||
Text(header, 0, 100, name);
|
||||
Octal(header, 100, 8, Convert.ToInt32("640", 8));
|
||||
Octal(header, 108, 8, 0);
|
||||
Octal(header, 116, 8, 0);
|
||||
if (entry.Size < 1L << 33)
|
||||
Octal(header, 124, 12, entry.Size);
|
||||
else
|
||||
{
|
||||
header[124] = 0x80;
|
||||
for (var i = 0; i < 8; i++)
|
||||
header[135 - i] = (byte)(entry.Size >> (8 * i));
|
||||
}
|
||||
Octal(header, 136, 12, new DateTimeOffset(entry.ModifiedAt).ToUnixTimeSeconds());
|
||||
header[156] = (byte)'0';
|
||||
Text(header, 257, 6, "ustar");
|
||||
Text(header, 263, 2, "00");
|
||||
Text(header, 265, 32, "privapub");
|
||||
Text(header, 297, 32, "privapub");
|
||||
Text(header, 345, 155, prefix);
|
||||
for (var i = 148; i < 156; i++)
|
||||
header[i] = (byte)' ';
|
||||
var sum = header.Sum(b => (long)b);
|
||||
Text(header, 148, 7, Convert.ToString(sum, 8).PadLeft(6, '0'));
|
||||
header[155] = (byte)' ';
|
||||
return header;
|
||||
}
|
||||
|
||||
static void Text(byte[] header, int at, int length, string value)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
Array.Copy(bytes, 0, header, at, Math.Min(bytes.Length, length));
|
||||
}
|
||||
|
||||
static void Octal(byte[] header, int at, int length, long value) =>
|
||||
Text(header, at, length - 1, Convert.ToString(value, 8).PadLeft(length - 1, '0'));
|
||||
|
||||
[GeneratedRegex(@"^\d{8}-\d{6}-[a-z-]{1,20}$")]
|
||||
public static partial Regex BackupId();
|
||||
|
||||
/// <summary>
|
||||
/// A backup read from an uploaded tar into the backups' root: refused when it holds anything but plain files and
|
||||
/// folders under one backup's folder, a path leaving it, or more than the disk can take. The backup, or why not.
|
||||
/// </summary>
|
||||
public static async Task<(BackupInfo Backup, string Error)> Import(Stream tar, string backupsRoot, CancellationToken token)
|
||||
{
|
||||
const int MaxEntries = 2_000_000;
|
||||
var id = default(string);
|
||||
var partial = default(string);
|
||||
var entries = 0;
|
||||
var free = new DriveInfo(Path.GetFullPath(backupsRoot)).AvailableFreeSpace;
|
||||
var written = 0L;
|
||||
try
|
||||
{
|
||||
await using var reader = new TarReader(tar, leaveOpen: true);
|
||||
while (await reader.GetNextEntryAsync(copyData: false, token) is { } entry)
|
||||
{
|
||||
if (++entries > MaxEntries)
|
||||
return (default, "too many files");
|
||||
var name = entry.Name.TrimEnd('/');
|
||||
var slash = name.IndexOf('/');
|
||||
var top = slash < 0 ? name : name[..slash];
|
||||
if (id == default)
|
||||
{
|
||||
if (!BackupId().IsMatch(top))
|
||||
return (default, $"{top} is not a backup's folder");
|
||||
id = top;
|
||||
if (Directory.Exists(Path.Combine(backupsRoot, id)))
|
||||
return (default, $"{id} is already here");
|
||||
partial = Path.Combine(backupsRoot, id + ServerBackup.PartialSuffix);
|
||||
if (Directory.Exists(partial))
|
||||
Directory.Delete(partial, recursive: true);
|
||||
ServerBackup.MakeDirectory(partial);
|
||||
}
|
||||
if (top != id)
|
||||
return (default, $"{name} is outside {id}");
|
||||
if (entry.EntryType is TarEntryType.Directory)
|
||||
continue;
|
||||
if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile))
|
||||
return (default, $"{name} is not a plain file");
|
||||
if (slash < 0)
|
||||
return (default, $"{name} is not inside a folder");
|
||||
written += entry.Length;
|
||||
if (written > free - 512L * 1024 * 1024)
|
||||
return (default, "not enough free disk");
|
||||
var path = ServerBackup.Inside(partial, name[(slash + 1)..]);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
await using var file = OperatingSystem.IsWindows()
|
||||
? new FileStream(path, FileMode.CreateNew, FileAccess.Write)
|
||||
: new FileStream(path, new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write,
|
||||
UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead });
|
||||
if (entry.DataStream != default)
|
||||
await entry.DataStream.CopyToAsync(file, token);
|
||||
}
|
||||
if (id == default)
|
||||
return (default, "the file holds nothing");
|
||||
var manifest = ArchiveManifest.Read(partial);
|
||||
if (manifest == default)
|
||||
return (default, "it has no manifest");
|
||||
manifest.Kind = "uploaded";
|
||||
manifest.Write(partial);
|
||||
Directory.Move(partial, Path.Combine(backupsRoot, id));
|
||||
partial = default;
|
||||
var problems = await ServerBackup.Verify(backupsRoot, id, token);
|
||||
if (problems.Count > 0)
|
||||
{
|
||||
ServerBackup.Delete(backupsRoot, id);
|
||||
return (default, string.Join("; ", problems.Take(5)));
|
||||
}
|
||||
return (ServerBackup.Find(backupsRoot, id), default);
|
||||
}
|
||||
catch (Exception ex) when (ex is FormatException or InvalidDataException or InvalidOperationException or EndOfStreamException or System.Text.Json.JsonException)
|
||||
{
|
||||
return (default, $"not a backup: {ex.Message}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (partial != default && Directory.Exists(partial))
|
||||
Directory.Delete(partial, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace PrivaPub.Infrastructure.Backup
|
||||
{
|
||||
// Backups leaving and arriving through the administrator's page, never through the client's memory:
|
||||
// - a download is a link with a ticket in its path, good for ten minutes for one backup (a browser downloads it, and
|
||||
// resumes it with Range), given for the administrator's password;
|
||||
// - an upload arrives in pieces appended at the offset already received (a piece that would leave a gap or overlap is
|
||||
// refused with what was received, so a broken upload resumes), into <backups>/.uploads, and is read into a backup
|
||||
// once whole. One left for a day is deleted.
|
||||
public class TransferStore(Backups backups)
|
||||
{
|
||||
public static readonly TimeSpan TicketLifetime = TimeSpan.FromMinutes(10);
|
||||
public const long ChunkBytes = 32L * 1024 * 1024;
|
||||
const string Uploads = ".uploads";
|
||||
|
||||
readonly ConcurrentDictionary<string, (string Backup, DateTime Expires)> _tickets = new();
|
||||
readonly ConcurrentDictionary<string, SemaphoreSlim> _appending = new();
|
||||
|
||||
public (string Ticket, DateTime Expires) Ticket(string backup)
|
||||
{
|
||||
foreach (var (ticket, stale) in _tickets.Where(t => t.Value.Expires < DateTime.UtcNow).ToList())
|
||||
_tickets.TryRemove(ticket, out _);
|
||||
var expires = DateTime.UtcNow + TicketLifetime;
|
||||
var token = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(32));
|
||||
_tickets[token] = (backup, expires);
|
||||
return (token, expires);
|
||||
}
|
||||
|
||||
/// <summary>The backup a ticket is for, while it is good.</summary>
|
||||
public string Redeem(string ticket) =>
|
||||
ticket != default && _tickets.TryGetValue(ticket, out var held) && held.Expires > DateTime.UtcNow ? held.Backup : default;
|
||||
|
||||
string Folder => Path.Combine(backups.Root, Uploads);
|
||||
|
||||
string File(string id) => Path.Combine(Folder, id + ".tar");
|
||||
|
||||
static bool IsId(string id) => id is { Length: 32 } && id.All(char.IsAsciiHexDigitLower);
|
||||
|
||||
public string Start(string by)
|
||||
{
|
||||
ServerBackup.MakeDirectory(backups.Root);
|
||||
ServerBackup.MakeDirectory(Folder);
|
||||
Forget(DateTime.UtcNow.AddDays(-1));
|
||||
var id = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16));
|
||||
using (OperatingSystem.IsWindows()
|
||||
? new FileStream(File(id), FileMode.CreateNew, FileAccess.Write)
|
||||
: new FileStream(File(id), new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write,
|
||||
UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead }))
|
||||
{
|
||||
}
|
||||
System.IO.File.WriteAllText(Path.Combine(Folder, id + ".json"), JsonSerializer.Serialize(new { by, at = DateTime.UtcNow }));
|
||||
return id;
|
||||
}
|
||||
|
||||
/// <summary>How much of an upload has arrived; -1 when there is no such upload.</summary>
|
||||
public long Received(string id) => IsId(id) && System.IO.File.Exists(File(id)) ? new FileInfo(File(id)).Length : -1;
|
||||
|
||||
/// <summary>A piece appended where the upload stands: what has arrived since, and whether it was taken.</summary>
|
||||
public async Task<(long Received, bool Taken)> Append(string id, long offset, Stream piece, CancellationToken token)
|
||||
{
|
||||
if (!IsId(id) || !System.IO.File.Exists(File(id)))
|
||||
return (-1, false);
|
||||
var gate = _appending.GetOrAdd(id, _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(token);
|
||||
try
|
||||
{
|
||||
await using var file = new FileStream(File(id), FileMode.Open, FileAccess.Write);
|
||||
if (file.Length != offset)
|
||||
return (file.Length, false);
|
||||
file.Seek(offset, SeekOrigin.Begin);
|
||||
var buffer = new byte[81920];
|
||||
var total = 0L;
|
||||
int read;
|
||||
while ((read = await piece.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
total += read;
|
||||
if (total > ChunkBytes)
|
||||
{
|
||||
file.SetLength(offset);
|
||||
return (offset, false);
|
||||
}
|
||||
await file.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
}
|
||||
await file.FlushAsync(token);
|
||||
return (file.Length, true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
//the piece broke off: what arrived of it is dropped, so the next one starts where this one did
|
||||
await using var file = new FileStream(File(id), FileMode.Open, FileAccess.Write);
|
||||
file.SetLength(offset);
|
||||
return (offset, false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The whole upload read into a backup: the backup, or why not. The upload goes either way.</summary>
|
||||
public async Task<(BackupInfo Backup, string Error)> Finish(string id, CancellationToken token)
|
||||
{
|
||||
if (!IsId(id) || !System.IO.File.Exists(File(id)))
|
||||
return (default, "no such upload");
|
||||
try
|
||||
{
|
||||
await using var tar = new FileStream(File(id), FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.SequentialScan);
|
||||
return await BackupTar.Import(tar, backups.Root, token);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Cancel(id);
|
||||
}
|
||||
}
|
||||
|
||||
public bool Cancel(string id)
|
||||
{
|
||||
if (!IsId(id) || !System.IO.File.Exists(File(id)))
|
||||
return false;
|
||||
System.IO.File.Delete(File(id));
|
||||
System.IO.File.Delete(Path.Combine(Folder, id + ".json"));
|
||||
_appending.TryRemove(id, out _);
|
||||
return true;
|
||||
}
|
||||
|
||||
// uploads left behind
|
||||
void Forget(DateTime before)
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(Folder).Where(f => System.IO.File.GetLastWriteTimeUtc(f) < before))
|
||||
System.IO.File.Delete(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user