import { encode } from "https://deno.land/std@0.126.0/encoding/base64.ts";import { walk } from "https://deno.land/std@0.132.0/fs/mod.ts";import { dirname, extname, join, relative, resolve,} from "https://deno.land/std@0.132.0/path/mod.ts";import { Crypto } from "../crypto/crypto.ts";import { EarthstarError, isErr } from "../util/errors.ts";import { FormatValidatorEs4 } from "../format-validators/format-validator-es4.ts";import { bytesExtensions, ES4_MAX_CONTENT_LENGTH, IGNORED_FILES, MANIFEST_FILE_NAME,} from "./constants.ts";import { FileInfoEntry, SyncFsManifest, SyncFsOptions,} from "./sync-fs-types.ts";import { dirBelongsToDifferentShare, getDirAssociatedShare, getTupleWinners, hasFilesButNoManifest, isAbsenceEntry, isFileInfoEntry, removeEmptyDir, writeDocToDir, writeEntryToReplica, writeManifest, zipByPath,} from "./util.ts";
const textEncoder = new TextEncoder();
export async function reconcileManifestWithDirContents( fsDirPath: string, forShare: string,): Promise<SyncFsManifest> { let manifest: SyncFsManifest = { share: forShare, entries: {}, };
try { const contents = await Deno.readTextFile( join(fsDirPath, MANIFEST_FILE_NAME), );
manifest = JSON.parse(contents); } catch { }
const fileEntries: Record<string, FileInfoEntry> = {};
for await (const entry of walk(fsDirPath)) { if (IGNORED_FILES.includes(entry.name)) { continue; }
if (entry.isFile) { const { path } = entry;
const stat = await Deno.stat(path); const extension = extname(path);
let contents = "";
if (bytesExtensions.includes(extension)) { const fileContents = await Deno.readFile(path); contents = encode(fileContents); } else { contents = await Deno.readTextFile(path); }
const hash = await Crypto.sha256base32(contents); const esPath = `/${relative(fsDirPath, path)}`;
const record: FileInfoEntry = { dirName: dirname(path), path: esPath, abspath: resolve(path), size: stat.size, contentsSize: textEncoder.encode(contents).length, mtimeMs: stat.mtime?.getTime() || null, birthtimeMs: stat.birthtime?.getTime() || null, hash, };
fileEntries[esPath] = record; } }
const zipped = zipByPath(manifest.entries, fileEntries);
const winners = getTupleWinners(zipped, (entryA, entryB) => { if (!entryA && !entryB) { return entryA as never; }
if (entryA && !entryB) { if (isAbsenceEntry(entryA)) { return entryA; }
return { fileLastSeenMs: entryA.mtimeMs || 0, path: entryA.path, }; }
if (!entryA && entryB) { return entryB; }
if (entryA && entryB) { if (isAbsenceEntry(entryA)) { return entryB; }
const latestA = Math.max( entryA.birthtimeMs || 0, entryA.mtimeMs || 0, ); const latestB = Math.max( entryB.birthtimeMs || 0, entryB.mtimeMs || 0, );
if (latestA > latestB || entryA.hash === entryB.hash) { return entryA; }
return entryB; }
return entryA as never; });
const nextManifest: SyncFsManifest = { share: forShare, entries: {}, };
for (const path in winners) { const entry = winners[path];
if (!entry) { console.error("This shouldn't happen!"); continue; }
nextManifest.entries[path] = entry; }
return nextManifest;}
export async function syncReplicaAndFsDir( opts: SyncFsOptions,) { if (await dirBelongsToDifferentShare(opts.dirPath, opts.replica.share)) { const manifestShare = await getDirAssociatedShare(opts.dirPath);
throw new EarthstarError( `Tried to sync a replica for ${opts.replica.share} with a directory which had been synced with ${manifestShare}`, ); }
if ( opts.allowDirtyDirWithoutManifest === false && await hasFilesButNoManifest(opts.dirPath) ) { throw new EarthstarError( "Tried to sync a directory for the first time, but it was not empty.", ); }
const reconciledManifest = await reconcileManifestWithDirContents( opts.dirPath, opts.replica.share, );
const errors = [];
for (const key in reconciledManifest.entries) { const entry = reconciledManifest.entries[key];
if (!entry) { continue; }
if (isAbsenceEntry(entry)) { const canWriteToPath = FormatValidatorEs4 ._checkAuthorCanWriteToPath(opts.keypair.address, entry.path);
if (isErr(canWriteToPath) && !opts.overwriteFilesAtOwnedPaths) { errors.push(canWriteToPath); } }
if (isFileInfoEntry(entry)) { const canWriteToPath = FormatValidatorEs4 ._checkAuthorCanWriteToPath(opts.keypair.address, entry.path);
const pathIsValid = FormatValidatorEs4._checkPathIsValid( entry.path, );
const sizeIsOkay = entry.contentsSize <= ES4_MAX_CONTENT_LENGTH;
if (isErr(canWriteToPath) && !opts.overwriteFilesAtOwnedPaths) { const correspondingDoc = await opts.replica.getLatestDocAtPath( entry.path, );
if (!correspondingDoc) { errors.push(canWriteToPath); }
if ( correspondingDoc && entry.mtimeMs && (entry.mtimeMs * 1000 > correspondingDoc.timestamp && correspondingDoc.contentHash !== entry.hash) ) { errors.push(canWriteToPath); } } else if ( isErr(canWriteToPath) && opts.overwriteFilesAtOwnedPaths === true ) { delete reconciledManifest.entries[key]; }
if (isErr(pathIsValid)) { errors.push(pathIsValid); }
if (!sizeIsOkay) { errors.push( new EarthstarError( `File too big for the es.4 format: ${entry.path}`, ), ); } } }
if (errors.length > 0) { throw errors[0]; }
for (const path in reconciledManifest.entries) { const entry = reconciledManifest.entries[path];
if (entry.path.indexOf("!") !== -1 && isFileInfoEntry(entry)) { const correspondingEphemeralDoc = await opts.replica.getLatestDocAtPath( path, );
if ( !correspondingEphemeralDoc || (correspondingEphemeralDoc && correspondingEphemeralDoc.deleteAfter && Date.now() * 1000 > correspondingEphemeralDoc.deleteAfter) ) { await Deno.remove(entry.abspath); await removeEmptyDir(entry.dirName, opts.dirPath); continue; } }
await writeEntryToReplica(entry, opts.replica, opts.keypair, opts.dirPath); }
const latestDocs = await opts.replica.getLatestDocs();
for (const doc of latestDocs) { if (doc.deleteAfter && Date.now() * 1000 > doc.deleteAfter) { return; }
await writeDocToDir(doc, opts.dirPath); }
try { for await (const entry of walk(opts.dirPath)) { if (entry.isDirectory) { await removeEmptyDir(entry.path, opts.dirPath); } } } catch { }
const manifestAfterOps = await reconcileManifestWithDirContents( opts.dirPath, opts.replica.share, );
await writeManifest(manifestAfterOps, opts.dirPath);}