import { walk } from "https://deno.land/std@0.154.0/fs/walk.ts";import { dirname, extname, join, relative, resolve,} from "https://deno.land/std@0.154.0/path/mod.ts";import { Crypto } from "../crypto/crypto.ts";import { EarthstarError, isErr } from "../util/errors.ts";import { IGNORED_FILES, MANIFEST_FILE_NAME, MIN_TIMESTAMP_MS,} from "./constants.ts";import { FileInfoEntry, SyncFsManifest, SyncFsOptions,} from "./sync-fs-types.ts";import { dirBelongsToDifferentShare, getDirAssociatedShare, getTupleWinners, hasFilesButNoManifest, isAbsenceEntry, isFileInfoEntry, removeEmptyDir, writeDocToDir, writeEntryToReplica, writeManifest, zipByPath,} from "./util.ts";import { AttachmentStreamInfo } from "../util/attachment_stream_info.ts";import { ConfigEs5, DocEs5, FormatEs5 } from "../formats/format_es5.ts";
const textEncoder = new TextEncoder();
export async function reconcileManifestWithDirContents( fsDirPath: string, forShare: string,): Promise<SyncFsManifest> { let manifest: SyncFsManifest = { share: forShare, entries: {}, };
try { const contents = await Deno.readTextFile( join(fsDirPath, MANIFEST_FILE_NAME), );
manifest = JSON.parse(contents); } catch { }
const fileEntries: Record<string, FileInfoEntry> = {};
for await (const entry of walk(fsDirPath)) { if (IGNORED_FILES.includes(entry.name)) { continue; }
if (entry.isFile) { const { path } = entry;
const stat = await Deno.stat(path); const extension = extname(path);
let exposedContentSize = 0; let exposedContentHash = "";
if (extension !== "") { const file = await Deno.open(path);
const streamInfo = new AttachmentStreamInfo();
await file.readable.pipeThrough(streamInfo).pipeTo( new WritableStream(), );
exposedContentSize = await streamInfo.size; exposedContentHash = await streamInfo.hash; } else { const contents = await Deno.readTextFile(path);
exposedContentHash = await Crypto.sha256base32(contents); exposedContentSize = textEncoder.encode(contents).byteLength; }
const esPath = `/${relative(fsDirPath, path)}`;
const record: FileInfoEntry = { dirName: dirname(path), path: esPath, abspath: resolve(path), exposedContentSize, mtimeMs: stat.mtime?.getTime() || MIN_TIMESTAMP_MS, birthtimeMs: stat.birthtime?.getTime() || MIN_TIMESTAMP_MS, exposedContentHash, };
fileEntries[esPath] = record; } }
const zipped = zipByPath(manifest.entries, fileEntries);
const winners = getTupleWinners(zipped, (entryA, entryB) => { if (!entryA && !entryB) { return entryA as never; }
if (entryA && !entryB) { if (isAbsenceEntry(entryA)) { return entryA; }
return { fileLastSeenMs: entryA.mtimeMs, path: entryA.path, }; }
if (!entryA && entryB) { return entryB; }
if (entryA && entryB) { if (isAbsenceEntry(entryA)) { return entryB; }
const latestA = Math.max( entryA.birthtimeMs || 0, entryA.mtimeMs || 0, ); const latestB = Math.max( entryB.birthtimeMs || 0, entryB.mtimeMs || 0, );
if ( latestA > latestB || entryA.exposedContentHash === entryB.exposedContentHash ) { return entryA; }
return entryB; }
return entryA as never; });
const nextManifest: SyncFsManifest = { share: forShare, entries: {}, };
for (const path in winners) { const entry = winners[path];
if (!entry) { console.error("This shouldn't happen!"); continue; }
nextManifest.entries[path] = entry; }
return nextManifest;}
export async function syncReplicaAndFsDir( opts: SyncFsOptions,) { if (await dirBelongsToDifferentShare(opts.dirPath, opts.replica.share)) { const manifestShare = await getDirAssociatedShare(opts.dirPath);
throw new EarthstarError( `Tried to sync a replica for ${opts.replica.share} with a directory which had been synced with ${manifestShare}`, ); }
if ( opts.allowDirtyDirWithoutManifest === false && await hasFilesButNoManifest(opts.dirPath) ) { throw new EarthstarError( "Tried to sync a directory for the first time, but it was not empty.", ); }
const reconciledManifest = await reconcileManifestWithDirContents( opts.dirPath, opts.replica.share, );
const errors = [];
for (const key in reconciledManifest.entries) { const entry = reconciledManifest.entries[key];
if (!entry) { continue; }
if (isAbsenceEntry(entry)) { const isAttachmentPath = extname(entry.path) !== "";
const result = await FormatEs5.generateDocument({ keypair: opts.keypair, share: opts.replica.share, timestamp: entry.fileLastSeenMs * 1000, input: { path: entry.path, text: "", format: "es.5", }, config: opts.replica.formatsConfig["es.5"] as ConfigEs5, });
if (isErr(result)) { errors.push(result); } else { let docToValidate = result.doc; if (isAttachmentPath) { docToValidate = await FormatEs5.updateAttachmentFields( opts.keypair, result.doc, 0, "b4oymiquy7qobjgx36tejs35zeqt24qpemsnzgtfeswmrw6csxbkq", opts.replica.formatsConfig["es.5"] as ConfigEs5, ) as DocEs5; }
const isValidDoc = await FormatEs5.checkDocumentIsValid(docToValidate);
if (isErr(isValidDoc) && !opts.overwriteFilesAtOwnedPaths) { errors.push(isValidDoc); } } }
if (isFileInfoEntry(entry)) { const isAttachmentPath = extname(entry.abspath) !== "";
const sizeIsOkay = isAttachmentPath || entry.exposedContentSize <= 8000;
if (!sizeIsOkay) { errors.push( new EarthstarError( `File too big for the es.5 format's text field: ${entry.path}`, ), ); }
const result = await FormatEs5.generateDocument({ keypair: opts.keypair, share: opts.replica.share, timestamp: Date.now() * 1000, input: { path: entry.path, text: "fake", format: "es.5", }, config: opts.replica.formatsConfig["es.5"] as ConfigEs5, });
if (isErr(result)) { errors.push(result); } else { let docToValidate = result.doc;
if (isAttachmentPath) { docToValidate = await FormatEs5.updateAttachmentFields( opts.keypair, result.doc, 1, "bwxkuyopgmzy4s4y3t5dr4wc5qjrm2t2usy7qzeyifwg46m2njr4a", opts.replica.formatsConfig["es.5"] as ConfigEs5, ) as DocEs5; }
const isValidDoc = await FormatEs5.checkDocumentIsValid(docToValidate);
if (isErr(isValidDoc)) { const cantWrite = isValidDoc.message.includes("can't write to path");
if (cantWrite && !opts.overwriteFilesAtOwnedPaths) { const correspondingDoc = await opts.replica.getLatestDocAtPath( entry.path, );
if (!correspondingDoc) { errors.push(isValidDoc); }
const hashToCompare = isAttachmentPath ? correspondingDoc?.attachmentHash : correspondingDoc?.textHash;
if ( correspondingDoc && entry.mtimeMs && ((entry.mtimeMs * 1000 > correspondingDoc.timestamp) && hashToCompare !== entry.exposedContentHash) ) { errors.push(isValidDoc); } } else if ( cantWrite && opts.overwriteFilesAtOwnedPaths === true ) { delete reconciledManifest.entries[key]; } else { errors.push(isValidDoc); } } } } }
if (errors.length > 0) { throw errors[0]; }
for (const path in reconciledManifest.entries) { const entry = reconciledManifest.entries[path];
if (entry.path.indexOf("!") !== -1 && isFileInfoEntry(entry)) { const correspondingEphemeralDoc = await opts.replica.getLatestDocAtPath( path, );
if ( !correspondingEphemeralDoc || (correspondingEphemeralDoc && correspondingEphemeralDoc.deleteAfter && Date.now() * 1000 > correspondingEphemeralDoc.deleteAfter) ) { await Deno.remove(entry.abspath); await removeEmptyDir(entry.dirName, opts.dirPath); continue; } }
await writeEntryToReplica( entry, opts.replica, opts.keypair, opts.dirPath, ); }
const latestDocs = await opts.replica.getLatestDocs();
for (const doc of latestDocs) { if (doc.deleteAfter && Date.now() * 1000 > doc.deleteAfter) { return; }
try { const newEntry = await writeDocToDir(doc, opts.replica, opts.dirPath);
reconciledManifest.entries[newEntry.path] = newEntry; } catch (err) { } }
try { for await (const entry of walk(opts.dirPath)) { if (entry.isDirectory) { await removeEmptyDir(entry.path, opts.dirPath); } } } catch { }
await writeManifest(reconciledManifest, opts.dirPath);}