123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731
import {  AuthorAddress,  Base32String,  DocBase,  DocInputBase,  LocalIndex,  Path,  ShareAddress,  Signature,  Timestamp,} from "../util/doc-types.ts";import { EarthstarError, isErr, ValidationError } from "../util/errors.ts";import { FormatterGenerateOpts, IFormat } from "./format_types.ts";import { Crypto } from "../crypto/crypto.ts";
import {  authorAddressChars,  b32chars,  pathChars,  workspaceAddressChars,} from "../core-validators/characters.ts";import {  checkInt,  checkLiteral,  checkObj,  CheckObjOpts,  checkString,  isPlainObject,} from "../core-validators/checkers.ts";import {  checkAuthorIsValid,  checkShareIsValid,} from "../core-validators/addresses.ts";
//--------------------------------------------------
import { Logger } from "../util/log.ts";import { AuthorKeypair } from "../crypto/crypto-types.ts";let logger = new Logger("validator es.5", "red");
//================================================================================
/** Contains data written and signed by an identity. */export interface DocEs5 extends DocBase<"es.5"> {  /** Which document format the doc adheres to, e.g. `es.5`. */  format: "es.5";  /** The address of the author keypair which wrote this document. */  author: AuthorAddress;  /** Text content. If the document has an attachment, this should be a description of that attachment. */  text: string;  /** Base32 SHA256 hash of this document's text. */  textHash: string;  /** When the document should be deleted, as a UNIX timestamp in microseconds. */  deleteAfter?: number;  /** The path this document was written to. */  path: Path;  /** Used to verify the authorship of the document. */  signature: Signature;  /** Used to verify the author knows the share's secret */  shareSignature: Signature;  /** When the document was written, as a UNIX timestamp in microseconds (millionths of a second, e.g. `Date.now() * 1000`).*/  timestamp: Timestamp;  /** The public address of the share this document is from. */  share: ShareAddress;  /** The size of the associated attachment in bytes, if any. */  attachmentSize?: number;  /** The sha256 hash of the associated attachment, if any. */  attachmentHash?: string;
  // Local Index:  // Our docs form a linear sequence with gaps.  // When a doc is updated (same author, same path, new content), it moves to the  // end of the sequence and gets a new, higher localIndex.  // This sequence is specific to this local storage, affected by the order it received  // documents.  //  // It's useful during syncing so that other peers can say "give me everything that's  // changed since your localIndex 23".  //  // This is sent over the wire as part of a Doc so the receiver knows what to ask for next time,  // but it's then moved into a separate data structure like:  //    knownPeerMaxLocalIndexes:  //        peer111: 77  //        peer222: 140  // ...which helps us continue syncing with that specific peer next time.  //  // When we upsert the doc into our own storage, we discard the other peer's value  // and replace it with our own localIndex.  //  // The localIndex is not included in the doc's signature.  _localIndex?: LocalIndex;}
/** A partial es.5 doc that is about to get written. The rest of the properties will be computed automatically. */export interface DocInputEs5 extends DocInputBase<"es.5"> {  /** The format the document adheres to, e.g. `es.5` */  format: "es.5";  path: Path;
  /** Can be left blank if there is a previous version of the document with an attachment. */  text?: string;
  /** Data as Uint8Array or ReadableStream, to be used as document's associated attachment. */  attachment?: Uint8Array | ReadableStream<Uint8Array>;
  /** A UNIX timestamp in microseconds indicating when the document was written. Determined automatically if omitted. */  timestamp?: number;  /** A UNIX timestamp in microseconds indicating when the document should be deleted by.*/  deleteAfter?: number | null;}
// Tolerance for accepting messages from the future (because of clock skew between peers)const FUTURE_CUTOFF_MINUTES = 10;const FUTURE_CUTOFF_MICROSECONDS = FUTURE_CUTOFF_MINUTES * 60 * 1000 * 1000;
// Allowed valid range of timestamps (in microseconds, not milliseconds)const MIN_TIMESTAMP = 10000000000000; // 10^13const MAX_TIMESTAMP = 9007199254740990; // Number.MAX_SAFE_INTEGER - 1
const MAX_TEXT_LENGTH = 8000; // 8 thousand bytes = 8 kilobytes (measured as bytes of utf-8, not normal string length)
const HASH_STR_LEN = 53; // number of base32 characters including leading 'b', which is 32 raw bytes when decodedconst SIG_STR_LEN = 104; // number of base32 characters including leading 'b', which is 64 raw bytes when decoded
const MIN_BLOB_SIZE = 0;const MAX_BLOB_SIZE = Number.MAX_SAFE_INTEGER;
const ES5_CORE_SCHEMA: CheckObjOpts = {  objSchema: {    format: checkLiteral("es.5"),    author: checkString({ allowedChars: authorAddressChars }),    text: checkString({ maxLen: MAX_TEXT_LENGTH }),    textHash: checkString({ allowedChars: b32chars, len: HASH_STR_LEN }),    deleteAfter: checkInt({      min: MIN_TIMESTAMP,      max: MAX_TIMESTAMP,      optional: true,    }),    path: checkString({ allowedChars: pathChars, minLen: 2, maxLen: 512 }),    signature: checkString({ allowedChars: b32chars, len: SIG_STR_LEN }),    shareSignature: checkString({ allowedChars: b32chars, len: SIG_STR_LEN }),    timestamp: checkInt({ min: MIN_TIMESTAMP, max: MAX_TIMESTAMP }),    share: checkString({ allowedChars: workspaceAddressChars }),    attachmentSize: checkInt({      min: MIN_BLOB_SIZE,      max: MAX_BLOB_SIZE,      optional: true,    }),    attachmentHash: checkString({      allowedChars: b32chars,      len: HASH_STR_LEN,      optional: true,    }),  },  allowLiteralUndefined: false,  allowExtraKeys: false,};
export type ConfigEs5 = {  shareSecret: string | undefined;};
/** * Format for 'es.5' documents. Supports attachments and share keypairs. * @link https://earthstar-project.org/specs/data-spec-es5 */export const FormatEs5: IFormat<  "es.5",  DocInputEs5,  DocEs5,  ConfigEs5> = class {  static id: "es.5" = "es.5";
  /** Deterministic hash of this version of the document */  static hashDocument(    doc: DocEs5,  ): Promise<Base32String | ValidationError> {    // Deterministic hash of the document.    // Can return a ValidationError, but only checks for very basic document validity.
    // The hash of the document is used for signatures and references to specific docs.    // We use the hash of the content in case we want to drop the actual content    // and only keep the hash around for verifying signatures.    // None of these fields are allowed to contain tabs or newlines    // (except content, but we use contentHash instead).
    // to check the basic validity it needs a signature of the correct length and characters,    // but the actual content of the signature is not checked here.    // so let's fake it.    const docWithFakeSig = {      ...doc,      signature:        "bthisisafakesignatureusedtofillintheobjectwhenvalidatingitforhashingaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",      shareSignature:        "bthisisafakesignatureusedtofillintheobjectwhenvalidatingitforhashingaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",    };    const err = this._checkBasicDocumentValidity(docWithFakeSig);    if (isErr(err)) return Promise.resolve(err);
    // Sort fields in lexicographic order by field name.    // let result = ''    // For each field,    //     skip "text" and "signature" and "shareSignature" fields.    //     skip fields with value === null.    //     result += fieldname + "\t" + convertToString(value) + "\n"    // return base32encode(sha256(result).binaryDigest())    return Crypto.sha256base32(      (doc.attachmentHash === undefined        ? ""        : `attachmentHash\t${doc.attachmentHash}\n`) +        (doc.attachmentSize === undefined          ? ""          : `attachmentSize\t${doc.attachmentSize}\n`) +        `author\t${doc.author}\n` +        (doc.deleteAfter === undefined          ? ""          : `deleteAfter\t${doc.deleteAfter}\n`) +        `format\t${doc.format}\n` +        `path\t${doc.path}\n` +        `textHash\t${doc.textHash}\n` +        `timestamp\t${doc.timestamp}\n` +        `share\t${doc.share}\n`, // \n at the end also, not just between    );  }
  /**   * Generate a signed document from the input format the validator expects.   */  static async generateDocument(    { input, keypair, share, timestamp, prevLatestDoc, config }:      FormatterGenerateOpts<        "es.5",        DocInputEs5,        DocEs5,        ConfigEs5      >,  ): Promise<    | { doc: DocEs5; attachment?: ReadableStream<Uint8Array> | Uint8Array }    | ValidationError  > {    if (input.text === undefined && prevLatestDoc?.text === undefined) {      return new ValidationError(        "Couldn't determine document text from given input or previous version's text.",      );    }
    const nextText = input.text !== undefined      ? input.text      : prevLatestDoc?.text as string;
    const doc: DocEs5 = {      ...prevLatestDoc,      format: "es.5",      author: keypair.address,      text: nextText,      textHash: await Crypto.sha256base32(nextText),      path: input.path,      timestamp,      share,      signature: "?", // signature will be added in just a moment      shareSignature: "?", // ditto      // _localIndex will be added during upsert.  it's not needed for the signature.    };
    if (input.deleteAfter) {      doc["deleteAfter"] = input.deleteAfter;    }
    const signed = await this.signDocument(keypair, doc, config);
    if (isErr(signed)) {      return signed;    }
    if (input.attachment) {      return { doc: signed, attachment: input.attachment };    }
    return { doc: signed };  }
  /**   * Generate a signed document from the input format the validator expects.   */  static async signDocument(    credentials: AuthorKeypair,    doc: DocEs5,    config: ConfigEs5,  ): Promise<DocEs5 | ValidationError> {    const hash = await this.hashDocument(doc);    if (isErr(hash)) return hash;
    const sig = await Crypto.sign(credentials, hash);
    if (config.shareSecret === undefined) {      return new EarthstarError(        `Tried to write a document to ${doc.share} without the secret.`,      );    }
    const shareSig = await Crypto.sign({      shareAddress: doc.share,      secret: config.shareSecret,    }, hash);
    if (isErr(sig)) return sig;    if (isErr(shareSig)) return shareSig;
    return { ...doc, signature: sig, shareSignature: shareSig };  }
  /**   * Overwrite the user-written contents of a document, wipe any associated data, and return the signed document.   */  static async wipeDocument(    credentials: AuthorKeypair,    doc: DocEs5,    config: ConfigEs5,  ): Promise<DocEs5 | ValidationError> {    if (doc.text.length === 0) {      return doc;    }
    const cleanedResult = this.removeExtraFields(doc);    if (isErr(cleanedResult)) return cleanedResult;    const cleanedDoc = cleanedResult.doc;
    if (cleanedDoc.attachmentHash) {      const emptyDoc: DocEs5 = {        ...cleanedDoc,        text: "",        textHash: await Crypto.sha256base32(""),        signature: "?",        shareSignature: "?",        attachmentHash: await Crypto.sha256base32(""),        attachmentSize: 0,      };
      return this.signDocument(credentials, emptyDoc, config);    }
    // make new doc which is empty and just barely newer than the original    const emptyDoc: DocEs5 = {      ...cleanedDoc,      text: "",      textHash: await Crypto.sha256base32(""),      signature: "?",      shareSignature: "?",    };
    return this.signDocument(credentials, emptyDoc, config);  }
  /**   * Return a copy of the doc without extra fields, plus the extra fields   * as a separate object.   * If the input is not a plain javascript object, return a ValidationError.   * This should be run before checkDocumentIsValid.  The output doc will be   * more likely to be valid once the extra fields have been removed.   */  static removeExtraFields(    doc: DocEs5,  ): { doc: DocEs5; extras: Record<string, any> } | ValidationError {    if (!isPlainObject(doc)) {      return new ValidationError("doc is not a plain javascript object");    }    const validKeys = new Set(Object.keys(ES5_CORE_SCHEMA.objSchema || {}));
    const doc2: Record<string, any> = {};    const extras: Record<string, any> = {};    for (const [key, val] of Object.entries(doc)) {      if (validKeys.has(key)) {        doc2[key] = val;      } else {        if (!key.startsWith("_")) {          return new ValidationError(            "extra document fields must have names starting with an underscore",          );        }        extras[key] = val;      }    }    return {      doc: doc2 as DocEs5,      extras,    };  }
  /**   * This calls all the more detailed functions which start with underscores.   * Returns true if the document is ok, or returns a ValidationError if anything is wrong.   * Normally `now` should be omitted so that it defaults to the current time,   * or you can override it for testing purposes.   */  static async checkDocumentIsValid(    doc: DocEs5,    now?: number,  ): Promise<true | ValidationError> {    if (now === undefined) now = Date.now() * 1000;    // do this first to ensure we have all the right datatypes in the right fields
    const errBV = this._checkBasicDocumentValidity(doc);    if (isErr(errBV)) return errBV;
    // this is the most likely to fail under regular conditions, so do it next    // (because of clock skew and expired ephemeral documents)    const errT = this._checkTimestampIsOk(      doc.timestamp,      doc.deleteAfter ? doc.deleteAfter : null,      now,    );    if (isErr(errT)) return errT;
    const errW = this._checkAuthorCanWriteToPath(doc.author, doc.path);    if (isErr(errW)) return errW;
    // Check that all attachment fields are defined    const errBFC = this._checkAttachmentFieldsConsistent(doc);    if (isErr(errBFC)) return errBFC;
    const errP = this._checkPathIsValid(      doc.path,      !!doc.attachmentHash,      doc.deleteAfter,    );    if (isErr(errP)) return errP;
    const errAA = checkAuthorIsValid(doc.author);    if (isErr(errAA)) return errAA;
    const errWA = checkShareIsValid(doc.share);    if (isErr(errWA)) return errWA;
    // do this after validating that the author address is well-formed    // so we don't pass garbage into the crypto signature code
    const errS = await this._checkAuthorSignatureIsValid(doc);    if (isErr(errS)) return errS;
    const errSS = await this._checkShareSignatureIsValid(doc);    if (isErr(errSS)) return errSS;
    const errCH = await this._checkContentMatchesHash(doc.text, doc.textHash);    if (isErr(errCH)) return errCH;    return true;  }
  // These are broken out for easier unit testing.  // They will not normally be used directly; use the main assertDocumentIsValid instead.  // Return true on success.  static _checkBasicDocumentValidity(doc: DocEs5): true | ValidationError { // check for correct fields and datatypes    const err = checkObj(ES5_CORE_SCHEMA)(doc);    if (err !== null) return new ValidationError(err);
    return true; // TODO: is there more to check?  }
  static _checkAttachmentFieldsConsistent(    doc: DocEs5,  ): true | ValidationError {    if (      doc.text.length === 0 && doc.attachmentSize && doc.attachmentSize > 0    ) {      return new ValidationError(        "Documents with attachments must have text.",      );    }
    if (      doc.text.length > 0 && doc.attachmentSize && doc.attachmentSize === 0    ) {      "Documents with deleted attachments must have no text.";    }
    if (doc.attachmentHash && doc.attachmentSize === undefined) {      return new ValidationError(        "Attachment size is undefined while attachment hash is defined",      );    }
    if (doc.attachmentSize && doc.attachmentHash === undefined) {      return new ValidationError(        "Attachment hash is undefined while attachment size is defined",      );    }
    return true;  }
  static _checkAuthorCanWriteToPath(    author: AuthorAddress,    path: Path,  ): true | ValidationError {    // Can the author write to the path?    // return a ValidationError, or return true on success.
    // no tilde: it's public, anyone can write    if (path.indexOf("~") === -1) return true;    // path contains "~" + author.  the author can write here.    if (path.indexOf("~" + author) !== -1) return true;    // else, path contains at least one tilde but not ~@author.  The author can't write here.    return new ValidationError(      `author ${author} can't write to path ${path}`,    );  }  static _checkTimestampIsOk(    timestamp: number,    deleteAfter: number | null,    now: number,  ): true | ValidationError {    // Check for valid timestamp, and expired ephemeral documents.    // return a ValidationError, or return true on success.
    // timestamp and deleteAfter are already verified as good numbers by the schema checker:    // - in the right range of min and max allowed timestamps    // - integers, and not NaN or infinity
    // Timestamp must not be from the future.    if (timestamp > now + FUTURE_CUTOFF_MICROSECONDS) {      return new ValidationError("timestamp too far in the future");    }
    // Ephemeral documents    if (deleteAfter !== null) {      // Only valid if expiration date is in the future      if (now > deleteAfter) {        return new ValidationError("ephemeral doc has expired");      }      // Can't expire before it was created, that makes no sense      if (deleteAfter <= timestamp) {        return new ValidationError(          "ephemeral doc expired before it was created",        );      }    }    return true;  }  static _checkPathIsValid(    path: Path,    hasAttachment: boolean,    deleteAfter?: number,  ): true | ValidationError {    // Ensure the path matches the spec for allowed path strings.    //    // Path validity depends on if the document is ephemeral or not.  To check    // that rule, supply deleteAfter.  Omit deleteAfter to skip checking that rule    // (e.g. to just check if a path is potentially valid, ephemeral or not).    //    // return a ValidationError, or return true on success.
    // A path is a series of one or more path segments.    // A path segment is '/' followed by one or more allowed characters.
    // the schema already checked that this    // - is a string    // - length between 2 and 512 characters inclusive    // - onlyHasChars(pathChars)
    if (!path.startsWith("/")) {      return new ValidationError("invalid path: must start with /");    }    if (path.endsWith("/")) {      return new ValidationError("invalid path: must not end with /");    }    if (path.startsWith("/@")) {      // This is disallowed so that we can tell paths and authors apart      // when joining a workspace and a path/author in a URL:      // +gardening.xxxxx/@aaaa.xxxx      // +gardening.xxxxx/wiki/shared/Bumblebee      return new ValidationError(        'invalid path: must not start with "/@"',      );    }    if (path.indexOf("//") !== -1) {      return new ValidationError(        "invalid path: must not contain two consecutive slashes",      );    }
    if (deleteAfter !== undefined) {      // path must contain at least one '!', if and only if the document is ephemeral      if (path.indexOf("!") === -1 && deleteAfter !== null) {        return new ValidationError(          "when deleteAfter is set, path must contain '!'",        );      }      if (path.indexOf("!") !== -1 && deleteAfter === null) {        return new ValidationError(          "when deleteAfter is null, path must not contain '!'",        );      }    }
    // path must contain at least one '.', if and only if the document is a attachment    if (!pathHasFileExtension(path) && hasAttachment) {      return new ValidationError(        "when a attachment is provided, the path must end with a file extension",      );    }    if (pathHasFileExtension(path) && hasAttachment === false) {      return new ValidationError(        "when no attachment is provided, the path cannot end with a file extension",      );    }
    return true;  }  static async _checkAuthorSignatureIsValid(    doc: DocEs5,  ): Promise<true | ValidationError> {    // Check if the signature is good.    // return a ValidationError, or return true on success.    try {      const hash = await this.hashDocument(doc);      if (isErr(hash)) return hash;      const verified = await Crypto.verify(doc.author, doc.signature, hash);      if (verified !== true) {        return new ValidationError("signature is invalid");      }      return true;    } catch {      return new ValidationError(        "signature is invalid (unexpected exception)",      );    }  }
  static async _checkShareSignatureIsValid(    doc: DocEs5,  ): Promise<true | ValidationError> {    // Check if the signature is good.    // return a ValidationError, or return true on success.    try {      const hash = await this.hashDocument(doc);      if (isErr(hash)) return hash;      const verified = await Crypto.verify(        doc.share,        doc.shareSignature,        hash,      );      if (verified !== true) {        return new ValidationError("share signature is invalid");      }      return true;    } catch {      return new ValidationError(        "share signature is invalid (unexpected exception)",      );    }  }
  static async _checkContentMatchesHash(    content: string,    contentHash: Base32String,  ): Promise<true | ValidationError> {    // Ensure the contentHash matches the actual content.    // return a ValidationError, or return true on success.
    // TODO: if content is null, skip this check    if (await Crypto.sha256base32(content) !== contentHash) {      return new ValidationError("content does not match contentHash");    }    return true;  }
  static getAttachmentInfo(    doc: DocEs5,  ): { size: number; hash: string } | ValidationError {    if (doc.attachmentHash && doc.attachmentSize === 0) {      return new ValidationError(        "This document has had its attachment wiped",      );    }
    if (!doc.attachmentHash || !doc.attachmentSize) {      return new ValidationError("This document has no attachment.");    }
    return {      size: doc.attachmentSize,      hash: doc.attachmentHash,    };  }
  static updateAttachmentFields(    credentials: AuthorKeypair,    doc: DocEs5,    size: number,    hash: string,    config: ConfigEs5,  ): Promise<DocEs5 | ValidationError> {    const updatedDoc = {      ...doc,      attachmentHash: hash,      attachmentSize: size,    };
    return this.signDocument(credentials, updatedDoc, config);  }
  static authorFromCredentials(credentials: AuthorKeypair): AuthorAddress {    return credentials.address;  }};
const fileExtensionRegex = /^.*\.(\w+)$/;const endingWithKeypairAddrRegex = /^.*~@\w{4}\.\w{53}$/;
function pathHasFileExtension(path: string): boolean {  if (path.indexOf(".") === -1) {    return false;  }
  // Check that it's in the right position.  const matches = path.match(fileExtensionRegex);
  if (matches === null) {    return false;  }
  const extension = matches[1];
  // Is this part of a keypair address?  if (extension.length === 53 && path.match(endingWithKeypairAddrRegex)) {    return false;  }
  return true;}