123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
import { DocBase, ShareAddress } from "../../util/doc-types.ts";import {  EarthstarError,  isErr,  ReplicaIsClosedError,} from "../../util/errors.ts";import { IReplicaDocDriver } from "../replica-types.ts";import {  CREATE_CONFIG_TABLE_QUERY,  CREATE_DOCS_TABLE_QUERY,  CREATE_INDEXES_QUERY,  DELETE_CONFIG_QUERY,  DELETE_EXPIRED_DOC_QUERY,  GET_ENCODING_QUERY,  makeDocQuerySql,  MAX_LOCAL_INDEX_QUERY,  ReplicaSqliteOpts,  SELECT_CONFIG_CONTENT_QUERY,  SELECT_EXPIRED_DOC_QUERY,  SELECT_KEY_CONFIG_QUERY,  SET_ENCODING_QUERY,  UPSERT_CONFIG_QUERY,  UPSERT_DOC_QUERY,} from "./sqlite.shared.ts";import * as Sqlite from "https://deno.land/x/sqlite@v3.2.0/mod.ts";
//--------------------------------------------------
import { Logger } from "../../util/log.ts";import { Query } from "../../query/query-types.ts";import { cleanUpQuery } from "../../query/query.ts";import { sortedInPlace } from "../compare.ts";import { checkShareIsValid } from "../../core-validators/addresses.ts";
const logger = new Logger("storage driver sqlite node", "yellow");
interface ConfigObject extends Sqlite.RowObject {  key: string;  content: string;}
interface DocObject extends Sqlite.RowObject {  doc: string;  format: string;  path: string;  author: string;  timestamp: number;  signature: string;  deleteAfter: number | null;  localIndex?: number;  toSortWithinPath?: number;  pathAuthor?: string;}
/** A strorage driver which persists to SQLite. Works in Deno and browsers. */export class DocDriverSqlite implements IReplicaDocDriver {  share: ShareAddress;  _filename: string;  _isClosed = false;  _db: Sqlite.DB = null as unknown as Sqlite.DB;  _maxLocalIndex: number;
  //--------------------------------------------------  // LIFECYCLE
  async close(erase: boolean): Promise<void> {    logger.debug("close");    if (this._isClosed) {      throw new ReplicaIsClosedError();    }    if (this._db) {      this._db.close();    }    // delete the sqlite file    if (erase === true && this._filename !== ":memory:") {      logger.log(`...close: and erase`);      try {        await Deno.remove(this._filename);      } catch (err) {        logger.error("Failed to delete Sqlite file.");        logger.error(err);      }    }    this._isClosed = true;    logger.debug("...close is done.");
    return Promise.resolve();  }
  isClosed(): boolean {    return this._isClosed;  }
  constructor(opts: ReplicaSqliteOpts) {    this._filename = opts.filename;    this.share = "NOT_INITIALIZED";
    // check if file exists    if (opts.mode === "create") {      if (opts.filename !== ":memory:") {        try {          // If no file is found, this will throw.          Deno.lstatSync(opts.filename);
          throw new EarthstarError(            `Tried to create an sqlite file but it already exists: ${opts.filename}`,          );        } catch (err) {          // Only throw if the error was an Earthstar error thrown by us.          // Otherwise it's the error thrown by the file not being found. Which is good.          if (isErr(err)) {            this.close(false);            throw err;          }        }      }    } else if (opts.mode === "open") {      if (opts.filename === ":memory:") {        this.close(false);        throw new EarthstarError(          `Tried to open :memory: as though it was a file`,        );      }
      try {        Deno.lstatSync(opts.filename);      } catch {        this.close(false);        throw new EarthstarError(          `Tried to open an sqlite file but it doesn't exist: ${opts.filename}`,        );      }    } else if (opts.mode === "create-or-open") {      // file can exist or not.    } else {      // unknown mode      this.close(false);      throw new EarthstarError(        `sqlite unrecognized opts.mode: ${(opts as any).mode}`,      );    }
    const addressIsValidResult = opts.share      ? checkShareIsValid(opts.share)      : true;
    if (isErr(addressIsValidResult)) {      throw addressIsValidResult;    }
    this._db = new Sqlite.DB(this._filename, {      memory: this._filename === ":memory:",    });    this.ensureTables();
    const maxLocalIndexQuery = this._db.prepareQuery<[number]>(      MAX_LOCAL_INDEX_QUERY,    );
    const [maxLocalIndexFromDb] = maxLocalIndexQuery.one();    maxLocalIndexQuery.finalize();
    // We have to do this because the maxLocalIndexDb could be 0, which is falsy.    this._maxLocalIndex = maxLocalIndexFromDb !== null      ? maxLocalIndexFromDb      : -1;
    // check share    if (opts.mode === "create") {      // share is provided; set it into the file which we know didn't exist until just now      this.share = opts.share;      this.setConfig("share", this.share);    } else if (opts.mode === "open") {      // load existing share from file, which we know already existed...      const existingShare = this._getConfigSync("share");      if (existingShare === undefined) {        this.close(false);        throw new EarthstarError(          `can't open sqlite file with opts.mode="open" because the file doesn't have a share saved in its config table. ${opts.filename}`,        );      }      // if it was also provided in opts, assert that it matches the file      if (        opts.share !== null &&        opts.share !== this._getConfigSync("share")      ) {        this.close(false);        throw new EarthstarError(          `sqlite with opts.mode="open" wanted share ${opts.share} but found ${existingShare} in the file ${opts.filename}`,        );      }      this.share = existingShare;    } else if (opts.mode === "create-or-open") {      // share must be provided      if (opts.share === null) {        this.close(false);        throw new EarthstarError(          'sqlite with opts.mode="create-or-open" must have a share provided, not null',        );      }      this.share = opts.share;
      // existing share can be undefined (file may not have existed yet)      const existingShare = this._getConfigSync("share");
      // if there is an existing share, it has to match the one given in opts      if (        existingShare !== undefined &&        opts.share !== existingShare      ) {        this.close(false);        throw new EarthstarError(          `sqlite file had existing share ${existingShare} but opts wanted it to be ${opts.share} in file ${opts.filename}`,        );      }
      // set share if it's not set yet      if (existingShare === undefined) {        this.setConfig("share", opts.share);      }
      this.share = opts.share;    }  }
  //--------------------------------------------------  // CONFIG
  setConfig(key: string, content: string): Promise<void> {    logger.debug(      `setConfig(${JSON.stringify(key)} = ${JSON.stringify(content)})`,    );    if (this._isClosed) {      throw new ReplicaIsClosedError();    }    this._db.query(UPSERT_CONFIG_QUERY, { key: key, content: content });
    return Promise.resolve();  }
  _getConfigSync(key: string): string | undefined {    const configQuery = this._db.prepareQuery<Sqlite.Row, ConfigObject>(      SELECT_CONFIG_CONTENT_QUERY,    );
    try {      const row = configQuery.oneEntry({ key });      const result = row.content;
      logger.debug(        `getConfig(${JSON.stringify(key)}) = ${JSON.stringify(result)}`,      );
      return result;    } catch {      return undefined;    } finally {      configQuery.finalize();    }  }
  _listConfigKeysSync(): string[] {    const keysQuery = this._db.prepareQuery<string[]>(SELECT_KEY_CONFIG_QUERY);
    const rows = keysQuery.all();
    keysQuery.finalize();
    return sortedInPlace(rows.map(([key]) => key));  }
  getConfig(key: string): Promise<string | undefined> {    if (this._isClosed) {      throw new ReplicaIsClosedError();    }    return Promise.resolve(this._getConfigSync(key));  }
  listConfigKeys(): Promise<string[]> {    if (this._isClosed) {      throw new ReplicaIsClosedError();    }    return Promise.resolve(this._listConfigKeysSync());  }
  deleteConfig(key: string): Promise<boolean> {    logger.debug(`deleteConfig(${JSON.stringify(key)})`);    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    this._db.query(DELETE_CONFIG_QUERY, { key: key });
    return Promise.resolve(this._db.changes > 0);  }
  //--------------------------------------------------  // GET
  getMaxLocalIndex(): Promise<number> {    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    return Promise.resolve(this._maxLocalIndex);  }
  queryDocs(queryToClean: Query<string[]>): Promise<DocBase<string>[]> {    return Promise.resolve(this.queryDocsSync(queryToClean));  }
  private queryDocsSync(queryToClean: Query<string[]>): DocBase<string>[] {    // Query the documents
    logger.debug("queryDocs", queryToClean);    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    // clean up the query and exit early if possible.    const { query, willMatch } = cleanUpQuery(queryToClean);    logger.debug(`    cleanUpQuery.  willMatch = ${willMatch}`);    if (willMatch === "nothing") {      return [];    }    const now = Date.now() * 1000;
    const { sql, params } = makeDocQuerySql(query, now, "documents");    logger.debug("  sql:", sql);    logger.debug("  params:", params);
    const docsQuery = this._db.prepareQuery<Sqlite.Row, DocObject>(sql);
    const docRows = docsQuery.allEntries(params);
    logger.debug(`  result: ${docRows.length} docs`);
    docsQuery.finalize();
    const docs = [];
    for (const row of docRows) {      const doc = JSON.parse(row.doc);      docs.push({ ...doc, _localIndex: row.localIndex });    }
    return docs;  }
  //--------------------------------------------------  // SET
  upsert<DocType extends DocBase<string>>(    doc: DocType,  ): Promise<DocType> {    return Promise.resolve(this.upsertSync(doc));  }
  private upsertSync<DocType extends DocBase<string>>(    doc: DocType,  ): DocType {    // Insert new doc, replacing old doc if there is one    logger.debug(`upsertDocument(doc.path: ${JSON.stringify(doc.path)})`);
    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    Object.freeze(doc);
    const row = {      doc: JSON.stringify(doc),      localIndex: this._maxLocalIndex + 1,      pathAuthor: `${doc.path} ${doc.author}`,    };
    this._maxLocalIndex += 1;
    this._db.query(UPSERT_DOC_QUERY, row);
    return { ...doc, _localIndex: row.localIndex };  }
  eraseExpiredDocs() {    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    const now = Date.now() * 1000;
    const toDeleteQuery = this._db.prepareQuery<Sqlite.Row, DocObject>(      SELECT_EXPIRED_DOC_QUERY,    );
    const docsToWipe = toDeleteQuery.allEntries({ now });
    toDeleteQuery.finalize();
    this._db.query(DELETE_EXPIRED_DOC_QUERY, { now });
    const docs = [];
    for (const row of docsToWipe) {      docs.push(JSON.parse(row.doc));    }
    return Promise.resolve(docs);  }
  //--------------------------------------------------  // SQL STUFF
  private ensureTables() {    if (this._isClosed) {      throw new ReplicaIsClosedError();    }
    // make sure sqlite is using utf-8    this._db.query(SET_ENCODING_QUERY);    this._db.query("pragma synchronous = NORMAL");    const encoding = this._db.query(GET_ENCODING_QUERY);    this._db.query(CREATE_CONFIG_TABLE_QUERY);
    // check and set schemaVersion    let schemaVersion = this._getConfigSync("schemaVersion");    logger.log(`constructor    schemaVersion: ${schemaVersion}`);
    const docsToMigrate: DocBase<string>[] = [];
    if (schemaVersion === undefined) {      this.setConfig("schemaVersion", "2");    } else if (schemaVersion !== "2") {      console.log("migrating...");      // MIGRATE.      const now = Date.now() * 1000;
      const { sql, params } = makeDocQuerySql(        { historyMode: "all", orderBy: "localIndex ASC" },        now,        "documents",      );      logger.debug("  sql:", sql);      logger.debug("  params:", params);
      const docsQuery = this._db.prepareQuery(sql);
      const docRows = docsQuery.allEntries(params);
      logger.debug(`  result: ${docRows.length} docs`);
      docsQuery.finalize();
      for (const row of docRows) {        docsToMigrate.push(          row as unknown as DocBase<string>,        );      }
      this._db.query(`DROP TABLE docs;`);      this.setConfig("schemaVersion", "2");    }
    this._db.query(CREATE_DOCS_TABLE_QUERY);    this._db.query(CREATE_INDEXES_QUERY);
    for (const doc of docsToMigrate) {      this.upsertSync(doc);    }  }}