123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::args::CliOptions;use crate::args::DenoSubcommand;use crate::args::Flags;use crate::args::TypeCheckMode;use crate::cache;use crate::compat;use crate::compat::NodeEsmResolver;use crate::deno_dir;use crate::emit;use crate::emit::EmitCache;use crate::emit::TsConfigType;use crate::emit::TsTypeLib;use crate::file_fetcher::FileFetcher;use crate::graph_util::graph_lock_or_exit;use crate::graph_util::GraphData;use crate::graph_util::ModuleEntry;use crate::http_cache;use crate::lockfile::as_maybe_locker;use crate::lockfile::Lockfile;use crate::resolver::ImportMapResolver;use crate::resolver::JsxResolver;
use deno_ast::MediaType;use deno_core::anyhow::anyhow;use deno_core::anyhow::Context;use deno_core::error::custom_error;use deno_core::error::AnyError;use deno_core::futures;use deno_core::parking_lot::Mutex;use deno_core::parking_lot::RwLock;use deno_core::resolve_url;use deno_core::url::Url;use deno_core::CompiledWasmModuleStore;use deno_core::ModuleSource;use deno_core::ModuleSpecifier;use deno_core::ModuleType;use deno_core::SharedArrayBufferStore;use deno_core::SourceMapGetter;use deno_graph::create_graph;use deno_graph::source::CacheInfo;use deno_graph::source::LoadFuture;use deno_graph::source::Loader;use deno_graph::source::ResolveResponse;use deno_graph::ModuleKind;use deno_graph::Resolved;use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;use deno_runtime::deno_tls::rustls::RootCertStore;use deno_runtime::deno_web::BlobStore;use deno_runtime::inspector_server::InspectorServer;use deno_runtime::permissions::Permissions;use import_map::ImportMap;use log::warn;use std::collections::HashSet;use std::env;use std::ops::Deref;use std::path::PathBuf;use std::sync::Arc;
/// This structure represents state of single "deno" program.////// It is shared by all created workers (thus V8 isolates).#[derive(Clone)]pub struct ProcState(Arc<Inner>);
pub struct Inner {  pub dir: deno_dir::DenoDir,  pub coverage_dir: Option<String>,  pub file_fetcher: FileFetcher,  pub options: Arc<CliOptions>,  graph_data: Arc<RwLock<GraphData>>,  pub lockfile: Option<Arc<Mutex<Lockfile>>>,  pub maybe_import_map: Option<Arc<ImportMap>>,  pub maybe_inspector_server: Option<Arc<InspectorServer>>,  pub root_cert_store: RootCertStore,  pub blob_store: BlobStore,  pub broadcast_channel: InMemoryBroadcastChannel,  pub shared_array_buffer_store: SharedArrayBufferStore,  pub compiled_wasm_module_store: CompiledWasmModuleStore,  maybe_resolver: Option<Arc<dyn deno_graph::source::Resolver + Send + Sync>>,  maybe_file_watcher_reporter: Option<FileWatcherReporter>,}
impl Deref for ProcState {  type Target = Arc<Inner>;  fn deref(&self) -> &Self::Target {    &self.0  }}
impl ProcState {  pub async fn build(flags: Flags) -> Result<Self, AnyError> {    Self::from_options(Arc::new(CliOptions::from_flags(flags)?)).await  }
  pub async fn from_options(    options: Arc<CliOptions>,  ) -> Result<Self, AnyError> {    Self::build_with_sender(options, None).await  }
  pub async fn build_for_file_watcher(    flags: Flags,    files_to_watch_sender: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,  ) -> Result<Self, AnyError> {    // resolve the config each time    let cli_options = Arc::new(CliOptions::from_flags(flags)?);    let ps =      Self::build_with_sender(cli_options, Some(files_to_watch_sender.clone()))        .await?;
    // Add the extra files listed in the watch flag    if let Some(watch_paths) = ps.options.watch_paths() {      files_to_watch_sender.send(watch_paths.clone()).unwrap();    }
    if let Ok(Some(import_map_path)) = ps      .options      .resolve_import_map_specifier()      .map(|ms| ms.and_then(|ref s| s.to_file_path().ok()))    {      files_to_watch_sender.send(vec![import_map_path]).unwrap();    }
    Ok(ps)  }
  async fn build_with_sender(    cli_options: Arc<CliOptions>,    maybe_sender: Option<tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>>,  ) -> Result<Self, AnyError> {    let blob_store = BlobStore::default();    let broadcast_channel = InMemoryBroadcastChannel::default();    let shared_array_buffer_store = SharedArrayBufferStore::default();    let compiled_wasm_module_store = CompiledWasmModuleStore::default();    let dir = cli_options.resolve_deno_dir()?;    let deps_cache_location = dir.root.join("deps");    let http_cache = http_cache::HttpCache::new(&deps_cache_location);    let root_cert_store = cli_options.resolve_root_cert_store()?;    let cache_usage = cli_options.cache_setting();    let file_fetcher = FileFetcher::new(      http_cache,      cache_usage,      !cli_options.no_remote(),      Some(root_cert_store.clone()),      blob_store.clone(),      cli_options        .unsafely_ignore_certificate_errors()        .map(ToOwned::to_owned),    )?;
    let lockfile = cli_options      .resolve_lock_file()?      .map(|f| Arc::new(Mutex::new(f)));    let maybe_import_map_specifier =      cli_options.resolve_import_map_specifier()?;
    let maybe_import_map =      if let Some(import_map_specifier) = maybe_import_map_specifier {        let file = file_fetcher          .fetch(&import_map_specifier, &mut Permissions::allow_all())          .await          .context(format!(            "Unable to load '{}' import map",            import_map_specifier          ))?;        let import_map =          import_map_from_text(&import_map_specifier, &file.source)?;        Some(Arc::new(import_map))      } else {        None      };
    let maybe_inspector_server =      cli_options.resolve_inspector_server().map(Arc::new);
    let coverage_dir = cli_options      .coverage_dir()      .map(ToOwned::to_owned)      .or_else(|| env::var("DENO_UNSTABLE_COVERAGE_DIR").ok());
    // FIXME(bartlomieju): `NodeEsmResolver` is not aware of JSX resolver    // created below    let node_resolver = NodeEsmResolver::new(      maybe_import_map.clone().map(ImportMapResolver::new),    );    let maybe_import_map_resolver =      maybe_import_map.clone().map(ImportMapResolver::new);    let maybe_jsx_resolver = cli_options      .to_maybe_jsx_import_source_module()      .map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone()));    let maybe_resolver: Option<      Arc<dyn deno_graph::source::Resolver + Send + Sync>,    > = if cli_options.compat() {      Some(Arc::new(node_resolver))    } else if let Some(jsx_resolver) = maybe_jsx_resolver {      // the JSX resolver offloads to the import map if present, otherwise uses      // the default Deno explicit import resolution.      Some(Arc::new(jsx_resolver))    } else if let Some(import_map_resolver) = maybe_import_map_resolver {      Some(Arc::new(import_map_resolver))    } else {      None    };
    let maybe_file_watcher_reporter =      maybe_sender.map(|sender| FileWatcherReporter {        sender,        file_paths: Arc::new(Mutex::new(vec![])),      });
    Ok(ProcState(Arc::new(Inner {      dir,      coverage_dir,      options: cli_options,      file_fetcher,      graph_data: Default::default(),      lockfile,      maybe_import_map,      maybe_inspector_server,      root_cert_store,      blob_store,      broadcast_channel,      shared_array_buffer_store,      compiled_wasm_module_store,      maybe_resolver,      maybe_file_watcher_reporter,    })))  }
  /// This method must be called for a module or a static importer of that  /// module before attempting to `load()` it from a `JsRuntime`. It will  /// populate `self.graph_data` in memory with the necessary source code, write  /// emits where necessary or report any module graph / type checking errors.  pub async fn prepare_module_load(    &self,    roots: Vec<ModuleSpecifier>,    is_dynamic: bool,    lib: TsTypeLib,    root_permissions: Permissions,    dynamic_permissions: Permissions,    reload_on_watch: bool,  ) -> Result<(), AnyError> {    let maybe_resolver: Option<&dyn deno_graph::source::Resolver> =      if let Some(resolver) = &self.maybe_resolver {        Some(resolver.as_ref())      } else {        None      };
    // NOTE(@bartlomieju):    // Even though `roots` are fully resolved at this point, we are going    // to resolve them through `maybe_resolver` to get module kind for the graph    // or default to ESM.    //    // One might argue that this is a code smell, and I would agree. However    // due to flux in "Node compatibility" it's not clear where it should be    // decided what `ModuleKind` is decided for root specifier.    let roots = roots      .into_iter()      .map(|r| {        if let Some(resolver) = &maybe_resolver {          let response =            resolver.resolve(r.as_str(), &Url::parse("unused:").unwrap());          // TODO(bartlomieju): this should be implemented in `deno_graph`          match response {            ResolveResponse::CommonJs(_) => (r, ModuleKind::CommonJs),            ResolveResponse::Err(_) => unreachable!(),            _ => (r, ModuleKind::Esm),          }        } else {          (r, ModuleKind::Esm)        }      })      .collect();
    // TODO(bartlomieju): this is very make-shift, is there an existing API    // that we could include it like with "maybe_imports"?    let roots = if self.options.compat() {      let mut r = vec![(compat::GLOBAL_URL.clone(), ModuleKind::Esm)];      r.extend(roots);      r    } else {      roots    };    if !reload_on_watch {      let graph_data = self.graph_data.read();      if self.options.type_check_mode() == TypeCheckMode::None        || graph_data.is_type_checked(&roots, &lib)      {        if let Some(result) = graph_data.check(          &roots,          self.options.type_check_mode() != TypeCheckMode::None,          false,        ) {          return result;        }      }    }    let mut cache = cache::FetchCacher::new(      self.dir.gen_cache.clone(),      self.file_fetcher.clone(),      root_permissions.clone(),      dynamic_permissions.clone(),    );    let maybe_locker = as_maybe_locker(self.lockfile.clone());    let maybe_imports = self.options.to_maybe_imports()?;
    struct ProcStateLoader<'a> {      inner: &'a mut cache::FetchCacher,      graph_data: Arc<RwLock<GraphData>>,      reload: bool,    }    impl Loader for ProcStateLoader<'_> {      fn get_cache_info(        &self,        specifier: &ModuleSpecifier,      ) -> Option<CacheInfo> {        self.inner.get_cache_info(specifier)      }      fn load(        &mut self,        specifier: &ModuleSpecifier,        is_dynamic: bool,      ) -> LoadFuture {        let graph_data = self.graph_data.read();        let found_specifier = graph_data.follow_redirect(specifier);        match graph_data.get(&found_specifier) {          Some(_) if !self.reload => {            Box::pin(futures::future::ready(Err(anyhow!(""))))          }          _ => self.inner.load(specifier, is_dynamic),        }      }    }    let mut loader = ProcStateLoader {      inner: &mut cache,      graph_data: self.graph_data.clone(),      reload: reload_on_watch,    };
    let maybe_file_watcher_reporter: Option<&dyn deno_graph::source::Reporter> =      if let Some(reporter) = &self.maybe_file_watcher_reporter {        Some(reporter)      } else {        None      };
    let graph = create_graph(      roots.clone(),      is_dynamic,      maybe_imports,      &mut loader,      maybe_resolver,      maybe_locker,      None,      maybe_file_watcher_reporter,    )    .await;
    let needs_cjs_esm_translation = graph      .modules()      .iter()      .any(|m| m.kind == ModuleKind::CommonJs);
    if needs_cjs_esm_translation {      for module in graph.modules() {        // TODO(bartlomieju): this is overly simplistic heuristic, once we are        // in compat mode, all files ending with plain `.js` extension are        // considered CommonJs modules. Which leads to situation where valid        // ESM modules with `.js` extension might undergo translation (it won't        // work in this situation).        if module.kind == ModuleKind::CommonJs {          let translated_source = compat::translate_cjs_to_esm(            &self.file_fetcher,            &module.specifier,            module.maybe_source.as_ref().unwrap().to_string(),            module.media_type,          )          .await?;          let mut graph_data = self.graph_data.write();          graph_data            .add_cjs_esm_translation(&module.specifier, translated_source);        }      }    }
    // If there was a locker, validate the integrity of all the modules in the    // locker.    graph_lock_or_exit(&graph);
    // Determine any modules that have already been emitted this session and    // should be skipped.    let reload_exclusions: HashSet<ModuleSpecifier> = {      let graph_data = self.graph_data.read();      graph_data.entries().into_keys().cloned().collect()    };
    {      let mut graph_data = self.graph_data.write();      graph_data.add_graph(&graph, reload_on_watch);      let check_js = self.options.check_js();      graph_data        .check(          &roots,          self.options.type_check_mode() != TypeCheckMode::None,          check_js,        )        .unwrap()?;    }
    let config_type = if self.options.type_check_mode() == TypeCheckMode::None {      TsConfigType::Emit    } else {      TsConfigType::Check {        tsc_emit: true,        lib,      }    };
    let ts_config_result =      self.options.resolve_ts_config_for_emit(config_type)?;
    if let Some(ignored_options) = ts_config_result.maybe_ignored_options {      log::warn!("{}", ignored_options);    }
    if self.options.type_check_mode() == TypeCheckMode::None {      let options = emit::EmitOptions {        ts_config: ts_config_result.ts_config,        reload: self.options.reload_flag(),        reload_exclusions,      };      let emit_result = emit::emit(&graph, &self.dir.gen_cache, options)?;      log::debug!("{}", emit_result.stats);    } else {      let maybe_config_specifier = self.options.maybe_config_file_specifier();      let options = emit::CheckOptions {        type_check_mode: self.options.type_check_mode(),        debug: self.options.log_level() == Some(log::Level::Debug),        emit_with_diagnostics: false,        maybe_config_specifier,        ts_config: ts_config_result.ts_config,        log_checks: true,        reload: self.options.reload_flag(),        reload_exclusions,      };      let emit_result = emit::check_and_maybe_emit(        &roots,        self.graph_data.clone(),        &self.dir.gen_cache,        options,      )?;      if !emit_result.diagnostics.is_empty() {        return Err(anyhow!(emit_result.diagnostics));      }      log::debug!("{}", emit_result.stats);    }
    if self.options.type_check_mode() != TypeCheckMode::None {      let mut graph_data = self.graph_data.write();      graph_data.set_type_checked(&roots, lib);    }
    // any updates to the lockfile should be updated now    if let Some(ref lockfile) = self.lockfile {      let g = lockfile.lock();      g.write()?;    }
    Ok(())  }
  pub fn resolve(    &self,    specifier: &str,    referrer: &str,  ) -> Result<ModuleSpecifier, AnyError> {    if let Ok(referrer) = deno_core::resolve_url_or_path(referrer) {      let graph_data = self.graph_data.read();      let found_referrer = graph_data.follow_redirect(&referrer);      let maybe_resolved = match graph_data.get(&found_referrer) {        Some(ModuleEntry::Module { dependencies, .. }) => {          dependencies.get(specifier).map(|d| &d.maybe_code)        }        _ => None,      };
      match maybe_resolved {        Some(Resolved::Ok { specifier, .. }) => return Ok(specifier.clone()),        Some(Resolved::Err(err)) => {          return Err(custom_error(            "TypeError",            format!("{}\n", err.to_string_with_range()),          ))        }        Some(Resolved::None) | None => {}      }    }
    // FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL    // and `Deno.core.evalContext` API. Ideally we should always have a referrer filled    // but sadly that's not the case due to missing APIs in V8.    let referrer = if referrer.is_empty()      && matches!(self.options.sub_command(), DenoSubcommand::Repl(_))    {      deno_core::resolve_url_or_path("./$deno$repl.ts").unwrap()    } else {      deno_core::resolve_url_or_path(referrer).unwrap()    };
    let maybe_resolver: Option<&dyn deno_graph::source::Resolver> =      if let Some(resolver) = &self.maybe_resolver {        Some(resolver.as_ref())      } else {        None      };    if let Some(resolver) = &maybe_resolver {      resolver.resolve(specifier, &referrer).to_result()    } else {      deno_core::resolve_import(specifier, referrer.as_str())        .map_err(|err| err.into())    }  }
  pub fn load(    &self,    specifier: ModuleSpecifier,    maybe_referrer: Option<ModuleSpecifier>,    is_dynamic: bool,  ) -> Result<ModuleSource, AnyError> {    log::debug!(      "specifier: {} maybe_referrer: {} is_dynamic: {}",      specifier,      maybe_referrer        .as_ref()        .map(|s| s.to_string())        .unwrap_or_else(|| "<none>".to_string()),      is_dynamic    );
    let graph_data = self.graph_data.read();    let found_url = graph_data.follow_redirect(&specifier);    match graph_data.get(&found_url) {      Some(ModuleEntry::Module {        code, media_type, ..      }) => {        let code = match media_type {          MediaType::JavaScript          | MediaType::Unknown          | MediaType::Cjs          | MediaType::Mjs          | MediaType::Json => {            if let Some(source) = graph_data.get_cjs_esm_translation(&specifier)            {              source.to_owned()            } else {              code.to_string()            }          }          MediaType::Dts | MediaType::Dcts | MediaType::Dmts => "".to_string(),          MediaType::TypeScript          | MediaType::Mts          | MediaType::Cts          | MediaType::Jsx          | MediaType::Tsx => {            let cached_text = self.dir.gen_cache.get_emit_text(&found_url);            match cached_text {              Some(text) => text,              None => unreachable!("Unexpected missing emit: {}\n\nTry reloading with the --reload CLI flag or deleting your DENO_DIR.", found_url),            }          }          MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {            panic!("Unexpected media type {} for {}", media_type, found_url)          }        };        Ok(ModuleSource {          code: code.into_bytes().into_boxed_slice(),          module_url_specified: specifier.to_string(),          module_url_found: found_url.to_string(),          module_type: match media_type {            MediaType::Json => ModuleType::Json,            _ => ModuleType::JavaScript,          },        })      }      _ => Err(anyhow!(        "Loading unprepared module: {}",        specifier.to_string()      )),    }  }}
// TODO(@kitsonk) this is only temporary, but should be refactored to somewhere// else, like a refactored file_fetcher.impl SourceMapGetter for ProcState {  fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {    if let Ok(specifier) = resolve_url(file_name) {      match specifier.scheme() {        // we should only be looking for emits for schemes that denote external        // modules, which the disk_cache supports        "wasm" | "file" | "http" | "https" | "data" | "blob" => (),        _ => return None,      }      if let Some(cache_data) = self.dir.gen_cache.get_emit_data(&specifier) {        source_map_from_code(cache_data.text.as_bytes())          .or_else(|| cache_data.map.map(|t| t.into_bytes()))      } else if let Ok(source) = self.load(specifier, None, false) {        source_map_from_code(&source.code)      } else {        None      }    } else {      None    }  }
  fn get_source_line(    &self,    file_name: &str,    line_number: usize,  ) -> Option<String> {    if let Ok(specifier) = resolve_url(file_name) {      self.file_fetcher.get_source(&specifier).map(|out| {        // Do NOT use .lines(): it skips the terminating empty line.        // (due to internally using_terminator() instead of .split())        let lines: Vec<&str> = out.source.split('\n').collect();        if line_number >= lines.len() {          format!(            "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)",            crate::colors::yellow("Warning"), line_number + 1,          )        } else {          lines[line_number].to_string()        }      })    } else {      None    }  }}
pub fn import_map_from_text(  specifier: &Url,  json_text: &str,) -> Result<ImportMap, AnyError> {  debug_assert!(    !specifier.as_str().contains("../"),    "Import map specifier incorrectly contained ../: {}",    specifier.as_str()  );  let result = import_map::parse_from_json(specifier, json_text)?;  if !result.diagnostics.is_empty() {    warn!(      "Import map diagnostics:\n{}",      result        .diagnostics        .into_iter()        .map(|d| format!("  - {}", d))        .collect::<Vec<_>>()        .join("\n")    );  }  Ok(result.import_map)}
fn source_map_from_code(code: &[u8]) -> Option<Vec<u8>> {  static PREFIX: &[u8] = b"//# sourceMappingURL=data:application/json;base64,";  let last_line = code.rsplitn(2, |u| u == &b'\n').next().unwrap();  if last_line.starts_with(PREFIX) {    let input = last_line.split_at(PREFIX.len()).1;    let decoded_map = base64::decode(input)      .expect("Unable to decode source map from emitted file.");    Some(decoded_map)  } else {    None  }}
#[derive(Debug)]struct FileWatcherReporter {  sender: tokio::sync::mpsc::UnboundedSender<Vec<PathBuf>>,  file_paths: Arc<Mutex<Vec<PathBuf>>>,}
impl deno_graph::source::Reporter for FileWatcherReporter {  fn on_load(    &self,    specifier: &ModuleSpecifier,    modules_done: usize,    modules_total: usize,  ) {    let mut file_paths = self.file_paths.lock();    if specifier.scheme() == "file" {      file_paths.push(specifier.to_file_path().unwrap());    }
    if modules_done == modules_total {      self.sender.send(file_paths.drain(..).collect()).unwrap();    }  }}