Skip to main content
Module

x/deno/cli/http_util.rs

A modern runtime for JavaScript and TypeScript.
denoland/deno
Go to Latest
File
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.use crate::auth_tokens::AuthToken;
use cache_control::Cachability;use cache_control::CacheControl;use chrono::DateTime;use deno_core::error::custom_error;use deno_core::error::generic_error;use deno_core::error::AnyError;use deno_core::url::Url;use deno_runtime::deno_fetch::reqwest::header::HeaderValue;use deno_runtime::deno_fetch::reqwest::header::ACCEPT;use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;use deno_runtime::deno_fetch::reqwest::header::LOCATION;use deno_runtime::deno_fetch::reqwest::Client;use deno_runtime::deno_fetch::reqwest::StatusCode;use log::debug;use std::collections::HashMap;use std::time::Duration;use std::time::SystemTime;
/// Construct the next uri based on base uri and location header fragment/// See <https://tools.ietf.org/html/rfc3986#section-4.2>fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {  if location.starts_with("http://") || location.starts_with("https://") {    // absolute uri    Url::parse(location).expect("provided redirect url should be a valid url")  } else if location.starts_with("//") {    // "//" authority path-abempty    Url::parse(&format!("{}:{}", base_url.scheme(), location))      .expect("provided redirect url should be a valid url")  } else if location.starts_with('/') {    // path-absolute    base_url      .join(location)      .expect("provided redirect url should be a valid url")  } else {    // assuming path-noscheme | path-empty    let base_url_path_str = base_url.path().to_owned();    // Pop last part or url (after last slash)    let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect();    let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location);    base_url      .join(&new_path)      .expect("provided redirect url should be a valid url")  }}
// TODO(ry) HTTP headers are not unique key, value pairs. There may be more than// one header line with the same key. This should be changed to something like// Vec<(String, String)>pub type HeadersMap = HashMap<String, String>;
/// A structure used to determine if a entity in the http cache can be used.////// This is heavily influenced by/// https://github.com/kornelski/rusty-http-cache-semantics which is BSD/// 2-Clause Licensed and copyright Kornel Lesińskipub struct CacheSemantics {  cache_control: CacheControl,  cached: SystemTime,  headers: HashMap<String, String>,  now: SystemTime,}
impl CacheSemantics {  pub fn new(    headers: HashMap<String, String>,    cached: SystemTime,    now: SystemTime,  ) -> Self {    let cache_control = headers      .get("cache-control")      .map(|v| CacheControl::from_value(v).unwrap_or_default())      .unwrap_or_default();    Self {      cache_control,      cached,      headers,      now,    }  }
  fn age(&self) -> Duration {    let mut age = self.age_header_value();
    if let Ok(resident_time) = self.now.duration_since(self.cached) {      age += resident_time;    }
    age  }
  fn age_header_value(&self) -> Duration {    Duration::from_secs(      self        .headers        .get("age")        .and_then(|v| v.parse().ok())        .unwrap_or(0),    )  }
  fn is_stale(&self) -> bool {    self.max_age() <= self.age()  }
  fn max_age(&self) -> Duration {    if self.cache_control.cachability == Some(Cachability::NoCache) {      return Duration::from_secs(0);    }
    if self.headers.get("vary").map(|s| s.trim()) == Some("*") {      return Duration::from_secs(0);    }
    if let Some(max_age) = self.cache_control.max_age {      return max_age;    }
    let default_min_ttl = Duration::from_secs(0);
    let server_date = self.raw_server_date();    if let Some(expires) = self.headers.get("expires") {      return match DateTime::parse_from_rfc2822(expires) {        Err(_) => Duration::from_secs(0),        Ok(expires) => {          let expires = SystemTime::UNIX_EPOCH            + Duration::from_secs(expires.timestamp().max(0) as _);          return default_min_ttl            .max(expires.duration_since(server_date).unwrap_or_default());        }      };    }
    if let Some(last_modified) = self.headers.get("last-modified") {      if let Ok(last_modified) = DateTime::parse_from_rfc2822(last_modified) {        let last_modified = SystemTime::UNIX_EPOCH          + Duration::from_secs(last_modified.timestamp().max(0) as _);        if let Ok(diff) = server_date.duration_since(last_modified) {          let secs_left = diff.as_secs() as f64 * 0.1;          return default_min_ttl.max(Duration::from_secs(secs_left as _));        }      }    }
    default_min_ttl  }
  fn raw_server_date(&self) -> SystemTime {    self      .headers      .get("date")      .and_then(|d| DateTime::parse_from_rfc2822(d).ok())      .and_then(|d| {        SystemTime::UNIX_EPOCH          .checked_add(Duration::from_secs(d.timestamp() as _))      })      .unwrap_or(self.cached)  }
  /// Returns true if the cached value is "fresh" respecting cached headers,  /// otherwise returns false.  pub fn should_use(&self) -> bool {    if self.cache_control.cachability == Some(Cachability::NoCache) {      return false;    }
    if let Some(max_age) = self.cache_control.max_age {      if self.age() > max_age {        return false;      }    }
    if let Some(min_fresh) = self.cache_control.min_fresh {      if self.time_to_live() < min_fresh {        return false;      }    }
    if self.is_stale() {      let has_max_stale = self.cache_control.max_stale.is_some();      let allows_stale = has_max_stale        && self          .cache_control          .max_stale          .map_or(true, |val| val > self.age() - self.max_age());      if !allows_stale {        return false;      }    }
    true  }
  fn time_to_live(&self) -> Duration {    self.max_age().checked_sub(self.age()).unwrap_or_default()  }}
#[derive(Debug, PartialEq)]pub enum FetchOnceResult {  Code(Vec<u8>, HeadersMap),  NotModified,  Redirect(Url, HeadersMap),}
#[derive(Debug)]pub struct FetchOnceArgs {  pub client: Client,  pub url: Url,  pub maybe_accept: Option<String>,  pub maybe_etag: Option<String>,  pub maybe_auth_token: Option<AuthToken>,}
/// Asynchronously fetches the given HTTP URL one pass only./// If no redirect is present and no error occurs,/// yields Code(ResultPayload)./// If redirect occurs, does not follow and/// yields Redirect(url).pub async fn fetch_once(  args: FetchOnceArgs,) -> Result<FetchOnceResult, AnyError> {  let mut request = args.client.get(args.url.clone());
  if let Some(etag) = args.maybe_etag {    let if_none_match_val = HeaderValue::from_str(&etag)?;    request = request.header(IF_NONE_MATCH, if_none_match_val);  }  if let Some(auth_token) = args.maybe_auth_token {    let authorization_val = HeaderValue::from_str(&auth_token.to_string())?;    request = request.header(AUTHORIZATION, authorization_val);  }  if let Some(accept) = args.maybe_accept {    let accepts_val = HeaderValue::from_str(&accept)?;    request = request.header(ACCEPT, accepts_val);  }  let response = request.send().await?;
  if response.status() == StatusCode::NOT_MODIFIED {    return Ok(FetchOnceResult::NotModified);  }
  let mut headers_: HashMap<String, String> = HashMap::new();  let headers = response.headers();
  if let Some(warning) = headers.get("X-Deno-Warning") {    eprintln!(      "{} {}",      crate::colors::yellow("Warning"),      warning.to_str().unwrap()    );  }
  for key in headers.keys() {    let key_str = key.to_string();    let values = headers.get_all(key);    let values_str = values      .iter()      .map(|e| e.to_str().unwrap().to_string())      .collect::<Vec<String>>()      .join(",");    headers_.insert(key_str, values_str);  }
  if response.status().is_redirection() {    if let Some(location) = response.headers().get(LOCATION) {      let location_string = location.to_str().unwrap();      debug!("Redirecting to {:?}...", &location_string);      let new_url = resolve_url_from_location(&args.url, location_string);      return Ok(FetchOnceResult::Redirect(new_url, headers_));    } else {      return Err(generic_error(format!(        "Redirection from '{}' did not provide location header",        args.url      )));    }  }
  if response.status().is_client_error() || response.status().is_server_error()  {    let err = if response.status() == StatusCode::NOT_FOUND {      custom_error(        "NotFound",        format!("Import '{}' failed, not found.", args.url),      )    } else {      generic_error(format!(        "Import '{}' failed: {}",        args.url,        response.status()      ))    };    return Err(err);  }
  let body = response.bytes().await?.to_vec();
  Ok(FetchOnceResult::Code(body, headers_))}
#[cfg(test)]mod tests {  use super::*;  use crate::version;  use deno_runtime::deno_fetch::create_http_client;  use std::fs::read;
  fn create_test_client() -> Client {    create_http_client(      "test_client".to_string(),      None,      vec![],      None,      None,      None,    )    .unwrap()  }
  #[tokio::test]  async fn test_fetch_string() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("http://127.0.0.1:4545/fixture.json").unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(headers.get("content-type").unwrap(), "application/json");      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_gzip() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("http://127.0.0.1:4545/053_import_compression/gziped")      .unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");      assert_eq!(        headers.get("content-type").unwrap(),        "application/javascript"      );      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_with_etag() {    let _http_server_guard = test_util::http_server();    let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client: client.clone(),      url: url.clone(),      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");      assert_eq!(        headers.get("content-type").unwrap(),        "application/typescript"      );      assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e");    } else {      panic!();    }
    let res = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: Some("33a64df551425fcc55e".to_string()),      maybe_auth_token: None,    })    .await;    assert_eq!(res.unwrap(), FetchOnceResult::NotModified);  }
  #[tokio::test]  async fn test_fetch_brotli() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("http://127.0.0.1:4545/053_import_compression/brotli")      .unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");      assert_eq!(        headers.get("content-type").unwrap(),        "application/javascript"      );      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_accept() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: Some("application/json".to_string()),      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, _)) = result {      assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes());    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_once_with_redirect() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("http://127.0.0.1:4546/fixture.json").unwrap();    // Dns resolver substitutes `127.0.0.1` with `localhost`    let target_url = Url::parse("http://localhost:4545/fixture.json").unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Redirect(url, _)) = result {      assert_eq!(url, target_url);    } else {      panic!();    }  }
  #[test]  fn test_resolve_url_from_location_full_1() {    let url = "http://deno.land".parse::<Url>().unwrap();    let new_uri = resolve_url_from_location(&url, "http://golang.org");    assert_eq!(new_uri.host_str().unwrap(), "golang.org");  }
  #[test]  fn test_resolve_url_from_location_full_2() {    let url = "https://deno.land".parse::<Url>().unwrap();    let new_uri = resolve_url_from_location(&url, "https://golang.org");    assert_eq!(new_uri.host_str().unwrap(), "golang.org");  }
  #[test]  fn test_resolve_url_from_location_relative_1() {    let url = "http://deno.land/x".parse::<Url>().unwrap();    let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US");    assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org");    assert_eq!(new_uri.path(), "/en-US");  }
  #[test]  fn test_resolve_url_from_location_relative_2() {    let url = "http://deno.land/x".parse::<Url>().unwrap();    let new_uri = resolve_url_from_location(&url, "/y");    assert_eq!(new_uri.host_str().unwrap(), "deno.land");    assert_eq!(new_uri.path(), "/y");  }
  #[test]  fn test_resolve_url_from_location_relative_3() {    let url = "http://deno.land/x".parse::<Url>().unwrap();    let new_uri = resolve_url_from_location(&url, "z");    assert_eq!(new_uri.host_str().unwrap(), "deno.land");    assert_eq!(new_uri.path(), "/z");  }
  #[tokio::test]  async fn test_fetch_with_cafile_string() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url = Url::parse("https://localhost:5545/fixture.json").unwrap();
    let client = create_http_client(      version::get_user_agent(),      None,      vec![read(        test_util::testdata_path()          .join("tls/RootCA.pem")          .to_str()          .unwrap(),      )      .unwrap()],      None,      None,      None,    )    .unwrap();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(headers.get("content-type").unwrap(), "application/json");      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_with_default_certificate_store() {    let _http_server_guard = test_util::http_server();    // Relies on external http server with a valid mozilla root CA cert.    let url = Url::parse("https://deno.land").unwrap();    let client = create_http_client(      version::get_user_agent(),      None, // This will load mozilla certs by default      vec![],      None,      None,      None,    )    .unwrap();
    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;
    println!("{:?}", result);    if let Ok(FetchOnceResult::Code(body, _headers)) = result {      assert!(!body.is_empty());    } else {      panic!();    }  }
  // TODO(@justinmchase): Windows should verify certs too and fail to make this request without ca certs  #[cfg(not(windows))]  #[tokio::test]  #[ignore] // https://github.com/denoland/deno/issues/12561  async fn test_fetch_with_empty_certificate_store() {    use deno_runtime::deno_tls::rustls::RootCertStore;
    let _http_server_guard = test_util::http_server();    // Relies on external http server with a valid mozilla root CA cert.    let url = Url::parse("https://deno.land").unwrap();    let client = create_http_client(      version::get_user_agent(),      Some(RootCertStore::empty()), // no certs loaded at all      vec![],      None,      None,      None,    )    .unwrap();
    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;
    if let Ok(FetchOnceResult::Code(_body, _headers)) = result {      // This test is expected to fail since to CA certs have been loaded      panic!();    }  }
  #[tokio::test]  async fn test_fetch_with_cafile_gzip() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url =      Url::parse("https://localhost:5545/053_import_compression/gziped")        .unwrap();    let client = create_http_client(      version::get_user_agent(),      None,      vec![read(        test_util::testdata_path()          .join("tls/RootCA.pem")          .to_str()          .unwrap(),      )      .unwrap()],      None,      None,      None,    )    .unwrap();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");      assert_eq!(        headers.get("content-type").unwrap(),        "application/javascript"      );      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn test_fetch_with_cafile_with_etag() {    let _http_server_guard = test_util::http_server();    let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();    let client = create_http_client(      version::get_user_agent(),      None,      vec![read(        test_util::testdata_path()          .join("tls/RootCA.pem")          .to_str()          .unwrap(),      )      .unwrap()],      None,      None,      None,    )    .unwrap();    let result = fetch_once(FetchOnceArgs {      client: client.clone(),      url: url.clone(),      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");      assert_eq!(        headers.get("content-type").unwrap(),        "application/typescript"      );      assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e");      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }
    let res = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: Some("33a64df551425fcc55e".to_string()),      maybe_auth_token: None,    })    .await;    assert_eq!(res.unwrap(), FetchOnceResult::NotModified);  }
  #[tokio::test]  async fn test_fetch_with_cafile_brotli() {    let _http_server_guard = test_util::http_server();    // Relies on external http server. See target/debug/test_server    let url =      Url::parse("https://localhost:5545/053_import_compression/brotli")        .unwrap();    let client = create_http_client(      version::get_user_agent(),      None,      vec![read(        test_util::testdata_path()          .join("tls/RootCA.pem")          .to_str()          .unwrap(),      )      .unwrap()],      None,      None,      None,    )    .unwrap();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    if let Ok(FetchOnceResult::Code(body, headers)) = result {      assert!(!body.is_empty());      assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");      assert_eq!(        headers.get("content-type").unwrap(),        "application/javascript"      );      assert_eq!(headers.get("etag"), None);      assert_eq!(headers.get("x-typescript-types"), None);    } else {      panic!();    }  }
  #[tokio::test]  async fn bad_redirect() {    let _g = test_util::http_server();    let url_str = "http://127.0.0.1:4545/bad_redirect";    let url = Url::parse(url_str).unwrap();    let client = create_test_client();    let result = fetch_once(FetchOnceArgs {      client,      url,      maybe_accept: None,      maybe_etag: None,      maybe_auth_token: None,    })    .await;    assert!(result.is_err());    let err = result.unwrap_err();    // Check that the error message contains the original URL    assert!(err.to_string().contains(url_str));  }}