Latest
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::args::CacheSetting;use crate::auth_tokens::AuthToken;use crate::auth_tokens::AuthTokens;use crate::cache::HttpCache;use crate::colors;use crate::http_util;use crate::http_util::resolve_redirect_from_response;use crate::http_util::CacheSemantics;use crate::http_util::HeadersMap;use crate::http_util::HttpClient;use crate::util::progress_bar::ProgressBar;use crate::util::progress_bar::UpdateGuard;
use deno_ast::MediaType;use deno_core::anyhow::Context;use deno_core::error::custom_error;use deno_core::error::generic_error;use deno_core::error::uri_error;use deno_core::error::AnyError;use deno_core::futures;use deno_core::futures::future::FutureExt;use deno_core::parking_lot::Mutex;use deno_core::url::Url;use deno_core::ModuleSpecifier;use deno_graph::source::LoaderChecksum;use deno_runtime::deno_fetch::reqwest::header::HeaderValue;use deno_runtime::deno_fetch::reqwest::header::ACCEPT;use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;use deno_runtime::deno_fetch::reqwest::StatusCode;use deno_runtime::deno_web::BlobStore;use deno_runtime::permissions::PermissionsContainer;use log::debug;use std::collections::HashMap;use std::env;use std::fs;use std::future::Future;use std::path::PathBuf;use std::pin::Pin;use std::sync::Arc;use std::time::SystemTime;
pub const SUPPORTED_SCHEMES: [&str; 5] = ["data", "blob", "file", "http", "https"];
#[derive(Debug, Clone, Eq, PartialEq)]pub struct TextDecodedFile { pub media_type: MediaType, /// The _final_ specifier for the file. The requested specifier and the final /// specifier maybe different for remote files that have been redirected. pub specifier: ModuleSpecifier, /// The source of the file. pub source: Arc<str>,}
/// A structure representing a source file.#[derive(Debug, Clone, Eq, PartialEq)]pub struct File { /// The _final_ specifier for the file. The requested specifier and the final /// specifier maybe different for remote files that have been redirected. pub specifier: ModuleSpecifier, pub maybe_headers: Option<HashMap<String, String>>, /// The source of the file. pub source: Arc<[u8]>,}
impl File { pub fn resolve_media_type_and_charset(&self) -> (MediaType, Option<&str>) { deno_graph::source::resolve_media_type_and_charset_from_headers( &self.specifier, self.maybe_headers.as_ref(), ) }
/// Decodes the source bytes into a string handling any encoding rules /// for local vs remote files and dealing with the charset. pub fn into_text_decoded(self) -> Result<TextDecodedFile, AnyError> { // lots of borrow checker fighting here let (media_type, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( &self.specifier, self.maybe_headers.as_ref(), ); let specifier = self.specifier; match deno_graph::source::decode_source( &specifier, self.source, maybe_charset, ) { Ok(source) => Ok(TextDecodedFile { media_type, specifier, source, }), Err(err) => { Err(err).with_context(|| format!("Failed decoding \"{}\".", specifier)) } } }}
#[derive(Debug, Clone, Default)]struct MemoryFiles(Arc<Mutex<HashMap<ModuleSpecifier, File>>>);
impl MemoryFiles { pub fn get(&self, specifier: &ModuleSpecifier) -> Option<File> { self.0.lock().get(specifier).cloned() }
pub fn insert(&self, specifier: ModuleSpecifier, file: File) -> Option<File> { self.0.lock().insert(specifier, file) }}
/// Fetch a source file from the local file system.fn fetch_local(specifier: &ModuleSpecifier) -> Result<File, AnyError> { let local = specifier.to_file_path().map_err(|_| { uri_error(format!("Invalid file path.\n Specifier: {specifier}")) })?; let bytes = fs::read(local)?;
Ok(File { specifier: specifier.clone(), maybe_headers: None, source: bytes.into(), })}
/// Return a validated scheme for a given module specifier.fn get_validated_scheme( specifier: &ModuleSpecifier,) -> Result<String, AnyError> { let scheme = specifier.scheme(); if !SUPPORTED_SCHEMES.contains(&scheme) { Err(generic_error(format!( "Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes: {SUPPORTED_SCHEMES:#?}" ))) } else { Ok(scheme.to_string()) }}
pub struct FetchOptions<'a> { pub specifier: &'a ModuleSpecifier, pub permissions: PermissionsContainer, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, pub maybe_checksum: Option<LoaderChecksum>,}
/// A structure for resolving, fetching and caching source files.#[derive(Debug, Clone)]pub struct FileFetcher { auth_tokens: AuthTokens, allow_remote: bool, memory_files: MemoryFiles, cache_setting: CacheSetting, http_cache: Arc<dyn HttpCache>, http_client: Arc<HttpClient>, blob_store: Arc<BlobStore>, download_log_level: log::Level, progress_bar: Option<ProgressBar>,}
impl FileFetcher { pub fn new( http_cache: Arc<dyn HttpCache>, cache_setting: CacheSetting, allow_remote: bool, http_client: Arc<HttpClient>, blob_store: Arc<BlobStore>, progress_bar: Option<ProgressBar>, ) -> Self { Self { auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), allow_remote, memory_files: Default::default(), cache_setting, http_cache, http_client, blob_store, download_log_level: log::Level::Info, progress_bar, } }
pub fn cache_setting(&self) -> &CacheSetting { &self.cache_setting }
/// Sets the log level to use when outputting the download message. pub fn set_download_log_level(&mut self, level: log::Level) { self.download_log_level = level; }
/// Fetch cached remote file. /// /// This is a recursive operation if source file has redirections. pub fn fetch_cached( &self, specifier: &ModuleSpecifier, maybe_checksum: Option<LoaderChecksum>, redirect_limit: i64, ) -> Result<Option<File>, AnyError> { debug!("FileFetcher::fetch_cached - specifier: {}", specifier); if redirect_limit < 0 { return Err(custom_error("Http", "Too many redirects.")); }
let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once let Some(headers) = self.http_cache.read_headers(&cache_key)? else { return Ok(None); }; if let Some(redirect_to) = headers.get("location") { let redirect = deno_core::resolve_import(redirect_to, specifier.as_str())?; return self.fetch_cached(&redirect, maybe_checksum, redirect_limit - 1); } let Some(bytes) = self.http_cache.read_file_bytes( &cache_key, maybe_checksum .as_ref() .map(|c| deno_cache_dir::Checksum::new(c.as_str())), deno_cache_dir::GlobalToLocalCopy::Allow, )? else { return Ok(None); };
Ok(Some(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), })) }
/// Convert a data URL into a file, resulting in an error if the URL is /// invalid. fn fetch_data_url( &self, specifier: &ModuleSpecifier, ) -> Result<File, AnyError> { debug!("FileFetcher::fetch_data_url() - specifier: {}", specifier); let data_url = deno_graph::source::RawDataUrl::parse(specifier)?; let (bytes, headers) = data_url.into_bytes_and_headers(); Ok(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), }) }
/// Get a blob URL. async fn fetch_blob_url( &self, specifier: &ModuleSpecifier, ) -> Result<File, AnyError> { debug!("FileFetcher::fetch_blob_url() - specifier: {}", specifier); let blob = self .blob_store .get_object_url(specifier.clone()) .ok_or_else(|| { custom_error( "NotFound", format!("Blob URL not found: \"{specifier}\"."), ) })?;
let bytes = blob.read_all().await?; let headers = HashMap::from([("content-type".to_string(), blob.media_type.clone())]);
Ok(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), }) }
/// Asynchronously fetch remote source file specified by the URL following /// redirects. /// /// **Note** this is a recursive method so it can't be "async", but needs to /// return a `Pin<Box<..>>`. fn fetch_remote( &self, specifier: &ModuleSpecifier, permissions: PermissionsContainer, redirect_limit: i64, maybe_accept: Option<String>, cache_setting: &CacheSetting, maybe_checksum: Option<LoaderChecksum>, ) -> Pin<Box<dyn Future<Output = Result<File, AnyError>> + Send>> { debug!("FileFetcher::fetch_remote() - specifier: {}", specifier); if redirect_limit < 0 { return futures::future::err(custom_error("Http", "Too many redirects.")) .boxed(); }
if let Err(err) = permissions.check_specifier(specifier) { return futures::future::err(err).boxed(); }
if self.should_use_cache(specifier, cache_setting) { match self.fetch_cached(specifier, maybe_checksum.clone(), redirect_limit) { Ok(Some(file)) => { return futures::future::ok(file).boxed(); } Ok(None) => {} Err(err) => { return futures::future::err(err).boxed(); } } }
if *cache_setting == CacheSetting::Only { return futures::future::err(custom_error( "NotCached", format!( "Specifier not found in cache: \"{specifier}\", --cached-only is specified." ), )) .boxed(); }
let mut maybe_progress_guard = None; if let Some(pb) = self.progress_bar.as_ref() { maybe_progress_guard = Some(pb.update(specifier.as_str())); } else { log::log!( self.download_log_level, "{} {}", colors::green("Download"), specifier ); }
let maybe_etag = self .http_cache .cache_item_key(specifier) .ok() .and_then(|key| self.http_cache.read_headers(&key).ok().flatten()) .and_then(|headers| headers.get("etag").cloned()); let maybe_auth_token = self.auth_tokens.get(specifier); let specifier = specifier.clone(); let client = self.http_client.clone(); let file_fetcher = self.clone(); let cache_setting = cache_setting.clone(); // A single pass of fetch either yields code or yields a redirect, server // error causes a single retry to avoid crashing hard on intermittent failures.
async fn handle_request_or_server_error( retried: &mut bool, specifier: &Url, err_str: String, ) -> Result<(), AnyError> { // Retry once, and bail otherwise. if !*retried { *retried = true; log::debug!("Import '{}' failed: {}. Retrying...", specifier, err_str); tokio::time::sleep(std::time::Duration::from_millis(50)).await; Ok(()) } else { Err(generic_error(format!( "Import '{}' failed: {}", specifier, err_str ))) } }
async move { let mut retried = false; let result = loop { let result = match fetch_once( &client, FetchOnceArgs { url: specifier.clone(), maybe_accept: maybe_accept.clone(), maybe_etag: maybe_etag.clone(), maybe_auth_token: maybe_auth_token.clone(), maybe_progress_guard: maybe_progress_guard.as_ref(), }, ) .await? { FetchOnceResult::NotModified => { let file = file_fetcher .fetch_cached(&specifier, maybe_checksum, 10)? .unwrap(); Ok(file) } FetchOnceResult::Redirect(redirect_url, headers) => { file_fetcher.http_cache.set(&specifier, headers, &[])?; file_fetcher .fetch_remote( &redirect_url, permissions, redirect_limit - 1, maybe_accept, &cache_setting, maybe_checksum, ) .await } FetchOnceResult::Code(bytes, headers) => { file_fetcher .http_cache .set(&specifier, headers.clone(), &bytes)?; if let Some(checksum) = &maybe_checksum { checksum.check_source(&bytes)?; } Ok(File { specifier, maybe_headers: Some(headers), source: Arc::from(bytes), }) } FetchOnceResult::RequestError(err) => { handle_request_or_server_error(&mut retried, &specifier, err) .await?; continue; } FetchOnceResult::ServerError(status) => { handle_request_or_server_error( &mut retried, &specifier, status.to_string(), ) .await?; continue; } }; break result; };
drop(maybe_progress_guard); result } .boxed() }
/// Returns if the cache should be used for a given specifier. fn should_use_cache( &self, specifier: &ModuleSpecifier, cache_setting: &CacheSetting, ) -> bool { match cache_setting { CacheSetting::ReloadAll => false, CacheSetting::Use | CacheSetting::Only => true, CacheSetting::RespectHeaders => { let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { return false; }; let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { return false; }; let Ok(Some(download_time)) = self.http_cache.read_download_time(&cache_key) else { return false; }; let cache_semantics = CacheSemantics::new(headers, download_time, SystemTime::now()); cache_semantics.should_use() } CacheSetting::ReloadSome(list) => { let mut url = specifier.clone(); url.set_fragment(None); if list.iter().any(|x| x == url.as_str()) { return false; } url.set_query(None); let mut path = PathBuf::from(url.as_str()); loop { if list.contains(&path.to_str().unwrap().to_string()) { return false; } if !path.pop() { break; } } true } } }
/// Fetch a source file and asynchronously return it. pub async fn fetch( &self, specifier: &ModuleSpecifier, permissions: PermissionsContainer, ) -> Result<File, AnyError> { self .fetch_with_options(FetchOptions { specifier, permissions, maybe_accept: None, maybe_cache_setting: None, maybe_checksum: None, }) .await }
pub async fn fetch_with_options( &self, options: FetchOptions<'_>, ) -> Result<File, AnyError> { let specifier = options.specifier; debug!("FileFetcher::fetch() - specifier: {}", specifier); let scheme = get_validated_scheme(specifier)?; options.permissions.check_specifier(specifier)?; if let Some(file) = self.memory_files.get(specifier) { Ok(file) } else if scheme == "file" { // we do not in memory cache files, as this would prevent files on the // disk changing effecting things like workers and dynamic imports. fetch_local(specifier) } else if scheme == "data" { self.fetch_data_url(specifier) } else if scheme == "blob" { self.fetch_blob_url(specifier).await } else if !self.allow_remote { Err(custom_error( "NoRemote", format!("A remote specifier was requested: \"{specifier}\", but --no-remote is specified."), )) } else { self .fetch_remote( specifier, options.permissions, 10, options.maybe_accept.map(String::from), options.maybe_cache_setting.unwrap_or(&self.cache_setting), options.maybe_checksum, ) .await } }
/// A synchronous way to retrieve a source file, where if the file has already /// been cached in memory it will be returned, otherwise for local files will /// be read from disk. pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option<File> { let maybe_file = self.memory_files.get(specifier); if maybe_file.is_none() { let is_local = specifier.scheme() == "file"; if is_local { if let Ok(file) = fetch_local(specifier) { return Some(file); } } None } else { maybe_file } }
/// Insert a temporary module for the file fetcher. pub fn insert_memory_files(&self, file: File) -> Option<File> { self.memory_files.insert(file.specifier.clone(), file) }}
#[derive(Debug, Eq, PartialEq)]enum FetchOnceResult { Code(Vec<u8>, HeadersMap), NotModified, Redirect(Url, HeadersMap), RequestError(String), ServerError(StatusCode),}
#[derive(Debug)]struct FetchOnceArgs<'a> { pub url: Url, pub maybe_accept: Option<String>, pub maybe_etag: Option<String>, pub maybe_auth_token: Option<AuthToken>, pub maybe_progress_guard: Option<&'a UpdateGuard>,}
/// Asynchronously fetches the given HTTP URL one pass only./// If no redirect is present and no error occurs,/// yields Code(ResultPayload)./// If redirect occurs, does not follow and/// yields Redirect(url).async fn fetch_once<'a>( http_client: &HttpClient, args: FetchOnceArgs<'a>,) -> Result<FetchOnceResult, AnyError> { let mut request = http_client.get_no_redirect(args.url.clone())?;
if let Some(etag) = args.maybe_etag { let if_none_match_val = HeaderValue::from_str(&etag)?; request = request.header(IF_NONE_MATCH, if_none_match_val); } if let Some(auth_token) = args.maybe_auth_token { let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; request = request.header(AUTHORIZATION, authorization_val); } if let Some(accept) = args.maybe_accept { let accepts_val = HeaderValue::from_str(&accept)?; request = request.header(ACCEPT, accepts_val); } let response = match request.send().await { Ok(resp) => resp, Err(err) => { if err.is_connect() || err.is_timeout() { return Ok(FetchOnceResult::RequestError(err.to_string())); } return Err(err.into()); } };
if response.status() == StatusCode::NOT_MODIFIED { return Ok(FetchOnceResult::NotModified); }
let mut result_headers = HashMap::new(); let response_headers = response.headers();
if let Some(warning) = response_headers.get("X-Deno-Warning") { log::warn!( "{} {}", crate::colors::yellow("Warning"), warning.to_str().unwrap() ); }
for key in response_headers.keys() { let key_str = key.to_string(); let values = response_headers.get_all(key); let values_str = values .iter() .map(|e| e.to_str().unwrap().to_string()) .collect::<Vec<String>>() .join(","); result_headers.insert(key_str, values_str); }
if response.status().is_redirection() { let new_url = resolve_redirect_from_response(&args.url, &response)?; return Ok(FetchOnceResult::Redirect(new_url, result_headers)); }
let status = response.status();
if status.is_server_error() { return Ok(FetchOnceResult::ServerError(status)); }
if status.is_client_error() { let err = if response.status() == StatusCode::NOT_FOUND { custom_error( "NotFound", format!("Import '{}' failed, not found.", args.url), ) } else { generic_error(format!( "Import '{}' failed: {}", args.url, response.status() )) }; return Err(err); }
let body = http_util::get_response_body_with_progress( response, args.maybe_progress_guard, ) .await?;
Ok(FetchOnceResult::Code(body, result_headers))}
#[cfg(test)]mod tests { use crate::cache::GlobalHttpCache; use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClient; use crate::version;
use super::*; use deno_core::error::get_custom_error_class; use deno_core::resolve_url; use deno_core::url::Url; use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_web::Blob; use deno_runtime::deno_web::InMemoryBlobPart; use std::collections::hash_map::RandomState; use std::collections::HashSet; use std::fs::read; use test_util::TempDir;
fn setup( cache_setting: CacheSetting, maybe_temp_dir: Option<TempDir>, ) -> (FileFetcher, TempDir) { let (file_fetcher, temp_dir, _) = setup_with_blob_store(cache_setting, maybe_temp_dir); (file_fetcher, temp_dir) }
fn setup_with_blob_store( cache_setting: CacheSetting, maybe_temp_dir: Option<TempDir>, ) -> (FileFetcher, TempDir, Arc<BlobStore>) { let temp_dir = maybe_temp_dir.unwrap_or_default(); let location = temp_dir.path().join("deps").to_path_buf(); let blob_store: Arc<BlobStore> = Default::default(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), cache_setting, true, Arc::new(HttpClient::new(None, None)), blob_store.clone(), None, ); (file_fetcher, temp_dir, blob_store) }
async fn test_fetch(specifier: &ModuleSpecifier) -> (File, FileFetcher) { let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); let result = file_fetcher .fetch(specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); (result.unwrap(), file_fetcher) }
async fn test_fetch_remote( specifier: &ModuleSpecifier, ) -> (File, HashMap<String, String>) { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); let result: Result<File, AnyError> = file_fetcher .fetch_remote( specifier, PermissionsContainer::allow_all(), 1, None, &file_fetcher.cache_setting, None, ) .await; let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); ( result.unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), ) }
// this test used to test how the file fetcher decoded strings, but // now we're using it as a bit of an integration test with the functionality // in deno_graph async fn test_fetch_remote_encoded( fixture: &str, charset: &str, expected: &str, ) { let url_str = format!("http://127.0.0.1:4545/encoding/{fixture}"); let specifier = resolve_url(&url_str).unwrap(); let (file, headers) = test_fetch_remote(&specifier).await; let (media_type, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( &specifier, Some(&headers), ); assert_eq!( deno_graph::source::decode_source(&specifier, file.source, maybe_charset) .unwrap() .as_ref(), expected ); assert_eq!(media_type, MediaType::TypeScript); assert_eq!( headers.get("content-type").unwrap(), &format!("application/typescript;charset={charset}") ); }
async fn test_fetch_local_encoded(charset: &str, expected: String) { let p = test_util::testdata_path().join(format!("encoding/{charset}.ts")); let specifier = ModuleSpecifier::from_file_path(p).unwrap(); let (file, _) = test_fetch(&specifier).await; assert_eq!( deno_graph::source::decode_source(&specifier, file.source, None) .unwrap() .as_ref(), expected ); }
#[test] fn test_get_validated_scheme() { let fixtures = vec![ ("https://deno.land/x/mod.ts", true, "https"), ("http://deno.land/x/mod.ts", true, "http"), ("file:///a/b/c.ts", true, "file"), ("file:///C:/a/b/c.ts", true, "file"), ("data:,some%20text", true, "data"), ("ftp://a/b/c.ts", false, ""), ("mailto:dino@deno.land", false, ""), ];
for (specifier, is_ok, expected) in fixtures { let specifier = ModuleSpecifier::parse(specifier).unwrap(); let actual = get_validated_scheme(&specifier); assert_eq!(actual.is_ok(), is_ok); if is_ok { assert_eq!(actual.unwrap(), expected); } } }
#[tokio::test] async fn test_insert_cached() { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let local = temp_dir.path().join("a.ts"); let specifier = ModuleSpecifier::from_file_path(&local).unwrap(); let file = File { source: Arc::from("some source code".as_bytes()), specifier: specifier.clone(), maybe_headers: Some(HashMap::from([( "content-type".to_string(), "application/javascript".to_string(), )])), }; file_fetcher.insert_memory_files(file.clone());
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let result_file = result.unwrap(); assert_eq!(result_file, file); }
#[tokio::test] async fn test_fetch_data_url() { let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" ); assert_eq!(file.media_type, MediaType::TypeScript); assert_eq!(file.specifier, specifier); }
#[tokio::test] async fn test_fetch_blob_url() { let (file_fetcher, _, blob_store) = setup_with_blob_store(CacheSetting::Use, None);
let bytes = "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" .as_bytes() .to_vec();
let specifier = blob_store.insert_object_url( Blob { media_type: "application/typescript".to_string(), parts: vec![Arc::new(InMemoryBlobPart::from(bytes))], }, None, );
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" ); assert_eq!(file.media_type, MediaType::TypeScript); assert_eq!(file.specifier, specifier); }
#[tokio::test] async fn test_fetch_complex() { let _http_server_guard = test_util::http_server(); let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let (file_fetcher_01, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); let (file_fetcher_02, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/subdir/mod2.ts").unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::TypeScript);
let cache_item_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); let mut headers = HashMap::new(); headers.insert("content-type".to_string(), "text/javascript".to_string()); file_fetcher .http_cache .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap();
let result = file_fetcher_01 .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); // This validates that when using the cached value, because we modified // the value above. assert_eq!(file.media_type, MediaType::JavaScript);
let headers2 = file_fetcher_02 .http_cache .read_headers(&cache_item_key) .unwrap() .unwrap(); assert_eq!(headers2.get("content-type").unwrap(), "text/javascript"); headers = HashMap::new(); headers.insert("content-type".to_string(), "application/json".to_string()); file_fetcher_02 .http_cache .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap();
let result = file_fetcher_02 .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::Json);
// This creates a totally new instance, simulating another Deno process // invocation and indicates to "cache bust". let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::ReloadAll, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::TypeScript); }
#[tokio::test] async fn test_fetch_uses_cache() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let specifier = resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap();
let file_modified_01 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location.clone(), crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, );
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let cache_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) };
let file_modified_02 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok());
let cache_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) };
assert_eq!(file_modified_01, file_modified_02); }
#[tokio::test] async fn test_fetch_redirected() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4546/subdir/redirects/redirect1.js") .unwrap(); let redirected_specifier = resolve_url("http://localhost:4545/subdir/redirects/redirect1.js") .unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier);
assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), );
assert_eq!( get_text_from_cache(&file_fetcher, &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_specifier), None, ); }
#[tokio::test] async fn test_fetch_multiple_redirects() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4548/subdir/redirects/redirect1.js") .unwrap(); let redirected_01_specifier = resolve_url("http://localhost:4546/subdir/redirects/redirect1.js") .unwrap(); let redirected_02_specifier = resolve_url("http://localhost:4545/subdir/redirects/redirect1.js") .unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_02_specifier);
assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("http://localhost:4546/subdir/redirects/redirect1.js".to_string()), );
assert_eq!( get_text_from_cache(&file_fetcher, &redirected_01_specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_01_specifier), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), );
assert_eq!( get_text_from_cache(&file_fetcher, &redirected_02_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_02_specifier), None, ); }
#[tokio::test] async fn test_fetch_uses_cache_with_redirects() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let specifier = resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap(); let redirected_specifier = resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap();
let metadata_file_modified_01 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location.clone(), crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, );
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok());
let cache_key = file_fetcher .http_cache .cache_item_key(&redirected_specifier) .unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) };
let metadata_file_modified_02 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&redirected_specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok());
let cache_key = file_fetcher .http_cache .cache_item_key(&redirected_specifier) .unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) };
assert_eq!(metadata_file_modified_01, metadata_file_modified_02); }
#[tokio::test] async fn test_fetcher_limits_redirects() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4548/subdir/redirects/redirect1.js") .unwrap();
let result = file_fetcher .fetch_remote( &specifier, PermissionsContainer::allow_all(), 2, None, &file_fetcher.cache_setting, None, ) .await; assert!(result.is_ok());
let result = file_fetcher .fetch_remote( &specifier, PermissionsContainer::allow_all(), 1, None, &file_fetcher.cache_setting, None, ) .await; assert!(result.is_err());
let result = file_fetcher.fetch_cached(&specifier, None, 2); assert!(result.is_ok());
let result = file_fetcher.fetch_cached(&specifier, None, 1); assert!(result.is_err()); }
#[tokio::test] async fn test_fetch_same_host_redirect() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url( "http://localhost:4550/REDIRECT/subdir/redirects/redirect1.js", ) .unwrap(); let redirected_specifier = resolve_url("http://localhost:4550/subdir/redirects/redirect1.js") .unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier);
assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("/subdir/redirects/redirect1.js".to_string()), );
assert_eq!( get_text_from_cache(&file_fetcher, &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_specifier), None ); }
#[tokio::test] async fn test_fetch_no_remote() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, false, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(get_custom_error_class(&err), Some("NoRemote")); assert_eq!(err.to_string(), "A remote specifier was requested: \"http://localhost:4545/run/002_hello.ts\", but --no-remote is specified."); }
#[tokio::test] async fn test_fetch_cache_only() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher_01 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), CacheSetting::Only, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let file_fetcher_02 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), Default::default(), None, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
let result = file_fetcher_01 .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.to_string(), "Specifier not found in cache: \"http://localhost:4545/run/002_hello.ts\", --cached-only is specified."); assert_eq!(get_custom_error_class(&err), Some("NotCached"));
let result = file_fetcher_02 .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok());
let result = file_fetcher_01 .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); }
#[tokio::test] async fn test_fetch_local_bypasses_file_cache() { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let fixture_path = temp_dir.path().join("mod.ts"); let specifier = ModuleSpecifier::from_file_path(&fixture_path).unwrap(); fs::write(fixture_path.clone(), r#"console.log("hello deno");"#).unwrap(); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!(&*file.source, r#"console.log("hello deno");"#);
fs::write(fixture_path, r#"console.log("goodbye deno");"#).unwrap(); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!(&*file.source, r#"console.log("goodbye deno");"#); }
#[tokio::test] async fn test_respect_cache_revalidates() { let _g = test_util::http_server(); let temp_dir = TempDir::new(); let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/dynamic").unwrap(); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let first = file.source;
let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let second = file.source;
assert_ne!(first, second); }
#[tokio::test] async fn test_respect_cache_still_fresh() { let _g = test_util::http_server(); let temp_dir = TempDir::new(); let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/dynamic_cache").unwrap(); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let first = file.source;
let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let result = file_fetcher .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let second = file.source;
assert_eq!(first, second); }
#[tokio::test] async fn test_fetch_local_utf_16be() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-16be", expected).await; }
#[tokio::test] async fn test_fetch_local_utf_16le() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-16le", expected).await; }
#[tokio::test] async fn test_fetch_local_utf8_with_bom() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-8", expected).await; }
#[tokio::test] async fn test_fetch_remote_utf16_le() { let expected = std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap(); test_fetch_remote_encoded("utf-16le.ts", "utf-16le", expected).await; }
#[tokio::test] async fn test_fetch_remote_utf16_be() { let expected = std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap(); test_fetch_remote_encoded("utf-16be.ts", "utf-16be", expected).await; }
#[tokio::test] async fn test_fetch_remote_window_1255() { let expected = "console.log(\"\u{5E9}\u{5DC}\u{5D5}\u{5DD} \ \u{5E2}\u{5D5}\u{5DC}\u{5DD}\");\u{A}"; test_fetch_remote_encoded("windows-1255", "windows-1255", expected).await; }
fn create_test_client() -> HttpClient { HttpClient::from_client( create_http_client("test_client", CreateHttpClientOptions::default()) .unwrap(), ) }
#[tokio::test] async fn test_fetch_string() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(headers.get("content-type").unwrap(), "application/json"); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
#[tokio::test] async fn test_fetch_gzip() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped") .unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( headers.get("content-type").unwrap(), "application/javascript" ); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
#[tokio::test] async fn test_fetch_with_etag() { let _http_server_guard = test_util::http_server(); let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url: url.clone(), maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); assert_eq!( headers.get("content-type").unwrap(), "application/typescript" ); assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); } else { panic!(); }
let res = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); }
#[tokio::test] async fn test_fetch_brotli() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli") .unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!( headers.get("content-type").unwrap(), "application/javascript" ); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
#[tokio::test] async fn test_fetch_accept() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: Some("application/json".to_string()), maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, _)) = result { assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes()); } else { panic!(); } }
#[tokio::test] async fn test_fetch_once_with_redirect() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4546/assets/fixture.json").unwrap(); // Dns resolver substitutes `127.0.0.1` with `localhost` let target_url = Url::parse("http://localhost:4545/assets/fixture.json").unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Redirect(url, _)) = result { assert_eq!(url, target_url); } else { panic!(); } }
#[tokio::test] async fn test_fetch_with_cafile_string() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap();
let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions { ca_certs: vec![read( test_util::testdata_path().join("tls/RootCA.pem"), ) .unwrap()], ..Default::default() }, ) .unwrap(), ); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(headers.get("content-type").unwrap(), "application/json"); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
static PUBLIC_HTTPS_URLS: &[&str] = &[ "https://deno.com/", "https://example.com/", "https://github.com/", "https://www.w3.org/", ];
/// This test depends on external servers, so we need to be careful to avoid mistaking an offline machine with a /// test failure. #[tokio::test] async fn test_fetch_with_default_certificate_store() { let urls: HashSet<_, RandomState> = HashSet::from_iter(PUBLIC_HTTPS_URLS.iter());
// Rely on the randomization of hashset iteration for url in urls { // Relies on external http server with a valid mozilla root CA cert. let url = Url::parse(url).unwrap(); eprintln!("Attempting to fetch {url}...");
let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions::default(), ) .unwrap(), );
let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await;
match result { Err(_) => { eprintln!("Fetch error: {result:?}"); continue; } Ok( FetchOnceResult::Code(..) | FetchOnceResult::NotModified | FetchOnceResult::Redirect(..), ) => return, Ok( FetchOnceResult::RequestError(_) | FetchOnceResult::ServerError(_), ) => { eprintln!("HTTP error: {result:?}"); continue; } }; }
// Use 1.1.1.1 and 8.8.8.8 as our last-ditch internet check if std::net::TcpStream::connect("8.8.8.8:80").is_err() && std::net::TcpStream::connect("1.1.1.1:80").is_err() { return; }
panic!("None of the expected public URLs were available but internet appears to be available"); }
#[tokio::test] async fn test_fetch_with_empty_certificate_store() { let root_cert_store = RootCertStore::empty(); let urls: HashSet<_, RandomState> = HashSet::from_iter(PUBLIC_HTTPS_URLS.iter());
// Rely on the randomization of hashset iteration let url = urls.into_iter().next().unwrap(); // Relies on external http server with a valid mozilla root CA cert. let url = Url::parse(url).unwrap(); eprintln!("Attempting to fetch {url}...");
let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions { root_cert_store: Some(root_cert_store), ..Default::default() }, ) .unwrap(), );
let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await;
match result { Err(_) => { eprintln!("Fetch error (expected): {result:?}"); return; } Ok( FetchOnceResult::Code(..) | FetchOnceResult::NotModified | FetchOnceResult::Redirect(..), ) => { panic!("Should not have successfully fetched a URL"); } Ok( FetchOnceResult::RequestError(_) | FetchOnceResult::ServerError(_), ) => { eprintln!("HTTP error (expected): {result:?}"); return; } }; }
#[tokio::test] async fn test_fetch_with_cafile_gzip() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/run/import_compression/gziped") .unwrap(); let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions { ca_certs: vec![read( test_util::testdata_path() .join("tls/RootCA.pem") .to_string(), ) .unwrap()], ..Default::default() }, ) .unwrap(), ); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( headers.get("content-type").unwrap(), "application/javascript" ); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
#[tokio::test] async fn test_fetch_with_cafile_with_etag() { let _http_server_guard = test_util::http_server(); let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions { ca_certs: vec![read( test_util::testdata_path() .join("tls/RootCA.pem") .to_string(), ) .unwrap()], ..Default::default() }, ) .unwrap(), ); let result = fetch_once( &client, FetchOnceArgs { url: url.clone(), maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); assert_eq!( headers.get("content-type").unwrap(), "application/typescript" ); assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); }
let res = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); }
#[tokio::test] async fn test_fetch_with_cafile_brotli() { let _http_server_guard = test_util::http_server(); // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/run/import_compression/brotli") .unwrap(); let client = HttpClient::from_client( create_http_client( version::get_user_agent(), CreateHttpClientOptions { ca_certs: vec![read( test_util::testdata_path() .join("tls/RootCA.pem") .to_string(), ) .unwrap()], ..Default::default() }, ) .unwrap(), ); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!( headers.get("content-type").unwrap(), "application/javascript" ); assert_eq!(headers.get("etag"), None); assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } }
#[tokio::test] async fn bad_redirect() { let _g = test_util::http_server(); let url_str = "http://127.0.0.1:4545/bad_redirect"; let url = Url::parse(url_str).unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await; assert!(result.is_err()); let err = result.unwrap_err(); // Check that the error message contains the original URL assert!(err.to_string().contains(url_str)); }
#[tokio::test] async fn server_error() { let _g = test_util::http_server(); let url_str = "http://127.0.0.1:4545/server_error"; let url = Url::parse(url_str).unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await;
if let Ok(FetchOnceResult::ServerError(status)) = result { assert_eq!(status, 500); } else { panic!(); } }
#[tokio::test] async fn request_error() { let _g = test_util::http_server(); let url_str = "http://127.0.0.1:9999/"; let url = Url::parse(url_str).unwrap(); let client = create_test_client(); let result = fetch_once( &client, FetchOnceArgs { url, maybe_accept: None, maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, }, ) .await;
assert!(matches!(result, Ok(FetchOnceResult::RequestError(_)))); }
#[track_caller] fn get_text_from_cache( file_fetcher: &FileFetcher, url: &ModuleSpecifier, ) -> String { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); let bytes = file_fetcher .http_cache .read_file_bytes( &cache_key, None, deno_cache_dir::GlobalToLocalCopy::Allow, ) .unwrap() .unwrap(); String::from_utf8(bytes).unwrap() }
#[track_caller] fn get_location_header_from_cache( file_fetcher: &FileFetcher, url: &ModuleSpecifier, ) -> Option<String> { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap() .remove("location") }}