std@0.139.0/archive/tar.ts

Deno standard library
GitHub Stars
2218
Go to latest
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license./** * Ported and modified from: https://github.com/beatgammit/tar-js and * licensed as: * * (The MIT License) * * Copyright (c) 2011 T. Jameson Little * Copyright (c) 2019 Jun Kato * Copyright (c) 2018-2022 the Deno authors * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */
/** * Provides a `Tar` and `Untar` classes for compressing and decompressing * arbitrary data. * * ## Examples * * ### Tar * * ```ts * import { Tar } from "https://deno.land/std@$STD_VERSION/archive/tar.ts"; * import { Buffer } from "https://deno.land/std@$STD_VERSION/io/buffer.ts"; * import { copy } from "https://deno.land/std@$STD_VERSION/streams/conversion.ts"; * * const tar = new Tar(); * const content = new TextEncoder().encode("Deno.land"); * await tar.append("deno.txt", { * reader: new Buffer(content), * contentSize: content.byteLength, * }); * * // Or specifying a filePath. * await tar.append("land.txt", { * filePath: "./land.txt", * }); * * // use tar.getReader() to read the contents. * * const writer = await Deno.open("./out.tar", { write: true, create: true }); * await copy(tar.getReader(), writer); * writer.close(); * ``` * * ### Untar * * ```ts * import { Untar } from "https://deno.land/std@$STD_VERSION/archive/tar.ts"; * import { ensureFile } from "https://deno.land/std@$STD_VERSION/fs/ensure_file.ts"; * import { ensureDir } from "https://deno.land/std@$STD_VERSION/fs/ensure_dir.ts"; * import { copy } from "https://deno.land/std@$STD_VERSION/streams/conversion.ts"; * * const reader = await Deno.open("./out.tar", { read: true }); * const untar = new Untar(reader); * * for await (const entry of untar) { * console.log(entry); // metadata * * if (entry.type === "directory") { * await ensureDir(entry.fileName); * continue; * } * * await ensureFile(entry.fileName); * const file = await Deno.open(entry.fileName, { write: true }); * // <entry> is a reader. * await copy(entry, file); * } * reader.close(); * ``` * * @module */
import { MultiReader } from "../io/readers.ts";import { Buffer, PartialReadError } from "../io/buffer.ts";import { assert } from "../_util/assert.ts";import { readAll } from "../streams/conversion.ts";
type Reader = Deno.Reader;type Seeker = Deno.Seeker;
const recordSize = 512;const ustar = "ustar\u000000";
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06// eight checksum bytes taken to be ascii spaces (decimal value 32)const initialChecksum = 8 * 32;
async function readBlock( reader: Deno.Reader, p: Uint8Array,): Promise<number | null> { let bytesRead = 0; while (bytesRead < p.length) { const rr = await reader.read(p.subarray(bytesRead)); if (rr === null) { if (bytesRead === 0) { return null; } else { throw new PartialReadError(); } } bytesRead += rr; } return bytesRead;}
/** * Simple file reader */class FileReader implements Reader { private file?: Deno.FsFile;
constructor(private filePath: string) {}
public async read(p: Uint8Array): Promise<number | null> { if (!this.file) { this.file = await Deno.open(this.filePath, { read: true }); } const res = await Deno.read(this.file.rid, p); if (res === null) { Deno.close(this.file.rid); this.file = undefined; } return res; }}
/** * Remove the trailing null codes * @param buffer */function trim(buffer: Uint8Array): Uint8Array { const index = buffer.findIndex((v): boolean => v === 0); if (index < 0) return buffer; return buffer.subarray(0, index);}
/** * Initialize Uint8Array of the specified length filled with 0 * @param length */function clean(length: number): Uint8Array { const buffer = new Uint8Array(length); buffer.fill(0, 0, length - 1); return buffer;}
function pad(num: number, bytes: number, base = 8): string { const numString = num.toString(base); return "000000000000".substr(numString.length + 12 - bytes) + numString;}
enum FileTypes { "file" = 0, "link" = 1, "symlink" = 2, "character-device" = 3, "block-device" = 4, "directory" = 5, "fifo" = 6, "contiguous-file" = 7,}
/*struct posix_header { // byte offset char name[100]; // 0 char mode[8]; // 100 char uid[8]; // 108 char gid[8]; // 116 char size[12]; // 124 char mtime[12]; // 136 char chksum[8]; // 148 char typeflag; // 156 char linkname[100]; // 157 char magic[6]; // 257 char version[2]; // 263 char uname[32]; // 265 char gname[32]; // 297 char devmajor[8]; // 329 char devminor[8]; // 337 char prefix[155]; // 345 // 500};*/
const ustarStructure: Array<{ field: string; length: number }> = [ { field: "fileName", length: 100, }, { field: "fileMode", length: 8, }, { field: "uid", length: 8, }, { field: "gid", length: 8, }, { field: "fileSize", length: 12, }, { field: "mtime", length: 12, }, { field: "checksum", length: 8, }, { field: "type", length: 1, }, { field: "linkName", length: 100, }, { field: "ustar", length: 8, }, { field: "owner", length: 32, }, { field: "group", length: 32, }, { field: "majorNumber", length: 8, }, { field: "minorNumber", length: 8, }, { field: "fileNamePrefix", length: 155, }, { field: "padding", length: 12, },];
/** * Create header for a file in a tar archive */function formatHeader(data: TarData): Uint8Array { const encoder = new TextEncoder(), buffer = clean(512); let offset = 0; ustarStructure.forEach(function (value): void { const entry = encoder.encode(data[value.field as keyof TarData] || ""); buffer.set(entry, offset); offset += value.length; // space it out with nulls }); return buffer;}
/** * Parse file header in a tar archive * @param length */function parseHeader(buffer: Uint8Array): { [key: string]: Uint8Array } { const data: { [key: string]: Uint8Array } = {}; let offset = 0; ustarStructure.forEach(function (value): void { const arr = buffer.subarray(offset, offset + value.length); data[value.field] = arr; offset += value.length; }); return data;}
interface TarHeader { [key: string]: Uint8Array;}
export interface TarData { fileName?: string; fileNamePrefix?: string; fileMode?: string; uid?: string; gid?: string; fileSize?: string; mtime?: string; checksum?: string; type?: string; ustar?: string; owner?: string; group?: string;}
export interface TarDataWithSource extends TarData { /** * file to read */ filePath?: string; /** * buffer to read */ reader?: Reader;}
export interface TarInfo { fileMode?: number; mtime?: number; uid?: number; gid?: number; owner?: string; group?: string; type?: string;}
export interface TarOptions extends TarInfo { /** * append file */ filePath?: string;
/** * append any arbitrary content */ reader?: Reader;
/** * size of the content to be appended */ contentSize?: number;}
export interface TarMeta extends TarInfo { fileName: string; fileSize?: number;}
// deno-lint-ignore no-empty-interfaceinterface TarEntry extends TarMeta {}
/** * A class to create a tar archive */export class Tar { data: TarDataWithSource[];
constructor() { this.data = []; }
/** * Append a file to this tar archive * @param fn file name * e.g., test.txt; use slash for directory separators * @param opts options */ async append(fn: string, opts: TarOptions) { if (typeof fn !== "string") { throw new Error("file name not specified"); } let fileName = fn; // separate file name into two parts if needed let fileNamePrefix: string | undefined; if (fileName.length > 100) { let i = fileName.length; while (i >= 0) { i = fileName.lastIndexOf("/", i); if (i <= 155) { fileNamePrefix = fileName.substr(0, i); fileName = fileName.substr(i + 1); break; } i--; } const errMsg = "ustar format does not allow a long file name (length of [file name" + "prefix] + / + [file name] must be shorter than 256 bytes)"; if (i < 0 || fileName.length > 100) { throw new Error(errMsg); } else { assert(fileNamePrefix != null); if (fileNamePrefix.length > 155) { throw new Error(errMsg); } } }
opts = opts || {};
// set meta data let info: Deno.FileInfo | undefined; if (opts.filePath) { info = await Deno.stat(opts.filePath); if (info.isDirectory) { info.size = 0; opts.reader = new Buffer(); } }
const mode = opts.fileMode || (info && info.mode) || parseInt("777", 8) & 0xfff, mtime = Math.floor( opts.mtime ?? (info?.mtime ?? new Date()).valueOf() / 1000, ), uid = opts.uid || 0, gid = opts.gid || 0; if (typeof opts.owner === "string" && opts.owner.length >= 32) { throw new Error( "ustar format does not allow owner name length >= 32 bytes", ); } if (typeof opts.group === "string" && opts.group.length >= 32) { throw new Error( "ustar format does not allow group name length >= 32 bytes", ); }
const fileSize = info?.size ?? opts.contentSize; assert(fileSize != null, "fileSize must be set");
const type = opts.type ? FileTypes[opts.type as keyof typeof FileTypes] : (info?.isDirectory ? FileTypes.directory : FileTypes.file); const tarData: TarDataWithSource = { fileName, fileNamePrefix, fileMode: pad(mode, 7), uid: pad(uid, 7), gid: pad(gid, 7), fileSize: pad(fileSize, 11), mtime: pad(mtime, 11), checksum: " ", type: type.toString(), ustar, owner: opts.owner || "", group: opts.group || "", filePath: opts.filePath, reader: opts.reader, };
// calculate the checksum let checksum = 0; const encoder = new TextEncoder(); Object.keys(tarData) .filter((key): boolean => ["filePath", "reader"].indexOf(key) < 0) .forEach(function (key): void { checksum += encoder .encode(tarData[key as keyof TarData]) .reduce((p, c): number => p + c, 0); });
tarData.checksum = pad(checksum, 6) + "\u0000 "; this.data.push(tarData); }
/** * Get a Reader instance for this tar data */ getReader(): Reader { const readers: Reader[] = []; this.data.forEach((tarData): void => { let { reader } = tarData; const { filePath } = tarData; const headerArr = formatHeader(tarData); readers.push(new Buffer(headerArr)); if (!reader) { assert(filePath != null); reader = new FileReader(filePath); } readers.push(reader);
// to the nearest multiple of recordSize assert(tarData.fileSize != null, "fileSize must be set"); readers.push( new Buffer( clean( recordSize - (parseInt(tarData.fileSize, 8) % recordSize || recordSize), ), ), ); });
// append 2 empty records readers.push(new Buffer(clean(recordSize * 2))); return new MultiReader(readers); }}
class TarEntry implements Reader { #header: TarHeader; #reader: Reader | (Reader & Deno.Seeker); #size: number; #read = 0; #consumed = false; #entrySize: number; constructor( meta: TarMeta, header: TarHeader, reader: Reader | (Reader & Deno.Seeker), ) { Object.assign(this, meta); this.#header = header; this.#reader = reader;
// File Size this.#size = this.fileSize || 0; // Entry Size const blocks = Math.ceil(this.#size / recordSize); this.#entrySize = blocks * recordSize; }
get consumed(): boolean { return this.#consumed; }
async read(p: Uint8Array): Promise<number | null> { // Bytes left for entry const entryBytesLeft = this.#entrySize - this.#read; const bufSize = Math.min( // bufSize can't be greater than p.length nor bytes left in the entry p.length, entryBytesLeft, );
if (entryBytesLeft <= 0) { this.#consumed = true; return null; }
const block = new Uint8Array(bufSize); const n = await readBlock(this.#reader, block); const bytesLeft = this.#size - this.#read;
this.#read += n || 0; if (n === null || bytesLeft <= 0) { if (n === null) this.#consumed = true; return null; }
// Remove zero filled const offset = bytesLeft < n ? bytesLeft : n; p.set(block.subarray(0, offset), 0);
return offset < 0 ? n - Math.abs(offset) : offset; }
async discard() { // Discard current entry if (this.#consumed) return; this.#consumed = true;
if (typeof (this.#reader as Seeker).seek === "function") { await (this.#reader as Seeker).seek( this.#entrySize - this.#read, Deno.SeekMode.Current, ); this.#read = this.#entrySize; } else { await readAll(this); } }}
/** * A class to extract a tar archive */export class Untar { reader: Reader; block: Uint8Array; #entry: TarEntry | undefined;
constructor(reader: Reader) { this.reader = reader; this.block = new Uint8Array(recordSize); }
#checksum(header: Uint8Array): number { let sum = initialChecksum; for (let i = 0; i < 512; i++) { if (i >= 148 && i < 156) { // Ignore checksum header continue; } sum += header[i]; } return sum; }
async #getHeader(): Promise<TarHeader | null> { await readBlock(this.reader, this.block); const header = parseHeader(this.block);
// calculate the checksum const decoder = new TextDecoder(); const checksum = this.#checksum(this.block);
if (parseInt(decoder.decode(header.checksum), 8) !== checksum) { if (checksum === initialChecksum) { // EOF return null; } throw new Error("checksum error"); }
const magic = decoder.decode(header.ustar);
if (magic.indexOf("ustar")) { throw new Error(`unsupported archive format: ${magic}`); }
return header; }
#getMetadata(header: TarHeader): TarMeta { const decoder = new TextDecoder(); // get meta data const meta: TarMeta = { fileName: decoder.decode(trim(header.fileName)), }; const fileNamePrefix = trim(header.fileNamePrefix); if (fileNamePrefix.byteLength > 0) { meta.fileName = decoder.decode(fileNamePrefix) + "/" + meta.fileName; } (["fileMode", "mtime", "uid", "gid"] as [ "fileMode", "mtime", "uid", "gid", ]).forEach((key): void => { const arr = trim(header[key]); if (arr.byteLength > 0) { meta[key] = parseInt(decoder.decode(arr), 8); } }); (["owner", "group", "type"] as ["owner", "group", "type"]).forEach( (key): void => { const arr = trim(header[key]); if (arr.byteLength > 0) { meta[key] = decoder.decode(arr); } }, );
meta.fileSize = parseInt(decoder.decode(header.fileSize), 8); meta.type = FileTypes[parseInt(meta.type!)] ?? meta.type;
return meta; }
async extract(): Promise<TarEntry | null> { if (this.#entry && !this.#entry.consumed) { // If entry body was not read, discard the body // so we can read the next entry. await this.#entry.discard(); }
const header = await this.#getHeader(); if (header === null) return null;
const meta = this.#getMetadata(header);
this.#entry = new TarEntry(meta, header, this.reader);
return this.#entry; }
async *[Symbol.asyncIterator](): AsyncIterableIterator<TarEntry> { while (true) { const entry = await this.extract();
if (entry === null) return;
yield entry; } }}