commit d1fbb6eefd282ab77dd93728bd184f8c3b9ec12a Author: Fabian Stamm Date: Sun Jul 26 13:18:06 2020 +0200 First commit containing tar package diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e5c48e1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root=true +[*] +charset = utf-8 +end_of_line = lf +indent_size = 3 +indent_style = space +insert_final_newline = true diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..aeb028d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "deno.enable": true +} \ No newline at end of file diff --git a/tar/.gitignore b/tar/.gitignore new file mode 100644 index 0000000..04d8d3d --- /dev/null +++ b/tar/.gitignore @@ -0,0 +1,4 @@ +demo.tar +demo2.tar +demo-unpacked/ +demo2-unpacked/ diff --git a/tar/demo/file1.ts b/tar/demo/file1.ts new file mode 100644 index 0000000..e69de29 diff --git a/tar/demo/folder/file2.ts b/tar/demo/folder/file2.ts new file mode 100644 index 0000000..e69de29 diff --git a/tar/dev.sh b/tar/dev.sh new file mode 100755 index 0000000..1a820a6 --- /dev/null +++ b/tar/dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +deno run -A https://raw.githubusercontent.com/hibas123/denovamon/master/denovamon.ts start --ignore="demo.tar,demo" --command="deno run -A --unstable src/mod_test.ts" diff --git a/tar/mod.ts b/tar/mod.ts new file mode 100644 index 0000000..a1ca67a --- /dev/null +++ b/tar/mod.ts @@ -0,0 +1 @@ +export * from "./src/mod.ts"; diff --git a/tar/src/deps.ts b/tar/src/deps.ts new file mode 100644 index 0000000..693598b --- /dev/null +++ b/tar/src/deps.ts @@ -0,0 +1,6 @@ +export * as Tar from "https://deno.land/std@0.62.0/archive/tar.ts"; +export * as Path from "https://deno.land/std@0.62.0/path/mod.ts"; +export * as FS from "https://deno.land/std@0.62.0/fs/mod.ts"; +export * as IO_Readers from "https://deno.land/std@0.62.0/io/readers.ts"; +export * as IO_BufIO from "https://deno.land/std@0.62.0/io/bufio.ts"; +export * as Assert from "https://deno.land/std@0.62.0/_util/assert.ts"; diff --git a/tar/src/mod.ts b/tar/src/mod.ts new file mode 100644 index 0000000..7c288bf --- /dev/null +++ b/tar/src/mod.ts @@ -0,0 +1,73 @@ +import { FS, Path } from "./deps.ts"; + +import { Tar, Untar } from "./tar.ts"; + +export async function uncompress(src: string, dest: string): Promise { + const tarFile = await Deno.open(src, { read: true }); + const untar = new Untar(tarFile); + + for await (const entry of untar) { + if (entry.type === "directory") { + await FS.ensureDir(Path.join(dest, entry.fileName)); + } else if (entry.type === "file") { + const path = Path.join(dest, entry.fileName); + await FS.ensureFile(path); + const file = await Deno.open(path, { write: true, create: true }); + await Deno.copy(entry, file); + file.close(); + } else { + //TODO: Error/Warning + } + } + tarFile.close(); +} + +export interface ICompressOptions { + excludeSrc?: boolean; +} + +export async function compress( + src: string, + dest: string, + options?: ICompressOptions +): Promise { + src = Path.resolve(src); + const tar = new Tar(); + + const stat = await Deno.lstat(src); + if (stat.isFile) { + tar.append(Path.basename(src), { + filePath: src, + }); + } else { + let root = src; + if (!options?.excludeSrc) { + root = Path.dirname(src); + await tar.append(Path.basename(src), { + type: "directory", + reader: new Deno.Buffer(), + contentSize: 0, + }); + } + const walker = FS.walk(src, { includeDirs: true, includeFiles: true }); + for await (const file of walker) { + const relativePath = Path.relative(root, file.path); + if (file.isDirectory) { + await tar.append(relativePath, { + type: "directory", + reader: new Deno.Buffer(), + contentSize: 0, + }); + } else { + await tar.append(relativePath, { + filePath: file.path, + type: "file", + }); + } + } + } + + const writer = await Deno.open(dest, { write: true, create: true }); + await Deno.copy(tar.getReader(), writer); + writer.close(); +} diff --git a/tar/src/mod_test.ts b/tar/src/mod_test.ts new file mode 100644 index 0000000..e5e7e57 --- /dev/null +++ b/tar/src/mod_test.ts @@ -0,0 +1,13 @@ +import { compress, uncompress } from "./mod.ts"; + +console.log("Compressing (excludeSrc:false)"); +await compress("demo", "demo.tar"); + +console.log("Decompressing (excludeSrc:false)"); +await uncompress("demo.tar", "demo-unpacked"); + +console.log("Compressing (excludeSrc:true)"); +await compress("demo", "demo2.tar", { excludeSrc: true }); + +console.log("Decompressing (excludeSrc:true)"); +await uncompress("demo2.tar", "demo2-unpacked"); diff --git a/tar/src/tar.ts b/tar/src/tar.ts new file mode 100644 index 0000000..46ce97c --- /dev/null +++ b/tar/src/tar.ts @@ -0,0 +1,639 @@ +/** + * Ported and modified from: https://github.com/beatgammit/tar-js and + * licensed as: + * + * (The MIT License) + * + * Copyright (c) 2011 T. Jameson Little + * Copyright (c) 2019 Jun Kato + * Copyright (c) 2020 the Deno authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +import { IO_Readers, IO_BufIO, Assert } from "./deps.ts"; + +const { MultiReader } = IO_Readers; +const { PartialReadError } = IO_BufIO; + +type Reader = Deno.Reader; +type Seeker = Deno.Seeker; + +const recordSize = 512; +const ustar = "ustar\u000000"; + +// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 +// eight checksum bytes taken to be ascii spaces (decimal value 32) +const initialChecksum = 8 * 32; + +async function readBlock( + reader: Deno.Reader, + p: Uint8Array +): Promise { + let bytesRead = 0; + while (bytesRead < p.length) { + const rr = await reader.read(p.subarray(bytesRead)); + if (rr === null) { + if (bytesRead === 0) { + return null; + } else { + throw new PartialReadError(); + } + } + bytesRead += rr; + } + return bytesRead; +} + +/** + * Simple file reader + */ +class FileReader implements Reader { + private file?: Deno.File; + + constructor(private filePath: string) {} + + public async read(p: Uint8Array): Promise { + if (!this.file) { + this.file = await Deno.open(this.filePath, { read: true }); + } + const res = await Deno.read(this.file.rid, p); + if (res === null) { + Deno.close(this.file.rid); + this.file = undefined; + } + return res; + } +} + +/** + * Remove the trailing null codes + * @param buffer + */ +function trim(buffer: Uint8Array): Uint8Array { + const index = buffer.findIndex((v): boolean => v === 0); + if (index < 0) return buffer; + return buffer.subarray(0, index); +} + +/** + * Initialize Uint8Array of the specified length filled with 0 + * @param length + */ +function clean(length: number): Uint8Array { + const buffer = new Uint8Array(length); + buffer.fill(0, 0, length - 1); + return buffer; +} + +function pad(num: number, bytes: number, base?: number): string { + const numString = num.toString(base || 8); + return "000000000000".substr(numString.length + 12 - bytes) + numString; +} + +const types: { [key: string]: string } = { + "": "file", + "0": "file", + "1": "link", + "2": "symlink", + "3": "character-device", + "4": "block-device", + "5": "directory", +}; + +/* +struct posix_header { // byte offset + char name[100]; // 0 + char mode[8]; // 100 + char uid[8]; // 108 + char gid[8]; // 116 + char size[12]; // 124 + char mtime[12]; // 136 + char chksum[8]; // 148 + char typeflag; // 156 + char linkname[100]; // 157 + char magic[6]; // 257 + char version[2]; // 263 + char uname[32]; // 265 + char gname[32]; // 297 + char devmajor[8]; // 329 + char devminor[8]; // 337 + char prefix[155]; // 345 + // 500 +}; +*/ + +const ustarStructure: Array<{ field: string; length: number }> = [ + { + field: "fileName", + length: 100, + }, + { + field: "fileMode", + length: 8, + }, + { + field: "uid", + length: 8, + }, + { + field: "gid", + length: 8, + }, + { + field: "fileSize", + length: 12, + }, + { + field: "mtime", + length: 12, + }, + { + field: "checksum", + length: 8, + }, + { + field: "type", + length: 1, + }, + { + field: "linkName", + length: 100, + }, + { + field: "ustar", + length: 8, + }, + { + field: "owner", + length: 32, + }, + { + field: "group", + length: 32, + }, + { + field: "majorNumber", + length: 8, + }, + { + field: "minorNumber", + length: 8, + }, + { + field: "fileNamePrefix", + length: 155, + }, + { + field: "padding", + length: 12, + }, +]; + +/** + * Create header for a file in a tar archive + */ +function formatHeader(data: TarData): Uint8Array { + const encoder = new TextEncoder(), + buffer = clean(512); + let offset = 0; + ustarStructure.forEach(function (value): void { + const entry = encoder.encode(data[value.field as keyof TarData] || ""); + buffer.set(entry, offset); + offset += value.length; // space it out with nulls + }); + return buffer; +} + +/** + * Parse file header in a tar archive + * @param length + */ +function parseHeader(buffer: Uint8Array): { [key: string]: Uint8Array } { + const data: { [key: string]: Uint8Array } = {}; + let offset = 0; + ustarStructure.forEach(function (value): void { + const arr = buffer.subarray(offset, offset + value.length); + data[value.field] = arr; + offset += value.length; + }); + return data; +} + +interface TarHeader { + [key: string]: Uint8Array; +} + +export interface TarData { + fileName?: string; + fileNamePrefix?: string; + fileMode?: string; + uid?: string; + gid?: string; + fileSize?: string; + mtime?: string; + checksum?: string; + type?: string; + ustar?: string; + owner?: string; + group?: string; +} + +export interface TarDataWithSource extends TarData { + /** + * file to read + */ + filePath?: string; + /** + * buffer to read + */ + reader?: Reader; +} + +export interface TarInfo { + fileMode?: number; + mtime?: number; + uid?: number; + gid?: number; + owner?: string; + group?: string; + type?: string; +} + +export interface TarOptions extends TarInfo { + /** + * append file + */ + filePath?: string; + + /** + * append any arbitrary content + */ + reader?: Reader; + + /** + * size of the content to be appended + */ + contentSize?: number; +} + +export interface TarMeta extends TarInfo { + fileName: string; + fileSize?: number; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +interface TarEntry extends TarMeta {} + +/** + * A class to create a tar archive + */ +export class Tar { + data: TarDataWithSource[]; + + constructor() { + this.data = []; + } + + /** + * Append a file to this tar archive + * @param fn file name + * e.g., test.txt; use slash for directory separators + * @param opts options + */ + async append(fn: string, opts: TarOptions): Promise { + if (typeof fn !== "string") { + throw new Error("file name not specified"); + } + let fileName = fn; + // separate file name into two parts if needed + let fileNamePrefix: string | undefined; + if (fileName.length > 100) { + let i = fileName.length; + while (i >= 0) { + i = fileName.lastIndexOf("/", i); + if (i <= 155) { + fileNamePrefix = fileName.substr(0, i); + fileName = fileName.substr(i + 1); + break; + } + i--; + } + const errMsg = + "ustar format does not allow a long file name (length of [file name" + + "prefix] + / + [file name] must be shorter than 256 bytes)"; + if (i < 0 || fileName.length > 100) { + throw new Error(errMsg); + } else { + Assert.assert(fileNamePrefix != null); + if (fileNamePrefix.length > 155) { + throw new Error(errMsg); + } + } + } + + opts = opts || {}; + + // set meta data + let info: Deno.FileInfo | undefined; + if (opts.filePath) { + info = await Deno.stat(opts.filePath); + } + + const mode = + opts.fileMode || (info && info.mode) || parseInt("777", 8) & 0xfff, + mtime = Math.floor( + opts.mtime ?? (info?.mtime ?? new Date()).valueOf() / 1000 + ), + uid = opts.uid || 0, + gid = opts.gid || 0; + if (typeof opts.owner === "string" && opts.owner.length >= 32) { + throw new Error( + "ustar format does not allow owner name length >= 32 bytes" + ); + } + if (typeof opts.group === "string" && opts.group.length >= 32) { + throw new Error( + "ustar format does not allow group name length >= 32 bytes" + ); + } + + const fileSize = info?.size ?? opts.contentSize; + Assert.assert(fileSize != null, "fileSize must be set"); + + let type = "0"; + if (opts.type) { + if (types[opts.type]) type = opts.type; + else { + type = + Object.keys(types).find((e) => types[e] === opts.type) || "0"; + } + } + + const tarData: TarDataWithSource = { + fileName, + fileNamePrefix, + fileMode: pad(mode, 7), + uid: pad(uid, 7), + gid: pad(gid, 7), + fileSize: pad(fileSize, 11), + mtime: pad(mtime, 11), + checksum: " ", + type, // just a file + ustar, + owner: opts.owner || "", + group: opts.group || "", + filePath: opts.filePath, + reader: opts.reader, + }; + + // calculate the checksum + let checksum = 0; + const encoder = new TextEncoder(); + Object.keys(tarData) + .filter((key): boolean => ["filePath", "reader"].indexOf(key) < 0) + .forEach(function (key): void { + checksum += encoder + .encode(tarData[key as keyof TarData]) + .reduce((p, c): number => p + c, 0); + }); + + tarData.checksum = pad(checksum, 6) + "\u0000 "; + this.data.push(tarData); + } + + /** + * Get a Reader instance for this tar data + */ + getReader(): Reader { + const readers: Reader[] = []; + this.data.forEach((tarData): void => { + let { reader } = tarData; + const { filePath } = tarData; + const headerArr = formatHeader(tarData); + readers.push(new Deno.Buffer(headerArr)); + if (!reader) { + Assert.assert(filePath != null); + reader = new FileReader(filePath); + } + readers.push(reader); + + // to the nearest multiple of recordSize + Assert.assert(tarData.fileSize != null, "fileSize must be set"); + readers.push( + new Deno.Buffer( + clean( + recordSize - + (parseInt(tarData.fileSize, 8) % recordSize || recordSize) + ) + ) + ); + }); + + // append 2 empty records + readers.push(new Deno.Buffer(clean(recordSize * 2))); + return new MultiReader(...readers); + } +} + +class TarEntry implements Reader { + #header: TarHeader; + #reader: Reader | (Reader & Deno.Seeker); + #size: number; + #read = 0; + #consumed = false; + #entrySize: number; + constructor( + meta: TarMeta, + header: TarHeader, + reader: Reader | (Reader & Deno.Seeker) + ) { + Object.assign(this, meta); + this.#header = header; + this.#reader = reader; + + // File Size + this.#size = this.fileSize || 0; + // Entry Size + const blocks = Math.ceil(this.#size / recordSize); + this.#entrySize = blocks * recordSize; + } + + get consumed(): boolean { + return this.#consumed; + } + + async read(p: Uint8Array): Promise { + // Bytes left for entry + const entryBytesLeft = this.#entrySize - this.#read; + const bufSize = Math.min( + // bufSize can't be greater than p.length nor bytes left in the entry + p.length, + entryBytesLeft + ); + + if (entryBytesLeft <= 0) return null; + + const block = new Uint8Array(bufSize); + const n = await readBlock(this.#reader, block); + const bytesLeft = this.#size - this.#read; + + this.#read += n || 0; + if (n === null || bytesLeft <= 0) { + if (null) this.#consumed = true; + return null; + } + + // Remove zero filled + const offset = bytesLeft < n ? bytesLeft : n; + p.set(block.subarray(0, offset), 0); + + return offset < 0 ? n - Math.abs(offset) : offset; + } + + async discard(): Promise { + // Discard current entry + if (this.#consumed) return; + this.#consumed = true; + + if (typeof (this.#reader as Seeker).seek === "function") { + await (this.#reader as Seeker).seek( + this.#entrySize - this.#read, + Deno.SeekMode.Current + ); + this.#read = this.#entrySize; + } else { + await Deno.readAll(this); + } + } +} + +/** + * A class to extract a tar archive + */ +export class Untar { + reader: Reader; + block: Uint8Array; + #entry: TarEntry | undefined; + + constructor(reader: Reader) { + this.reader = reader; + this.block = new Uint8Array(recordSize); + } + + #checksum = (header: Uint8Array): number => { + let sum = initialChecksum; + for (let i = 0; i < 512; i++) { + if (i >= 148 && i < 156) { + // Ignore checksum header + continue; + } + sum += header[i]; + } + return sum; + }; + + #getHeader = async (): Promise => { + await readBlock(this.reader, this.block); + const header = parseHeader(this.block); + + // calculate the checksum + const decoder = new TextDecoder(); + const checksum = this.#checksum(this.block); + + if (parseInt(decoder.decode(header.checksum), 8) !== checksum) { + if (checksum === initialChecksum) { + // EOF + return null; + } + throw new Error("checksum error"); + } + + const magic = decoder.decode(header.ustar); + + if (magic.indexOf("ustar")) { + throw new Error(`unsupported archive format: ${magic}`); + } + + return header; + }; + + #getMetadata = (header: TarHeader): TarMeta => { + const decoder = new TextDecoder(); + // get meta data + const meta: TarMeta = { + fileName: decoder.decode(trim(header.fileName)), + }; + const fileNamePrefix = trim(header.fileNamePrefix); + if (fileNamePrefix.byteLength > 0) { + meta.fileName = decoder.decode(fileNamePrefix) + "/" + meta.fileName; + } + (["fileMode", "mtime", "uid", "gid"] as [ + "fileMode", + "mtime", + "uid", + "gid" + ]).forEach((key): void => { + const arr = trim(header[key]); + if (arr.byteLength > 0) { + meta[key] = parseInt(decoder.decode(arr), 8); + } + }); + (["owner", "group", "type"] as ["owner", "group", "type"]).forEach( + (key): void => { + const arr = trim(header[key]); + if (arr.byteLength > 0) { + meta[key] = decoder.decode(arr); + } + } + ); + + meta.fileSize = parseInt(decoder.decode(header.fileSize), 8); + meta.type = types[meta.type as string] || meta.type; + + return meta; + }; + + async extract(): Promise { + if (this.#entry && !this.#entry.consumed) { + // If entry body was not read, discard the body + // so we can read the next entry. + await this.#entry.discard(); + } + + const header = await this.#getHeader(); + if (header === null) return null; + + const meta = this.#getMetadata(header); + + this.#entry = new TarEntry(meta, header, this.reader); + + return this.#entry; + } + + async *[Symbol.asyncIterator](): AsyncIterableIterator { + while (true) { + const entry = await this.extract(); + + if (entry === null) return; + + yield entry; + } + } +}