Popular
Latest
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443import { concat } from "./deps.ts";import { assertEquals, assertRejects } from "./dev_deps.ts";import { readCSV, readCSVObjects, readCSVRows, readCSVStream,} from "./reader.ts";import { asyncArrayFrom, asyncArrayFrom2 } from "./utils.ts";
class MyReader implements Deno.Reader { private buf: Uint8Array; private index: number;
constructor(content: string, options: { withBom?: boolean } = {}) { const opts = { withBom: false, ...options }; this.buf = new TextEncoder().encode(content); if (opts.withBom) { this.buf = concat(new Uint8Array([0xef, 0xbb, 0xbf]), this.buf); } this.index = 0; }
public read(p: Uint8Array): Promise<number | null> { const unread = this.buf.length - this.index;
if (unread <= 0) { return Promise.resolve(null); }
const toRead = Math.min(p.length, unread);
p.set(this.buf.subarray(this.index, this.index + toRead)); this.index += toRead;
return Promise.resolve(toRead); }
public reset() { this.index = 0; }}
Deno.test({ name: "readCSVObjects parses simple file", async fn() { const reader = new MyReader(`a,b,c1,2,3`);
const rows = await asyncArrayFrom(readCSVObjects(reader));
assertEquals(rows, [{ a: "1", b: "2", c: "3" }]); },});
Deno.test({ name: "readCSV parses simple file", async fn() { const reader = new MyReader(`1,2,3a,b,c`);
const rows = await asyncArrayFrom2(readCSV(reader));
assertEquals(rows, [ ["1", "2", "3"], ["a", "b", "c"], ]); },});
Deno.test({ name: "readCSV parses file with UTF BOM", async fn() { const reader = new MyReader(`"1","2"`, { withBom: true });
const rows = await asyncArrayFrom2(readCSV(reader));
assertEquals(rows, [["1", "2"]]); },});
Deno.test({ name: "readCSV skips empty lines", async fn() { const reader = new MyReader(`1,2,3
a,b,c`);
const rows = await asyncArrayFrom2(readCSV(reader));
assertEquals(rows, [ ["1", "2", "3"], ["a", "b", "c"], ]); },});
Deno.test({ name: "readCSV parses emoji", async fn() { const reader = new MyReader(`😀,2,3a,😀,c`);
const rows = await asyncArrayFrom2(readCSV(reader));
assertEquals(rows, [ ["😀", "2", "3"], ["a", "😀", "c"], ]); },});
Deno.test({ name: "readCSV parses file with quotes", async fn() { const reader = new MyReader(`1,"2",3a,"b""1",c`);
const rows = await asyncArrayFrom2(readCSV(reader));
assertEquals(rows, [ ["1", "2", "3"], ["a", 'b\n"1', "c"], ]); },});
Deno.test({ name: "readCSV parses file with custom separators", async fn() { const reader = new MyReader(`a\tb\tc\r\n1\t2\t$$$3$`);
const rows = await asyncArrayFrom2( readCSV(reader, { quote: "$", lineSeparator: new TextEncoder().encode("\r\n"), columnSeparator: "\t", }), );
assertEquals(rows, [ ["a", "b", "c"], ["1", "2", "$3"], ]); },});
Deno.test({ name: "readCSV throws when quote is unclosed", async fn() { const reader = new MyReader(`1,"2`);
await assertRejects( async () => { await asyncArrayFrom2(readCSV(reader)); }, Error, "Expected quote, received EOF (line 1, character 5)", ); },});
Deno.test({ name: "readCSV throws when quote is not last character in column", async fn() { const reader = new MyReader(`1,"2"3`);
await assertRejects( async () => { await asyncArrayFrom2(readCSV(reader)); }, Error, "Expected EOF, COLUMN_SEPARATOR, LINE_SEPARATOR; received 3 (line 1, character 6)", ); },});
Deno.test({ name: "readCSV throws when quote is found in unquoted field", async fn() { const reader = new MyReader(`1,2 "3",4`);
await assertRejects( async () => { await asyncArrayFrom2(readCSV(reader)); }, Error, "Unexpected quote in unquoted field (line 1, character 5)", ); },});
Deno.test({ name: "readCSV calculates error position", async fn() { const reader = new MyReader(`1,23,4
5,"123",,,"123
1"21,2`);
await assertRejects( async () => { await asyncArrayFrom2(readCSV(reader)); }, Error, "Expected EOF, COLUMN_SEPARATOR, LINE_SEPARATOR; received 2 (line 7, character 3)", ); },});
Deno.test({ name: "readCSV parses huge file", async fn() { const stats = { reads: 0, inputBufferShrinks: 0, columnBufferExpands: 0, }; const reader = new MyReader( `aaaaaaaaaaaaaaaaaaaa,bbbbbbbbbbbbbbbbbbbbb\n11111111111111111111,22222222222222222222`, );
const rows = await asyncArrayFrom2( readCSV(reader, { _readerIteratorBufferSize: 1, _columnBufferMinStepSize: 1, _inputBufferIndexLimit: 1, _columnBufferReserve: 1, _stats: stats, // deno-lint-ignore no-explicit-any } as any), );
assertEquals(rows, [ ["aaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbb"], ["11111111111111111111", "22222222222222222222"], ]); assertEquals(stats, { reads: 85, inputBufferShrinks: 84, columnBufferExpands: 11, }); },});
Deno.test({ name: "readCSV read rows correctly even when rowsIterator not read", async fn() { const reader = new MyReader(`a,b\n1,2\n3,4`);
let n = 0; for await (const _row of readCSV(reader)) { n++; }
assertEquals(n, 3); },});
Deno.test({ name: "readCSVStream couldn't be used twice", async fn() { const reader = new MyReader(`a,b\n1,2\n3,4`); const r = readCSVStream(reader);
let a = 0; for await (const _token of r) { a++; } reader.reset(); let b = 0; for await (const _token of r) { b++; }
assertEquals(a, 9); assertEquals(b, 0); },});
Deno.test({ name: "readCSVRows couldn't be used twice", async fn() { const reader = new MyReader(`a,b\n1,2\n3,4`); const r = readCSVRows(reader);
let a = 0; for await (const _row of r) { a++; } reader.reset(); let b = 0; for await (const _row of r) { b++; }
assertEquals(a, 3); assertEquals(b, 0); },});
Deno.test({ name: "readCSV couldn't be used twice", async fn() { const reader = new MyReader(`a,b\n1,2\n3,4`); const r = readCSV(reader);
let a = 0; for await (const _row of r) { a++; } reader.reset(); let b = 0; for await (const _row of r) { b++; }
assertEquals(a, 3); assertEquals(b, 0); },});
Deno.test({ name: "readCSVObjects couldn't be used twice", async fn() { const reader = new MyReader(`a,b\n1,2\n3,4`); const r = readCSVObjects(reader);
let a = 0; for await (const _obj of r) { a++; } reader.reset(); let b = 0; for await (const _obj of r) { b++; }
assertEquals(a, 2); assertEquals(b, 0); },});
Deno.test({ name: "readCSVRows should work with long cell", async fn() { const reader = new MyReader( `"{""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false, ""key1"": false}"`, );
let count = 0; for await (const _row of readCSVRows(reader)) { count++; }
assertEquals(count, 1); },});
Deno.test({ name: "readCSVRows should skip lines if options.fromLine and options.toLine passed", async fn() { const reader = new MyReader( `a,bc,de,fg,h`, );
const rows = await asyncArrayFrom( readCSVRows(reader, { fromLine: 1, toLine: 3 }), );
assertEquals(rows, [ ["c", "d"], ["e", "f"], ]); },});
Deno.test({ name: "readCSVRows options.toLine should be exclusive", async fn() { const reader = new MyReader( `a,bc,de,fg,h`, );
const rows = await asyncArrayFrom( readCSVRows(reader, { fromLine: 1, toLine: 2 }), );
assertEquals(rows, [["c", "d"]]); },});
Deno.test({ name: "readCSVRows can read only the first line", async fn() { const reader = new MyReader( `1,2,3a,b,c!,@,#`, );
const rows = await asyncArrayFrom( readCSVRows(reader, { fromLine: 0, toLine: 1, }), );
assertEquals(rows, [["1", "2", "3"]]); },});
Deno.test({ name: "readCSVRows can read empty lines (not prepends to the next line)", async fn() { const reader = new MyReader( `col1,col2,col3a,b,c,,d,e,f`, ); const rows = await asyncArrayFrom(readCSVRows(reader)); assertEquals(rows, [ ["col1", "col2", "col3"], ["a", "b", "c"], ["", "", ""], ["d", "e", "f"], ]); },});