typing fixes: (#63)

- ensure type IndexerOffsetLength is exported - ensure WARC.create() / readFully() also accepts Iterator as well as AsyncIterator for payload - bump to 2.2.1
webrecorder · Nov 1, 2023 · 397c2a6 · 397c2a6
1 parent c6dd0b8
commit 397c2a6
Show file tree

Hide file tree

Showing 12 changed files with 37 additions and 36 deletions.
diff --git a/dist/cli.cjs b/dist/cli.cjs
diff --git a/dist/cli.js b/dist/cli.js
diff --git a/dist/index.all.js b/dist/index.all.js
diff --git a/dist/index.all.js.map b/dist/index.all.js.map
diff --git a/dist/index.cjs b/dist/index.cjs
diff --git a/dist/index.d.ts b/dist/index.d.ts
@@ -36,7 +36,7 @@ declare class NoConcatInflator<T extends BaseAsyncIterReader> extends pako.Infla
     onEnd(status: pako.ReturnCodes): void;
 }
 declare abstract class BaseAsyncIterReader {
-    static readFully(iter: AsyncIterable<Uint8Array>): Promise<Uint8Array>;
+    static readFully(iter: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): Promise<Uint8Array>;
     abstract [Symbol.asyncIterator](): AsyncIterator<Uint8Array>;
     getReadableStream(): ReadableStream<any>;
     readFully(): Promise<Uint8Array>;
@@ -139,10 +139,10 @@ type WARCRecordOpts = {
     refersToDate?: string;
 };
 declare class WARCRecord extends BaseAsyncIterReader {
-    static create({ url, date, type, warcHeaders, filename, httpHeaders, statusline, warcVersion, keepHeadersCase, refersToUrl, refersToDate, }?: WARCRecordOpts, reader?: AsyncIterable<Uint8Array>): WARCRecord;
+    static create({ url, date, type, warcHeaders, filename, httpHeaders, statusline, warcVersion, keepHeadersCase, refersToUrl, refersToDate, }?: WARCRecordOpts, reader?: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): WARCRecord;
     static createWARCInfo(opts: WARCRecordOpts | undefined, info: Record<string, string>): WARCRecord;
     warcHeaders: StatusAndHeaders;
-    _reader: AsyncIterable<Uint8Array>;
+    _reader: AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
     _contentReader: BaseAsyncIterReader | null;
     payload: Uint8Array | null;
     httpHeaders: StatusAndHeaders | null;
@@ -154,7 +154,7 @@ declare class WARCRecord extends BaseAsyncIterReader {
     _urlkey: string;
     constructor({ warcHeaders, reader, }: {
         warcHeaders: StatusAndHeaders;
-        reader: AsyncIterable<Uint8Array>;
+        reader: AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
     });
     getResponseInfo(): {
         headers: Map<string, string> | Headers;
@@ -163,8 +163,8 @@ declare class WARCRecord extends BaseAsyncIterReader {
     } | null;
     fixUp(): void;
     readFully(isContent?: boolean): Promise<Uint8Array>;
-    get reader(): AsyncIterable<Uint8Array>;
-    get contentReader(): AsyncIterable<Uint8Array>;
+    get reader(): AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
+    get contentReader(): AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
     _createDecodingReader(source: Source): AsyncIterReader;
     readlineRaw(maxLength?: number): Promise<Uint8Array | null>;
     contentText(): Promise<string>;
@@ -299,4 +299,4 @@ declare function mfdToQueryString(mfd: string | Uint8Array, contentType: string)
 declare function concatChunks(chunks: Uint8Array[], size: number): Uint8Array;
 declare function splitChunk(chunk: Uint8Array, inx: number): [Uint8Array, Uint8Array];
 
-export { AsyncIterReader, AsyncIterReaderOpts, BaseAsyncIterReader, BaseSerializerBuffer, CDXAndRecordIndexer, CDXIndexer, Indexer, LimitReader, NoConcatInflator, Request, Source, SourceReadable, SourceReader, StatusAndHeaders, StatusAndHeadersParser, StreamResult, StreamResults, WARCParser, WARCParserOpts, WARCRecord, WARCRecordOpts, WARCSerializer, WARCSerializerOpts, WARCType, WARC_1_0, WARC_1_1, appendRequestQuery, concatChunks, getSurt, jsonToQueryParams, jsonToQueryString, mfdToQueryParams, mfdToQueryString, postToGetUrl, splitChunk };
+export { AsyncIterReader, AsyncIterReaderOpts, BaseAsyncIterReader, BaseSerializerBuffer, CDXAndRecordIndexer, CDXIndexer, Indexer, IndexerOffsetLength, LimitReader, NoConcatInflator, Request, Source, SourceReadable, SourceReader, StatusAndHeaders, StatusAndHeadersParser, StreamResult, StreamResults, WARCParser, WARCParserOpts, WARCRecord, WARCRecordOpts, WARCSerializer, WARCSerializerOpts, WARCType, WARC_1_0, WARC_1_1, appendRequestQuery, concatChunks, getSurt, jsonToQueryParams, jsonToQueryString, mfdToQueryParams, mfdToQueryString, postToGetUrl, splitChunk };
diff --git a/dist/index.js b/dist/index.js
diff --git a/dist/node/index.d.ts b/dist/node/index.d.ts
@@ -1,5 +1,5 @@
 import { WriteStream } from 'node:fs';
-import { f as WARCSerializerOpts$1, b as WARCSerializer$1, W as WARCRecord, j as SerializerInMemBuffer } from '../warcserializer-5e21cd02.js';
+import { f as WARCSerializerOpts$1, b as WARCSerializer$1, W as WARCRecord, j as SerializerInMemBuffer } from '../warcserializer-48f5a2a4.js';
 import 'hash-wasm/dist/lib/WASMInterface';
 import 'pako';
 import '../types-af1fe7fc.js';

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "warcio",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "keywords": [
     "WARC",
     "web archiving"

diff --git a/src/index.ts b/src/index.ts
@@ -40,4 +40,5 @@ export type {
   StreamResult,
   StreamResults,
   Request,
+  IndexerOffsetLength
 } from "./lib";
diff --git a/src/lib/readers.ts b/src/lib/readers.ts
@@ -26,7 +26,7 @@ export class NoConcatInflator<T extends BaseAsyncIterReader> extends pako.Inflat
 
 // ===========================================================================
 export abstract class BaseAsyncIterReader {
-  static async readFully(iter: AsyncIterable<Uint8Array>) : Promise<Uint8Array> {
+  static async readFully(iter: AsyncIterable<Uint8Array> | Iterable<Uint8Array>) : Promise<Uint8Array> {
     const chunks = [];
     let size = 0;
 

diff --git a/src/lib/warcrecord.ts b/src/lib/warcrecord.ts
@@ -63,7 +63,7 @@ export class WARCRecord extends BaseAsyncIterReader {
       refersToUrl = undefined,
       refersToDate = undefined,
     }: WARCRecordOpts = {},
-    reader?: AsyncIterable<Uint8Array>
+    reader?: AsyncIterable<Uint8Array> | Iterable<Uint8Array>
   ) {
     function checkDate(d: string) {
       const date = d;
@@ -163,7 +163,7 @@ export class WARCRecord extends BaseAsyncIterReader {
   }
 
   warcHeaders: StatusAndHeaders;
-  _reader: AsyncIterable<Uint8Array>;
+  _reader: AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
   _contentReader: BaseAsyncIterReader | null;
   payload: Uint8Array | null;
   httpHeaders: StatusAndHeaders | null;
@@ -181,7 +181,7 @@ export class WARCRecord extends BaseAsyncIterReader {
     reader,
   }: {
     warcHeaders: StatusAndHeaders;
-    reader: AsyncIterable<Uint8Array>;
+    reader: AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
   }) {
     super();
 
@@ -250,7 +250,7 @@ export class WARCRecord extends BaseAsyncIterReader {
       this.payload = await super.readFully();
       this.consumed = "content";
     } else {
-      this.payload = await WARCRecord.readFully(this._reader);
+      this.payload = await BaseAsyncIterReader.readFully(this._reader);
       this.consumed = "raw";
     }