From 34b8383e47daa767012fc790a8cdd9a58fbbe284 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 3 Mar 2021 13:29:58 -0800 Subject: [PATCH] LimitReader: check if _unread() exists before calling to make limitreader more flexible, work with any async iterator bump to 1.4.2 --- dist/warcio.js | 4 +++- dist/warcio.min.js | 2 +- package.json | 2 +- src/readers.js | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dist/warcio.js b/dist/warcio.js index 77d41e3..b7c041f 100644 --- a/dist/warcio.js +++ b/dist/warcio.js @@ -3676,7 +3676,9 @@ class LimitReader extends BaseAsyncIterReader const [first, remainder] = LimitReader.splitChunk(chunk, this.limit); chunk = first; - this.sourceIter._unread(remainder); + if (this.sourceIter._unread) { + this.sourceIter._unread(remainder); + } } if (chunk.length) { diff --git a/dist/warcio.min.js b/dist/warcio.min.js index d9d8063..8882b8b 100644 --- a/dist/warcio.min.js +++ b/dist/warcio.min.js @@ -3,7 +3,7 @@ * See https://github.com/webrecorder/warcio.js for more info. Copyright (C) 2020, Webrecorder Software */ -var t="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function e(t,e){return t(e={exports:{}},e.exports),e.exports}var a=e((function(t,e){var a="undefined"!=typeof Uint8Array&&"undefined"!=typeof Uint16Array&&"undefined"!=typeof Int32Array;function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}e.assign=function(t){for(var e=Array.prototype.slice.call(arguments,1);e.length;){var a=e.shift();if(a){if("object"!=typeof a)throw new TypeError(a+"must be non-object");for(var s in a)r(a,s)&&(t[s]=a[s])}}return t},e.shrinkBuf=function(t,e){return t.length===e?t:t.subarray?t.subarray(0,e):(t.length=e,t)};var s={arraySet:function(t,e,a,r,s){if(e.subarray&&t.subarray)t.set(e.subarray(a,a+r),s);else for(var i=0;i>>16&65535|0,n=0;0!==a;){a-=n=a>2e3?2e3:a;do{i=i+(s=s+e[r++]|0)|0}while(--n);s%=65521,i%=65521}return s|i<<16|0};var s=function(){for(var t,e=[],a=0;a<256;a++){t=a;for(var r=0;r<8;r++)t=1&t?3988292384^t>>>1:t>>>1;e[a]=t}return e}();var i=function(t,e,a,r){var i=s,n=r+a;t^=-1;for(var o=r;o>>8^i[255&(t^e[o])];return-1^t},n=function(t,e){var a,r,s,i,n,o,h,l,d,c,u,f,_,p,w,g,m,b,y,v,k,x,C,A,R;a=t.state,r=t.next_in,A=t.input,s=r+(t.avail_in-5),i=t.next_out,R=t.output,n=i-(e-t.avail_out),o=i+(t.avail_out-257),h=a.dmax,l=a.wsize,d=a.whave,c=a.wnext,u=a.window,f=a.hold,_=a.bits,p=a.lencode,w=a.distcode,g=(1<>>=y=b>>>24,_-=y,0===(y=b>>>16&255))R[i++]=65535&b;else{if(!(16&y)){if(0==(64&y)){b=p[(65535&b)+(f&(1<>>=y,_-=y),_<15&&(f+=A[r++]<<_,_+=8,f+=A[r++]<<_,_+=8),b=w[f&m];a:for(;;){if(f>>>=y=b>>>24,_-=y,!(16&(y=b>>>16&255))){if(0==(64&y)){b=w[(65535&b)+(f&(1<h){t.msg="invalid distance too far back",a.mode=30;break t}if(f>>>=y,_-=y,k>(y=i-n)){if((y=k-y)>d&&a.sane){t.msg="invalid distance too far back",a.mode=30;break t}if(x=0,C=u,0===c){if(x+=l-y,y2;)R[i++]=C[x++],R[i++]=C[x++],R[i++]=C[x++],v-=3;v&&(R[i++]=C[x++],v>1&&(R[i++]=C[x++]))}else{x=i-k;do{R[i++]=R[x++],R[i++]=R[x++],R[i++]=R[x++],v-=3}while(v>2);v&&(R[i++]=R[x++],v>1&&(R[i++]=R[x++]))}break}}break}}while(r>3,f&=(1<<(_-=v<<3))-1,t.next_in=r,t.next_out=i,t.avail_in=r=1&&0===L[R];R--);if(z>R&&(z=R),0===R)return i[n++]=20971520,i[n++]=20971520,u.bits=1,0;for(A=1;A0&&(0===t||1!==R))return-1;for(U[1]=0,x=1;x<15;x++)U[x+1]=U[x]+L[x];for(C=0;C852||2===t&&T>592)return 1;for(;;){b=x-B,c[C]m?(y=O[W+c[C]],v=D[E+c[C]]):(y=96,v=0),f=1<>B)+(_-=f)]=b<<24|y<<16|v|0}while(0!==_);for(f=1<>=1;if(0!==f?(I&=f-1,I+=f):I=0,C++,0==--L[x]){if(x===R)break;x=e[r+c[C]]}if(x>z&&(I&w)!==p){for(0===B&&(B=z),g+=A,H=1<<(S=x-B);S+B852||2===t&&T>592)return 1;i[p=I&w]=z<<24|S<<16|g-n|0}}return 0!==I&&(i[g+I]=x-B<<24|64<<16|0),u.bits=z,0};function u(t){return(t>>>24&255)+(t>>>8&65280)+((65280&t)<<8)+((255&t)<<24)}function f(){this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new a.Buf16(320),this.work=new a.Buf16(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}function _(t){var e;return t&&t.state?(e=t.state,t.total_in=t.total_out=e.total=0,t.msg="",e.wrap&&(t.adler=1&e.wrap),e.mode=1,e.last=0,e.havedict=0,e.dmax=32768,e.head=null,e.hold=0,e.bits=0,e.lencode=e.lendyn=new a.Buf32(852),e.distcode=e.distdyn=new a.Buf32(592),e.sane=1,e.back=-1,0):-2}function p(t){var e;return t&&t.state?((e=t.state).wsize=0,e.whave=0,e.wnext=0,_(t)):-2}function w(t,e){var a,r;return t&&t.state?(r=t.state,e<0?(a=0,e=-e):(a=1+(e>>4),e<48&&(e&=15)),e&&(e<8||e>15)?-2:(null!==r.window&&r.wbits!==e&&(r.window=null),r.wrap=a,r.wbits=e,p(t))):-2}function g(t,e){var a,r;return t?(r=new f,t.state=r,r.window=null,0!==(a=w(t,e))&&(t.state=null),a):-2}var m,b,y=!0;function v(t){if(y){var e;for(m=new a.Buf32(512),b=new a.Buf32(32),e=0;e<144;)t.lens[e++]=8;for(;e<256;)t.lens[e++]=9;for(;e<280;)t.lens[e++]=7;for(;e<288;)t.lens[e++]=8;for(c(1,t.lens,0,288,m,0,t.work,{bits:9}),e=0;e<32;)t.lens[e++]=5;c(2,t.lens,0,32,b,0,t.work,{bits:5}),y=!1}t.lencode=m,t.lenbits=9,t.distcode=b,t.distbits=5}function k(t,e,r,s){var i,n=t.state;return null===n.window&&(n.wsize=1<=n.wsize?(a.arraySet(n.window,e,r-n.wsize,n.wsize,0),n.wnext=0,n.whave=n.wsize):((i=n.wsize-n.wnext)>s&&(i=s),a.arraySet(n.window,e,r-s,i,n.wnext),(s-=i)?(a.arraySet(n.window,e,r-s,s,0),n.wnext=s,n.whave=n.wsize):(n.wnext+=i,n.wnext===n.wsize&&(n.wnext=0),n.whave>>8&255,s.check=i(s.check,L,2,0),p=0,w=0,s.mode=2;break}if(s.flags=0,s.head&&(s.head.done=!1),!(1&s.wrap)||(((255&p)<<8)+(p>>8))%31){t.msg="incorrect header check",s.mode=30;break}if(8!=(15&p)){t.msg="unknown compression method",s.mode=30;break}if(w-=4,H=8+(15&(p>>>=4)),0===s.wbits)s.wbits=H;else if(H>s.wbits){t.msg="invalid window size",s.mode=30;break}s.dmax=1<>8&1),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=3;case 3:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>8&255,L[2]=p>>>16&255,L[3]=p>>>24&255,s.check=i(s.check,L,4,0)),p=0,w=0,s.mode=4;case 4:for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>8),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=5;case 5:if(1024&s.flags){for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0}else s.head&&(s.head.extra=null);s.mode=6;case 6:if(1024&s.flags&&((b=s.length)>f&&(b=f),b&&(s.head&&(H=s.head.extra_len-s.length,s.head.extra||(s.head.extra=new Array(s.head.extra_len)),a.arraySet(s.head.extra,o,l,b,H)),512&s.flags&&(s.check=i(s.check,o,b,l)),f-=b,l+=b,s.length-=b),s.length))break t;s.length=0,s.mode=7;case 7:if(2048&s.flags){if(0===f)break t;b=0;do{H=o[l+b++],s.head&&H&&s.length<65536&&(s.head.name+=String.fromCharCode(H))}while(H&&b>9&1,s.head.done=!0),t.adler=s.check=0,s.mode=12;break;case 10:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>=7&w,w-=7&w,s.mode=27;break}for(;w<3;){if(0===f)break t;f--,p+=o[l++]<>>=1)){case 0:s.mode=14;break;case 1:if(v(s),s.mode=20,6===e){p>>>=2,w-=2;break t}break;case 2:s.mode=17;break;case 3:t.msg="invalid block type",s.mode=30}p>>>=2,w-=2;break;case 14:for(p>>>=7&w,w-=7&w;w<32;){if(0===f)break t;f--,p+=o[l++]<>>16^65535)){t.msg="invalid stored block lengths",s.mode=30;break}if(s.length=65535&p,p=0,w=0,s.mode=15,6===e)break t;case 15:s.mode=16;case 16:if(b=s.length){if(b>f&&(b=f),b>_&&(b=_),0===b)break t;a.arraySet(h,o,l,b,d),f-=b,l+=b,_-=b,d+=b,s.length-=b;break}s.mode=12;break;case 17:for(;w<14;){if(0===f)break t;f--,p+=o[l++]<>>=5,w-=5,s.ndist=1+(31&p),p>>>=5,w-=5,s.ncode=4+(15&p),p>>>=4,w-=4,s.nlen>286||s.ndist>30){t.msg="too many length or distance symbols",s.mode=30;break}s.have=0,s.mode=18;case 18:for(;s.have>>=3,w-=3}for(;s.have<19;)s.lens[U[s.have++]]=0;if(s.lencode=s.lendyn,s.lenbits=7,I={bits:s.lenbits},T=c(0,s.lens,0,19,s.lencode,0,s.work,I),s.lenbits=I.bits,T){t.msg="invalid code lengths set",s.mode=30;break}s.have=0,s.mode=19;case 19:for(;s.have>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=C,w-=C,s.lens[s.have++]=R;else{if(16===R){for(D=C+2;w>>=C,w-=C,0===s.have){t.msg="invalid bit length repeat",s.mode=30;break}H=s.lens[s.have-1],b=3+(3&p),p>>>=2,w-=2}else if(17===R){for(D=C+3;w>>=C)),p>>>=3,w-=3}else{for(D=C+7;w>>=C)),p>>>=7,w-=7}if(s.have+b>s.nlen+s.ndist){t.msg="invalid bit length repeat",s.mode=30;break}for(;b--;)s.lens[s.have++]=H}}if(30===s.mode)break;if(0===s.lens[256]){t.msg="invalid code -- missing end-of-block",s.mode=30;break}if(s.lenbits=9,I={bits:s.lenbits},T=c(1,s.lens,0,s.nlen,s.lencode,0,s.work,I),s.lenbits=I.bits,T){t.msg="invalid literal/lengths set",s.mode=30;break}if(s.distbits=6,s.distcode=s.distdyn,I={bits:s.distbits},T=c(2,s.lens,s.nlen,s.ndist,s.distcode,0,s.work,I),s.distbits=I.bits,T){t.msg="invalid distances set",s.mode=30;break}if(s.mode=20,6===e)break t;case 20:s.mode=21;case 21:if(f>=6&&_>=258){t.next_out=d,t.avail_out=_,t.next_in=l,t.avail_in=f,s.hold=p,s.bits=w,n(t,m),d=t.next_out,h=t.output,_=t.avail_out,l=t.next_in,o=t.input,f=t.avail_in,p=s.hold,w=s.bits,12===s.mode&&(s.back=-1);break}for(s.back=0;A=(E=s.lencode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,s.length=R,0===A){s.mode=26;break}if(32&A){s.back=-1,s.mode=12;break}if(64&A){t.msg="invalid literal/length code",s.mode=30;break}s.extra=15&A,s.mode=22;case 22:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}s.was=s.length,s.mode=23;case 23:for(;A=(E=s.distcode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,64&A){t.msg="invalid distance code",s.mode=30;break}s.offset=R,s.extra=15&A,s.mode=24;case 24:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}if(s.offset>s.dmax){t.msg="invalid distance too far back",s.mode=30;break}s.mode=25;case 25:if(0===_)break t;if(b=m-_,s.offset>b){if((b=s.offset-b)>s.whave&&s.sane){t.msg="invalid distance too far back",s.mode=30;break}b>s.wnext?(b-=s.wnext,y=s.wsize-b):y=s.wnext-b,b>s.length&&(b=s.length),x=s.window}else x=h,y=d-s.offset,b=s.length;b>_&&(b=_),_-=b,s.length-=b;do{h[d++]=x[y++]}while(--b);0===s.length&&(s.mode=21);break;case 26:if(0===_)break t;h[d++]=s.length,_--,s.mode=21;break;case 27:if(s.wrap){for(;w<32;){if(0===f)break t;f--,p|=o[l++]<=252?6:z>=248?5:z>=240?4:z>=224?3:z>=192?2:1;R[254]=R[254]=1;function S(t,e){if(e<65534&&(t.subarray&&A||!t.subarray&&C))return String.fromCharCode.apply(null,a.shrinkBuf(t,e));for(var r="",s=0;s>>6,e[n++]=128|63&r):r<65536?(e[n++]=224|r>>>12,e[n++]=128|r>>>6&63,e[n++]=128|63&r):(e[n++]=240|r>>>18,e[n++]=128|r>>>12&63,e[n++]=128|r>>>6&63,e[n++]=128|63&r);return e},H=function(t){return S(t,t.length)},T=function(t){for(var e=new a.Buf8(t.length),r=0,s=e.length;r4)o[r++]=65533,a+=i-1;else{for(s&=2===i?31:3===i?15:7;i>1&&a1?o[r++]=65533:s<65536?o[r++]=s:(s-=65536,o[r++]=55296|s>>10&1023,o[r++]=56320|1023&s)}return S(o,r)},D=function(t,e){var a;for((e=e||t.length)>t.length&&(e=t.length),a=e-1;a>=0&&128==(192&t[a]);)a--;return a<0||0===a?e:a+R[t[a]]>e?a:e},E=0,L=2,U=4,O=0,W=1,F=2,j=-5,P={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"};var N=function(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0};var q=function(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1},$=Object.prototype.toString;function M(t){if(!(this instanceof M))return new M(t);this.options=a.assign({chunkSize:16384,windowBits:0,to:""},t||{});var e=this.options;e.raw&&e.windowBits>=0&&e.windowBits<16&&(e.windowBits=-e.windowBits,0===e.windowBits&&(e.windowBits=-15)),!(e.windowBits>=0&&e.windowBits<16)||t&&t.windowBits||(e.windowBits+=32),e.windowBits>15&&e.windowBits<48&&0==(15&e.windowBits)&&(e.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new N,this.strm.avail_out=0;var r=x.inflateInit2(this.strm,e.windowBits);if(r!==O)throw new Error(P[r]);if(this.header=new q,x.inflateGetHeader(this.strm,this.header),e.dictionary&&("string"==typeof e.dictionary?e.dictionary=B(e.dictionary):"[object ArrayBuffer]"===$.call(e.dictionary)&&(e.dictionary=new Uint8Array(e.dictionary)),e.raw&&(r=x.inflateSetDictionary(this.strm,e.dictionary))!==O))throw new Error(P[r])}M.prototype.push=function(t,e){var r,s,i,n,o,h=this.strm,l=this.options.chunkSize,d=this.options.dictionary,c=!1;if(this.ended)return!1;s=e===~~e?e:!0===e?U:E,"string"==typeof t?h.input=T(t):"[object ArrayBuffer]"===$.call(t)?h.input=new Uint8Array(t):h.input=t,h.next_in=0,h.avail_in=h.input.length;do{if(0===h.avail_out&&(h.output=new a.Buf8(l),h.next_out=0,h.avail_out=l),(r=x.inflate(h,E))===F&&d&&(r=x.inflateSetDictionary(this.strm,d)),r===j&&!0===c&&(r=O,c=!1),r!==W&&r!==O)return this.onEnd(r),this.ended=!0,!1;h.next_out&&(0!==h.avail_out&&r!==W&&(0!==h.avail_in||s!==U&&s!==L)||("string"===this.options.to?(i=D(h.output,h.next_out),n=h.next_out-i,o=I(h.output,i),h.next_out=n,h.avail_out=l-n,n&&a.arraySet(h.output,h.output,i,n,0),this.onData(o)):this.onData(a.shrinkBuf(h.output,h.next_out)))),0===h.avail_in&&0===h.avail_out&&(c=!0)}while((h.avail_in>0||0===h.avail_out)&&r!==W);return r===W&&(s=U),s===U?(r=x.inflateEnd(this.strm),this.onEnd(r),this.ended=!0,r===O):s!==L||(this.onEnd(O),h.avail_out=0,!0)},M.prototype.onData=function(t){this.chunks.push(t)},M.prototype.onEnd=function(t){t===O&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var J=M;const V=new TextDecoder("utf-8");class Z extends J{constructor(t,e){super(t),this.reader=e}onEnd(t){this.err=t,this.err||(this.reader._rawOffset+=this.strm.total_in)}}class G{static concatChunks(t,e){if(1===t.length)return t[0];const a=new Uint8Array(e);let r=0;for(const e of t)a.set(e,r),r+=e.byteLength;return a}static splitChunk(t,e){return[t.slice(0,e),t.slice(e)]}static async readFully(t){const e=[];let a=0;for await(const r of t)e.push(r),a+=r.byteLength;return G.concatChunks(e,a)}getReadableStream(){const t=this[Symbol.asyncIterator]();return new ReadableStream({pull:e=>t.next().then(t=>{t.done||!t.value?e.close():e.enqueue(t.value)})})}readFully(){return G.readFully(this)}async readline(t=0){const e=await this.readlineRaw(t);return e?V.decode(e):""}async*iterLines(t=0){let e=null;for(;e=await this.readline(t);)yield e}}class X extends G{constructor(t,e="gzip",a=!1){if(super(),this.compressed=e,this.opts={raw:"deflateRaw"===e},this.inflator=e?new Z(this.opts,this):null,"function"!=typeof t[Symbol.asyncIterator])if("function"==typeof t.getReader)t=X.fromReadable(t.getReader());else if("function"==typeof t.read)t=X.fromReadable(t);else{if("function"!=typeof t[Symbol.iterator])throw new TypeError("Invalid Stream Source");t=X.fromIter(t)}this._sourceIter=a?this.dechunk(t):t[Symbol.asyncIterator](),this.lastValue=null,this.errored=!1,this._savedChunk=null,this._rawOffset=0,this._readOffset=0,this.numChunks=0}async _loadNext(){const t=await this._sourceIter.next();return t.done?null:t.value}async*dechunk(t){const e=t instanceof X?t:new X(t,null);let a=-1,r=!0;for(;0!=a;){const t=await e.readlineRaw(64);let s=null;if(a=parseInt(V.decode(t),16),!a||a>2**32){if(Number.isNaN(a)||a>2**32){r||(this.errored=!0),yield t;break}}else if(s=await e.readSize(a),s.length!=a){r?yield t:this.errored=!0,yield s;break}const i=await e.readSize(2);if(13!=i[0]||10!=i[1]){r?yield t:this.errored=!0,yield s,yield i;break}if(r=!1,!s||0===a)return;yield s}yield*e}_unread(t){t.length&&(this._readOffset-=t.length,this._savedChunk&&console.log("Already have chunk!"),this._savedChunk=t)}async _next(){if(this._savedChunk){const t=this._savedChunk;return this._savedChunk=null,t}if(this.compressed){const t=this._getNextChunk();if(t)return t}let t=await this._loadNext();for(;this.compressed&&t;){this._push(t);const e=this._getNextChunk(t);if(e)return e;t=await this._loadNext()}return t}_push(t){this.lastValue=t,this.inflator.ended&&(this.inflator=new Z(this.opts,this)),this.inflator.push(t),this.inflator.err&&this.inflator.ended&&"deflate"===this.compressed&&!1===this.opts.raw&&0===this.numChunks&&(this.opts.raw=!0,this.compressed="deflateRaw",this.inflator=new Z(this.opts,this),this.inflator.push(t))}_getNextChunk(t){for(;;){if(this.inflator.chunks.length>0)return this.numChunks++,this.inflator.chunks.shift();if(this.inflator.ended){if(0!==this.inflator.err)return this.compressed=null,t;const e=this.inflator.strm.avail_in;if(e&&this.lastValue){this._push(this.lastValue.slice(-e));continue}}return null}}async*[Symbol.asyncIterator](){let t=null;for(;t=await this._next();)this._readOffset+=t.length,yield t}async readlineRaw(t){const e=[];let a,r=0,s=null;for await(const i of this){if(t&&r+i.byteLength>t){s=i,a=t-r-1;const e=i.slice(0,a+1).indexOf(10);e>=0&&(a=e);break}if(a=i.indexOf(10),a>=0){s=i;break}e.push(i),r+=i.byteLength}if(s){const[t,i]=X.splitChunk(s,a+1);e.push(t),r+=t.byteLength,this._unread(i)}else if(!e.length)return null;return X.concatChunks(e,r)}readFully(){return this.readSize()}async readSize(t=-1,e=!1){const a=[];let r=0;for await(const s of this){if(t>=0){if(s.length>t){const[i,n]=X.splitChunk(s,t);e||a.push(i),r+=i.byteLength,this._unread(n);break}if(s.length===t){e||a.push(s),r+=s.byteLength,t=0;break}t-=s.length}e||a.push(s),r+=s.byteLength}return e?r:X.concatChunks(a,r)}getReadOffset(){return this._readOffset}getRawOffset(){return this.compressed?this._rawOffset:this._readOffset}getRawLength(t){return this.compressed?this.inflator.strm.total_in:this._readOffset-t}static fromReadable(t){return{async*[Symbol.asyncIterator](){let e=null;for(;(e=await t.read())&&!e.done;)yield e.value}}}static fromIter(t){return{async*[Symbol.asyncIterator](){for(const e of t)yield e}}}}class K extends G{constructor(t,e,a=0){super(),this.sourceIter=t,this.length=e,this.limit=e,this.skip=a}setLimitSkip(t,e=0){this.limit=t,this.skip=e}async*[Symbol.asyncIterator](){if(!(this.limit<=0))for await(let t of this.sourceIter){if(this.skip>0){if(!(t.length>=this.skip)){this.skip-=t.length;continue}{const[e,a]=K.splitChunk(t,this.skip);t=a,this.skip=0}}if(t.length>this.limit){const[e,a]=K.splitChunk(t,this.limit);t=e,this.sourceIter._unread(a)}if(t.length&&(this.limit-=t.length,yield t),this.limit<=0)break}}async readlineRaw(t){if(this.limit<=0)return null;const e=await this.sourceIter.readlineRaw(t?Math.min(t,this.limit):this.limit);return this.limit-=e.length,e}async skipFully(){const t=this.limit;for(;this.limit>0;)this.limit-=await this.sourceIter.readSize(this.limit,!0);return t}}const Q=new Uint8Array([13,10]),Y=new Uint8Array([13,10,13,10]);class tt{constructor({statusline:t,headers:e}){this.statusline=t,this.headers=e}toString(){const t=[this.statusline];for(const[e,a]of this.headers)t.push(`${e}: ${a}`);return t.join("\r\n")+"\r\n"}async*iterSerialize(t){yield t.encode(this.statusline),yield Q;for(const[e,a]of this.headers)yield t.encode(`${e}: ${a}\r\n`)}_parseResponseStatusLine(){const t=at(this.statusline," ",2);this._protocol=t[0],this._statusCode=t.length>1?Number(t[1]):"",this._statusText=t.length>2?t[2]:""}get statusCode(){return void 0===this._statusCode&&this._parseResponseStatusLine(),this._statusCode}get protocol(){return void 0===this._protocol&&this._parseResponseStatusLine(),this._protocol}get statusText(){return void 0===this._statusText&&this._parseResponseStatusLine(),this._statusText}_parseRequestStatusLine(){const t=this.statusline.split(" ",2);this._method=t[0],this._requestPath=t.length>1?t[1]:""}get method(){return void 0===this._method&&this._parseRequestStatusLine(),this._method}get requestPath(){return void 0===this._requestPath&&this._parseRequestStatusLine(),this._requestPath}}class et{startsWithSpace(t){const e=t.charAt(0);return" "===e||"\t"===e}async parse(t,{headersClass:e=Map,firstLine:a}={}){const r=a||await t.readline();if(!r)return null;let s=r.trimEnd();const i=new e;if(!s)return null;let n=(await t.readline()).trimEnd();for(;n;){let[e,a]=at(n,":",1);a&&(e=e.trimStart(),a=a.trim());let r=(await t.readline()).trimEnd();for(;this.startsWithSpace(r);)a&&(a+=r),r=(await t.readline()).trimEnd();if(a)try{i.set(e,a)}catch(t){}n=r}return new tt({statusline:s,headers:i,totalRead:this.totalRead})}}function at(t,e,a){const r=t.split(e),s=r.slice(0,a);return r.slice(a).length>0&&s.push(r.slice(a).join(e)),s}var rt,st=(rt=Object.freeze({__proto__:null,default:{}}))&&rt.default||rt,it=e((function(t){!function(){var e,a,r,s=0,i=[];for(a=0;a<256;a++)i[a]=(a+256).toString(16).substr(1);function n(){var t=function(t){return(!e||s+t>o.BUFFER_SIZE)&&(s=0,e=o.randomBytes(o.BUFFER_SIZE)),e.slice(s,s+=t)}(16);return t[6]=15&t[6]|64,t[8]=63&t[8]|128,t}function o(){var t=n();return i[t[0]]+i[t[1]]+i[t[2]]+i[t[3]]+"-"+i[t[4]]+i[t[5]]+"-"+i[t[6]]+i[t[7]]+"-"+i[t[8]]+i[t[9]]+"-"+i[t[10]]+i[t[11]]+i[t[12]]+i[t[13]]+i[t[14]]+i[t[15]]}o.BUFFER_SIZE=4096,o.bin=n,o.clearBuffer=function(){e=null,s=0},o.test=function(t){return"string"==typeof t&&/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(t)},"undefined"!=typeof crypto?r=crypto:"undefined"!=typeof window&&void 0!==window.msCrypto&&(r=window.msCrypto),r=r||st,t.exports=o,o.randomBytes=function(){if(r){if(r.randomBytes)return r.randomBytes;if(r.getRandomValues)return function(t){var e=new Uint8Array(t);return r.getRandomValues(e),e}}return function(t){var e,a=[];for(e=0;e`),!r.headers.get("Content-Type")&&ht[a]&&r.headers.set("Content-Type",ht[a]),!c){c=async function*(){}()}const f=new lt({warcHeaders:r,reader:c});switch(a){case"response":case"request":case"revisit":f.httpHeaders=new tt({statusline:n,headers:h?new Map(Object.entries(i)):new Headers(i)})}return f}static createWARCInfo(t={},e){return t.type="warcinfo",lt.create(t,async function*(){for(const[t,a]of Object.entries(e))yield ot.encode(`${t}: ${a}\r\n`)}())}constructor({warcHeaders:t,reader:e}){super(),this.warcHeaders=t,this._reader=e,this._contentReader=null,this.payload=null,this.httpHeaders=null,this.consumed=!1,this.fixUp()}getResponseInfo(){const t=this.httpHeaders;return t?{headers:t.headers,status:t.statusCode,statusText:t.statusText}:null}fixUp(){const t=this.warcHeaders.headers.get("WARC-Target-URI");t&&t.startsWith("<")&&t.endsWith(">")&&this.warcHeaders.headers.set("WARC-Target-URI",t.slice(1,-1))}async readFully(t=!1){if(this.httpHeaders){if(this._contentReader&&!t)throw new TypeError("WARC Record decoding already started, but requesting raw payload");if(t&&"raw"===this.consumed&&this.payload)return await this._createDecodingReader([this.payload]).readFully()}return this.payload||(t?(this.payload=await super.readFully(),this.consumed="content"):(this.payload=await lt.readFully(this._reader),this.consumed="raw")),this.payload}get reader(){if(this._contentReader)throw new TypeError("WARC Record decoding already started, but requesting raw payload");return this._reader}get contentReader(){return this.httpHeaders?(this._contentReader||(this._contentReader=this._createDecodingReader(this._reader)),this._contentReader):this._reader}_createDecodingReader(t){let e=this.httpHeaders.headers.get("content-encoding"),a=this.httpHeaders.headers.get("transfer-encoding");const r="chunked"===a;return e||r||(e=a),new X(t,e,r)}async readlineRaw(t){if(this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");return this.contentReader.readlineRaw(t)}async contentText(){const t=await this.readFully(!0);return nt.decode(t)}async*[Symbol.asyncIterator](){for await(const t of this.contentReader)if(yield t,this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");this.consumed="content"}async skipFully(){if(this.consumed)return;const t=await this._reader.skipFully();return this.consumed="skipped",t}warcHeader(t){return this.warcHeaders.headers.get(t)}get warcType(){return this.warcHeaders.headers.get("WARC-Type")}get warcTargetURI(){return this.warcHeaders.headers.get("WARC-Target-URI")}get warcDate(){return this.warcHeaders.headers.get("WARC-Date")}get warcRefersToTargetURI(){return this.warcHeaders.headers.get("WARC-Refers-To-Target-URI")}get warcRefersToDate(){return this.warcHeaders.headers.get("WARC-Refers-To-Date")}get warcPayloadDigest(){return this.warcHeaders.headers.get("WARC-Payload-Digest")}get warcBlockDigest(){return this.warcHeaders.headers.get("WARC-Block-Digest")}get warcContentType(){return this.warcHeaders.headers.get("Content-Type")}get warcContentLength(){return Number(this.warcHeaders.headers.get("Content-Length"))}}class dt{static parse(t,e){return new dt(t,e).parse()}static iterRecords(t,e){return new dt(t,e)[Symbol.asyncIterator]()}constructor(t,{keepHeadersCase:e=!1,parseHttp:a=!0}={}){this._offset=0,this._warcHeadersLength=0,this._headersClass=e?Map:Headers,this._parseHttp=a,this._atRecordBoundary=!0,t instanceof X||(t=new X(t)),this._reader=t,this._record=null}async readToNextRecord(){let t="";if(!this._atRecordBoundary&&this._reader&&this._record){await this._record.skipFully(),t=await this._reader.readline();const e=t.trim().length;if(e&&console.warn(`Content-Length Too Small: Record not followed by newline, Remainder Length: ${e}, Offset: ${this._reader.getRawOffset()-t.length}`),this._reader.compressed)await this._reader.readSize(2,!0),t="";else for(t=await this._reader.readline();2===t.length;)t=await this._reader.readline()}return this._atRecordBoundary=!0,t}_initRecordReader(t){return new K(this._reader,Number(t.headers.get("Content-Length")||0))}async parse(){const t=await this.readToNextRecord();this._offset=this._reader.getRawOffset()-t.length;const e=new et,a=await e.parse(this._reader,{firstLine:t,headersClass:this._headersClass});if(!a)return null;this._warcHeadersLength=this._reader.getReadOffset();const r=new lt({warcHeaders:a,reader:this._initRecordReader(a)});if(this._atRecordBoundary=!1,this._record=r,this._parseHttp)switch(r.warcType){case"response":case"request":await this._addHttpHeaders(r,e,this._reader);break;case"revisit":r.warcContentLength>0&&await this._addHttpHeaders(r,e,this._reader)}return r}get offset(){return this._offset}get recordLength(){return this._reader.getRawLength(this._offset)}async*[Symbol.asyncIterator](){let t=null;for(;t=await this.parse(this._reader);)yield t;this._record=null}async _addHttpHeaders(t,e){const a=await e.parse(this._reader,{headersClass:this._headersClass});t.httpHeaders=a;const r=this._reader.getReadOffset()-this._warcHeadersLength;t.reader.setLimitSkip&&t.reader.setLimitSkip(t.warcContentLength-r)}}var ct=e((function(e){ +var t="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function e(t,e){return t(e={exports:{}},e.exports),e.exports}var a=e((function(t,e){var a="undefined"!=typeof Uint8Array&&"undefined"!=typeof Uint16Array&&"undefined"!=typeof Int32Array;function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}e.assign=function(t){for(var e=Array.prototype.slice.call(arguments,1);e.length;){var a=e.shift();if(a){if("object"!=typeof a)throw new TypeError(a+"must be non-object");for(var s in a)r(a,s)&&(t[s]=a[s])}}return t},e.shrinkBuf=function(t,e){return t.length===e?t:t.subarray?t.subarray(0,e):(t.length=e,t)};var s={arraySet:function(t,e,a,r,s){if(e.subarray&&t.subarray)t.set(e.subarray(a,a+r),s);else for(var i=0;i>>16&65535|0,n=0;0!==a;){a-=n=a>2e3?2e3:a;do{i=i+(s=s+e[r++]|0)|0}while(--n);s%=65521,i%=65521}return s|i<<16|0};var s=function(){for(var t,e=[],a=0;a<256;a++){t=a;for(var r=0;r<8;r++)t=1&t?3988292384^t>>>1:t>>>1;e[a]=t}return e}();var i=function(t,e,a,r){var i=s,n=r+a;t^=-1;for(var o=r;o>>8^i[255&(t^e[o])];return-1^t},n=function(t,e){var a,r,s,i,n,o,h,l,d,c,u,f,_,p,w,g,m,b,y,v,k,x,C,A,R;a=t.state,r=t.next_in,A=t.input,s=r+(t.avail_in-5),i=t.next_out,R=t.output,n=i-(e-t.avail_out),o=i+(t.avail_out-257),h=a.dmax,l=a.wsize,d=a.whave,c=a.wnext,u=a.window,f=a.hold,_=a.bits,p=a.lencode,w=a.distcode,g=(1<>>=y=b>>>24,_-=y,0===(y=b>>>16&255))R[i++]=65535&b;else{if(!(16&y)){if(0==(64&y)){b=p[(65535&b)+(f&(1<>>=y,_-=y),_<15&&(f+=A[r++]<<_,_+=8,f+=A[r++]<<_,_+=8),b=w[f&m];a:for(;;){if(f>>>=y=b>>>24,_-=y,!(16&(y=b>>>16&255))){if(0==(64&y)){b=w[(65535&b)+(f&(1<h){t.msg="invalid distance too far back",a.mode=30;break t}if(f>>>=y,_-=y,k>(y=i-n)){if((y=k-y)>d&&a.sane){t.msg="invalid distance too far back",a.mode=30;break t}if(x=0,C=u,0===c){if(x+=l-y,y2;)R[i++]=C[x++],R[i++]=C[x++],R[i++]=C[x++],v-=3;v&&(R[i++]=C[x++],v>1&&(R[i++]=C[x++]))}else{x=i-k;do{R[i++]=R[x++],R[i++]=R[x++],R[i++]=R[x++],v-=3}while(v>2);v&&(R[i++]=R[x++],v>1&&(R[i++]=R[x++]))}break}}break}}while(r>3,f&=(1<<(_-=v<<3))-1,t.next_in=r,t.next_out=i,t.avail_in=r=1&&0===L[R];R--);if(z>R&&(z=R),0===R)return i[n++]=20971520,i[n++]=20971520,u.bits=1,0;for(A=1;A0&&(0===t||1!==R))return-1;for(U[1]=0,x=1;x<15;x++)U[x+1]=U[x]+L[x];for(C=0;C852||2===t&&T>592)return 1;for(;;){b=x-B,c[C]m?(y=O[W+c[C]],v=D[E+c[C]]):(y=96,v=0),f=1<>B)+(_-=f)]=b<<24|y<<16|v|0}while(0!==_);for(f=1<>=1;if(0!==f?(I&=f-1,I+=f):I=0,C++,0==--L[x]){if(x===R)break;x=e[r+c[C]]}if(x>z&&(I&w)!==p){for(0===B&&(B=z),g+=A,H=1<<(S=x-B);S+B852||2===t&&T>592)return 1;i[p=I&w]=z<<24|S<<16|g-n|0}}return 0!==I&&(i[g+I]=x-B<<24|64<<16|0),u.bits=z,0};function u(t){return(t>>>24&255)+(t>>>8&65280)+((65280&t)<<8)+((255&t)<<24)}function f(){this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new a.Buf16(320),this.work=new a.Buf16(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}function _(t){var e;return t&&t.state?(e=t.state,t.total_in=t.total_out=e.total=0,t.msg="",e.wrap&&(t.adler=1&e.wrap),e.mode=1,e.last=0,e.havedict=0,e.dmax=32768,e.head=null,e.hold=0,e.bits=0,e.lencode=e.lendyn=new a.Buf32(852),e.distcode=e.distdyn=new a.Buf32(592),e.sane=1,e.back=-1,0):-2}function p(t){var e;return t&&t.state?((e=t.state).wsize=0,e.whave=0,e.wnext=0,_(t)):-2}function w(t,e){var a,r;return t&&t.state?(r=t.state,e<0?(a=0,e=-e):(a=1+(e>>4),e<48&&(e&=15)),e&&(e<8||e>15)?-2:(null!==r.window&&r.wbits!==e&&(r.window=null),r.wrap=a,r.wbits=e,p(t))):-2}function g(t,e){var a,r;return t?(r=new f,t.state=r,r.window=null,0!==(a=w(t,e))&&(t.state=null),a):-2}var m,b,y=!0;function v(t){if(y){var e;for(m=new a.Buf32(512),b=new a.Buf32(32),e=0;e<144;)t.lens[e++]=8;for(;e<256;)t.lens[e++]=9;for(;e<280;)t.lens[e++]=7;for(;e<288;)t.lens[e++]=8;for(c(1,t.lens,0,288,m,0,t.work,{bits:9}),e=0;e<32;)t.lens[e++]=5;c(2,t.lens,0,32,b,0,t.work,{bits:5}),y=!1}t.lencode=m,t.lenbits=9,t.distcode=b,t.distbits=5}function k(t,e,r,s){var i,n=t.state;return null===n.window&&(n.wsize=1<=n.wsize?(a.arraySet(n.window,e,r-n.wsize,n.wsize,0),n.wnext=0,n.whave=n.wsize):((i=n.wsize-n.wnext)>s&&(i=s),a.arraySet(n.window,e,r-s,i,n.wnext),(s-=i)?(a.arraySet(n.window,e,r-s,s,0),n.wnext=s,n.whave=n.wsize):(n.wnext+=i,n.wnext===n.wsize&&(n.wnext=0),n.whave>>8&255,s.check=i(s.check,L,2,0),p=0,w=0,s.mode=2;break}if(s.flags=0,s.head&&(s.head.done=!1),!(1&s.wrap)||(((255&p)<<8)+(p>>8))%31){t.msg="incorrect header check",s.mode=30;break}if(8!=(15&p)){t.msg="unknown compression method",s.mode=30;break}if(w-=4,H=8+(15&(p>>>=4)),0===s.wbits)s.wbits=H;else if(H>s.wbits){t.msg="invalid window size",s.mode=30;break}s.dmax=1<>8&1),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=3;case 3:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>8&255,L[2]=p>>>16&255,L[3]=p>>>24&255,s.check=i(s.check,L,4,0)),p=0,w=0,s.mode=4;case 4:for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>8),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=5;case 5:if(1024&s.flags){for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0}else s.head&&(s.head.extra=null);s.mode=6;case 6:if(1024&s.flags&&((b=s.length)>f&&(b=f),b&&(s.head&&(H=s.head.extra_len-s.length,s.head.extra||(s.head.extra=new Array(s.head.extra_len)),a.arraySet(s.head.extra,o,l,b,H)),512&s.flags&&(s.check=i(s.check,o,b,l)),f-=b,l+=b,s.length-=b),s.length))break t;s.length=0,s.mode=7;case 7:if(2048&s.flags){if(0===f)break t;b=0;do{H=o[l+b++],s.head&&H&&s.length<65536&&(s.head.name+=String.fromCharCode(H))}while(H&&b>9&1,s.head.done=!0),t.adler=s.check=0,s.mode=12;break;case 10:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>=7&w,w-=7&w,s.mode=27;break}for(;w<3;){if(0===f)break t;f--,p+=o[l++]<>>=1)){case 0:s.mode=14;break;case 1:if(v(s),s.mode=20,6===e){p>>>=2,w-=2;break t}break;case 2:s.mode=17;break;case 3:t.msg="invalid block type",s.mode=30}p>>>=2,w-=2;break;case 14:for(p>>>=7&w,w-=7&w;w<32;){if(0===f)break t;f--,p+=o[l++]<>>16^65535)){t.msg="invalid stored block lengths",s.mode=30;break}if(s.length=65535&p,p=0,w=0,s.mode=15,6===e)break t;case 15:s.mode=16;case 16:if(b=s.length){if(b>f&&(b=f),b>_&&(b=_),0===b)break t;a.arraySet(h,o,l,b,d),f-=b,l+=b,_-=b,d+=b,s.length-=b;break}s.mode=12;break;case 17:for(;w<14;){if(0===f)break t;f--,p+=o[l++]<>>=5,w-=5,s.ndist=1+(31&p),p>>>=5,w-=5,s.ncode=4+(15&p),p>>>=4,w-=4,s.nlen>286||s.ndist>30){t.msg="too many length or distance symbols",s.mode=30;break}s.have=0,s.mode=18;case 18:for(;s.have>>=3,w-=3}for(;s.have<19;)s.lens[U[s.have++]]=0;if(s.lencode=s.lendyn,s.lenbits=7,I={bits:s.lenbits},T=c(0,s.lens,0,19,s.lencode,0,s.work,I),s.lenbits=I.bits,T){t.msg="invalid code lengths set",s.mode=30;break}s.have=0,s.mode=19;case 19:for(;s.have>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=C,w-=C,s.lens[s.have++]=R;else{if(16===R){for(D=C+2;w>>=C,w-=C,0===s.have){t.msg="invalid bit length repeat",s.mode=30;break}H=s.lens[s.have-1],b=3+(3&p),p>>>=2,w-=2}else if(17===R){for(D=C+3;w>>=C)),p>>>=3,w-=3}else{for(D=C+7;w>>=C)),p>>>=7,w-=7}if(s.have+b>s.nlen+s.ndist){t.msg="invalid bit length repeat",s.mode=30;break}for(;b--;)s.lens[s.have++]=H}}if(30===s.mode)break;if(0===s.lens[256]){t.msg="invalid code -- missing end-of-block",s.mode=30;break}if(s.lenbits=9,I={bits:s.lenbits},T=c(1,s.lens,0,s.nlen,s.lencode,0,s.work,I),s.lenbits=I.bits,T){t.msg="invalid literal/lengths set",s.mode=30;break}if(s.distbits=6,s.distcode=s.distdyn,I={bits:s.distbits},T=c(2,s.lens,s.nlen,s.ndist,s.distcode,0,s.work,I),s.distbits=I.bits,T){t.msg="invalid distances set",s.mode=30;break}if(s.mode=20,6===e)break t;case 20:s.mode=21;case 21:if(f>=6&&_>=258){t.next_out=d,t.avail_out=_,t.next_in=l,t.avail_in=f,s.hold=p,s.bits=w,n(t,m),d=t.next_out,h=t.output,_=t.avail_out,l=t.next_in,o=t.input,f=t.avail_in,p=s.hold,w=s.bits,12===s.mode&&(s.back=-1);break}for(s.back=0;A=(E=s.lencode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,s.length=R,0===A){s.mode=26;break}if(32&A){s.back=-1,s.mode=12;break}if(64&A){t.msg="invalid literal/length code",s.mode=30;break}s.extra=15&A,s.mode=22;case 22:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}s.was=s.length,s.mode=23;case 23:for(;A=(E=s.distcode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,64&A){t.msg="invalid distance code",s.mode=30;break}s.offset=R,s.extra=15&A,s.mode=24;case 24:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}if(s.offset>s.dmax){t.msg="invalid distance too far back",s.mode=30;break}s.mode=25;case 25:if(0===_)break t;if(b=m-_,s.offset>b){if((b=s.offset-b)>s.whave&&s.sane){t.msg="invalid distance too far back",s.mode=30;break}b>s.wnext?(b-=s.wnext,y=s.wsize-b):y=s.wnext-b,b>s.length&&(b=s.length),x=s.window}else x=h,y=d-s.offset,b=s.length;b>_&&(b=_),_-=b,s.length-=b;do{h[d++]=x[y++]}while(--b);0===s.length&&(s.mode=21);break;case 26:if(0===_)break t;h[d++]=s.length,_--,s.mode=21;break;case 27:if(s.wrap){for(;w<32;){if(0===f)break t;f--,p|=o[l++]<=252?6:z>=248?5:z>=240?4:z>=224?3:z>=192?2:1;R[254]=R[254]=1;function S(t,e){if(e<65534&&(t.subarray&&A||!t.subarray&&C))return String.fromCharCode.apply(null,a.shrinkBuf(t,e));for(var r="",s=0;s>>6,e[n++]=128|63&r):r<65536?(e[n++]=224|r>>>12,e[n++]=128|r>>>6&63,e[n++]=128|63&r):(e[n++]=240|r>>>18,e[n++]=128|r>>>12&63,e[n++]=128|r>>>6&63,e[n++]=128|63&r);return e},H=function(t){return S(t,t.length)},T=function(t){for(var e=new a.Buf8(t.length),r=0,s=e.length;r4)o[r++]=65533,a+=i-1;else{for(s&=2===i?31:3===i?15:7;i>1&&a1?o[r++]=65533:s<65536?o[r++]=s:(s-=65536,o[r++]=55296|s>>10&1023,o[r++]=56320|1023&s)}return S(o,r)},D=function(t,e){var a;for((e=e||t.length)>t.length&&(e=t.length),a=e-1;a>=0&&128==(192&t[a]);)a--;return a<0||0===a?e:a+R[t[a]]>e?a:e},E=0,L=2,U=4,O=0,W=1,F=2,j=-5,P={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"};var N=function(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0};var q=function(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1},$=Object.prototype.toString;function M(t){if(!(this instanceof M))return new M(t);this.options=a.assign({chunkSize:16384,windowBits:0,to:""},t||{});var e=this.options;e.raw&&e.windowBits>=0&&e.windowBits<16&&(e.windowBits=-e.windowBits,0===e.windowBits&&(e.windowBits=-15)),!(e.windowBits>=0&&e.windowBits<16)||t&&t.windowBits||(e.windowBits+=32),e.windowBits>15&&e.windowBits<48&&0==(15&e.windowBits)&&(e.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new N,this.strm.avail_out=0;var r=x.inflateInit2(this.strm,e.windowBits);if(r!==O)throw new Error(P[r]);if(this.header=new q,x.inflateGetHeader(this.strm,this.header),e.dictionary&&("string"==typeof e.dictionary?e.dictionary=B(e.dictionary):"[object ArrayBuffer]"===$.call(e.dictionary)&&(e.dictionary=new Uint8Array(e.dictionary)),e.raw&&(r=x.inflateSetDictionary(this.strm,e.dictionary))!==O))throw new Error(P[r])}M.prototype.push=function(t,e){var r,s,i,n,o,h=this.strm,l=this.options.chunkSize,d=this.options.dictionary,c=!1;if(this.ended)return!1;s=e===~~e?e:!0===e?U:E,"string"==typeof t?h.input=T(t):"[object ArrayBuffer]"===$.call(t)?h.input=new Uint8Array(t):h.input=t,h.next_in=0,h.avail_in=h.input.length;do{if(0===h.avail_out&&(h.output=new a.Buf8(l),h.next_out=0,h.avail_out=l),(r=x.inflate(h,E))===F&&d&&(r=x.inflateSetDictionary(this.strm,d)),r===j&&!0===c&&(r=O,c=!1),r!==W&&r!==O)return this.onEnd(r),this.ended=!0,!1;h.next_out&&(0!==h.avail_out&&r!==W&&(0!==h.avail_in||s!==U&&s!==L)||("string"===this.options.to?(i=D(h.output,h.next_out),n=h.next_out-i,o=I(h.output,i),h.next_out=n,h.avail_out=l-n,n&&a.arraySet(h.output,h.output,i,n,0),this.onData(o)):this.onData(a.shrinkBuf(h.output,h.next_out)))),0===h.avail_in&&0===h.avail_out&&(c=!0)}while((h.avail_in>0||0===h.avail_out)&&r!==W);return r===W&&(s=U),s===U?(r=x.inflateEnd(this.strm),this.onEnd(r),this.ended=!0,r===O):s!==L||(this.onEnd(O),h.avail_out=0,!0)},M.prototype.onData=function(t){this.chunks.push(t)},M.prototype.onEnd=function(t){t===O&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var J=M;const V=new TextDecoder("utf-8");class Z extends J{constructor(t,e){super(t),this.reader=e}onEnd(t){this.err=t,this.err||(this.reader._rawOffset+=this.strm.total_in)}}class G{static concatChunks(t,e){if(1===t.length)return t[0];const a=new Uint8Array(e);let r=0;for(const e of t)a.set(e,r),r+=e.byteLength;return a}static splitChunk(t,e){return[t.slice(0,e),t.slice(e)]}static async readFully(t){const e=[];let a=0;for await(const r of t)e.push(r),a+=r.byteLength;return G.concatChunks(e,a)}getReadableStream(){const t=this[Symbol.asyncIterator]();return new ReadableStream({pull:e=>t.next().then(t=>{t.done||!t.value?e.close():e.enqueue(t.value)})})}readFully(){return G.readFully(this)}async readline(t=0){const e=await this.readlineRaw(t);return e?V.decode(e):""}async*iterLines(t=0){let e=null;for(;e=await this.readline(t);)yield e}}class X extends G{constructor(t,e="gzip",a=!1){if(super(),this.compressed=e,this.opts={raw:"deflateRaw"===e},this.inflator=e?new Z(this.opts,this):null,"function"!=typeof t[Symbol.asyncIterator])if("function"==typeof t.getReader)t=X.fromReadable(t.getReader());else if("function"==typeof t.read)t=X.fromReadable(t);else{if("function"!=typeof t[Symbol.iterator])throw new TypeError("Invalid Stream Source");t=X.fromIter(t)}this._sourceIter=a?this.dechunk(t):t[Symbol.asyncIterator](),this.lastValue=null,this.errored=!1,this._savedChunk=null,this._rawOffset=0,this._readOffset=0,this.numChunks=0}async _loadNext(){const t=await this._sourceIter.next();return t.done?null:t.value}async*dechunk(t){const e=t instanceof X?t:new X(t,null);let a=-1,r=!0;for(;0!=a;){const t=await e.readlineRaw(64);let s=null;if(a=parseInt(V.decode(t),16),!a||a>2**32){if(Number.isNaN(a)||a>2**32){r||(this.errored=!0),yield t;break}}else if(s=await e.readSize(a),s.length!=a){r?yield t:this.errored=!0,yield s;break}const i=await e.readSize(2);if(13!=i[0]||10!=i[1]){r?yield t:this.errored=!0,yield s,yield i;break}if(r=!1,!s||0===a)return;yield s}yield*e}_unread(t){t.length&&(this._readOffset-=t.length,this._savedChunk&&console.log("Already have chunk!"),this._savedChunk=t)}async _next(){if(this._savedChunk){const t=this._savedChunk;return this._savedChunk=null,t}if(this.compressed){const t=this._getNextChunk();if(t)return t}let t=await this._loadNext();for(;this.compressed&&t;){this._push(t);const e=this._getNextChunk(t);if(e)return e;t=await this._loadNext()}return t}_push(t){this.lastValue=t,this.inflator.ended&&(this.inflator=new Z(this.opts,this)),this.inflator.push(t),this.inflator.err&&this.inflator.ended&&"deflate"===this.compressed&&!1===this.opts.raw&&0===this.numChunks&&(this.opts.raw=!0,this.compressed="deflateRaw",this.inflator=new Z(this.opts,this),this.inflator.push(t))}_getNextChunk(t){for(;;){if(this.inflator.chunks.length>0)return this.numChunks++,this.inflator.chunks.shift();if(this.inflator.ended){if(0!==this.inflator.err)return this.compressed=null,t;const e=this.inflator.strm.avail_in;if(e&&this.lastValue){this._push(this.lastValue.slice(-e));continue}}return null}}async*[Symbol.asyncIterator](){let t=null;for(;t=await this._next();)this._readOffset+=t.length,yield t}async readlineRaw(t){const e=[];let a,r=0,s=null;for await(const i of this){if(t&&r+i.byteLength>t){s=i,a=t-r-1;const e=i.slice(0,a+1).indexOf(10);e>=0&&(a=e);break}if(a=i.indexOf(10),a>=0){s=i;break}e.push(i),r+=i.byteLength}if(s){const[t,i]=X.splitChunk(s,a+1);e.push(t),r+=t.byteLength,this._unread(i)}else if(!e.length)return null;return X.concatChunks(e,r)}readFully(){return this.readSize()}async readSize(t=-1,e=!1){const a=[];let r=0;for await(const s of this){if(t>=0){if(s.length>t){const[i,n]=X.splitChunk(s,t);e||a.push(i),r+=i.byteLength,this._unread(n);break}if(s.length===t){e||a.push(s),r+=s.byteLength,t=0;break}t-=s.length}e||a.push(s),r+=s.byteLength}return e?r:X.concatChunks(a,r)}getReadOffset(){return this._readOffset}getRawOffset(){return this.compressed?this._rawOffset:this._readOffset}getRawLength(t){return this.compressed?this.inflator.strm.total_in:this._readOffset-t}static fromReadable(t){return{async*[Symbol.asyncIterator](){let e=null;for(;(e=await t.read())&&!e.done;)yield e.value}}}static fromIter(t){return{async*[Symbol.asyncIterator](){for(const e of t)yield e}}}}class K extends G{constructor(t,e,a=0){super(),this.sourceIter=t,this.length=e,this.limit=e,this.skip=a}setLimitSkip(t,e=0){this.limit=t,this.skip=e}async*[Symbol.asyncIterator](){if(!(this.limit<=0))for await(let t of this.sourceIter){if(this.skip>0){if(!(t.length>=this.skip)){this.skip-=t.length;continue}{const[e,a]=K.splitChunk(t,this.skip);t=a,this.skip=0}}if(t.length>this.limit){const[e,a]=K.splitChunk(t,this.limit);t=e,this.sourceIter._unread&&this.sourceIter._unread(a)}if(t.length&&(this.limit-=t.length,yield t),this.limit<=0)break}}async readlineRaw(t){if(this.limit<=0)return null;const e=await this.sourceIter.readlineRaw(t?Math.min(t,this.limit):this.limit);return this.limit-=e.length,e}async skipFully(){const t=this.limit;for(;this.limit>0;)this.limit-=await this.sourceIter.readSize(this.limit,!0);return t}}const Q=new Uint8Array([13,10]),Y=new Uint8Array([13,10,13,10]);class tt{constructor({statusline:t,headers:e}){this.statusline=t,this.headers=e}toString(){const t=[this.statusline];for(const[e,a]of this.headers)t.push(`${e}: ${a}`);return t.join("\r\n")+"\r\n"}async*iterSerialize(t){yield t.encode(this.statusline),yield Q;for(const[e,a]of this.headers)yield t.encode(`${e}: ${a}\r\n`)}_parseResponseStatusLine(){const t=at(this.statusline," ",2);this._protocol=t[0],this._statusCode=t.length>1?Number(t[1]):"",this._statusText=t.length>2?t[2]:""}get statusCode(){return void 0===this._statusCode&&this._parseResponseStatusLine(),this._statusCode}get protocol(){return void 0===this._protocol&&this._parseResponseStatusLine(),this._protocol}get statusText(){return void 0===this._statusText&&this._parseResponseStatusLine(),this._statusText}_parseRequestStatusLine(){const t=this.statusline.split(" ",2);this._method=t[0],this._requestPath=t.length>1?t[1]:""}get method(){return void 0===this._method&&this._parseRequestStatusLine(),this._method}get requestPath(){return void 0===this._requestPath&&this._parseRequestStatusLine(),this._requestPath}}class et{startsWithSpace(t){const e=t.charAt(0);return" "===e||"\t"===e}async parse(t,{headersClass:e=Map,firstLine:a}={}){const r=a||await t.readline();if(!r)return null;let s=r.trimEnd();const i=new e;if(!s)return null;let n=(await t.readline()).trimEnd();for(;n;){let[e,a]=at(n,":",1);a&&(e=e.trimStart(),a=a.trim());let r=(await t.readline()).trimEnd();for(;this.startsWithSpace(r);)a&&(a+=r),r=(await t.readline()).trimEnd();if(a)try{i.set(e,a)}catch(t){}n=r}return new tt({statusline:s,headers:i,totalRead:this.totalRead})}}function at(t,e,a){const r=t.split(e),s=r.slice(0,a);return r.slice(a).length>0&&s.push(r.slice(a).join(e)),s}var rt,st=(rt=Object.freeze({__proto__:null,default:{}}))&&rt.default||rt,it=e((function(t){!function(){var e,a,r,s=0,i=[];for(a=0;a<256;a++)i[a]=(a+256).toString(16).substr(1);function n(){var t=function(t){return(!e||s+t>o.BUFFER_SIZE)&&(s=0,e=o.randomBytes(o.BUFFER_SIZE)),e.slice(s,s+=t)}(16);return t[6]=15&t[6]|64,t[8]=63&t[8]|128,t}function o(){var t=n();return i[t[0]]+i[t[1]]+i[t[2]]+i[t[3]]+"-"+i[t[4]]+i[t[5]]+"-"+i[t[6]]+i[t[7]]+"-"+i[t[8]]+i[t[9]]+"-"+i[t[10]]+i[t[11]]+i[t[12]]+i[t[13]]+i[t[14]]+i[t[15]]}o.BUFFER_SIZE=4096,o.bin=n,o.clearBuffer=function(){e=null,s=0},o.test=function(t){return"string"==typeof t&&/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(t)},"undefined"!=typeof crypto?r=crypto:"undefined"!=typeof window&&void 0!==window.msCrypto&&(r=window.msCrypto),r=r||st,t.exports=o,o.randomBytes=function(){if(r){if(r.randomBytes)return r.randomBytes;if(r.getRandomValues)return function(t){var e=new Uint8Array(t);return r.getRandomValues(e),e}}return function(t){var e,a=[];for(e=0;e`),!r.headers.get("Content-Type")&&ht[a]&&r.headers.set("Content-Type",ht[a]),!c){c=async function*(){}()}const f=new lt({warcHeaders:r,reader:c});switch(a){case"response":case"request":case"revisit":f.httpHeaders=new tt({statusline:n,headers:h?new Map(Object.entries(i)):new Headers(i)})}return f}static createWARCInfo(t={},e){return t.type="warcinfo",lt.create(t,async function*(){for(const[t,a]of Object.entries(e))yield ot.encode(`${t}: ${a}\r\n`)}())}constructor({warcHeaders:t,reader:e}){super(),this.warcHeaders=t,this._reader=e,this._contentReader=null,this.payload=null,this.httpHeaders=null,this.consumed=!1,this.fixUp()}getResponseInfo(){const t=this.httpHeaders;return t?{headers:t.headers,status:t.statusCode,statusText:t.statusText}:null}fixUp(){const t=this.warcHeaders.headers.get("WARC-Target-URI");t&&t.startsWith("<")&&t.endsWith(">")&&this.warcHeaders.headers.set("WARC-Target-URI",t.slice(1,-1))}async readFully(t=!1){if(this.httpHeaders){if(this._contentReader&&!t)throw new TypeError("WARC Record decoding already started, but requesting raw payload");if(t&&"raw"===this.consumed&&this.payload)return await this._createDecodingReader([this.payload]).readFully()}return this.payload||(t?(this.payload=await super.readFully(),this.consumed="content"):(this.payload=await lt.readFully(this._reader),this.consumed="raw")),this.payload}get reader(){if(this._contentReader)throw new TypeError("WARC Record decoding already started, but requesting raw payload");return this._reader}get contentReader(){return this.httpHeaders?(this._contentReader||(this._contentReader=this._createDecodingReader(this._reader)),this._contentReader):this._reader}_createDecodingReader(t){let e=this.httpHeaders.headers.get("content-encoding"),a=this.httpHeaders.headers.get("transfer-encoding");const r="chunked"===a;return e||r||(e=a),new X(t,e,r)}async readlineRaw(t){if(this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");return this.contentReader.readlineRaw(t)}async contentText(){const t=await this.readFully(!0);return nt.decode(t)}async*[Symbol.asyncIterator](){for await(const t of this.contentReader)if(yield t,this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");this.consumed="content"}async skipFully(){if(this.consumed)return;const t=await this._reader.skipFully();return this.consumed="skipped",t}warcHeader(t){return this.warcHeaders.headers.get(t)}get warcType(){return this.warcHeaders.headers.get("WARC-Type")}get warcTargetURI(){return this.warcHeaders.headers.get("WARC-Target-URI")}get warcDate(){return this.warcHeaders.headers.get("WARC-Date")}get warcRefersToTargetURI(){return this.warcHeaders.headers.get("WARC-Refers-To-Target-URI")}get warcRefersToDate(){return this.warcHeaders.headers.get("WARC-Refers-To-Date")}get warcPayloadDigest(){return this.warcHeaders.headers.get("WARC-Payload-Digest")}get warcBlockDigest(){return this.warcHeaders.headers.get("WARC-Block-Digest")}get warcContentType(){return this.warcHeaders.headers.get("Content-Type")}get warcContentLength(){return Number(this.warcHeaders.headers.get("Content-Length"))}}class dt{static parse(t,e){return new dt(t,e).parse()}static iterRecords(t,e){return new dt(t,e)[Symbol.asyncIterator]()}constructor(t,{keepHeadersCase:e=!1,parseHttp:a=!0}={}){this._offset=0,this._warcHeadersLength=0,this._headersClass=e?Map:Headers,this._parseHttp=a,this._atRecordBoundary=!0,t instanceof X||(t=new X(t)),this._reader=t,this._record=null}async readToNextRecord(){let t="";if(!this._atRecordBoundary&&this._reader&&this._record){await this._record.skipFully(),t=await this._reader.readline();const e=t.trim().length;if(e&&console.warn(`Content-Length Too Small: Record not followed by newline, Remainder Length: ${e}, Offset: ${this._reader.getRawOffset()-t.length}`),this._reader.compressed)await this._reader.readSize(2,!0),t="";else for(t=await this._reader.readline();2===t.length;)t=await this._reader.readline()}return this._atRecordBoundary=!0,t}_initRecordReader(t){return new K(this._reader,Number(t.headers.get("Content-Length")||0))}async parse(){const t=await this.readToNextRecord();this._offset=this._reader.getRawOffset()-t.length;const e=new et,a=await e.parse(this._reader,{firstLine:t,headersClass:this._headersClass});if(!a)return null;this._warcHeadersLength=this._reader.getReadOffset();const r=new lt({warcHeaders:a,reader:this._initRecordReader(a)});if(this._atRecordBoundary=!1,this._record=r,this._parseHttp)switch(r.warcType){case"response":case"request":await this._addHttpHeaders(r,e,this._reader);break;case"revisit":r.warcContentLength>0&&await this._addHttpHeaders(r,e,this._reader)}return r}get offset(){return this._offset}get recordLength(){return this._reader.getRawLength(this._offset)}async*[Symbol.asyncIterator](){let t=null;for(;t=await this.parse(this._reader);)yield t;this._record=null}async _addHttpHeaders(t,e){const a=await e.parse(this._reader,{headersClass:this._headersClass});t.httpHeaders=a;const r=this._reader.getReadOffset()-this._warcHeadersLength;t.reader.setLimitSkip&&t.reader.setLimitSkip(t.warcContentLength-r)}}var ct=e((function(e){ /* * [hi-base32]{@link https://github.com/emn178/hi-base32} * diff --git a/package.json b/package.json index 1f056ba..cfb40c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "warcio", - "version": "1.4.1", + "version": "1.4.2", "main": "index.js", "module": "main.js", "license": "Apache-2.0", diff --git a/src/readers.js b/src/readers.js index da7fa3b..bcd0fe5 100644 --- a/src/readers.js +++ b/src/readers.js @@ -457,7 +457,9 @@ class LimitReader extends BaseAsyncIterReader const [first, remainder] = LimitReader.splitChunk(chunk, this.limit); chunk = first; - this.sourceIter._unread(remainder); + if (this.sourceIter._unread) { + this.sourceIter._unread(remainder); + } } if (chunk.length) {