diff --git a/dist/warcio.js b/dist/warcio.js index 80374dd..c90c58b 100644 --- a/dist/warcio.js +++ b/dist/warcio.js @@ -8402,6 +8402,124 @@ class WARCSerializer extends BaseAsyncIterReader } } +function getSurt(url) { + try { + if (!url.startsWith("https:") && !url.startsWith("http:")) { + return url; + } + url = url.replace(/www\d*\./, ''); + const urlObj = new URL(url.toLowerCase()); + + const hostParts = urlObj.hostname.split(".").reverse(); + let surt = hostParts.join(","); + if (urlObj.port) { + surt += ":" + urlObj.port; + } + surt += ")"; + surt += urlObj.pathname; + if (urlObj.search) { + urlObj.searchParams.sort(); + surt += urlObj.search; + } + return surt; + } catch (e) { + return url; + } +} + +function postToGetUrl(request) { + let {url, method, headers, postData} = request; + + if (method === "GET") { + return false; + } + + const requestMime = (headers.get("content-type") || "").split(";")[0]; + + function decodeIfNeeded(postData) { + if (postData instanceof Uint8Array) { + postData = new TextDecoder().decode(postData); + } + return postData; + } + + let query = null; + + switch (requestMime) { + case "application/x-www-form-urlencoded": + query = decodeIfNeeded(postData); + break; + + case "text/plain": + case "application/json": + query = jsonToQueryString(decodeIfNeeded(postData)); + break; + + case "multipart/form-data": + query = mfdToQueryString(decodeIfNeeded(postData), headers.get("content-type")); + break; + + default: + return false; + } + + if (query) { + const start = (url.indexOf("?") > 0 ? "&" : "?"); + request.url += `${start}__wb_method=${method}&${query}`; + request.method = "GET"; + return true; + } + + return false; +} + +function jsonToQueryString(json) { + if (typeof(json) === "string") { + try { + json = JSON.parse(json); + } catch(e) { + json = {}; + } + } + + const q = new URLSearchParams(); + + try { + JSON.stringify(json, (k, v) => { + if (!["object", "function"].includes(typeof(v))) { + q.set(k, v); + } + return v; + }); + } catch (e) {} + + return q.toString(); +} + +function mfdToQueryString(mfd, contentType) { + const params = new URLSearchParams(); + + if (mfd instanceof Uint8Array) { + mfd = new TextDecoder().decode(mfd); + } + + try { + const boundary = contentType.split("boundary=")[1]; + + const parts = mfd.split(new RegExp("-*" + boundary + "-*", "mi")); + + for (let i = 0; i < parts.length; i++) { + const m = parts[i].trim().match(/name="([^"]+)"\r\n\r\n(.*)/mi); + if (m) { + params.set(m[1], m[2]); + } + } + + } catch (e) {} + + return params.toString(); +} + const DEFAULT_FIELDS = 'offset,warc-type,warc-target-uri'.split(','); @@ -8437,12 +8555,16 @@ class BaseIndexer const parser = new WARCParser(reader, params); - for await (const record of parser) { - await record.skipFully(); - const result = this.indexRecord(record, parser, filename); - if (result) { - yield result; - } + yield* this.iterRecords(parser, filename); + } + } + + async* iterRecords(parser, filename) { + for await (const record of parser) { + await record.skipFully(); + const result = this.indexRecord(record, parser, filename); + if (result) { + yield result; } } } @@ -8534,6 +8656,7 @@ class CDXIndexer extends Indexer this.includeAll = opts.all; this.fields = DEFAULT_CDX_FIELDS; this.parseHttp = true; + this._lastRecord = null; switch (opts.format) { case "cdxj": @@ -8546,6 +8669,23 @@ class CDXIndexer extends Indexer } } + async* iterRecords(parser, filename) { + this._lastRecord = null; + + for await (const record of parser) { + await record.readFully(); + const result = this.indexRecord(record, parser, filename); + if (result) { + yield result; + } + } + + const result = this.indexRecord(null, parser, filename); + if (result) { + yield result; + } + } + filterRecord(record) { if (this.includeAll) { return true; @@ -8559,6 +8699,76 @@ class CDXIndexer extends Indexer return true; } + indexRecord(record, parser, filename) { + if (this.includeAll) { + if (!record) { + return; + } + return super.indexRecord(record, parser, filename); + } + + const lastRecord = this._lastRecord; + + if (record) { + record._offset = parser.offset; + record._length = parser.recordLength; + } + + if (!lastRecord) { + this._lastRecord = record; + return null; + } + + if (!record || lastRecord.warcTargetURI != record.warcTargetURI) { + this._lastRecord = record; + return this.indexRecordPair(lastRecord, null, parser, filename); + } + + if (record.warcType === "request" && lastRecord.warcType === "response") { + this._lastRecord = null; + return this.indexRecordPair(lastRecord, record, parser, filename); + } else if (record.warcType === "response" && lastRecord.warcType === "request") { + this._lastRecord = null; + return this.indexRecordPair(record, lastRecord, parser, filename); + } else { + this._lastRecord = record; + return this.indexRecordPair(lastRecord, null, parser, filename); + } + } + + indexRecordPair(record, reqRecord, parser, filename) { + let method; + let requestBody; + + if (reqRecord && reqRecord.httpHeaders.method !== "GET") { + const request = { + url: record.warcTargetURI, + method: reqRecord.httpHeaders.method, + headers: reqRecord.httpHeaders.headers, + postData: reqRecord.payload, + }; + + method = request.method; + + if (postToGetUrl(request)) { + requestBody = request.url.slice(record.warcTargetURI.length); + } + } + + const res = super.indexRecord(record, parser, filename); + if (res && record && record._offset !== undefined) { + res.offset = record._offset; + res.length = record._length; + } + if (method) { + res.method = method; + } + if (requestBody) { + res.requestBody = requestBody; + } + return res; + } + serializeCDXJ(result) { const { urlkey, timestamp } = result; delete result.urlkey; @@ -8582,7 +8792,7 @@ class CDXIndexer extends Indexer switch (field) { case "urlkey": - return this.getSurt(record.warcTargetURI); + return getSurt(record.updatedURL ? record.updatedURL : record.warcTargetURI); case "timestamp": value = record.warcDate; @@ -8616,26 +8826,6 @@ class CDXIndexer extends Indexer return value ? value.split(":", 2)[1] : null; } } - - getSurt(url) { - try { - const urlObj = new URL(url); - if (urlObj.protocol !== "http:" && urlObj.protocol !== "https:") { - return url; - } - - const hostParts = urlObj.hostname.split(".").reverse(); - let surt = hostParts.join(","); - if (urlObj.port) { - surt += ":" + urlObj.port; - } - surt += ")"; - surt += urlObj.pathname; - return surt.toLowerCase(); - } catch (e) { - return url; - } - } } -export { AsyncIterReader, BaseAsyncIterReader, CDXIndexer, Indexer, LimitReader, StatusAndHeaders, StatusAndHeadersParser, WARCParser, WARCRecord, WARCSerializer }; +export { AsyncIterReader, BaseAsyncIterReader, CDXIndexer, Indexer, LimitReader, StatusAndHeaders, StatusAndHeadersParser, WARCParser, WARCRecord, WARCSerializer, getSurt, postToGetUrl }; diff --git a/dist/warcio.min.js b/dist/warcio.min.js index 5a008c7..7c6067b 100644 --- a/dist/warcio.min.js +++ b/dist/warcio.min.js @@ -3,7 +3,7 @@ * See https://github.com/webrecorder/warcio.js for more info. Copyright (C) 2020, Webrecorder Software */ -var t="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function e(t,e){return t(e={exports:{}},e.exports),e.exports}var a=e((function(t,e){var a="undefined"!=typeof Uint8Array&&"undefined"!=typeof Uint16Array&&"undefined"!=typeof Int32Array;function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}e.assign=function(t){for(var e=Array.prototype.slice.call(arguments,1);e.length;){var a=e.shift();if(a){if("object"!=typeof a)throw new TypeError(a+"must be non-object");for(var i in a)r(a,i)&&(t[i]=a[i])}}return t},e.shrinkBuf=function(t,e){return t.length===e?t:t.subarray?t.subarray(0,e):(t.length=e,t)};var i={arraySet:function(t,e,a,r,i){if(e.subarray&&t.subarray)t.set(e.subarray(a,a+r),i);else for(var s=0;s>>16&65535|0,n=0;0!==a;){a-=n=a>2e3?2e3:a;do{s=s+(i=i+e[r++]|0)|0}while(--n);i%=65521,s%=65521}return i|s<<16|0};var i=function(){for(var t,e=[],a=0;a<256;a++){t=a;for(var r=0;r<8;r++)t=1&t?3988292384^t>>>1:t>>>1;e[a]=t}return e}();var s=function(t,e,a,r){var s=i,n=r+a;t^=-1;for(var o=r;o>>8^s[255&(t^e[o])];return-1^t},n=function(t,e){var a,r,i,s,n,o,h,l,d,c,u,f,_,p,g,w,m,b,y,v,k,x,C,A,z;a=t.state,r=t.next_in,A=t.input,i=r+(t.avail_in-5),s=t.next_out,z=t.output,n=s-(e-t.avail_out),o=s+(t.avail_out-257),h=a.dmax,l=a.wsize,d=a.whave,c=a.wnext,u=a.window,f=a.hold,_=a.bits,p=a.lencode,g=a.distcode,w=(1<>>=y=b>>>24,_-=y,0===(y=b>>>16&255))z[s++]=65535&b;else{if(!(16&y)){if(0==(64&y)){b=p[(65535&b)+(f&(1<>>=y,_-=y),_<15&&(f+=A[r++]<<_,_+=8,f+=A[r++]<<_,_+=8),b=g[f&m];a:for(;;){if(f>>>=y=b>>>24,_-=y,!(16&(y=b>>>16&255))){if(0==(64&y)){b=g[(65535&b)+(f&(1<h){t.msg="invalid distance too far back",a.mode=30;break t}if(f>>>=y,_-=y,k>(y=s-n)){if((y=k-y)>d&&a.sane){t.msg="invalid distance too far back",a.mode=30;break t}if(x=0,C=u,0===c){if(x+=l-y,y2;)z[s++]=C[x++],z[s++]=C[x++],z[s++]=C[x++],v-=3;v&&(z[s++]=C[x++],v>1&&(z[s++]=C[x++]))}else{x=s-k;do{z[s++]=z[x++],z[s++]=z[x++],z[s++]=z[x++],v-=3}while(v>2);v&&(z[s++]=z[x++],v>1&&(z[s++]=z[x++]))}break}}break}}while(r>3,f&=(1<<(_-=v<<3))-1,t.next_in=r,t.next_out=s,t.avail_in=r=1&&0===L[z];z--);if(R>z&&(R=z),0===z)return s[n++]=20971520,s[n++]=20971520,u.bits=1,0;for(A=1;A0&&(0===t||1!==z))return-1;for(O[1]=0,x=1;x<15;x++)O[x+1]=O[x]+L[x];for(C=0;C852||2===t&&T>592)return 1;for(;;){b=x-B,c[C]m?(y=U[W+c[C]],v=D[E+c[C]]):(y=96,v=0),f=1<>B)+(_-=f)]=b<<24|y<<16|v|0}while(0!==_);for(f=1<>=1;if(0!==f?(I&=f-1,I+=f):I=0,C++,0==--L[x]){if(x===z)break;x=e[r+c[C]]}if(x>R&&(I&g)!==p){for(0===B&&(B=R),w+=A,H=1<<(S=x-B);S+B852||2===t&&T>592)return 1;s[p=I&g]=R<<24|S<<16|w-n|0}}return 0!==I&&(s[w+I]=x-B<<24|64<<16|0),u.bits=R,0};function u(t){return(t>>>24&255)+(t>>>8&65280)+((65280&t)<<8)+((255&t)<<24)}function f(){this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new a.Buf16(320),this.work=new a.Buf16(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}function _(t){var e;return t&&t.state?(e=t.state,t.total_in=t.total_out=e.total=0,t.msg="",e.wrap&&(t.adler=1&e.wrap),e.mode=1,e.last=0,e.havedict=0,e.dmax=32768,e.head=null,e.hold=0,e.bits=0,e.lencode=e.lendyn=new a.Buf32(852),e.distcode=e.distdyn=new a.Buf32(592),e.sane=1,e.back=-1,0):-2}function p(t){var e;return t&&t.state?((e=t.state).wsize=0,e.whave=0,e.wnext=0,_(t)):-2}function g(t,e){var a,r;return t&&t.state?(r=t.state,e<0?(a=0,e=-e):(a=1+(e>>4),e<48&&(e&=15)),e&&(e<8||e>15)?-2:(null!==r.window&&r.wbits!==e&&(r.window=null),r.wrap=a,r.wbits=e,p(t))):-2}function w(t,e){var a,r;return t?(r=new f,t.state=r,r.window=null,0!==(a=g(t,e))&&(t.state=null),a):-2}var m,b,y=!0;function v(t){if(y){var e;for(m=new a.Buf32(512),b=new a.Buf32(32),e=0;e<144;)t.lens[e++]=8;for(;e<256;)t.lens[e++]=9;for(;e<280;)t.lens[e++]=7;for(;e<288;)t.lens[e++]=8;for(c(1,t.lens,0,288,m,0,t.work,{bits:9}),e=0;e<32;)t.lens[e++]=5;c(2,t.lens,0,32,b,0,t.work,{bits:5}),y=!1}t.lencode=m,t.lenbits=9,t.distcode=b,t.distbits=5}function k(t,e,r,i){var s,n=t.state;return null===n.window&&(n.wsize=1<=n.wsize?(a.arraySet(n.window,e,r-n.wsize,n.wsize,0),n.wnext=0,n.whave=n.wsize):((s=n.wsize-n.wnext)>i&&(s=i),a.arraySet(n.window,e,r-i,s,n.wnext),(i-=s)?(a.arraySet(n.window,e,r-i,i,0),n.wnext=i,n.whave=n.wsize):(n.wnext+=s,n.wnext===n.wsize&&(n.wnext=0),n.whave>>8&255,i.check=s(i.check,L,2,0),p=0,g=0,i.mode=2;break}if(i.flags=0,i.head&&(i.head.done=!1),!(1&i.wrap)||(((255&p)<<8)+(p>>8))%31){t.msg="incorrect header check",i.mode=30;break}if(8!=(15&p)){t.msg="unknown compression method",i.mode=30;break}if(g-=4,H=8+(15&(p>>>=4)),0===i.wbits)i.wbits=H;else if(H>i.wbits){t.msg="invalid window size",i.mode=30;break}i.dmax=1<>8&1),512&i.flags&&(L[0]=255&p,L[1]=p>>>8&255,i.check=s(i.check,L,2,0)),p=0,g=0,i.mode=3;case 3:for(;g<32;){if(0===f)break t;f--,p+=o[l++]<>>8&255,L[2]=p>>>16&255,L[3]=p>>>24&255,i.check=s(i.check,L,4,0)),p=0,g=0,i.mode=4;case 4:for(;g<16;){if(0===f)break t;f--,p+=o[l++]<>8),512&i.flags&&(L[0]=255&p,L[1]=p>>>8&255,i.check=s(i.check,L,2,0)),p=0,g=0,i.mode=5;case 5:if(1024&i.flags){for(;g<16;){if(0===f)break t;f--,p+=o[l++]<>>8&255,i.check=s(i.check,L,2,0)),p=0,g=0}else i.head&&(i.head.extra=null);i.mode=6;case 6:if(1024&i.flags&&((b=i.length)>f&&(b=f),b&&(i.head&&(H=i.head.extra_len-i.length,i.head.extra||(i.head.extra=new Array(i.head.extra_len)),a.arraySet(i.head.extra,o,l,b,H)),512&i.flags&&(i.check=s(i.check,o,b,l)),f-=b,l+=b,i.length-=b),i.length))break t;i.length=0,i.mode=7;case 7:if(2048&i.flags){if(0===f)break t;b=0;do{H=o[l+b++],i.head&&H&&i.length<65536&&(i.head.name+=String.fromCharCode(H))}while(H&&b>9&1,i.head.done=!0),t.adler=i.check=0,i.mode=12;break;case 10:for(;g<32;){if(0===f)break t;f--,p+=o[l++]<>>=7&g,g-=7&g,i.mode=27;break}for(;g<3;){if(0===f)break t;f--,p+=o[l++]<>>=1)){case 0:i.mode=14;break;case 1:if(v(i),i.mode=20,6===e){p>>>=2,g-=2;break t}break;case 2:i.mode=17;break;case 3:t.msg="invalid block type",i.mode=30}p>>>=2,g-=2;break;case 14:for(p>>>=7&g,g-=7&g;g<32;){if(0===f)break t;f--,p+=o[l++]<>>16^65535)){t.msg="invalid stored block lengths",i.mode=30;break}if(i.length=65535&p,p=0,g=0,i.mode=15,6===e)break t;case 15:i.mode=16;case 16:if(b=i.length){if(b>f&&(b=f),b>_&&(b=_),0===b)break t;a.arraySet(h,o,l,b,d),f-=b,l+=b,_-=b,d+=b,i.length-=b;break}i.mode=12;break;case 17:for(;g<14;){if(0===f)break t;f--,p+=o[l++]<>>=5,g-=5,i.ndist=1+(31&p),p>>>=5,g-=5,i.ncode=4+(15&p),p>>>=4,g-=4,i.nlen>286||i.ndist>30){t.msg="too many length or distance symbols",i.mode=30;break}i.have=0,i.mode=18;case 18:for(;i.have>>=3,g-=3}for(;i.have<19;)i.lens[O[i.have++]]=0;if(i.lencode=i.lendyn,i.lenbits=7,I={bits:i.lenbits},T=c(0,i.lens,0,19,i.lencode,0,i.work,I),i.lenbits=I.bits,T){t.msg="invalid code lengths set",i.mode=30;break}i.have=0,i.mode=19;case 19:for(;i.have>>16&255,z=65535&E,!((C=E>>>24)<=g);){if(0===f)break t;f--,p+=o[l++]<>>=C,g-=C,i.lens[i.have++]=z;else{if(16===z){for(D=C+2;g>>=C,g-=C,0===i.have){t.msg="invalid bit length repeat",i.mode=30;break}H=i.lens[i.have-1],b=3+(3&p),p>>>=2,g-=2}else if(17===z){for(D=C+3;g>>=C)),p>>>=3,g-=3}else{for(D=C+7;g>>=C)),p>>>=7,g-=7}if(i.have+b>i.nlen+i.ndist){t.msg="invalid bit length repeat",i.mode=30;break}for(;b--;)i.lens[i.have++]=H}}if(30===i.mode)break;if(0===i.lens[256]){t.msg="invalid code -- missing end-of-block",i.mode=30;break}if(i.lenbits=9,I={bits:i.lenbits},T=c(1,i.lens,0,i.nlen,i.lencode,0,i.work,I),i.lenbits=I.bits,T){t.msg="invalid literal/lengths set",i.mode=30;break}if(i.distbits=6,i.distcode=i.distdyn,I={bits:i.distbits},T=c(2,i.lens,i.nlen,i.ndist,i.distcode,0,i.work,I),i.distbits=I.bits,T){t.msg="invalid distances set",i.mode=30;break}if(i.mode=20,6===e)break t;case 20:i.mode=21;case 21:if(f>=6&&_>=258){t.next_out=d,t.avail_out=_,t.next_in=l,t.avail_in=f,i.hold=p,i.bits=g,n(t,m),d=t.next_out,h=t.output,_=t.avail_out,l=t.next_in,o=t.input,f=t.avail_in,p=i.hold,g=i.bits,12===i.mode&&(i.back=-1);break}for(i.back=0;A=(E=i.lencode[p&(1<>>16&255,z=65535&E,!((C=E>>>24)<=g);){if(0===f)break t;f--,p+=o[l++]<>R)])>>>16&255,z=65535&E,!(R+(C=E>>>24)<=g);){if(0===f)break t;f--,p+=o[l++]<>>=R,g-=R,i.back+=R}if(p>>>=C,g-=C,i.back+=C,i.length=z,0===A){i.mode=26;break}if(32&A){i.back=-1,i.mode=12;break}if(64&A){t.msg="invalid literal/length code",i.mode=30;break}i.extra=15&A,i.mode=22;case 22:if(i.extra){for(D=i.extra;g>>=i.extra,g-=i.extra,i.back+=i.extra}i.was=i.length,i.mode=23;case 23:for(;A=(E=i.distcode[p&(1<>>16&255,z=65535&E,!((C=E>>>24)<=g);){if(0===f)break t;f--,p+=o[l++]<>R)])>>>16&255,z=65535&E,!(R+(C=E>>>24)<=g);){if(0===f)break t;f--,p+=o[l++]<>>=R,g-=R,i.back+=R}if(p>>>=C,g-=C,i.back+=C,64&A){t.msg="invalid distance code",i.mode=30;break}i.offset=z,i.extra=15&A,i.mode=24;case 24:if(i.extra){for(D=i.extra;g>>=i.extra,g-=i.extra,i.back+=i.extra}if(i.offset>i.dmax){t.msg="invalid distance too far back",i.mode=30;break}i.mode=25;case 25:if(0===_)break t;if(b=m-_,i.offset>b){if((b=i.offset-b)>i.whave&&i.sane){t.msg="invalid distance too far back",i.mode=30;break}b>i.wnext?(b-=i.wnext,y=i.wsize-b):y=i.wnext-b,b>i.length&&(b=i.length),x=i.window}else x=h,y=d-i.offset,b=i.length;b>_&&(b=_),_-=b,i.length-=b;do{h[d++]=x[y++]}while(--b);0===i.length&&(i.mode=21);break;case 26:if(0===_)break t;h[d++]=i.length,_--,i.mode=21;break;case 27:if(i.wrap){for(;g<32;){if(0===f)break t;f--,p|=o[l++]<=252?6:R>=248?5:R>=240?4:R>=224?3:R>=192?2:1;z[254]=z[254]=1;function S(t,e){if(e<65534&&(t.subarray&&A||!t.subarray&&C))return String.fromCharCode.apply(null,a.shrinkBuf(t,e));for(var r="",i=0;i>>6,e[n++]=128|63&r):r<65536?(e[n++]=224|r>>>12,e[n++]=128|r>>>6&63,e[n++]=128|63&r):(e[n++]=240|r>>>18,e[n++]=128|r>>>12&63,e[n++]=128|r>>>6&63,e[n++]=128|63&r);return e},H=function(t){return S(t,t.length)},T=function(t){for(var e=new a.Buf8(t.length),r=0,i=e.length;r4)o[r++]=65533,a+=s-1;else{for(i&=2===s?31:3===s?15:7;s>1&&a1?o[r++]=65533:i<65536?o[r++]=i:(i-=65536,o[r++]=55296|i>>10&1023,o[r++]=56320|1023&i)}return S(o,r)},D=function(t,e){var a;for((e=e||t.length)>t.length&&(e=t.length),a=e-1;a>=0&&128==(192&t[a]);)a--;return a<0||0===a?e:a+z[t[a]]>e?a:e},E=0,L=2,O=4,U=0,W=1,F=2,j=-5,N={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"};var P=function(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0};var q=function(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1},M=Object.prototype.toString;function $(t){if(!(this instanceof $))return new $(t);this.options=a.assign({chunkSize:16384,windowBits:0,to:""},t||{});var e=this.options;e.raw&&e.windowBits>=0&&e.windowBits<16&&(e.windowBits=-e.windowBits,0===e.windowBits&&(e.windowBits=-15)),!(e.windowBits>=0&&e.windowBits<16)||t&&t.windowBits||(e.windowBits+=32),e.windowBits>15&&e.windowBits<48&&0==(15&e.windowBits)&&(e.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new P,this.strm.avail_out=0;var r=x.inflateInit2(this.strm,e.windowBits);if(r!==U)throw new Error(N[r]);if(this.header=new q,x.inflateGetHeader(this.strm,this.header),e.dictionary&&("string"==typeof e.dictionary?e.dictionary=B(e.dictionary):"[object ArrayBuffer]"===M.call(e.dictionary)&&(e.dictionary=new Uint8Array(e.dictionary)),e.raw&&(r=x.inflateSetDictionary(this.strm,e.dictionary))!==U))throw new Error(N[r])}$.prototype.push=function(t,e){var r,i,s,n,o,h=this.strm,l=this.options.chunkSize,d=this.options.dictionary,c=!1;if(this.ended)return!1;i=e===~~e?e:!0===e?O:E,"string"==typeof t?h.input=T(t):"[object ArrayBuffer]"===M.call(t)?h.input=new Uint8Array(t):h.input=t,h.next_in=0,h.avail_in=h.input.length;do{if(0===h.avail_out&&(h.output=new a.Buf8(l),h.next_out=0,h.avail_out=l),(r=x.inflate(h,E))===F&&d&&(r=x.inflateSetDictionary(this.strm,d)),r===j&&!0===c&&(r=U,c=!1),r!==W&&r!==U)return this.onEnd(r),this.ended=!0,!1;h.next_out&&(0!==h.avail_out&&r!==W&&(0!==h.avail_in||i!==O&&i!==L)||("string"===this.options.to?(s=D(h.output,h.next_out),n=h.next_out-s,o=I(h.output,s),h.next_out=n,h.avail_out=l-n,n&&a.arraySet(h.output,h.output,s,n,0),this.onData(o)):this.onData(a.shrinkBuf(h.output,h.next_out)))),0===h.avail_in&&0===h.avail_out&&(c=!0)}while((h.avail_in>0||0===h.avail_out)&&r!==W);return r===W&&(i=O),i===O?(r=x.inflateEnd(this.strm),this.onEnd(r),this.ended=!0,r===U):i!==L||(this.onEnd(U),h.avail_out=0,!0)},$.prototype.onData=function(t){this.chunks.push(t)},$.prototype.onEnd=function(t){t===U&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var V=$;const Z=new TextDecoder("utf-8");class J extends V{constructor(t,e){super(t),this.reader=e}onEnd(t){this.err=t,this.err||(this.reader._rawOffset+=this.strm.total_in)}}class X{static concatChunks(t,e){if(1===t.length)return t[0];const a=new Uint8Array(e);let r=0;for(const e of t)a.set(e,r),r+=e.byteLength;return a}static splitChunk(t,e){return[t.slice(0,e),t.slice(e)]}static async readFully(t){const e=[];let a=0;for await(const r of t)e.push(r),a+=r.byteLength;return X.concatChunks(e,a)}getReadableStream(){const t=this[Symbol.asyncIterator]();return new ReadableStream({pull:e=>t.next().then(t=>{t.done||!t.value?e.close():e.enqueue(t.value)})})}readFully(){return X.readFully(this)}async readline(t=0){const e=await this.readlineRaw(t);return e?Z.decode(e):""}async*iterLines(t=0){let e=null;for(;e=await this.readline(t);)yield e}}class K extends X{constructor(t,e="gzip",a=!1){if(super(),this.compressed=e,this.opts={raw:"deflateRaw"===e},this.inflator=e?new J(this.opts,this):null,"function"!=typeof t[Symbol.asyncIterator])if("function"==typeof t.getReader)t=K.fromReadable(t.getReader());else if("function"==typeof t.read)t=K.fromReadable(t);else{if("function"!=typeof t[Symbol.iterator])throw new TypeError("Invalid Stream Source");t=K.fromIter(t)}this._sourceIter=a?this.dechunk(t):t[Symbol.asyncIterator](),this.lastValue=null,this.errored=!1,this._savedChunk=null,this._rawOffset=0,this._readOffset=0,this.numChunks=0}async _loadNext(){const t=await this._sourceIter.next();return t.done?null:t.value}async*dechunk(t){const e=t instanceof K?t:new K(t,null);let a=-1,r=!0;for(;0!=a;){const t=await e.readlineRaw(64);let i=null;if(a=parseInt(Z.decode(t),16),!a||a>2**32){if(Number.isNaN(a)||a>2**32){r||(this.errored=!0),yield t;break}}else if(i=await e.readSize(a),i.length!=a){r?yield t:this.errored=!0,yield i;break}const s=await e.readSize(2);if(13!=s[0]||10!=s[1]){r?yield t:this.errored=!0,yield i,yield s;break}if(r=!1,!i||0===a)return;yield i}yield*e}_unread(t){t.length&&(this._readOffset-=t.length,this._savedChunk&&console.log("Already have chunk!"),this._savedChunk=t)}async _next(){if(this._savedChunk){const t=this._savedChunk;return this._savedChunk=null,t}if(this.compressed){const t=this._getNextChunk();if(t)return t}let t=await this._loadNext();for(;this.compressed&&t;){this._push(t);const e=this._getNextChunk(t);if(e)return e;t=await this._loadNext()}return t}_push(t){this.lastValue=t,this.inflator.ended&&(this.inflator=new J(this.opts,this)),this.inflator.push(t),this.inflator.err&&this.inflator.ended&&"deflate"===this.compressed&&!1===this.opts.raw&&0===this.numChunks&&(this.opts.raw=!0,this.compressed="deflateRaw",this.inflator=new J(this.opts,this),this.inflator.push(t))}_getNextChunk(t){for(;;){if(this.inflator.chunks.length>0)return this.numChunks++,this.inflator.chunks.shift();if(this.inflator.ended){if(0!==this.inflator.err)return this.compressed=null,t;const e=this.inflator.strm.avail_in;if(e&&this.lastValue){this._push(this.lastValue.slice(-e));continue}}return null}}async*[Symbol.asyncIterator](){let t=null;for(;t=await this._next();)this._readOffset+=t.length,yield t}async readlineRaw(t){const e=[];let a,r=0,i=null;for await(const s of this){if(t&&r+s.byteLength>t){i=s,a=t-r-1;const e=s.slice(0,a+1).indexOf(10);e>=0&&(a=e);break}if(a=s.indexOf(10),a>=0){i=s;break}e.push(s),r+=s.byteLength}if(i){const[t,s]=K.splitChunk(i,a+1);e.push(t),r+=t.byteLength,this._unread(s)}else if(!e.length)return null;return K.concatChunks(e,r)}readFully(){return this.readSize()}async readSize(t=-1,e=!1){const a=[];let r=0;for await(const i of this){if(t>=0){if(i.length>t){const[s,n]=K.splitChunk(i,t);e||a.push(s),r+=s.byteLength,this._unread(n);break}if(i.length===t){e||a.push(i),r+=i.byteLength,t=0;break}t-=i.length}e||a.push(i),r+=i.byteLength}return e?r:K.concatChunks(a,r)}getReadOffset(){return this._readOffset}getRawOffset(){return this.compressed?this._rawOffset:this._readOffset}getRawLength(t){return this.compressed?this.inflator.strm.total_in:this._readOffset-t}static fromReadable(t){return{async*[Symbol.asyncIterator](){let e=null;for(;(e=await t.read())&&!e.done;)yield e.value}}}static fromIter(t){return{async*[Symbol.asyncIterator](){for(const e of t)yield e}}}}class G extends X{constructor(t,e,a=0){super(),this.sourceIter=t,this.length=e,this.limit=e,this.skip=a}setLimitSkip(t,e=0){this.limit=t,this.skip=e}async*[Symbol.asyncIterator](){if(!(this.limit<=0))for await(let t of this.sourceIter){if(this.skip>0){if(!(t.length>=this.skip)){this.skip-=t.length;continue}{const[e,a]=G.splitChunk(t,this.skip);t=a,this.skip=0}}if(t.length>this.limit){const[e,a]=G.splitChunk(t,this.limit);t=e,this.sourceIter._unread(a)}if(t.length&&(this.limit-=t.length,yield t),this.limit<=0)break}}async readlineRaw(t){if(this.limit<=0)return null;const e=await this.sourceIter.readlineRaw(t?Math.min(t,this.limit):this.limit);return this.limit-=e.length,e}async skipFully(){const t=this.limit;for(;this.limit>0;)this.limit-=await this.sourceIter.readSize(this.limit,!0);return t}}const Q=new Uint8Array([13,10]),Y=new Uint8Array([13,10,13,10]);class tt{constructor({statusline:t,headers:e}){this.statusline=t,this.headers=e}toString(){const t=[this.statusline];for(const[e,a]of this.headers)t.push(`${e}: ${a}`);return t.join("\r\n")+"\r\n"}async*iterSerialize(t){yield t.encode(this.statusline),yield Q;for(const[e,a]of this.headers)yield t.encode(`${e}: ${a}\r\n`)}_parseResponseStatusLine(){const t=at(this.statusline," ",2);this._protocol=t[0],this._statusCode=t.length>1?Number(t[1]):"",this._statusText=t.length>2?t[2]:""}get statusCode(){return void 0===this._statusCode&&this._parseResponseStatusLine(),this._statusCode}get protocol(){return void 0===this._protocol&&this._parseResponseStatusLine(),this._protocol}get statusText(){return void 0===this._statusText&&this._parseResponseStatusLine(),this._statusText}_parseRequestStatusLine(){const t=this.statusline.split(" ",2);this._method=t[0],this._requestPath=t.length>1?t[1]:""}get method(){return void 0===this._method&&this._parseRequestStatusLine(),this._method}get requestPath(){return void 0===this._requestPath&&this._parseRequestStatusLine(),this._requestPath}}class et{startsWithSpace(t){const e=t.charAt(0);return" "===e||"\t"===e}async parse(t,{headersClass:e=Map,firstLine:a}={}){const r=a||await t.readline();if(!r)return null;let i=r.trimEnd();const s=new e;if(!i)return null;let n=(await t.readline()).trimEnd();for(;n;){let[e,a]=at(n,":",1);a&&(e=e.trimStart(),a=a.trim());let r=(await t.readline()).trimEnd();for(;this.startsWithSpace(r);)a&&(a+=r),r=(await t.readline()).trimEnd();if(a)try{s.set(e,a)}catch(t){}n=r}return new tt({statusline:i,headers:s,totalRead:this.totalRead})}}function at(t,e,a){const r=t.split(e),i=r.slice(0,a);return r.slice(a).length>0&&i.push(r.slice(a).join(e)),i}var rt,it=(rt=Object.freeze({__proto__:null,default:{}}))&&rt.default||rt,st=e((function(t){!function(){var e,a,r,i=0,s=[];for(a=0;a<256;a++)s[a]=(a+256).toString(16).substr(1);function n(){var t=function(t){return(!e||i+t>o.BUFFER_SIZE)&&(i=0,e=o.randomBytes(o.BUFFER_SIZE)),e.slice(i,i+=t)}(16);return t[6]=15&t[6]|64,t[8]=63&t[8]|128,t}function o(){var t=n();return s[t[0]]+s[t[1]]+s[t[2]]+s[t[3]]+"-"+s[t[4]]+s[t[5]]+"-"+s[t[6]]+s[t[7]]+"-"+s[t[8]]+s[t[9]]+"-"+s[t[10]]+s[t[11]]+s[t[12]]+s[t[13]]+s[t[14]]+s[t[15]]}o.BUFFER_SIZE=4096,o.bin=n,o.clearBuffer=function(){e=null,i=0},o.test=function(t){return"string"==typeof t&&/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(t)},"undefined"!=typeof crypto?r=crypto:"undefined"!=typeof window&&void 0!==window.msCrypto&&(r=window.msCrypto),r=r||it,t.exports=o,o.randomBytes=function(){if(r){if(r.randomBytes)return r.randomBytes;if(r.getRandomValues)return function(t){var e=new Uint8Array(t);return r.getRandomValues(e),e}}return function(t){var e,a=[];for(e=0;e`),!r.headers.get("Content-Type")&&ht[a]&&r.headers.set("Content-Type",ht[a]),!c){c=async function*(){}()}const f=new lt({warcHeaders:r,reader:c});switch(a){case"response":case"request":case"revisit":f.httpHeaders=new tt({statusline:n,headers:h?new Map(Object.entries(s)):new Headers(s)})}return f}static createWARCInfo(t={},e){return t.type="warcinfo",lt.create(t,async function*(){for(const[t,a]of Object.entries(e))yield ot.encode(`${t}: ${a}\r\n`)}())}constructor({warcHeaders:t,reader:e}){super(),this.warcHeaders=t,this._reader=e,this._contentReader=null,this.payload=null,this.httpHeaders=null,this.consumed=!1,this.fixUp()}getResponseInfo(){const t=this.httpHeaders;return t?{headers:t.headers,status:t.statusCode,statusText:t.statusText}:null}fixUp(){const t=this.warcHeaders.headers.get("WARC-Target-URI");t&&t.startsWith("<")&&t.endsWith(">")&&this.warcHeaders.headers.set("WARC-Target-URI",t.slice(1,-1))}async readFully(t=!1){if(this.httpHeaders){if(this._contentReader&&!t)throw new TypeError("WARC Record decoding already started, but requesting raw payload");if(t&&"raw"===this.consumed&&this.payload)return await this._createDecodingReader([this.payload]).readFully()}return this.payload||(t?(this.payload=await super.readFully(),this.consumed="content"):(this.payload=await lt.readFully(this._reader),this.consumed="raw")),this.payload}get reader(){if(this._contentReader)throw new TypeError("WARC Record decoding already started, but requesting raw payload");return this._reader}get contentReader(){return this.httpHeaders?(this._contentReader||(this._contentReader=this._createDecodingReader(this._reader)),this._contentReader):this._reader}_createDecodingReader(t){let e=this.httpHeaders.headers.get("content-encoding"),a=this.httpHeaders.headers.get("transfer-encoding");const r="chunked"===a;return e||r||(e=a),new K(t,e,r)}async readlineRaw(t){if(this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");return this.contentReader.readlineRaw(t)}async contentText(){const t=await this.readFully(!0);return nt.decode(t)}async*[Symbol.asyncIterator](){for await(const t of this.contentReader)if(yield t,this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");this.consumed="content"}async skipFully(){if(this.consumed)return;const t=await this._reader.skipFully();return this.consumed="skipped",t}warcHeader(t){return this.warcHeaders.headers.get(t)}get warcType(){return this.warcHeaders.headers.get("WARC-Type")}get warcTargetURI(){return this.warcHeaders.headers.get("WARC-Target-URI")}get warcDate(){return this.warcHeaders.headers.get("WARC-Date")}get warcRefersToTargetURI(){return this.warcHeaders.headers.get("WARC-Refers-To-Target-URI")}get warcRefersToDate(){return this.warcHeaders.headers.get("WARC-Refers-To-Date")}get warcPayloadDigest(){return this.warcHeaders.headers.get("WARC-Payload-Digest")}get warcBlockDigest(){return this.warcHeaders.headers.get("WARC-Block-Digest")}get warcContentType(){return this.warcHeaders.headers.get("Content-Type")}get warcContentLength(){return Number(this.warcHeaders.headers.get("Content-Length"))}}class dt{static parse(t,e){return new dt(t,e).parse()}static iterRecords(t,e){return new dt(t,e)[Symbol.asyncIterator]()}constructor(t,{keepHeadersCase:e=!1,parseHttp:a=!0}={}){this._offset=0,this._warcHeadersLength=0,this._headersClass=e?Map:Headers,this._parseHttp=a,this._atRecordBoundary=!0,t instanceof K||(t=new K(t)),this._reader=t,this._record=null}async readToNextRecord(){let t="";if(!this._atRecordBoundary&&this._reader&&this._record){await this._record.skipFully(),t=await this._reader.readline();const e=t.trim().length;if(e&&console.warn(`Content-Length Too Small: Record not followed by newline, Remainder Length: ${e}, Offset: ${this._reader.getRawOffset()-t.length}`),this._reader.compressed)await this._reader.readSize(2,!0),t="";else for(t=await this._reader.readline();2===t.length;)t=await this._reader.readline()}return this._atRecordBoundary=!0,t}_initRecordReader(t){return new G(this._reader,Number(t.headers.get("Content-Length")||0))}async parse(){const t=await this.readToNextRecord();this._offset=this._reader.getRawOffset()-t.length;const e=new et,a=await e.parse(this._reader,{firstLine:t,headersClass:this._headersClass});if(!a)return null;this._warcHeadersLength=this._reader.getReadOffset();const r=new lt({warcHeaders:a,reader:this._initRecordReader(a)});if(this._atRecordBoundary=!1,this._record=r,this._parseHttp)switch(r.warcType){case"response":case"request":await this._addHttpHeaders(r,e,this._reader);break;case"revisit":r.warcContentLength>0&&await this._addHttpHeaders(r,e,this._reader)}return r}get offset(){return this._offset}get recordLength(){return this._reader.getRawLength(this._offset)}async*[Symbol.asyncIterator](){let t=null;for(;t=await this.parse(this._reader);)yield t;this._record=null}async _addHttpHeaders(t,e){const a=await e.parse(this._reader,{headersClass:this._headersClass});t.httpHeaders=a;const r=this._reader.getReadOffset()-this._warcHeadersLength;t.reader.setLimitSkip&&t.reader.setLimitSkip(t.warcContentLength-r)}}var ct=e((function(e){ +var t="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function e(t,e){return t(e={exports:{}},e.exports),e.exports}var a=e((function(t,e){var a="undefined"!=typeof Uint8Array&&"undefined"!=typeof Uint16Array&&"undefined"!=typeof Int32Array;function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}e.assign=function(t){for(var e=Array.prototype.slice.call(arguments,1);e.length;){var a=e.shift();if(a){if("object"!=typeof a)throw new TypeError(a+"must be non-object");for(var s in a)r(a,s)&&(t[s]=a[s])}}return t},e.shrinkBuf=function(t,e){return t.length===e?t:t.subarray?t.subarray(0,e):(t.length=e,t)};var s={arraySet:function(t,e,a,r,s){if(e.subarray&&t.subarray)t.set(e.subarray(a,a+r),s);else for(var i=0;i>>16&65535|0,n=0;0!==a;){a-=n=a>2e3?2e3:a;do{i=i+(s=s+e[r++]|0)|0}while(--n);s%=65521,i%=65521}return s|i<<16|0};var s=function(){for(var t,e=[],a=0;a<256;a++){t=a;for(var r=0;r<8;r++)t=1&t?3988292384^t>>>1:t>>>1;e[a]=t}return e}();var i=function(t,e,a,r){var i=s,n=r+a;t^=-1;for(var o=r;o>>8^i[255&(t^e[o])];return-1^t},n=function(t,e){var a,r,s,i,n,o,h,l,d,c,u,f,_,p,w,g,m,b,y,v,k,x,C,A,R;a=t.state,r=t.next_in,A=t.input,s=r+(t.avail_in-5),i=t.next_out,R=t.output,n=i-(e-t.avail_out),o=i+(t.avail_out-257),h=a.dmax,l=a.wsize,d=a.whave,c=a.wnext,u=a.window,f=a.hold,_=a.bits,p=a.lencode,w=a.distcode,g=(1<>>=y=b>>>24,_-=y,0===(y=b>>>16&255))R[i++]=65535&b;else{if(!(16&y)){if(0==(64&y)){b=p[(65535&b)+(f&(1<>>=y,_-=y),_<15&&(f+=A[r++]<<_,_+=8,f+=A[r++]<<_,_+=8),b=w[f&m];a:for(;;){if(f>>>=y=b>>>24,_-=y,!(16&(y=b>>>16&255))){if(0==(64&y)){b=w[(65535&b)+(f&(1<h){t.msg="invalid distance too far back",a.mode=30;break t}if(f>>>=y,_-=y,k>(y=i-n)){if((y=k-y)>d&&a.sane){t.msg="invalid distance too far back",a.mode=30;break t}if(x=0,C=u,0===c){if(x+=l-y,y2;)R[i++]=C[x++],R[i++]=C[x++],R[i++]=C[x++],v-=3;v&&(R[i++]=C[x++],v>1&&(R[i++]=C[x++]))}else{x=i-k;do{R[i++]=R[x++],R[i++]=R[x++],R[i++]=R[x++],v-=3}while(v>2);v&&(R[i++]=R[x++],v>1&&(R[i++]=R[x++]))}break}}break}}while(r>3,f&=(1<<(_-=v<<3))-1,t.next_in=r,t.next_out=i,t.avail_in=r=1&&0===L[R];R--);if(z>R&&(z=R),0===R)return i[n++]=20971520,i[n++]=20971520,u.bits=1,0;for(A=1;A0&&(0===t||1!==R))return-1;for(U[1]=0,x=1;x<15;x++)U[x+1]=U[x]+L[x];for(C=0;C852||2===t&&H>592)return 1;for(;;){b=x-B,c[C]m?(y=O[W+c[C]],v=D[E+c[C]]):(y=96,v=0),f=1<>B)+(_-=f)]=b<<24|y<<16|v|0}while(0!==_);for(f=1<>=1;if(0!==f?(I&=f-1,I+=f):I=0,C++,0==--L[x]){if(x===R)break;x=e[r+c[C]]}if(x>z&&(I&w)!==p){for(0===B&&(B=z),g+=A,T=1<<(S=x-B);S+B852||2===t&&H>592)return 1;i[p=I&w]=z<<24|S<<16|g-n|0}}return 0!==I&&(i[g+I]=x-B<<24|64<<16|0),u.bits=z,0};function u(t){return(t>>>24&255)+(t>>>8&65280)+((65280&t)<<8)+((255&t)<<24)}function f(){this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new a.Buf16(320),this.work=new a.Buf16(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}function _(t){var e;return t&&t.state?(e=t.state,t.total_in=t.total_out=e.total=0,t.msg="",e.wrap&&(t.adler=1&e.wrap),e.mode=1,e.last=0,e.havedict=0,e.dmax=32768,e.head=null,e.hold=0,e.bits=0,e.lencode=e.lendyn=new a.Buf32(852),e.distcode=e.distdyn=new a.Buf32(592),e.sane=1,e.back=-1,0):-2}function p(t){var e;return t&&t.state?((e=t.state).wsize=0,e.whave=0,e.wnext=0,_(t)):-2}function w(t,e){var a,r;return t&&t.state?(r=t.state,e<0?(a=0,e=-e):(a=1+(e>>4),e<48&&(e&=15)),e&&(e<8||e>15)?-2:(null!==r.window&&r.wbits!==e&&(r.window=null),r.wrap=a,r.wbits=e,p(t))):-2}function g(t,e){var a,r;return t?(r=new f,t.state=r,r.window=null,0!==(a=w(t,e))&&(t.state=null),a):-2}var m,b,y=!0;function v(t){if(y){var e;for(m=new a.Buf32(512),b=new a.Buf32(32),e=0;e<144;)t.lens[e++]=8;for(;e<256;)t.lens[e++]=9;for(;e<280;)t.lens[e++]=7;for(;e<288;)t.lens[e++]=8;for(c(1,t.lens,0,288,m,0,t.work,{bits:9}),e=0;e<32;)t.lens[e++]=5;c(2,t.lens,0,32,b,0,t.work,{bits:5}),y=!1}t.lencode=m,t.lenbits=9,t.distcode=b,t.distbits=5}function k(t,e,r,s){var i,n=t.state;return null===n.window&&(n.wsize=1<=n.wsize?(a.arraySet(n.window,e,r-n.wsize,n.wsize,0),n.wnext=0,n.whave=n.wsize):((i=n.wsize-n.wnext)>s&&(i=s),a.arraySet(n.window,e,r-s,i,n.wnext),(s-=i)?(a.arraySet(n.window,e,r-s,s,0),n.wnext=s,n.whave=n.wsize):(n.wnext+=i,n.wnext===n.wsize&&(n.wnext=0),n.whave>>8&255,s.check=i(s.check,L,2,0),p=0,w=0,s.mode=2;break}if(s.flags=0,s.head&&(s.head.done=!1),!(1&s.wrap)||(((255&p)<<8)+(p>>8))%31){t.msg="incorrect header check",s.mode=30;break}if(8!=(15&p)){t.msg="unknown compression method",s.mode=30;break}if(w-=4,T=8+(15&(p>>>=4)),0===s.wbits)s.wbits=T;else if(T>s.wbits){t.msg="invalid window size",s.mode=30;break}s.dmax=1<>8&1),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=3;case 3:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>8&255,L[2]=p>>>16&255,L[3]=p>>>24&255,s.check=i(s.check,L,4,0)),p=0,w=0,s.mode=4;case 4:for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>8),512&s.flags&&(L[0]=255&p,L[1]=p>>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0,s.mode=5;case 5:if(1024&s.flags){for(;w<16;){if(0===f)break t;f--,p+=o[l++]<>>8&255,s.check=i(s.check,L,2,0)),p=0,w=0}else s.head&&(s.head.extra=null);s.mode=6;case 6:if(1024&s.flags&&((b=s.length)>f&&(b=f),b&&(s.head&&(T=s.head.extra_len-s.length,s.head.extra||(s.head.extra=new Array(s.head.extra_len)),a.arraySet(s.head.extra,o,l,b,T)),512&s.flags&&(s.check=i(s.check,o,b,l)),f-=b,l+=b,s.length-=b),s.length))break t;s.length=0,s.mode=7;case 7:if(2048&s.flags){if(0===f)break t;b=0;do{T=o[l+b++],s.head&&T&&s.length<65536&&(s.head.name+=String.fromCharCode(T))}while(T&&b>9&1,s.head.done=!0),t.adler=s.check=0,s.mode=12;break;case 10:for(;w<32;){if(0===f)break t;f--,p+=o[l++]<>>=7&w,w-=7&w,s.mode=27;break}for(;w<3;){if(0===f)break t;f--,p+=o[l++]<>>=1)){case 0:s.mode=14;break;case 1:if(v(s),s.mode=20,6===e){p>>>=2,w-=2;break t}break;case 2:s.mode=17;break;case 3:t.msg="invalid block type",s.mode=30}p>>>=2,w-=2;break;case 14:for(p>>>=7&w,w-=7&w;w<32;){if(0===f)break t;f--,p+=o[l++]<>>16^65535)){t.msg="invalid stored block lengths",s.mode=30;break}if(s.length=65535&p,p=0,w=0,s.mode=15,6===e)break t;case 15:s.mode=16;case 16:if(b=s.length){if(b>f&&(b=f),b>_&&(b=_),0===b)break t;a.arraySet(h,o,l,b,d),f-=b,l+=b,_-=b,d+=b,s.length-=b;break}s.mode=12;break;case 17:for(;w<14;){if(0===f)break t;f--,p+=o[l++]<>>=5,w-=5,s.ndist=1+(31&p),p>>>=5,w-=5,s.ncode=4+(15&p),p>>>=4,w-=4,s.nlen>286||s.ndist>30){t.msg="too many length or distance symbols",s.mode=30;break}s.have=0,s.mode=18;case 18:for(;s.have>>=3,w-=3}for(;s.have<19;)s.lens[U[s.have++]]=0;if(s.lencode=s.lendyn,s.lenbits=7,I={bits:s.lenbits},H=c(0,s.lens,0,19,s.lencode,0,s.work,I),s.lenbits=I.bits,H){t.msg="invalid code lengths set",s.mode=30;break}s.have=0,s.mode=19;case 19:for(;s.have>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=C,w-=C,s.lens[s.have++]=R;else{if(16===R){for(D=C+2;w>>=C,w-=C,0===s.have){t.msg="invalid bit length repeat",s.mode=30;break}T=s.lens[s.have-1],b=3+(3&p),p>>>=2,w-=2}else if(17===R){for(D=C+3;w>>=C)),p>>>=3,w-=3}else{for(D=C+7;w>>=C)),p>>>=7,w-=7}if(s.have+b>s.nlen+s.ndist){t.msg="invalid bit length repeat",s.mode=30;break}for(;b--;)s.lens[s.have++]=T}}if(30===s.mode)break;if(0===s.lens[256]){t.msg="invalid code -- missing end-of-block",s.mode=30;break}if(s.lenbits=9,I={bits:s.lenbits},H=c(1,s.lens,0,s.nlen,s.lencode,0,s.work,I),s.lenbits=I.bits,H){t.msg="invalid literal/lengths set",s.mode=30;break}if(s.distbits=6,s.distcode=s.distdyn,I={bits:s.distbits},H=c(2,s.lens,s.nlen,s.ndist,s.distcode,0,s.work,I),s.distbits=I.bits,H){t.msg="invalid distances set",s.mode=30;break}if(s.mode=20,6===e)break t;case 20:s.mode=21;case 21:if(f>=6&&_>=258){t.next_out=d,t.avail_out=_,t.next_in=l,t.avail_in=f,s.hold=p,s.bits=w,n(t,m),d=t.next_out,h=t.output,_=t.avail_out,l=t.next_in,o=t.input,f=t.avail_in,p=s.hold,w=s.bits,12===s.mode&&(s.back=-1);break}for(s.back=0;A=(E=s.lencode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,s.length=R,0===A){s.mode=26;break}if(32&A){s.back=-1,s.mode=12;break}if(64&A){t.msg="invalid literal/length code",s.mode=30;break}s.extra=15&A,s.mode=22;case 22:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}s.was=s.length,s.mode=23;case 23:for(;A=(E=s.distcode[p&(1<>>16&255,R=65535&E,!((C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>z)])>>>16&255,R=65535&E,!(z+(C=E>>>24)<=w);){if(0===f)break t;f--,p+=o[l++]<>>=z,w-=z,s.back+=z}if(p>>>=C,w-=C,s.back+=C,64&A){t.msg="invalid distance code",s.mode=30;break}s.offset=R,s.extra=15&A,s.mode=24;case 24:if(s.extra){for(D=s.extra;w>>=s.extra,w-=s.extra,s.back+=s.extra}if(s.offset>s.dmax){t.msg="invalid distance too far back",s.mode=30;break}s.mode=25;case 25:if(0===_)break t;if(b=m-_,s.offset>b){if((b=s.offset-b)>s.whave&&s.sane){t.msg="invalid distance too far back",s.mode=30;break}b>s.wnext?(b-=s.wnext,y=s.wsize-b):y=s.wnext-b,b>s.length&&(b=s.length),x=s.window}else x=h,y=d-s.offset,b=s.length;b>_&&(b=_),_-=b,s.length-=b;do{h[d++]=x[y++]}while(--b);0===s.length&&(s.mode=21);break;case 26:if(0===_)break t;h[d++]=s.length,_--,s.mode=21;break;case 27:if(s.wrap){for(;w<32;){if(0===f)break t;f--,p|=o[l++]<=252?6:z>=248?5:z>=240?4:z>=224?3:z>=192?2:1;R[254]=R[254]=1;function S(t,e){if(e<65534&&(t.subarray&&A||!t.subarray&&C))return String.fromCharCode.apply(null,a.shrinkBuf(t,e));for(var r="",s=0;s>>6,e[n++]=128|63&r):r<65536?(e[n++]=224|r>>>12,e[n++]=128|r>>>6&63,e[n++]=128|63&r):(e[n++]=240|r>>>18,e[n++]=128|r>>>12&63,e[n++]=128|r>>>6&63,e[n++]=128|63&r);return e},T=function(t){return S(t,t.length)},H=function(t){for(var e=new a.Buf8(t.length),r=0,s=e.length;r4)o[r++]=65533,a+=i-1;else{for(s&=2===i?31:3===i?15:7;i>1&&a1?o[r++]=65533:s<65536?o[r++]=s:(s-=65536,o[r++]=55296|s>>10&1023,o[r++]=56320|1023&s)}return S(o,r)},D=function(t,e){var a;for((e=e||t.length)>t.length&&(e=t.length),a=e-1;a>=0&&128==(192&t[a]);)a--;return a<0||0===a?e:a+R[t[a]]>e?a:e},E=0,L=2,U=4,O=0,W=1,F=2,j=-5,P={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"};var N=function(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0};var q=function(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1},$=Object.prototype.toString;function M(t){if(!(this instanceof M))return new M(t);this.options=a.assign({chunkSize:16384,windowBits:0,to:""},t||{});var e=this.options;e.raw&&e.windowBits>=0&&e.windowBits<16&&(e.windowBits=-e.windowBits,0===e.windowBits&&(e.windowBits=-15)),!(e.windowBits>=0&&e.windowBits<16)||t&&t.windowBits||(e.windowBits+=32),e.windowBits>15&&e.windowBits<48&&0==(15&e.windowBits)&&(e.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new N,this.strm.avail_out=0;var r=x.inflateInit2(this.strm,e.windowBits);if(r!==O)throw new Error(P[r]);if(this.header=new q,x.inflateGetHeader(this.strm,this.header),e.dictionary&&("string"==typeof e.dictionary?e.dictionary=B(e.dictionary):"[object ArrayBuffer]"===$.call(e.dictionary)&&(e.dictionary=new Uint8Array(e.dictionary)),e.raw&&(r=x.inflateSetDictionary(this.strm,e.dictionary))!==O))throw new Error(P[r])}M.prototype.push=function(t,e){var r,s,i,n,o,h=this.strm,l=this.options.chunkSize,d=this.options.dictionary,c=!1;if(this.ended)return!1;s=e===~~e?e:!0===e?U:E,"string"==typeof t?h.input=H(t):"[object ArrayBuffer]"===$.call(t)?h.input=new Uint8Array(t):h.input=t,h.next_in=0,h.avail_in=h.input.length;do{if(0===h.avail_out&&(h.output=new a.Buf8(l),h.next_out=0,h.avail_out=l),(r=x.inflate(h,E))===F&&d&&(r=x.inflateSetDictionary(this.strm,d)),r===j&&!0===c&&(r=O,c=!1),r!==W&&r!==O)return this.onEnd(r),this.ended=!0,!1;h.next_out&&(0!==h.avail_out&&r!==W&&(0!==h.avail_in||s!==U&&s!==L)||("string"===this.options.to?(i=D(h.output,h.next_out),n=h.next_out-i,o=I(h.output,i),h.next_out=n,h.avail_out=l-n,n&&a.arraySet(h.output,h.output,i,n,0),this.onData(o)):this.onData(a.shrinkBuf(h.output,h.next_out)))),0===h.avail_in&&0===h.avail_out&&(c=!0)}while((h.avail_in>0||0===h.avail_out)&&r!==W);return r===W&&(s=U),s===U?(r=x.inflateEnd(this.strm),this.onEnd(r),this.ended=!0,r===O):s!==L||(this.onEnd(O),h.avail_out=0,!0)},M.prototype.onData=function(t){this.chunks.push(t)},M.prototype.onEnd=function(t){t===O&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var J=M;const V=new TextDecoder("utf-8");class Z extends J{constructor(t,e){super(t),this.reader=e}onEnd(t){this.err=t,this.err||(this.reader._rawOffset+=this.strm.total_in)}}class G{static concatChunks(t,e){if(1===t.length)return t[0];const a=new Uint8Array(e);let r=0;for(const e of t)a.set(e,r),r+=e.byteLength;return a}static splitChunk(t,e){return[t.slice(0,e),t.slice(e)]}static async readFully(t){const e=[];let a=0;for await(const r of t)e.push(r),a+=r.byteLength;return G.concatChunks(e,a)}getReadableStream(){const t=this[Symbol.asyncIterator]();return new ReadableStream({pull:e=>t.next().then(t=>{t.done||!t.value?e.close():e.enqueue(t.value)})})}readFully(){return G.readFully(this)}async readline(t=0){const e=await this.readlineRaw(t);return e?V.decode(e):""}async*iterLines(t=0){let e=null;for(;e=await this.readline(t);)yield e}}class X extends G{constructor(t,e="gzip",a=!1){if(super(),this.compressed=e,this.opts={raw:"deflateRaw"===e},this.inflator=e?new Z(this.opts,this):null,"function"!=typeof t[Symbol.asyncIterator])if("function"==typeof t.getReader)t=X.fromReadable(t.getReader());else if("function"==typeof t.read)t=X.fromReadable(t);else{if("function"!=typeof t[Symbol.iterator])throw new TypeError("Invalid Stream Source");t=X.fromIter(t)}this._sourceIter=a?this.dechunk(t):t[Symbol.asyncIterator](),this.lastValue=null,this.errored=!1,this._savedChunk=null,this._rawOffset=0,this._readOffset=0,this.numChunks=0}async _loadNext(){const t=await this._sourceIter.next();return t.done?null:t.value}async*dechunk(t){const e=t instanceof X?t:new X(t,null);let a=-1,r=!0;for(;0!=a;){const t=await e.readlineRaw(64);let s=null;if(a=parseInt(V.decode(t),16),!a||a>2**32){if(Number.isNaN(a)||a>2**32){r||(this.errored=!0),yield t;break}}else if(s=await e.readSize(a),s.length!=a){r?yield t:this.errored=!0,yield s;break}const i=await e.readSize(2);if(13!=i[0]||10!=i[1]){r?yield t:this.errored=!0,yield s,yield i;break}if(r=!1,!s||0===a)return;yield s}yield*e}_unread(t){t.length&&(this._readOffset-=t.length,this._savedChunk&&console.log("Already have chunk!"),this._savedChunk=t)}async _next(){if(this._savedChunk){const t=this._savedChunk;return this._savedChunk=null,t}if(this.compressed){const t=this._getNextChunk();if(t)return t}let t=await this._loadNext();for(;this.compressed&&t;){this._push(t);const e=this._getNextChunk(t);if(e)return e;t=await this._loadNext()}return t}_push(t){this.lastValue=t,this.inflator.ended&&(this.inflator=new Z(this.opts,this)),this.inflator.push(t),this.inflator.err&&this.inflator.ended&&"deflate"===this.compressed&&!1===this.opts.raw&&0===this.numChunks&&(this.opts.raw=!0,this.compressed="deflateRaw",this.inflator=new Z(this.opts,this),this.inflator.push(t))}_getNextChunk(t){for(;;){if(this.inflator.chunks.length>0)return this.numChunks++,this.inflator.chunks.shift();if(this.inflator.ended){if(0!==this.inflator.err)return this.compressed=null,t;const e=this.inflator.strm.avail_in;if(e&&this.lastValue){this._push(this.lastValue.slice(-e));continue}}return null}}async*[Symbol.asyncIterator](){let t=null;for(;t=await this._next();)this._readOffset+=t.length,yield t}async readlineRaw(t){const e=[];let a,r=0,s=null;for await(const i of this){if(t&&r+i.byteLength>t){s=i,a=t-r-1;const e=i.slice(0,a+1).indexOf(10);e>=0&&(a=e);break}if(a=i.indexOf(10),a>=0){s=i;break}e.push(i),r+=i.byteLength}if(s){const[t,i]=X.splitChunk(s,a+1);e.push(t),r+=t.byteLength,this._unread(i)}else if(!e.length)return null;return X.concatChunks(e,r)}readFully(){return this.readSize()}async readSize(t=-1,e=!1){const a=[];let r=0;for await(const s of this){if(t>=0){if(s.length>t){const[i,n]=X.splitChunk(s,t);e||a.push(i),r+=i.byteLength,this._unread(n);break}if(s.length===t){e||a.push(s),r+=s.byteLength,t=0;break}t-=s.length}e||a.push(s),r+=s.byteLength}return e?r:X.concatChunks(a,r)}getReadOffset(){return this._readOffset}getRawOffset(){return this.compressed?this._rawOffset:this._readOffset}getRawLength(t){return this.compressed?this.inflator.strm.total_in:this._readOffset-t}static fromReadable(t){return{async*[Symbol.asyncIterator](){let e=null;for(;(e=await t.read())&&!e.done;)yield e.value}}}static fromIter(t){return{async*[Symbol.asyncIterator](){for(const e of t)yield e}}}}class K extends G{constructor(t,e,a=0){super(),this.sourceIter=t,this.length=e,this.limit=e,this.skip=a}setLimitSkip(t,e=0){this.limit=t,this.skip=e}async*[Symbol.asyncIterator](){if(!(this.limit<=0))for await(let t of this.sourceIter){if(this.skip>0){if(!(t.length>=this.skip)){this.skip-=t.length;continue}{const[e,a]=K.splitChunk(t,this.skip);t=a,this.skip=0}}if(t.length>this.limit){const[e,a]=K.splitChunk(t,this.limit);t=e,this.sourceIter._unread(a)}if(t.length&&(this.limit-=t.length,yield t),this.limit<=0)break}}async readlineRaw(t){if(this.limit<=0)return null;const e=await this.sourceIter.readlineRaw(t?Math.min(t,this.limit):this.limit);return this.limit-=e.length,e}async skipFully(){const t=this.limit;for(;this.limit>0;)this.limit-=await this.sourceIter.readSize(this.limit,!0);return t}}const Q=new Uint8Array([13,10]),Y=new Uint8Array([13,10,13,10]);class tt{constructor({statusline:t,headers:e}){this.statusline=t,this.headers=e}toString(){const t=[this.statusline];for(const[e,a]of this.headers)t.push(`${e}: ${a}`);return t.join("\r\n")+"\r\n"}async*iterSerialize(t){yield t.encode(this.statusline),yield Q;for(const[e,a]of this.headers)yield t.encode(`${e}: ${a}\r\n`)}_parseResponseStatusLine(){const t=at(this.statusline," ",2);this._protocol=t[0],this._statusCode=t.length>1?Number(t[1]):"",this._statusText=t.length>2?t[2]:""}get statusCode(){return void 0===this._statusCode&&this._parseResponseStatusLine(),this._statusCode}get protocol(){return void 0===this._protocol&&this._parseResponseStatusLine(),this._protocol}get statusText(){return void 0===this._statusText&&this._parseResponseStatusLine(),this._statusText}_parseRequestStatusLine(){const t=this.statusline.split(" ",2);this._method=t[0],this._requestPath=t.length>1?t[1]:""}get method(){return void 0===this._method&&this._parseRequestStatusLine(),this._method}get requestPath(){return void 0===this._requestPath&&this._parseRequestStatusLine(),this._requestPath}}class et{startsWithSpace(t){const e=t.charAt(0);return" "===e||"\t"===e}async parse(t,{headersClass:e=Map,firstLine:a}={}){const r=a||await t.readline();if(!r)return null;let s=r.trimEnd();const i=new e;if(!s)return null;let n=(await t.readline()).trimEnd();for(;n;){let[e,a]=at(n,":",1);a&&(e=e.trimStart(),a=a.trim());let r=(await t.readline()).trimEnd();for(;this.startsWithSpace(r);)a&&(a+=r),r=(await t.readline()).trimEnd();if(a)try{i.set(e,a)}catch(t){}n=r}return new tt({statusline:s,headers:i,totalRead:this.totalRead})}}function at(t,e,a){const r=t.split(e),s=r.slice(0,a);return r.slice(a).length>0&&s.push(r.slice(a).join(e)),s}var rt,st=(rt=Object.freeze({__proto__:null,default:{}}))&&rt.default||rt,it=e((function(t){!function(){var e,a,r,s=0,i=[];for(a=0;a<256;a++)i[a]=(a+256).toString(16).substr(1);function n(){var t=function(t){return(!e||s+t>o.BUFFER_SIZE)&&(s=0,e=o.randomBytes(o.BUFFER_SIZE)),e.slice(s,s+=t)}(16);return t[6]=15&t[6]|64,t[8]=63&t[8]|128,t}function o(){var t=n();return i[t[0]]+i[t[1]]+i[t[2]]+i[t[3]]+"-"+i[t[4]]+i[t[5]]+"-"+i[t[6]]+i[t[7]]+"-"+i[t[8]]+i[t[9]]+"-"+i[t[10]]+i[t[11]]+i[t[12]]+i[t[13]]+i[t[14]]+i[t[15]]}o.BUFFER_SIZE=4096,o.bin=n,o.clearBuffer=function(){e=null,s=0},o.test=function(t){return"string"==typeof t&&/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(t)},"undefined"!=typeof crypto?r=crypto:"undefined"!=typeof window&&void 0!==window.msCrypto&&(r=window.msCrypto),r=r||st,t.exports=o,o.randomBytes=function(){if(r){if(r.randomBytes)return r.randomBytes;if(r.getRandomValues)return function(t){var e=new Uint8Array(t);return r.getRandomValues(e),e}}return function(t){var e,a=[];for(e=0;e`),!r.headers.get("Content-Type")&&ht[a]&&r.headers.set("Content-Type",ht[a]),!c){c=async function*(){}()}const f=new lt({warcHeaders:r,reader:c});switch(a){case"response":case"request":case"revisit":f.httpHeaders=new tt({statusline:n,headers:h?new Map(Object.entries(i)):new Headers(i)})}return f}static createWARCInfo(t={},e){return t.type="warcinfo",lt.create(t,async function*(){for(const[t,a]of Object.entries(e))yield ot.encode(`${t}: ${a}\r\n`)}())}constructor({warcHeaders:t,reader:e}){super(),this.warcHeaders=t,this._reader=e,this._contentReader=null,this.payload=null,this.httpHeaders=null,this.consumed=!1,this.fixUp()}getResponseInfo(){const t=this.httpHeaders;return t?{headers:t.headers,status:t.statusCode,statusText:t.statusText}:null}fixUp(){const t=this.warcHeaders.headers.get("WARC-Target-URI");t&&t.startsWith("<")&&t.endsWith(">")&&this.warcHeaders.headers.set("WARC-Target-URI",t.slice(1,-1))}async readFully(t=!1){if(this.httpHeaders){if(this._contentReader&&!t)throw new TypeError("WARC Record decoding already started, but requesting raw payload");if(t&&"raw"===this.consumed&&this.payload)return await this._createDecodingReader([this.payload]).readFully()}return this.payload||(t?(this.payload=await super.readFully(),this.consumed="content"):(this.payload=await lt.readFully(this._reader),this.consumed="raw")),this.payload}get reader(){if(this._contentReader)throw new TypeError("WARC Record decoding already started, but requesting raw payload");return this._reader}get contentReader(){return this.httpHeaders?(this._contentReader||(this._contentReader=this._createDecodingReader(this._reader)),this._contentReader):this._reader}_createDecodingReader(t){let e=this.httpHeaders.headers.get("content-encoding"),a=this.httpHeaders.headers.get("transfer-encoding");const r="chunked"===a;return e||r||(e=a),new X(t,e,r)}async readlineRaw(t){if(this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");return this.contentReader.readlineRaw(t)}async contentText(){const t=await this.readFully(!0);return nt.decode(t)}async*[Symbol.asyncIterator](){for await(const t of this.contentReader)if(yield t,this.consumed)throw new Error("Record already consumed.. Perhaps a promise was not awaited?");this.consumed="content"}async skipFully(){if(this.consumed)return;const t=await this._reader.skipFully();return this.consumed="skipped",t}warcHeader(t){return this.warcHeaders.headers.get(t)}get warcType(){return this.warcHeaders.headers.get("WARC-Type")}get warcTargetURI(){return this.warcHeaders.headers.get("WARC-Target-URI")}get warcDate(){return this.warcHeaders.headers.get("WARC-Date")}get warcRefersToTargetURI(){return this.warcHeaders.headers.get("WARC-Refers-To-Target-URI")}get warcRefersToDate(){return this.warcHeaders.headers.get("WARC-Refers-To-Date")}get warcPayloadDigest(){return this.warcHeaders.headers.get("WARC-Payload-Digest")}get warcBlockDigest(){return this.warcHeaders.headers.get("WARC-Block-Digest")}get warcContentType(){return this.warcHeaders.headers.get("Content-Type")}get warcContentLength(){return Number(this.warcHeaders.headers.get("Content-Length"))}}class dt{static parse(t,e){return new dt(t,e).parse()}static iterRecords(t,e){return new dt(t,e)[Symbol.asyncIterator]()}constructor(t,{keepHeadersCase:e=!1,parseHttp:a=!0}={}){this._offset=0,this._warcHeadersLength=0,this._headersClass=e?Map:Headers,this._parseHttp=a,this._atRecordBoundary=!0,t instanceof X||(t=new X(t)),this._reader=t,this._record=null}async readToNextRecord(){let t="";if(!this._atRecordBoundary&&this._reader&&this._record){await this._record.skipFully(),t=await this._reader.readline();const e=t.trim().length;if(e&&console.warn(`Content-Length Too Small: Record not followed by newline, Remainder Length: ${e}, Offset: ${this._reader.getRawOffset()-t.length}`),this._reader.compressed)await this._reader.readSize(2,!0),t="";else for(t=await this._reader.readline();2===t.length;)t=await this._reader.readline()}return this._atRecordBoundary=!0,t}_initRecordReader(t){return new K(this._reader,Number(t.headers.get("Content-Length")||0))}async parse(){const t=await this.readToNextRecord();this._offset=this._reader.getRawOffset()-t.length;const e=new et,a=await e.parse(this._reader,{firstLine:t,headersClass:this._headersClass});if(!a)return null;this._warcHeadersLength=this._reader.getReadOffset();const r=new lt({warcHeaders:a,reader:this._initRecordReader(a)});if(this._atRecordBoundary=!1,this._record=r,this._parseHttp)switch(r.warcType){case"response":case"request":await this._addHttpHeaders(r,e,this._reader);break;case"revisit":r.warcContentLength>0&&await this._addHttpHeaders(r,e,this._reader)}return r}get offset(){return this._offset}get recordLength(){return this._reader.getRawLength(this._offset)}async*[Symbol.asyncIterator](){let t=null;for(;t=await this.parse(this._reader);)yield t;this._record=null}async _addHttpHeaders(t,e){const a=await e.parse(this._reader,{headersClass:this._headersClass});t.httpHeaders=a;const r=this._reader.getReadOffset()-this._warcHeadersLength;t.reader.setLimitSkip&&t.reader.setLimitSkip(t.warcContentLength-r)}}var ct=e((function(e){ /* * [hi-base32]{@link https://github.com/emn178/hi-base32} * @@ -12,4 +12,4 @@ var t="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?windo * @copyright Chen, Yi-Cyuan 2015-2018 * @license MIT */ -!function(){var a="object"==typeof window?window:{};!a.HI_BASE32_NO_NODE_JS&&"object"==typeof process&&process.versions&&process.versions.node&&(a=t);var r=!a.HI_BASE32_NO_COMMON_JS&&e.exports,i="ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".split(""),s={A:0,B:1,C:2,D:3,E:4,F:5,G:6,H:7,I:8,J:9,K:10,L:11,M:12,N:13,O:14,P:15,Q:16,R:17,S:18,T:19,U:20,V:21,W:22,X:23,Y:24,Z:25,2:26,3:27,4:28,5:29,6:30,7:31},n=[0,0,0,0,0,0,0,0],o=function(t,e){e.length>10&&(e="..."+e.substr(-10));var a=new Error("Decoded data is not valid UTF-8. Maybe try base32.decode.asBytes()? Partial data after reading "+t+" bytes: "+e+" <-");throw a.position=t,a},h=function(t){if(!/^[A-Z2-7=]+$/.test(t))throw new Error("Invalid base32 characters");for(var e,a,r,i,n,o,h,l,d=[],c=0,u=(t=t.replace(/=/g,"")).length,f=0,_=u>>3<<3;f<_;)e=s[t.charAt(f++)],a=s[t.charAt(f++)],r=s[t.charAt(f++)],i=s[t.charAt(f++)],n=s[t.charAt(f++)],o=s[t.charAt(f++)],h=s[t.charAt(f++)],l=s[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|i>>>4),d[c++]=255&(i<<4|n>>>1),d[c++]=255&(n<<7|o<<2|h>>>3),d[c++]=255&(h<<5|l);var p=u-_;return 2===p?(e=s[t.charAt(f++)],a=s[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2)):4===p?(e=s[t.charAt(f++)],a=s[t.charAt(f++)],r=s[t.charAt(f++)],i=s[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|i>>>4)):5===p?(e=s[t.charAt(f++)],a=s[t.charAt(f++)],r=s[t.charAt(f++)],i=s[t.charAt(f++)],n=s[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|i>>>4),d[c++]=255&(i<<4|n>>>1)):7===p&&(e=s[t.charAt(f++)],a=s[t.charAt(f++)],r=s[t.charAt(f++)],i=s[t.charAt(f++)],n=s[t.charAt(f++)],o=s[t.charAt(f++)],h=s[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|i>>>4),d[c++]=255&(i<<4|n>>>1),d[c++]=255&(n<<7|o<<2|h>>>3)),d},l=function(t,e){if(!e)return function(t){for(var e,a,r="",i=t.length,s=0,n=0;s191&&e<=223?(a=31&e,n=1):e<=239?(a=15&e,n=2):e<=247?(a=7&e,n=3):o(s,r);for(var h=0;h191)&&o(s,r),a<<=6,a+=63&e;a>=55296&&a<=57343&&o(s,r),a>1114111&&o(s,r),a<=65535?r+=String.fromCharCode(a):(a-=65536,r+=String.fromCharCode(55296+(a>>10)),r+=String.fromCharCode(56320+(1023&a)))}return r}(h(t));if(!/^[A-Z2-7=]+$/.test(t))throw new Error("Invalid base32 characters");var a,r,i,n,l,d,c,u,f="",_=t.indexOf("=");-1===_&&(_=t.length);for(var p=0,g=_>>3<<3;p>>2))+String.fromCharCode(255&(r<<6|i<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))+String.fromCharCode(255&(l<<7|d<<2|c>>>3))+String.fromCharCode(255&(c<<5|u));var w=_-g;return 2===w?(a=s[t.charAt(p++)],r=s[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))):4===w?(a=s[t.charAt(p++)],r=s[t.charAt(p++)],i=s[t.charAt(p++)],n=s[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|i<<1|n>>>4))):5===w?(a=s[t.charAt(p++)],r=s[t.charAt(p++)],i=s[t.charAt(p++)],n=s[t.charAt(p++)],l=s[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|i<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))):7===w&&(a=s[t.charAt(p++)],r=s[t.charAt(p++)],i=s[t.charAt(p++)],n=s[t.charAt(p++)],l=s[t.charAt(p++)],d=s[t.charAt(p++)],c=s[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|i<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))+String.fromCharCode(255&(l<<7|d<<2|c>>>3))),f},d={encode:function(t,e){var a="string"!=typeof t;return a&&t.constructor===ArrayBuffer&&(t=new Uint8Array(t)),a?function(t){for(var e,a,r,s,n,o="",h=t.length,l=0,d=5*parseInt(h/5);l>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[31&(s<<3|n>>>5)]+i[31&n];var c=h-d;return 1===c?(e=t[l],o+=i[e>>>3]+i[e<<2&31]+"======"):2===c?(e=t[l++],a=t[l],o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[a<<4&31]+"===="):3===c?(e=t[l++],a=t[l++],r=t[l],o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[r<<1&31]+"==="):4===c&&(e=t[l++],a=t[l++],r=t[l++],s=t[l],o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[s<<3&31]+"="),o}(t):e?function(t){for(var e,a,r,s,n,o="",h=t.length,l=0,d=5*parseInt(h/5);l>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[31&(s<<3|n>>>5)]+i[31&n];var c=h-d;return 1===c?(e=t.charCodeAt(l),o+=i[e>>>3]+i[e<<2&31]+"======"):2===c?(e=t.charCodeAt(l++),a=t.charCodeAt(l),o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[a<<4&31]+"===="):3===c?(e=t.charCodeAt(l++),a=t.charCodeAt(l++),r=t.charCodeAt(l),o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[r<<1&31]+"==="):4===c&&(e=t.charCodeAt(l++),a=t.charCodeAt(l++),r=t.charCodeAt(l++),s=t.charCodeAt(l),o+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[s<<3&31]+"="),o}(t):function(t){var e,a,r,s,o,h,l,d=!1,c="",u=0,f=0,_=t.length;do{for(n[0]=n[5],n[1]=n[6],n[2]=n[7],l=f;u<_&&l<5;++u)(h=t.charCodeAt(u))<128?n[l++]=h:h<2048?(n[l++]=192|h>>6,n[l++]=128|63&h):h<55296||h>=57344?(n[l++]=224|h>>12,n[l++]=128|h>>6&63,n[l++]=128|63&h):(h=65536+((1023&h)<<10|1023&t.charCodeAt(++u)),n[l++]=240|h>>18,n[l++]=128|h>>12&63,n[l++]=128|h>>6&63,n[l++]=128|63&h);f=l-5,u===_&&++u,u>_&&l<6&&(d=!0),e=n[0],l>4?(a=n[1],r=n[2],s=n[3],o=n[4],c+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[31&(s<<3|o>>>5)]+i[31&o]):1===l?c+=i[e>>>3]+i[e<<2&31]+"======":2===l?(a=n[1],c+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[a<<4&31]+"===="):3===l?(a=n[1],r=n[2],c+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[r<<1&31]+"==="):(a=n[1],r=n[2],s=n[3],c+=i[e>>>3]+i[31&(e<<2|a>>>6)]+i[a>>>1&31]+i[31&(a<<4|r>>>4)]+i[31&(r<<1|s>>>7)]+i[s>>>2&31]+i[s<<3&31]+"=")}while(!d);return c}(t)},decode:l};l.asBytes=h,r?e.exports=d:a.base32=d}()}));function ut(t){for(var e=t.length;--e>=0;)t[e]=0}var ft=[0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0],_t=[0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13],pt=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7],gt=[16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15],wt=new Array(576);ut(wt);var mt=new Array(60);ut(mt);var bt=new Array(512);ut(bt);var yt=new Array(256);ut(yt);var vt=new Array(29);ut(vt);var kt,xt,Ct,At=new Array(30);function zt(t,e,a,r,i){this.static_tree=t,this.extra_bits=e,this.extra_base=a,this.elems=r,this.max_length=i,this.has_stree=t&&t.length}function Rt(t,e){this.dyn_tree=t,this.max_code=0,this.stat_desc=e}function St(t){return t<256?bt[t]:bt[256+(t>>>7)]}function Bt(t,e){t.pending_buf[t.pending++]=255&e,t.pending_buf[t.pending++]=e>>>8&255}function Ht(t,e,a){t.bi_valid>16-a?(t.bi_buf|=e<>16-t.bi_valid,t.bi_valid+=a-16):(t.bi_buf|=e<>>=1,a<<=1}while(--e>0);return a>>>1}function Dt(t,e,a){var r,i,s=new Array(16),n=0;for(r=1;r<=15;r++)s[r]=n=n+a[r-1]<<1;for(i=0;i<=e;i++){var o=t[2*i+1];0!==o&&(t[2*i]=It(s[o]++,o))}}function Et(t){var e;for(e=0;e<286;e++)t.dyn_ltree[2*e]=0;for(e=0;e<30;e++)t.dyn_dtree[2*e]=0;for(e=0;e<19;e++)t.bl_tree[2*e]=0;t.dyn_ltree[512]=1,t.opt_len=t.static_len=0,t.last_lit=t.matches=0}function Lt(t){t.bi_valid>8?Bt(t,t.bi_buf):t.bi_valid>0&&(t.pending_buf[t.pending++]=t.bi_buf),t.bi_buf=0,t.bi_valid=0}function Ot(t,e,a,r){var i=2*e,s=2*a;return t[i]>1;a>=1;a--)Ut(t,s,a);i=h;do{a=t.heap[1],t.heap[1]=t.heap[t.heap_len--],Ut(t,s,1),r=t.heap[1],t.heap[--t.heap_max]=a,t.heap[--t.heap_max]=r,s[2*i]=s[2*a]+s[2*r],t.depth[i]=(t.depth[a]>=t.depth[r]?t.depth[a]:t.depth[r])+1,s[2*a+1]=s[2*r+1]=i,t.heap[1]=i++,Ut(t,s,1)}while(t.heap_len>=2);t.heap[--t.heap_max]=t.heap[1],function(t,e){var a,r,i,s,n,o,h=e.dyn_tree,l=e.max_code,d=e.stat_desc.static_tree,c=e.stat_desc.has_stree,u=e.stat_desc.extra_bits,f=e.stat_desc.extra_base,_=e.stat_desc.max_length,p=0;for(s=0;s<=15;s++)t.bl_count[s]=0;for(h[2*t.heap[t.heap_max]+1]=0,a=t.heap_max+1;a<573;a++)(s=h[2*h[2*(r=t.heap[a])+1]+1]+1)>_&&(s=_,p++),h[2*r+1]=s,r>l||(t.bl_count[s]++,n=0,r>=f&&(n=u[r-f]),o=h[2*r],t.opt_len+=o*(s+n),c&&(t.static_len+=o*(d[2*r+1]+n)));if(0!==p){do{for(s=_-1;0===t.bl_count[s];)s--;t.bl_count[s]--,t.bl_count[s+1]+=2,t.bl_count[_]--,p-=2}while(p>0);for(s=_;0!==s;s--)for(r=t.bl_count[s];0!==r;)(i=t.heap[--a])>l||(h[2*i+1]!==s&&(t.opt_len+=(s-h[2*i+1])*h[2*i],h[2*i+1]=s),r--)}}(t,e),Dt(s,l,t.bl_count)}function jt(t,e,a){var r,i,s=-1,n=e[1],o=0,h=7,l=4;for(0===n&&(h=138,l=3),e[2*(a+1)+1]=65535,r=0;r<=a;r++)i=n,n=e[2*(r+1)+1],++o>=7;r<30;r++)for(At[r]=i<<7,t=0;t<1<<_t[r]-7;t++)bt[256+i++]=r;for(e=0;e<=15;e++)s[e]=0;for(t=0;t<=143;)wt[2*t+1]=8,t++,s[8]++;for(;t<=255;)wt[2*t+1]=9,t++,s[9]++;for(;t<=279;)wt[2*t+1]=7,t++,s[7]++;for(;t<=287;)wt[2*t+1]=8,t++,s[8]++;for(Dt(wt,287,s),t=0;t<30;t++)mt[2*t+1]=5,mt[2*t]=It(t,5);kt=new zt(wt,ft,257,286,15),xt=new zt(mt,_t,0,30,15),Ct=new zt(new Array(0),pt,0,19,7)}(),Pt=!0),t.l_desc=new Rt(t.dyn_ltree,kt),t.d_desc=new Rt(t.dyn_dtree,xt),t.bl_desc=new Rt(t.bl_tree,Ct),t.bi_buf=0,t.bi_valid=0,Et(t)},_tr_stored_block:qt,_tr_flush_block:function(t,e,a,r){var i,s,n=0;t.level>0?(2===t.strm.data_type&&(t.strm.data_type=function(t){var e,a=4093624447;for(e=0;e<=31;e++,a>>>=1)if(1&a&&0!==t.dyn_ltree[2*e])return 0;if(0!==t.dyn_ltree[18]||0!==t.dyn_ltree[20]||0!==t.dyn_ltree[26])return 1;for(e=32;e<256;e++)if(0!==t.dyn_ltree[2*e])return 1;return 0}(t)),Ft(t,t.l_desc),Ft(t,t.d_desc),n=function(t){var e;for(jt(t,t.dyn_ltree,t.l_desc.max_code),jt(t,t.dyn_dtree,t.d_desc.max_code),Ft(t,t.bl_desc),e=18;e>=3&&0===t.bl_tree[2*gt[e]+1];e--);return t.opt_len+=3*(e+1)+5+5+4,e}(t),i=t.opt_len+3+7>>>3,(s=t.static_len+3+7>>>3)<=i&&(i=s)):i=s=a+5,a+4<=i&&-1!==e?qt(t,e,a,r):4===t.strategy||s===i?(Ht(t,2+(r?1:0),3),Wt(t,wt,mt)):(Ht(t,4+(r?1:0),3),function(t,e,a,r){var i;for(Ht(t,e-257,5),Ht(t,a-1,5),Ht(t,r-4,4),i=0;i>>8&255,t.pending_buf[t.d_buf+2*t.last_lit+1]=255&e,t.pending_buf[t.l_buf+t.last_lit]=255&a,t.last_lit++,0===e?t.dyn_ltree[2*a]++:(t.matches++,e--,t.dyn_ltree[2*(yt[a]+256+1)]++,t.dyn_dtree[2*St(e)]++),t.last_lit===t.lit_bufsize-1},_tr_align:function(t){Ht(t,2,3),Tt(t,256,wt),function(t){16===t.bi_valid?(Bt(t,t.bi_buf),t.bi_buf=0,t.bi_valid=0):t.bi_valid>=8&&(t.pending_buf[t.pending++]=255&t.bi_buf,t.bi_buf>>=8,t.bi_valid-=8)}(t)}};function Vt(t,e){return t.msg=N[e],e}function Zt(t){return(t<<1)-(t>4?9:0)}function Jt(t){for(var e=t.length;--e>=0;)t[e]=0}function Xt(t){var e=t.state,r=e.pending;r>t.avail_out&&(r=t.avail_out),0!==r&&(a.arraySet(t.output,e.pending_buf,e.pending_out,r,t.next_out),t.next_out+=r,e.pending_out+=r,t.total_out+=r,t.avail_out-=r,e.pending-=r,0===e.pending&&(e.pending_out=0))}function Kt(t,e){$t._tr_flush_block(t,t.block_start>=0?t.block_start:-1,t.strstart-t.block_start,e),t.block_start=t.strstart,Xt(t.strm)}function Gt(t,e){t.pending_buf[t.pending++]=e}function Qt(t,e){t.pending_buf[t.pending++]=e>>>8&255,t.pending_buf[t.pending++]=255&e}function Yt(t,e){var a,r,i=t.max_chain_length,s=t.strstart,n=t.prev_length,o=t.nice_match,h=t.strstart>t.w_size-262?t.strstart-(t.w_size-262):0,l=t.window,d=t.w_mask,c=t.prev,u=t.strstart+258,f=l[s+n-1],_=l[s+n];t.prev_length>=t.good_match&&(i>>=2),o>t.lookahead&&(o=t.lookahead);do{if(l[(a=e)+n]===_&&l[a+n-1]===f&&l[a]===l[s]&&l[++a]===l[s+1]){s+=2,a++;do{}while(l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&l[++s]===l[++a]&&sn){if(t.match_start=e,n=r,r>=o)break;f=l[s+n-1],_=l[s+n]}}}while((e=c[e&d])>h&&0!=--i);return n<=t.lookahead?n:t.lookahead}function te(t){var e,i,n,o,h,l,d,c,u,f,_=t.w_size;do{if(o=t.window_size-t.lookahead-t.strstart,t.strstart>=_+(_-262)){a.arraySet(t.window,t.window,_,_,0),t.match_start-=_,t.strstart-=_,t.block_start-=_,e=i=t.hash_size;do{n=t.head[--e],t.head[e]=n>=_?n-_:0}while(--i);e=i=_;do{n=t.prev[--e],t.prev[e]=n>=_?n-_:0}while(--i);o+=_}if(0===t.strm.avail_in)break;if(l=t.strm,d=t.window,c=t.strstart+t.lookahead,u=o,f=void 0,(f=l.avail_in)>u&&(f=u),i=0===f?0:(l.avail_in-=f,a.arraySet(d,l.input,l.next_in,f,c),1===l.state.wrap?l.adler=r(l.adler,d,f,c):2===l.state.wrap&&(l.adler=s(l.adler,d,f,c)),l.next_in+=f,l.total_in+=f,f),t.lookahead+=i,t.lookahead+t.insert>=3)for(h=t.strstart-t.insert,t.ins_h=t.window[h],t.ins_h=(t.ins_h<=3&&(t.ins_h=(t.ins_h<=3)if(r=$t._tr_tally(t,t.strstart-t.match_start,t.match_length-3),t.lookahead-=t.match_length,t.match_length<=t.max_lazy_match&&t.lookahead>=3){t.match_length--;do{t.strstart++,t.ins_h=(t.ins_h<=3&&(t.ins_h=(t.ins_h<4096)&&(t.match_length=2)),t.prev_length>=3&&t.match_length<=t.prev_length){i=t.strstart+t.lookahead-3,r=$t._tr_tally(t,t.strstart-1-t.prev_match,t.prev_length-3),t.lookahead-=t.prev_length-1,t.prev_length-=2;do{++t.strstart<=i&&(t.ins_h=(t.ins_h<15&&(o=2,i-=16),s<1||s>9||8!==r||i<8||i>15||e<0||e>9||n<0||n>4)return Vt(t,-2);8===i&&(i=9);var h=new ie;return t.state=h,h.strm=t,h.wrap=o,h.gzhead=null,h.w_bits=i,h.w_size=1<t.pending_buf_size-5&&(a=t.pending_buf_size-5);;){if(t.lookahead<=1){if(te(t),0===t.lookahead&&0===e)return 1;if(0===t.lookahead)break}t.strstart+=t.lookahead,t.lookahead=0;var r=t.block_start+a;if((0===t.strstart||t.strstart>=r)&&(t.lookahead=t.strstart-r,t.strstart=r,Kt(t,!1),0===t.strm.avail_out))return 1;if(t.strstart-t.block_start>=t.w_size-262&&(Kt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Kt(t,!0),0===t.strm.avail_out?3:4):(t.strstart>t.block_start&&(Kt(t,!1),t.strm.avail_out),1)})),new re(4,4,8,4,ee),new re(4,5,16,8,ee),new re(4,6,32,32,ee),new re(4,4,16,16,ae),new re(8,16,32,32,ae),new re(8,16,128,128,ae),new re(8,32,128,256,ae),new re(32,128,258,1024,ae),new re(32,258,258,4096,ae)];var he={deflateInit:function(t,e){return oe(t,e,8,15,8,0)},deflateInit2:oe,deflateReset:ne,deflateResetKeep:se,deflateSetHeader:function(t,e){return t&&t.state?2!==t.state.wrap?-2:(t.state.gzhead=e,0):-2},deflate:function(t,e){var a,r,i,n;if(!t||!t.state||e>5||e<0)return t?Vt(t,-2):-2;if(r=t.state,!t.output||!t.input&&0!==t.avail_in||666===r.status&&4!==e)return Vt(t,0===t.avail_out?-5:-2);if(r.strm=t,a=r.last_flush,r.last_flush=e,42===r.status)if(2===r.wrap)t.adler=0,Gt(r,31),Gt(r,139),Gt(r,8),r.gzhead?(Gt(r,(r.gzhead.text?1:0)+(r.gzhead.hcrc?2:0)+(r.gzhead.extra?4:0)+(r.gzhead.name?8:0)+(r.gzhead.comment?16:0)),Gt(r,255&r.gzhead.time),Gt(r,r.gzhead.time>>8&255),Gt(r,r.gzhead.time>>16&255),Gt(r,r.gzhead.time>>24&255),Gt(r,9===r.level?2:r.strategy>=2||r.level<2?4:0),Gt(r,255&r.gzhead.os),r.gzhead.extra&&r.gzhead.extra.length&&(Gt(r,255&r.gzhead.extra.length),Gt(r,r.gzhead.extra.length>>8&255)),r.gzhead.hcrc&&(t.adler=s(t.adler,r.pending_buf,r.pending,0)),r.gzindex=0,r.status=69):(Gt(r,0),Gt(r,0),Gt(r,0),Gt(r,0),Gt(r,0),Gt(r,9===r.level?2:r.strategy>=2||r.level<2?4:0),Gt(r,3),r.status=113);else{var o=8+(r.w_bits-8<<4)<<8;o|=(r.strategy>=2||r.level<2?0:r.level<6?1:6===r.level?2:3)<<6,0!==r.strstart&&(o|=32),o+=31-o%31,r.status=113,Qt(r,o),0!==r.strstart&&(Qt(r,t.adler>>>16),Qt(r,65535&t.adler)),t.adler=1}if(69===r.status)if(r.gzhead.extra){for(i=r.pending;r.gzindex<(65535&r.gzhead.extra.length)&&(r.pending!==r.pending_buf_size||(r.gzhead.hcrc&&r.pending>i&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),Xt(t),i=r.pending,r.pending!==r.pending_buf_size));)Gt(r,255&r.gzhead.extra[r.gzindex]),r.gzindex++;r.gzhead.hcrc&&r.pending>i&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),r.gzindex===r.gzhead.extra.length&&(r.gzindex=0,r.status=73)}else r.status=73;if(73===r.status)if(r.gzhead.name){i=r.pending;do{if(r.pending===r.pending_buf_size&&(r.gzhead.hcrc&&r.pending>i&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),Xt(t),i=r.pending,r.pending===r.pending_buf_size)){n=1;break}n=r.gzindexi&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),0===n&&(r.gzindex=0,r.status=91)}else r.status=91;if(91===r.status)if(r.gzhead.comment){i=r.pending;do{if(r.pending===r.pending_buf_size&&(r.gzhead.hcrc&&r.pending>i&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),Xt(t),i=r.pending,r.pending===r.pending_buf_size)){n=1;break}n=r.gzindexi&&(t.adler=s(t.adler,r.pending_buf,r.pending-i,i)),0===n&&(r.status=103)}else r.status=103;if(103===r.status&&(r.gzhead.hcrc?(r.pending+2>r.pending_buf_size&&Xt(t),r.pending+2<=r.pending_buf_size&&(Gt(r,255&t.adler),Gt(r,t.adler>>8&255),t.adler=0,r.status=113)):r.status=113),0!==r.pending){if(Xt(t),0===t.avail_out)return r.last_flush=-1,0}else if(0===t.avail_in&&Zt(e)<=Zt(a)&&4!==e)return Vt(t,-5);if(666===r.status&&0!==t.avail_in)return Vt(t,-5);if(0!==t.avail_in||0!==r.lookahead||0!==e&&666!==r.status){var h=2===r.strategy?function(t,e){for(var a;;){if(0===t.lookahead&&(te(t),0===t.lookahead)){if(0===e)return 1;break}if(t.match_length=0,a=$t._tr_tally(t,0,t.window[t.strstart]),t.lookahead--,t.strstart++,a&&(Kt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Kt(t,!0),0===t.strm.avail_out?3:4):t.last_lit&&(Kt(t,!1),0===t.strm.avail_out)?1:2}(r,e):3===r.strategy?function(t,e){for(var a,r,i,s,n=t.window;;){if(t.lookahead<=258){if(te(t),t.lookahead<=258&&0===e)return 1;if(0===t.lookahead)break}if(t.match_length=0,t.lookahead>=3&&t.strstart>0&&(r=n[i=t.strstart-1])===n[++i]&&r===n[++i]&&r===n[++i]){s=t.strstart+258;do{}while(r===n[++i]&&r===n[++i]&&r===n[++i]&&r===n[++i]&&r===n[++i]&&r===n[++i]&&r===n[++i]&&r===n[++i]&&it.lookahead&&(t.match_length=t.lookahead)}if(t.match_length>=3?(a=$t._tr_tally(t,1,t.match_length-3),t.lookahead-=t.match_length,t.strstart+=t.match_length,t.match_length=0):(a=$t._tr_tally(t,0,t.window[t.strstart]),t.lookahead--,t.strstart++),a&&(Kt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Kt(t,!0),0===t.strm.avail_out?3:4):t.last_lit&&(Kt(t,!1),0===t.strm.avail_out)?1:2}(r,e):Mt[r.level].func(r,e);if(3!==h&&4!==h||(r.status=666),1===h||3===h)return 0===t.avail_out&&(r.last_flush=-1),0;if(2===h&&(1===e?$t._tr_align(r):5!==e&&($t._tr_stored_block(r,0,0,!1),3===e&&(Jt(r.head),0===r.lookahead&&(r.strstart=0,r.block_start=0,r.insert=0))),Xt(t),0===t.avail_out))return r.last_flush=-1,0}return 4!==e?0:r.wrap<=0?1:(2===r.wrap?(Gt(r,255&t.adler),Gt(r,t.adler>>8&255),Gt(r,t.adler>>16&255),Gt(r,t.adler>>24&255),Gt(r,255&t.total_in),Gt(r,t.total_in>>8&255),Gt(r,t.total_in>>16&255),Gt(r,t.total_in>>24&255)):(Qt(r,t.adler>>>16),Qt(r,65535&t.adler)),Xt(t),r.wrap>0&&(r.wrap=-r.wrap),0!==r.pending?0:1)},deflateEnd:function(t){var e;return t&&t.state?42!==(e=t.state.status)&&69!==e&&73!==e&&91!==e&&103!==e&&113!==e&&666!==e?Vt(t,-2):(t.state=null,113===e?Vt(t,-3):0):-2},deflateSetDictionary:function(t,e){var i,s,n,o,h,l,d,c,u=e.length;if(!t||!t.state)return-2;if(2===(o=(i=t.state).wrap)||1===o&&42!==i.status||i.lookahead)return-2;for(1===o&&(t.adler=r(t.adler,e,u,0)),i.wrap=0,u>=i.w_size&&(0===o&&(Jt(i.head),i.strstart=0,i.block_start=0,i.insert=0),c=new a.Buf8(i.w_size),a.arraySet(c,e,u-i.w_size,i.w_size,0),e=c,u=i.w_size),h=t.avail_in,l=t.next_in,d=t.input,t.avail_in=u,t.next_in=0,t.input=e,te(i);i.lookahead>=3;){s=i.strstart,n=i.lookahead-2;do{i.ins_h=(i.ins_h<0?e.windowBits=-e.windowBits:e.gzip&&e.windowBits>0&&e.windowBits<16&&(e.windowBits+=16),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new P,this.strm.avail_out=0;var r=he.deflateInit2(this.strm,e.level,e.method,e.windowBits,e.memLevel,e.strategy);if(0!==r)throw new Error(N[r]);if(e.header&&he.deflateSetHeader(this.strm,e.header),e.dictionary){var i;if(i="string"==typeof e.dictionary?B(e.dictionary):"[object ArrayBuffer]"===le.call(e.dictionary)?new Uint8Array(e.dictionary):e.dictionary,0!==(r=he.deflateSetDictionary(this.strm,i)))throw new Error(N[r]);this._dict_set=!0}}de.prototype.push=function(t,e){var r,i,s=this.strm,n=this.options.chunkSize;if(this.ended)return!1;i=e===~~e?e:!0===e?4:0,"string"==typeof t?s.input=B(t):"[object ArrayBuffer]"===le.call(t)?s.input=new Uint8Array(t):s.input=t,s.next_in=0,s.avail_in=s.input.length;do{if(0===s.avail_out&&(s.output=new a.Buf8(n),s.next_out=0,s.avail_out=n),1!==(r=he.deflate(s,i))&&0!==r)return this.onEnd(r),this.ended=!0,!1;0!==s.avail_out&&(0!==s.avail_in||4!==i&&2!==i)||("string"===this.options.to?this.onData(H(a.shrinkBuf(s.output,s.next_out))):this.onData(a.shrinkBuf(s.output,s.next_out)))}while((s.avail_in>0||0===s.avail_out)&&1!==r);return 4===i?(r=he.deflateEnd(this.strm),this.onEnd(r),this.ended=!0,0===r):2!==i||(this.onEnd(0),s.avail_out=0,!0)},de.prototype.onData=function(t){this.chunks.push(t)},de.prototype.onEnd=function(t){0===t&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var ce=de;const ue=new TextEncoder;class fe extends X{static async serialize(t,e){const a=new fe(t,e);return await a.readFully()}static base16(t){return Array.from(new Uint8Array(t)).map(t=>t.toString(16).padStart(2,"0")).join("")}constructor(t,e={}){super(),this.record=t,this.gzip=e.gzip;const a=e&&e.digest||{};"revisit"===t.warcType||"warcinfo"===t.warcType||t.warcPayloadDigest&&t.warcBlockDigest?this.digestAlgo=null:(this.digestAlgo=a.algo||"sha-256",this.digestAlgoPrefix=a.prefix||"sha-256:",this.digestBase32=a.base32||!1)}async*[Symbol.asyncIterator](){if(!this.gzip)return void(yield*this.generateRecord());let t=null;try{t=new CompressionStream("gzip")}catch(t){}t?yield*this.streamCompress(t):yield*this.pakoCompress()}async*pakoCompress(){const t=new ce({gzip:!0});let e=null;for await(const a of this.generateRecord())for(e&&e.length>0&&t.push(e),e=a;t.chunks.length;)yield t.chunks.shift();t.push(e,!0),yield t.result}async*streamCompress(t){const e=this.generateRecord();new ReadableStream({async pull(t){const a=await e.next();a.done?t.close():t.enqueue(a.value)}}).pipeThrough(t);let a=null;const r=t.readable.getReader();for(;(a=await r.read())&&!a.done;)yield a.value}async digestMessage(t){const e=await crypto.subtle.digest(this.digestAlgo,t);return this.digestAlgoPrefix+(this.digestBase32?ct.encode(e):fe.base16(e))}async*generateRecord(){let t=0,e=null;this.record.httpHeaders&&(e=ue.encode(this.record.httpHeaders.toString()+"\r\n"),t+=e.length);const a=await this.record.readFully();if(t+=a.length,this.digestAlgo){const r=await this.digestMessage(a),i=e?await this.digestMessage(fe.concatChunks([e,a],t)):r;this.record.warcHeaders.headers.set("WARC-Payload-Digest",r),this.record.warcHeaders.headers.set("WARC-Block-Digest",i)}this.record.warcHeaders.headers.set("Content-Length",t);const r=ue.encode(this.record.warcHeaders.toString());yield r,yield Q,e&&(yield e),yield a,yield Y}}const _e="offset,warc-type,warc-target-uri".split(",");class pe extends class{constructor(t,e){this.opts=t,this.out=e}serialize(t){return JSON.stringify(t)+"\n"}write(t){this.out.write(this.serialize(t))}async run(t){for await(const e of this.iterIndex(t))this.write(e)}async*iterIndex(t){const e={strictHeaders:!0,parseHttp:this.parseHttp};for(const{filename:a,reader:r}of t){if(!a||!r)continue;const t=new dt(r,e);for await(const e of t){await e.skipFully();const r=this.indexRecord(e,t,a);r&&(yield r)}}}indexRecord(t,e,a){if(this.filterRecord&&!this.filterRecord(t))return null;const r={},i={offset:e.offset,length:e.recordLength,filename:a};for(const e of this.fields)null!=i[e]?r[e]=i[e]:this.setField(e,t,r);return r}setField(t,e,a){const r=this.getField(t,e);null!=r&&(a[t]=r)}getField(t,e){return"http:status"===t?!e.httpHeaders||"response"!==e.warcType&&"revisit"!==e.warcType?null:e.httpHeaders.statusCode:t.startsWith("http:")?e.httpHeaders?e.httpHeaders.headers.get(t.slice(5)):null:e.warcHeaders.headers.get(t)}}{constructor(t={},e=null){if(super(t,e),t.fields){this.fields=t.fields.split(","),this.parseHttp=!1;for(const t of this.fields)if(t.startsWith("http:")){this.parseHttp=!0;break}}else this.fields=_e,this.parseHttp=!1}}const ge="urlkey,timestamp,url,mime,status,digest,length,offset,filename".split(","),we="urlkey,timestamp,url,mime,status,digest,redirect,meta,length,offset,filename".split(",");class me extends pe{constructor(t={},e=null){switch(super(t,e),this.includeAll=t.all,this.fields=ge,this.parseHttp=!0,t.format){case"cdxj":this.serialize=this.serializeCDXJ;break;case"cdx":this.serialize=this.serializeCDX11}}filterRecord(t){if(this.includeAll)return!0;const e=t.warcType;return"request"!==e&&"warcinfo"!==e}serializeCDXJ(t){const{urlkey:e,timestamp:a}=t;return delete t.urlkey,delete t.timestamp,`${e} ${a} ${JSON.stringify(t)}\n`}serializeCDX11(t){const e=[];for(const a of we)e.push(null!=t[a]?t[a]:"-");return e.join(" ")+"\n"}getField(t,e){let a=null;switch(t){case"urlkey":return this.getSurt(e.warcTargetURI);case"timestamp":return a=e.warcDate,a.replace(/[-:T]/g,"").slice(0,14);case"url":return e.warcTargetURI;case"mime":switch(e.warcType){case"revisit":return"warc/revisit";case"response":case"request":t="http:content-type";break;default:t="content-type"}return a=super.getField(t,e),a?a.split(";",1)[0].trim():null;case"status":return super.getField("http:status",e);case"digest":return a=e.warcPayloadDigest,a?a.split(":",2)[1]:null}}getSurt(t){try{const e=new URL(t);if("http:"!==e.protocol&&"https:"!==e.protocol)return t;let a=e.hostname.split(".").reverse().join(",");return e.port&&(a+=":"+e.port),a+=")",a+=e.pathname,a.toLowerCase()}catch(e){return t}}}export{K as AsyncIterReader,X as BaseAsyncIterReader,me as CDXIndexer,pe as Indexer,G as LimitReader,tt as StatusAndHeaders,et as StatusAndHeadersParser,dt as WARCParser,lt as WARCRecord,fe as WARCSerializer}; +!function(){var a="object"==typeof window?window:{};!a.HI_BASE32_NO_NODE_JS&&"object"==typeof process&&process.versions&&process.versions.node&&(a=t);var r=!a.HI_BASE32_NO_COMMON_JS&&e.exports,s="ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".split(""),i={A:0,B:1,C:2,D:3,E:4,F:5,G:6,H:7,I:8,J:9,K:10,L:11,M:12,N:13,O:14,P:15,Q:16,R:17,S:18,T:19,U:20,V:21,W:22,X:23,Y:24,Z:25,2:26,3:27,4:28,5:29,6:30,7:31},n=[0,0,0,0,0,0,0,0],o=function(t,e){e.length>10&&(e="..."+e.substr(-10));var a=new Error("Decoded data is not valid UTF-8. Maybe try base32.decode.asBytes()? Partial data after reading "+t+" bytes: "+e+" <-");throw a.position=t,a},h=function(t){if(!/^[A-Z2-7=]+$/.test(t))throw new Error("Invalid base32 characters");for(var e,a,r,s,n,o,h,l,d=[],c=0,u=(t=t.replace(/=/g,"")).length,f=0,_=u>>3<<3;f<_;)e=i[t.charAt(f++)],a=i[t.charAt(f++)],r=i[t.charAt(f++)],s=i[t.charAt(f++)],n=i[t.charAt(f++)],o=i[t.charAt(f++)],h=i[t.charAt(f++)],l=i[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|s>>>4),d[c++]=255&(s<<4|n>>>1),d[c++]=255&(n<<7|o<<2|h>>>3),d[c++]=255&(h<<5|l);var p=u-_;return 2===p?(e=i[t.charAt(f++)],a=i[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2)):4===p?(e=i[t.charAt(f++)],a=i[t.charAt(f++)],r=i[t.charAt(f++)],s=i[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|s>>>4)):5===p?(e=i[t.charAt(f++)],a=i[t.charAt(f++)],r=i[t.charAt(f++)],s=i[t.charAt(f++)],n=i[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|s>>>4),d[c++]=255&(s<<4|n>>>1)):7===p&&(e=i[t.charAt(f++)],a=i[t.charAt(f++)],r=i[t.charAt(f++)],s=i[t.charAt(f++)],n=i[t.charAt(f++)],o=i[t.charAt(f++)],h=i[t.charAt(f++)],d[c++]=255&(e<<3|a>>>2),d[c++]=255&(a<<6|r<<1|s>>>4),d[c++]=255&(s<<4|n>>>1),d[c++]=255&(n<<7|o<<2|h>>>3)),d},l=function(t,e){if(!e)return function(t){for(var e,a,r="",s=t.length,i=0,n=0;i191&&e<=223?(a=31&e,n=1):e<=239?(a=15&e,n=2):e<=247?(a=7&e,n=3):o(i,r);for(var h=0;h191)&&o(i,r),a<<=6,a+=63&e;a>=55296&&a<=57343&&o(i,r),a>1114111&&o(i,r),a<=65535?r+=String.fromCharCode(a):(a-=65536,r+=String.fromCharCode(55296+(a>>10)),r+=String.fromCharCode(56320+(1023&a)))}return r}(h(t));if(!/^[A-Z2-7=]+$/.test(t))throw new Error("Invalid base32 characters");var a,r,s,n,l,d,c,u,f="",_=t.indexOf("=");-1===_&&(_=t.length);for(var p=0,w=_>>3<<3;p>>2))+String.fromCharCode(255&(r<<6|s<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))+String.fromCharCode(255&(l<<7|d<<2|c>>>3))+String.fromCharCode(255&(c<<5|u));var g=_-w;return 2===g?(a=i[t.charAt(p++)],r=i[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))):4===g?(a=i[t.charAt(p++)],r=i[t.charAt(p++)],s=i[t.charAt(p++)],n=i[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|s<<1|n>>>4))):5===g?(a=i[t.charAt(p++)],r=i[t.charAt(p++)],s=i[t.charAt(p++)],n=i[t.charAt(p++)],l=i[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|s<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))):7===g&&(a=i[t.charAt(p++)],r=i[t.charAt(p++)],s=i[t.charAt(p++)],n=i[t.charAt(p++)],l=i[t.charAt(p++)],d=i[t.charAt(p++)],c=i[t.charAt(p++)],f+=String.fromCharCode(255&(a<<3|r>>>2))+String.fromCharCode(255&(r<<6|s<<1|n>>>4))+String.fromCharCode(255&(n<<4|l>>>1))+String.fromCharCode(255&(l<<7|d<<2|c>>>3))),f},d={encode:function(t,e){var a="string"!=typeof t;return a&&t.constructor===ArrayBuffer&&(t=new Uint8Array(t)),a?function(t){for(var e,a,r,i,n,o="",h=t.length,l=0,d=5*parseInt(h/5);l>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[31&(i<<3|n>>>5)]+s[31&n];var c=h-d;return 1===c?(e=t[l],o+=s[e>>>3]+s[e<<2&31]+"======"):2===c?(e=t[l++],a=t[l],o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[a<<4&31]+"===="):3===c?(e=t[l++],a=t[l++],r=t[l],o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[r<<1&31]+"==="):4===c&&(e=t[l++],a=t[l++],r=t[l++],i=t[l],o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[i<<3&31]+"="),o}(t):e?function(t){for(var e,a,r,i,n,o="",h=t.length,l=0,d=5*parseInt(h/5);l>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[31&(i<<3|n>>>5)]+s[31&n];var c=h-d;return 1===c?(e=t.charCodeAt(l),o+=s[e>>>3]+s[e<<2&31]+"======"):2===c?(e=t.charCodeAt(l++),a=t.charCodeAt(l),o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[a<<4&31]+"===="):3===c?(e=t.charCodeAt(l++),a=t.charCodeAt(l++),r=t.charCodeAt(l),o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[r<<1&31]+"==="):4===c&&(e=t.charCodeAt(l++),a=t.charCodeAt(l++),r=t.charCodeAt(l++),i=t.charCodeAt(l),o+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[i<<3&31]+"="),o}(t):function(t){var e,a,r,i,o,h,l,d=!1,c="",u=0,f=0,_=t.length;do{for(n[0]=n[5],n[1]=n[6],n[2]=n[7],l=f;u<_&&l<5;++u)(h=t.charCodeAt(u))<128?n[l++]=h:h<2048?(n[l++]=192|h>>6,n[l++]=128|63&h):h<55296||h>=57344?(n[l++]=224|h>>12,n[l++]=128|h>>6&63,n[l++]=128|63&h):(h=65536+((1023&h)<<10|1023&t.charCodeAt(++u)),n[l++]=240|h>>18,n[l++]=128|h>>12&63,n[l++]=128|h>>6&63,n[l++]=128|63&h);f=l-5,u===_&&++u,u>_&&l<6&&(d=!0),e=n[0],l>4?(a=n[1],r=n[2],i=n[3],o=n[4],c+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[31&(i<<3|o>>>5)]+s[31&o]):1===l?c+=s[e>>>3]+s[e<<2&31]+"======":2===l?(a=n[1],c+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[a<<4&31]+"===="):3===l?(a=n[1],r=n[2],c+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[r<<1&31]+"==="):(a=n[1],r=n[2],i=n[3],c+=s[e>>>3]+s[31&(e<<2|a>>>6)]+s[a>>>1&31]+s[31&(a<<4|r>>>4)]+s[31&(r<<1|i>>>7)]+s[i>>>2&31]+s[i<<3&31]+"=")}while(!d);return c}(t)},decode:l};l.asBytes=h,r?e.exports=d:a.base32=d}()}));function ut(t){for(var e=t.length;--e>=0;)t[e]=0}var ft=[0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0],_t=[0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13],pt=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7],wt=[16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15],gt=new Array(576);ut(gt);var mt=new Array(60);ut(mt);var bt=new Array(512);ut(bt);var yt=new Array(256);ut(yt);var vt=new Array(29);ut(vt);var kt,xt,Ct,At=new Array(30);function Rt(t,e,a,r,s){this.static_tree=t,this.extra_bits=e,this.extra_base=a,this.elems=r,this.max_length=s,this.has_stree=t&&t.length}function zt(t,e){this.dyn_tree=t,this.max_code=0,this.stat_desc=e}function St(t){return t<256?bt[t]:bt[256+(t>>>7)]}function Bt(t,e){t.pending_buf[t.pending++]=255&e,t.pending_buf[t.pending++]=e>>>8&255}function Tt(t,e,a){t.bi_valid>16-a?(t.bi_buf|=e<>16-t.bi_valid,t.bi_valid+=a-16):(t.bi_buf|=e<>>=1,a<<=1}while(--e>0);return a>>>1}function Dt(t,e,a){var r,s,i=new Array(16),n=0;for(r=1;r<=15;r++)i[r]=n=n+a[r-1]<<1;for(s=0;s<=e;s++){var o=t[2*s+1];0!==o&&(t[2*s]=It(i[o]++,o))}}function Et(t){var e;for(e=0;e<286;e++)t.dyn_ltree[2*e]=0;for(e=0;e<30;e++)t.dyn_dtree[2*e]=0;for(e=0;e<19;e++)t.bl_tree[2*e]=0;t.dyn_ltree[512]=1,t.opt_len=t.static_len=0,t.last_lit=t.matches=0}function Lt(t){t.bi_valid>8?Bt(t,t.bi_buf):t.bi_valid>0&&(t.pending_buf[t.pending++]=t.bi_buf),t.bi_buf=0,t.bi_valid=0}function Ut(t,e,a,r){var s=2*e,i=2*a;return t[s]>1;a>=1;a--)Ot(t,i,a);s=h;do{a=t.heap[1],t.heap[1]=t.heap[t.heap_len--],Ot(t,i,1),r=t.heap[1],t.heap[--t.heap_max]=a,t.heap[--t.heap_max]=r,i[2*s]=i[2*a]+i[2*r],t.depth[s]=(t.depth[a]>=t.depth[r]?t.depth[a]:t.depth[r])+1,i[2*a+1]=i[2*r+1]=s,t.heap[1]=s++,Ot(t,i,1)}while(t.heap_len>=2);t.heap[--t.heap_max]=t.heap[1],function(t,e){var a,r,s,i,n,o,h=e.dyn_tree,l=e.max_code,d=e.stat_desc.static_tree,c=e.stat_desc.has_stree,u=e.stat_desc.extra_bits,f=e.stat_desc.extra_base,_=e.stat_desc.max_length,p=0;for(i=0;i<=15;i++)t.bl_count[i]=0;for(h[2*t.heap[t.heap_max]+1]=0,a=t.heap_max+1;a<573;a++)(i=h[2*h[2*(r=t.heap[a])+1]+1]+1)>_&&(i=_,p++),h[2*r+1]=i,r>l||(t.bl_count[i]++,n=0,r>=f&&(n=u[r-f]),o=h[2*r],t.opt_len+=o*(i+n),c&&(t.static_len+=o*(d[2*r+1]+n)));if(0!==p){do{for(i=_-1;0===t.bl_count[i];)i--;t.bl_count[i]--,t.bl_count[i+1]+=2,t.bl_count[_]--,p-=2}while(p>0);for(i=_;0!==i;i--)for(r=t.bl_count[i];0!==r;)(s=t.heap[--a])>l||(h[2*s+1]!==i&&(t.opt_len+=(i-h[2*s+1])*h[2*s],h[2*s+1]=i),r--)}}(t,e),Dt(i,l,t.bl_count)}function jt(t,e,a){var r,s,i=-1,n=e[1],o=0,h=7,l=4;for(0===n&&(h=138,l=3),e[2*(a+1)+1]=65535,r=0;r<=a;r++)s=n,n=e[2*(r+1)+1],++o>=7;r<30;r++)for(At[r]=s<<7,t=0;t<1<<_t[r]-7;t++)bt[256+s++]=r;for(e=0;e<=15;e++)i[e]=0;for(t=0;t<=143;)gt[2*t+1]=8,t++,i[8]++;for(;t<=255;)gt[2*t+1]=9,t++,i[9]++;for(;t<=279;)gt[2*t+1]=7,t++,i[7]++;for(;t<=287;)gt[2*t+1]=8,t++,i[8]++;for(Dt(gt,287,i),t=0;t<30;t++)mt[2*t+1]=5,mt[2*t]=It(t,5);kt=new Rt(gt,ft,257,286,15),xt=new Rt(mt,_t,0,30,15),Ct=new Rt(new Array(0),pt,0,19,7)}(),Nt=!0),t.l_desc=new zt(t.dyn_ltree,kt),t.d_desc=new zt(t.dyn_dtree,xt),t.bl_desc=new zt(t.bl_tree,Ct),t.bi_buf=0,t.bi_valid=0,Et(t)},_tr_stored_block:qt,_tr_flush_block:function(t,e,a,r){var s,i,n=0;t.level>0?(2===t.strm.data_type&&(t.strm.data_type=function(t){var e,a=4093624447;for(e=0;e<=31;e++,a>>>=1)if(1&a&&0!==t.dyn_ltree[2*e])return 0;if(0!==t.dyn_ltree[18]||0!==t.dyn_ltree[20]||0!==t.dyn_ltree[26])return 1;for(e=32;e<256;e++)if(0!==t.dyn_ltree[2*e])return 1;return 0}(t)),Ft(t,t.l_desc),Ft(t,t.d_desc),n=function(t){var e;for(jt(t,t.dyn_ltree,t.l_desc.max_code),jt(t,t.dyn_dtree,t.d_desc.max_code),Ft(t,t.bl_desc),e=18;e>=3&&0===t.bl_tree[2*wt[e]+1];e--);return t.opt_len+=3*(e+1)+5+5+4,e}(t),s=t.opt_len+3+7>>>3,(i=t.static_len+3+7>>>3)<=s&&(s=i)):s=i=a+5,a+4<=s&&-1!==e?qt(t,e,a,r):4===t.strategy||i===s?(Tt(t,2+(r?1:0),3),Wt(t,gt,mt)):(Tt(t,4+(r?1:0),3),function(t,e,a,r){var s;for(Tt(t,e-257,5),Tt(t,a-1,5),Tt(t,r-4,4),s=0;s>>8&255,t.pending_buf[t.d_buf+2*t.last_lit+1]=255&e,t.pending_buf[t.l_buf+t.last_lit]=255&a,t.last_lit++,0===e?t.dyn_ltree[2*a]++:(t.matches++,e--,t.dyn_ltree[2*(yt[a]+256+1)]++,t.dyn_dtree[2*St(e)]++),t.last_lit===t.lit_bufsize-1},_tr_align:function(t){Tt(t,2,3),Ht(t,256,gt),function(t){16===t.bi_valid?(Bt(t,t.bi_buf),t.bi_buf=0,t.bi_valid=0):t.bi_valid>=8&&(t.pending_buf[t.pending++]=255&t.bi_buf,t.bi_buf>>=8,t.bi_valid-=8)}(t)}};function Jt(t,e){return t.msg=P[e],e}function Vt(t){return(t<<1)-(t>4?9:0)}function Zt(t){for(var e=t.length;--e>=0;)t[e]=0}function Gt(t){var e=t.state,r=e.pending;r>t.avail_out&&(r=t.avail_out),0!==r&&(a.arraySet(t.output,e.pending_buf,e.pending_out,r,t.next_out),t.next_out+=r,e.pending_out+=r,t.total_out+=r,t.avail_out-=r,e.pending-=r,0===e.pending&&(e.pending_out=0))}function Xt(t,e){Mt._tr_flush_block(t,t.block_start>=0?t.block_start:-1,t.strstart-t.block_start,e),t.block_start=t.strstart,Gt(t.strm)}function Kt(t,e){t.pending_buf[t.pending++]=e}function Qt(t,e){t.pending_buf[t.pending++]=e>>>8&255,t.pending_buf[t.pending++]=255&e}function Yt(t,e){var a,r,s=t.max_chain_length,i=t.strstart,n=t.prev_length,o=t.nice_match,h=t.strstart>t.w_size-262?t.strstart-(t.w_size-262):0,l=t.window,d=t.w_mask,c=t.prev,u=t.strstart+258,f=l[i+n-1],_=l[i+n];t.prev_length>=t.good_match&&(s>>=2),o>t.lookahead&&(o=t.lookahead);do{if(l[(a=e)+n]===_&&l[a+n-1]===f&&l[a]===l[i]&&l[++a]===l[i+1]){i+=2,a++;do{}while(l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&l[++i]===l[++a]&&in){if(t.match_start=e,n=r,r>=o)break;f=l[i+n-1],_=l[i+n]}}}while((e=c[e&d])>h&&0!=--s);return n<=t.lookahead?n:t.lookahead}function te(t){var e,s,n,o,h,l,d,c,u,f,_=t.w_size;do{if(o=t.window_size-t.lookahead-t.strstart,t.strstart>=_+(_-262)){a.arraySet(t.window,t.window,_,_,0),t.match_start-=_,t.strstart-=_,t.block_start-=_,e=s=t.hash_size;do{n=t.head[--e],t.head[e]=n>=_?n-_:0}while(--s);e=s=_;do{n=t.prev[--e],t.prev[e]=n>=_?n-_:0}while(--s);o+=_}if(0===t.strm.avail_in)break;if(l=t.strm,d=t.window,c=t.strstart+t.lookahead,u=o,f=void 0,(f=l.avail_in)>u&&(f=u),s=0===f?0:(l.avail_in-=f,a.arraySet(d,l.input,l.next_in,f,c),1===l.state.wrap?l.adler=r(l.adler,d,f,c):2===l.state.wrap&&(l.adler=i(l.adler,d,f,c)),l.next_in+=f,l.total_in+=f,f),t.lookahead+=s,t.lookahead+t.insert>=3)for(h=t.strstart-t.insert,t.ins_h=t.window[h],t.ins_h=(t.ins_h<=3&&(t.ins_h=(t.ins_h<=3)if(r=Mt._tr_tally(t,t.strstart-t.match_start,t.match_length-3),t.lookahead-=t.match_length,t.match_length<=t.max_lazy_match&&t.lookahead>=3){t.match_length--;do{t.strstart++,t.ins_h=(t.ins_h<=3&&(t.ins_h=(t.ins_h<4096)&&(t.match_length=2)),t.prev_length>=3&&t.match_length<=t.prev_length){s=t.strstart+t.lookahead-3,r=Mt._tr_tally(t,t.strstart-1-t.prev_match,t.prev_length-3),t.lookahead-=t.prev_length-1,t.prev_length-=2;do{++t.strstart<=s&&(t.ins_h=(t.ins_h<15&&(o=2,s-=16),i<1||i>9||8!==r||s<8||s>15||e<0||e>9||n<0||n>4)return Jt(t,-2);8===s&&(s=9);var h=new se;return t.state=h,h.strm=t,h.wrap=o,h.gzhead=null,h.w_bits=s,h.w_size=1<t.pending_buf_size-5&&(a=t.pending_buf_size-5);;){if(t.lookahead<=1){if(te(t),0===t.lookahead&&0===e)return 1;if(0===t.lookahead)break}t.strstart+=t.lookahead,t.lookahead=0;var r=t.block_start+a;if((0===t.strstart||t.strstart>=r)&&(t.lookahead=t.strstart-r,t.strstart=r,Xt(t,!1),0===t.strm.avail_out))return 1;if(t.strstart-t.block_start>=t.w_size-262&&(Xt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Xt(t,!0),0===t.strm.avail_out?3:4):(t.strstart>t.block_start&&(Xt(t,!1),t.strm.avail_out),1)})),new re(4,4,8,4,ee),new re(4,5,16,8,ee),new re(4,6,32,32,ee),new re(4,4,16,16,ae),new re(8,16,32,32,ae),new re(8,16,128,128,ae),new re(8,32,128,256,ae),new re(32,128,258,1024,ae),new re(32,258,258,4096,ae)];var he={deflateInit:function(t,e){return oe(t,e,8,15,8,0)},deflateInit2:oe,deflateReset:ne,deflateResetKeep:ie,deflateSetHeader:function(t,e){return t&&t.state?2!==t.state.wrap?-2:(t.state.gzhead=e,0):-2},deflate:function(t,e){var a,r,s,n;if(!t||!t.state||e>5||e<0)return t?Jt(t,-2):-2;if(r=t.state,!t.output||!t.input&&0!==t.avail_in||666===r.status&&4!==e)return Jt(t,0===t.avail_out?-5:-2);if(r.strm=t,a=r.last_flush,r.last_flush=e,42===r.status)if(2===r.wrap)t.adler=0,Kt(r,31),Kt(r,139),Kt(r,8),r.gzhead?(Kt(r,(r.gzhead.text?1:0)+(r.gzhead.hcrc?2:0)+(r.gzhead.extra?4:0)+(r.gzhead.name?8:0)+(r.gzhead.comment?16:0)),Kt(r,255&r.gzhead.time),Kt(r,r.gzhead.time>>8&255),Kt(r,r.gzhead.time>>16&255),Kt(r,r.gzhead.time>>24&255),Kt(r,9===r.level?2:r.strategy>=2||r.level<2?4:0),Kt(r,255&r.gzhead.os),r.gzhead.extra&&r.gzhead.extra.length&&(Kt(r,255&r.gzhead.extra.length),Kt(r,r.gzhead.extra.length>>8&255)),r.gzhead.hcrc&&(t.adler=i(t.adler,r.pending_buf,r.pending,0)),r.gzindex=0,r.status=69):(Kt(r,0),Kt(r,0),Kt(r,0),Kt(r,0),Kt(r,0),Kt(r,9===r.level?2:r.strategy>=2||r.level<2?4:0),Kt(r,3),r.status=113);else{var o=8+(r.w_bits-8<<4)<<8;o|=(r.strategy>=2||r.level<2?0:r.level<6?1:6===r.level?2:3)<<6,0!==r.strstart&&(o|=32),o+=31-o%31,r.status=113,Qt(r,o),0!==r.strstart&&(Qt(r,t.adler>>>16),Qt(r,65535&t.adler)),t.adler=1}if(69===r.status)if(r.gzhead.extra){for(s=r.pending;r.gzindex<(65535&r.gzhead.extra.length)&&(r.pending!==r.pending_buf_size||(r.gzhead.hcrc&&r.pending>s&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),Gt(t),s=r.pending,r.pending!==r.pending_buf_size));)Kt(r,255&r.gzhead.extra[r.gzindex]),r.gzindex++;r.gzhead.hcrc&&r.pending>s&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),r.gzindex===r.gzhead.extra.length&&(r.gzindex=0,r.status=73)}else r.status=73;if(73===r.status)if(r.gzhead.name){s=r.pending;do{if(r.pending===r.pending_buf_size&&(r.gzhead.hcrc&&r.pending>s&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),Gt(t),s=r.pending,r.pending===r.pending_buf_size)){n=1;break}n=r.gzindexs&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),0===n&&(r.gzindex=0,r.status=91)}else r.status=91;if(91===r.status)if(r.gzhead.comment){s=r.pending;do{if(r.pending===r.pending_buf_size&&(r.gzhead.hcrc&&r.pending>s&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),Gt(t),s=r.pending,r.pending===r.pending_buf_size)){n=1;break}n=r.gzindexs&&(t.adler=i(t.adler,r.pending_buf,r.pending-s,s)),0===n&&(r.status=103)}else r.status=103;if(103===r.status&&(r.gzhead.hcrc?(r.pending+2>r.pending_buf_size&&Gt(t),r.pending+2<=r.pending_buf_size&&(Kt(r,255&t.adler),Kt(r,t.adler>>8&255),t.adler=0,r.status=113)):r.status=113),0!==r.pending){if(Gt(t),0===t.avail_out)return r.last_flush=-1,0}else if(0===t.avail_in&&Vt(e)<=Vt(a)&&4!==e)return Jt(t,-5);if(666===r.status&&0!==t.avail_in)return Jt(t,-5);if(0!==t.avail_in||0!==r.lookahead||0!==e&&666!==r.status){var h=2===r.strategy?function(t,e){for(var a;;){if(0===t.lookahead&&(te(t),0===t.lookahead)){if(0===e)return 1;break}if(t.match_length=0,a=Mt._tr_tally(t,0,t.window[t.strstart]),t.lookahead--,t.strstart++,a&&(Xt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Xt(t,!0),0===t.strm.avail_out?3:4):t.last_lit&&(Xt(t,!1),0===t.strm.avail_out)?1:2}(r,e):3===r.strategy?function(t,e){for(var a,r,s,i,n=t.window;;){if(t.lookahead<=258){if(te(t),t.lookahead<=258&&0===e)return 1;if(0===t.lookahead)break}if(t.match_length=0,t.lookahead>=3&&t.strstart>0&&(r=n[s=t.strstart-1])===n[++s]&&r===n[++s]&&r===n[++s]){i=t.strstart+258;do{}while(r===n[++s]&&r===n[++s]&&r===n[++s]&&r===n[++s]&&r===n[++s]&&r===n[++s]&&r===n[++s]&&r===n[++s]&&st.lookahead&&(t.match_length=t.lookahead)}if(t.match_length>=3?(a=Mt._tr_tally(t,1,t.match_length-3),t.lookahead-=t.match_length,t.strstart+=t.match_length,t.match_length=0):(a=Mt._tr_tally(t,0,t.window[t.strstart]),t.lookahead--,t.strstart++),a&&(Xt(t,!1),0===t.strm.avail_out))return 1}return t.insert=0,4===e?(Xt(t,!0),0===t.strm.avail_out?3:4):t.last_lit&&(Xt(t,!1),0===t.strm.avail_out)?1:2}(r,e):$t[r.level].func(r,e);if(3!==h&&4!==h||(r.status=666),1===h||3===h)return 0===t.avail_out&&(r.last_flush=-1),0;if(2===h&&(1===e?Mt._tr_align(r):5!==e&&(Mt._tr_stored_block(r,0,0,!1),3===e&&(Zt(r.head),0===r.lookahead&&(r.strstart=0,r.block_start=0,r.insert=0))),Gt(t),0===t.avail_out))return r.last_flush=-1,0}return 4!==e?0:r.wrap<=0?1:(2===r.wrap?(Kt(r,255&t.adler),Kt(r,t.adler>>8&255),Kt(r,t.adler>>16&255),Kt(r,t.adler>>24&255),Kt(r,255&t.total_in),Kt(r,t.total_in>>8&255),Kt(r,t.total_in>>16&255),Kt(r,t.total_in>>24&255)):(Qt(r,t.adler>>>16),Qt(r,65535&t.adler)),Gt(t),r.wrap>0&&(r.wrap=-r.wrap),0!==r.pending?0:1)},deflateEnd:function(t){var e;return t&&t.state?42!==(e=t.state.status)&&69!==e&&73!==e&&91!==e&&103!==e&&113!==e&&666!==e?Jt(t,-2):(t.state=null,113===e?Jt(t,-3):0):-2},deflateSetDictionary:function(t,e){var s,i,n,o,h,l,d,c,u=e.length;if(!t||!t.state)return-2;if(2===(o=(s=t.state).wrap)||1===o&&42!==s.status||s.lookahead)return-2;for(1===o&&(t.adler=r(t.adler,e,u,0)),s.wrap=0,u>=s.w_size&&(0===o&&(Zt(s.head),s.strstart=0,s.block_start=0,s.insert=0),c=new a.Buf8(s.w_size),a.arraySet(c,e,u-s.w_size,s.w_size,0),e=c,u=s.w_size),h=t.avail_in,l=t.next_in,d=t.input,t.avail_in=u,t.next_in=0,t.input=e,te(s);s.lookahead>=3;){i=s.strstart,n=s.lookahead-2;do{s.ins_h=(s.ins_h<0?e.windowBits=-e.windowBits:e.gzip&&e.windowBits>0&&e.windowBits<16&&(e.windowBits+=16),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new N,this.strm.avail_out=0;var r=he.deflateInit2(this.strm,e.level,e.method,e.windowBits,e.memLevel,e.strategy);if(0!==r)throw new Error(P[r]);if(e.header&&he.deflateSetHeader(this.strm,e.header),e.dictionary){var s;if(s="string"==typeof e.dictionary?B(e.dictionary):"[object ArrayBuffer]"===le.call(e.dictionary)?new Uint8Array(e.dictionary):e.dictionary,0!==(r=he.deflateSetDictionary(this.strm,s)))throw new Error(P[r]);this._dict_set=!0}}de.prototype.push=function(t,e){var r,s,i=this.strm,n=this.options.chunkSize;if(this.ended)return!1;s=e===~~e?e:!0===e?4:0,"string"==typeof t?i.input=B(t):"[object ArrayBuffer]"===le.call(t)?i.input=new Uint8Array(t):i.input=t,i.next_in=0,i.avail_in=i.input.length;do{if(0===i.avail_out&&(i.output=new a.Buf8(n),i.next_out=0,i.avail_out=n),1!==(r=he.deflate(i,s))&&0!==r)return this.onEnd(r),this.ended=!0,!1;0!==i.avail_out&&(0!==i.avail_in||4!==s&&2!==s)||("string"===this.options.to?this.onData(T(a.shrinkBuf(i.output,i.next_out))):this.onData(a.shrinkBuf(i.output,i.next_out)))}while((i.avail_in>0||0===i.avail_out)&&1!==r);return 4===s?(r=he.deflateEnd(this.strm),this.onEnd(r),this.ended=!0,0===r):2!==s||(this.onEnd(0),i.avail_out=0,!0)},de.prototype.onData=function(t){this.chunks.push(t)},de.prototype.onEnd=function(t){0===t&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=a.flattenChunks(this.chunks)),this.chunks=[],this.err=t,this.msg=this.strm.msg};var ce=de;const ue=new TextEncoder;class fe extends G{static async serialize(t,e){const a=new fe(t,e);return await a.readFully()}static base16(t){return Array.from(new Uint8Array(t)).map(t=>t.toString(16).padStart(2,"0")).join("")}constructor(t,e={}){super(),this.record=t,this.gzip=e.gzip;const a=e&&e.digest||{};"revisit"===t.warcType||"warcinfo"===t.warcType||t.warcPayloadDigest&&t.warcBlockDigest?this.digestAlgo=null:(this.digestAlgo=a.algo||"sha-256",this.digestAlgoPrefix=a.prefix||"sha-256:",this.digestBase32=a.base32||!1)}async*[Symbol.asyncIterator](){if(!this.gzip)return void(yield*this.generateRecord());let t=null;try{t=new CompressionStream("gzip")}catch(t){}t?yield*this.streamCompress(t):yield*this.pakoCompress()}async*pakoCompress(){const t=new ce({gzip:!0});let e=null;for await(const a of this.generateRecord())for(e&&e.length>0&&t.push(e),e=a;t.chunks.length;)yield t.chunks.shift();t.push(e,!0),yield t.result}async*streamCompress(t){const e=this.generateRecord();new ReadableStream({async pull(t){const a=await e.next();a.done?t.close():t.enqueue(a.value)}}).pipeThrough(t);let a=null;const r=t.readable.getReader();for(;(a=await r.read())&&!a.done;)yield a.value}async digestMessage(t){const e=await crypto.subtle.digest(this.digestAlgo,t);return this.digestAlgoPrefix+(this.digestBase32?ct.encode(e):fe.base16(e))}async*generateRecord(){let t=0,e=null;this.record.httpHeaders&&(e=ue.encode(this.record.httpHeaders.toString()+"\r\n"),t+=e.length);const a=await this.record.readFully();if(t+=a.length,this.digestAlgo){const r=await this.digestMessage(a),s=e?await this.digestMessage(fe.concatChunks([e,a],t)):r;this.record.warcHeaders.headers.set("WARC-Payload-Digest",r),this.record.warcHeaders.headers.set("WARC-Block-Digest",s)}this.record.warcHeaders.headers.set("Content-Length",t);const r=ue.encode(this.record.warcHeaders.toString());yield r,yield Q,e&&(yield e),yield a,yield Y}}function _e(t){try{if(!t.startsWith("https:")&&!t.startsWith("http:"))return t;t=t.replace(/www\d*\./,"");const e=new URL(t.toLowerCase());let a=e.hostname.split(".").reverse().join(",");return e.port&&(a+=":"+e.port),a+=")",a+=e.pathname,e.search&&(e.searchParams.sort(),a+=e.search),a}catch(e){return t}}function pe(t){let{url:e,method:a,headers:r,postData:s}=t;if("GET"===a)return!1;function i(t){return t instanceof Uint8Array&&(t=(new TextDecoder).decode(t)),t}let n=null;switch((r.get("content-type")||"").split(";")[0]){case"application/x-www-form-urlencoded":n=i(s);break;case"text/plain":case"application/json":n=function(t){if("string"==typeof t)try{t=JSON.parse(t)}catch(e){t={}}const e=new URLSearchParams;try{JSON.stringify(t,(t,a)=>(["object","function"].includes(typeof a)||e.set(t,a),a))}catch(t){}return e.toString()}(i(s));break;case"multipart/form-data":n=function(t,e){const a=new URLSearchParams;t instanceof Uint8Array&&(t=(new TextDecoder).decode(t));try{const r=e.split("boundary=")[1],s=t.split(new RegExp("-*"+r+"-*","mi"));for(let t=0;t0?"&":"?";return t.url+=`${r}__wb_method=${a}&${n}`,t.method="GET",!0}return!1}const we="offset,warc-type,warc-target-uri".split(",");class ge extends class{constructor(t,e){this.opts=t,this.out=e}serialize(t){return JSON.stringify(t)+"\n"}write(t){this.out.write(this.serialize(t))}async run(t){for await(const e of this.iterIndex(t))this.write(e)}async*iterIndex(t){const e={strictHeaders:!0,parseHttp:this.parseHttp};for(const{filename:a,reader:r}of t){if(!a||!r)continue;const t=new dt(r,e);yield*this.iterRecords(t,a)}}async*iterRecords(t,e){for await(const a of t){await a.skipFully();const r=this.indexRecord(a,t,e);r&&(yield r)}}indexRecord(t,e,a){if(this.filterRecord&&!this.filterRecord(t))return null;const r={},s={offset:e.offset,length:e.recordLength,filename:a};for(const e of this.fields)null!=s[e]?r[e]=s[e]:this.setField(e,t,r);return r}setField(t,e,a){const r=this.getField(t,e);null!=r&&(a[t]=r)}getField(t,e){return"http:status"===t?!e.httpHeaders||"response"!==e.warcType&&"revisit"!==e.warcType?null:e.httpHeaders.statusCode:t.startsWith("http:")?e.httpHeaders?e.httpHeaders.headers.get(t.slice(5)):null:e.warcHeaders.headers.get(t)}}{constructor(t={},e=null){if(super(t,e),t.fields){this.fields=t.fields.split(","),this.parseHttp=!1;for(const t of this.fields)if(t.startsWith("http:")){this.parseHttp=!0;break}}else this.fields=we,this.parseHttp=!1}}const me="urlkey,timestamp,url,mime,status,digest,length,offset,filename".split(","),be="urlkey,timestamp,url,mime,status,digest,redirect,meta,length,offset,filename".split(",");class ye extends ge{constructor(t={},e=null){switch(super(t,e),this.includeAll=t.all,this.fields=me,this.parseHttp=!0,this._lastRecord=null,t.format){case"cdxj":this.serialize=this.serializeCDXJ;break;case"cdx":this.serialize=this.serializeCDX11}}async*iterRecords(t,e){this._lastRecord=null;for await(const a of t){await a.readFully();const r=this.indexRecord(a,t,e);r&&(yield r)}const a=this.indexRecord(null,t,e);a&&(yield a)}filterRecord(t){if(this.includeAll)return!0;const e=t.warcType;return"request"!==e&&"warcinfo"!==e}indexRecord(t,e,a){if(this.includeAll){if(!t)return;return super.indexRecord(t,e,a)}const r=this._lastRecord;return t&&(t._offset=e.offset,t._length=e.recordLength),r?t&&r.warcTargetURI==t.warcTargetURI?"request"===t.warcType&&"response"===r.warcType?(this._lastRecord=null,this.indexRecordPair(r,t,e,a)):"response"===t.warcType&&"request"===r.warcType?(this._lastRecord=null,this.indexRecordPair(t,r,e,a)):(this._lastRecord=t,this.indexRecordPair(r,null,e,a)):(this._lastRecord=t,this.indexRecordPair(r,null,e,a)):(this._lastRecord=t,null)}indexRecordPair(t,e,a,r){let s,i;if(e&&"GET"!==e.httpHeaders.method){const a={url:t.warcTargetURI,method:e.httpHeaders.method,headers:e.httpHeaders.headers,postData:e.payload};s=a.method,pe(a)&&(i=a.url.slice(t.warcTargetURI.length))}const n=super.indexRecord(t,a,r);return n&&t&&void 0!==t._offset&&(n.offset=t._offset,n.length=t._length),s&&(n.method=s),i&&(n.requestBody=i),n}serializeCDXJ(t){const{urlkey:e,timestamp:a}=t;return delete t.urlkey,delete t.timestamp,`${e} ${a} ${JSON.stringify(t)}\n`}serializeCDX11(t){const e=[];for(const a of be)e.push(null!=t[a]?t[a]:"-");return e.join(" ")+"\n"}getField(t,e){let a=null;switch(t){case"urlkey":return _e(e.updatedURL?e.updatedURL:e.warcTargetURI);case"timestamp":return a=e.warcDate,a.replace(/[-:T]/g,"").slice(0,14);case"url":return e.warcTargetURI;case"mime":switch(e.warcType){case"revisit":return"warc/revisit";case"response":case"request":t="http:content-type";break;default:t="content-type"}return a=super.getField(t,e),a?a.split(";",1)[0].trim():null;case"status":return super.getField("http:status",e);case"digest":return a=e.warcPayloadDigest,a?a.split(":",2)[1]:null}}}export{X as AsyncIterReader,G as BaseAsyncIterReader,ye as CDXIndexer,ge as Indexer,K as LimitReader,tt as StatusAndHeaders,et as StatusAndHeadersParser,dt as WARCParser,lt as WARCRecord,fe as WARCSerializer,_e as getSurt,pe as postToGetUrl}; diff --git a/main.js b/main.js index 1077838..b910f9f 100644 --- a/main.js +++ b/main.js @@ -10,3 +10,5 @@ export { WARCSerializer } from './src/warcserializer'; export { WARCRecord } from './src/warcrecord'; export { Indexer, CDXIndexer } from './src/indexer'; + +export { postToGetUrl, getSurt } from './src/utils'; diff --git a/package.json b/package.json index fc6d75f..bb4b8d0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "warcio", - "version": "1.3.3", + "version": "1.4.0", "main": "index.js", "module": "main.js", "license": "Apache-2.0", diff --git a/src/indexer.js b/src/indexer.js index 448f39c..05e8104 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -2,6 +2,8 @@ import { WARCParser } from './warcparser'; const DEFAULT_FIELDS = 'offset,warc-type,warc-target-uri'.split(','); +import { postToGetUrl, getSurt } from './utils'; + // =========================================================================== class BaseIndexer @@ -35,12 +37,16 @@ class BaseIndexer const parser = new WARCParser(reader, params); - for await (const record of parser) { - await record.skipFully() - const result = this.indexRecord(record, parser, filename); - if (result) { - yield result; - } + yield* this.iterRecords(parser, filename); + } + } + + async* iterRecords(parser, filename) { + for await (const record of parser) { + await record.skipFully() + const result = this.indexRecord(record, parser, filename); + if (result) { + yield result; } } } @@ -132,6 +138,7 @@ class CDXIndexer extends Indexer this.includeAll = opts.all; this.fields = DEFAULT_CDX_FIELDS; this.parseHttp = true; + this._lastRecord = null; switch (opts.format) { case "cdxj": @@ -149,6 +156,23 @@ class CDXIndexer extends Indexer } } + async* iterRecords(parser, filename) { + this._lastRecord = null; + + for await (const record of parser) { + await record.readFully() + const result = this.indexRecord(record, parser, filename); + if (result) { + yield result; + } + } + + const result = this.indexRecord(null, parser, filename); + if (result) { + yield result; + } + } + filterRecord(record) { if (this.includeAll) { return true; @@ -162,6 +186,76 @@ class CDXIndexer extends Indexer return true; } + indexRecord(record, parser, filename) { + if (this.includeAll) { + if (!record) { + return; + } + return super.indexRecord(record, parser, filename); + } + + const lastRecord = this._lastRecord; + + if (record) { + record._offset = parser.offset; + record._length = parser.recordLength; + } + + if (!lastRecord) { + this._lastRecord = record; + return null; + } + + if (!record || lastRecord.warcTargetURI != record.warcTargetURI) { + this._lastRecord = record; + return this.indexRecordPair(lastRecord, null, parser, filename); + } + + if (record.warcType === "request" && lastRecord.warcType === "response") { + this._lastRecord = null; + return this.indexRecordPair(lastRecord, record, parser, filename); + } else if (record.warcType === "response" && lastRecord.warcType === "request") { + this._lastRecord = null; + return this.indexRecordPair(record, lastRecord, parser, filename); + } else { + this._lastRecord = record; + return this.indexRecordPair(lastRecord, null, parser, filename); + } + } + + indexRecordPair(record, reqRecord, parser, filename) { + let method; + let requestBody; + + if (reqRecord && reqRecord.httpHeaders.method !== "GET") { + const request = { + url: record.warcTargetURI, + method: reqRecord.httpHeaders.method, + headers: reqRecord.httpHeaders.headers, + postData: reqRecord.payload, + }; + + method = request.method; + + if (postToGetUrl(request)) { + requestBody = request.url.slice(record.warcTargetURI.length); + } + } + + const res = super.indexRecord(record, parser, filename); + if (res && record && record._offset !== undefined) { + res.offset = record._offset; + res.length = record._length; + } + if (method) { + res.method = method; + } + if (requestBody) { + res.requestBody = requestBody; + } + return res; + } + serializeCDXJ(result) { const { urlkey, timestamp } = result; delete result.urlkey; @@ -185,7 +279,7 @@ class CDXIndexer extends Indexer switch (field) { case "urlkey": - return this.getSurt(record.warcTargetURI); + return getSurt(record.updatedURL ? record.updatedURL : record.warcTargetURI); case "timestamp": value = record.warcDate; @@ -219,26 +313,6 @@ class CDXIndexer extends Indexer return value ? value.split(":", 2)[1] : null; } } - - getSurt(url) { - try { - const urlObj = new URL(url); - if (urlObj.protocol !== "http:" && urlObj.protocol !== "https:") { - return url; - } - - const hostParts = urlObj.hostname.split(".").reverse(); - let surt = hostParts.join(","); - if (urlObj.port) { - surt += ":" + urlObj.port; - } - surt += ")"; - surt += urlObj.pathname; - return surt.toLowerCase(); - } catch (e) { - return url; - } - } } diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..a172762 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,120 @@ + +function getSurt(url) { + try { + if (!url.startsWith("https:") && !url.startsWith("http:")) { + return url; + } + url = url.replace(/www\d*\./, ''); + const urlObj = new URL(url.toLowerCase()); + + const hostParts = urlObj.hostname.split(".").reverse(); + let surt = hostParts.join(","); + if (urlObj.port) { + surt += ":" + urlObj.port; + } + surt += ")"; + surt += urlObj.pathname; + if (urlObj.search) { + urlObj.searchParams.sort(); + surt += urlObj.search; + } + return surt; + } catch (e) { + return url; + } +} + +function postToGetUrl(request) { + let {url, method, headers, postData} = request; + + if (method === "GET") { + return false; + } + + const requestMime = (headers.get("content-type") || "").split(";")[0]; + + function decodeIfNeeded(postData) { + if (postData instanceof Uint8Array) { + postData = new TextDecoder().decode(postData); + } + return postData; + } + + let query = null; + + switch (requestMime) { + case "application/x-www-form-urlencoded": + query = decodeIfNeeded(postData); + break; + + case "text/plain": + case "application/json": + query = jsonToQueryString(decodeIfNeeded(postData)); + break; + + case "multipart/form-data": + query = mfdToQueryString(decodeIfNeeded(postData), headers.get("content-type")); + break; + + default: + return false; + } + + if (query) { + const start = (url.indexOf("?") > 0 ? "&" : "?"); + request.url += `${start}__wb_method=${method}&${query}`; + request.method = "GET"; + return true; + } + + return false; +} + +function jsonToQueryString(json) { + if (typeof(json) === "string") { + try { + json = JSON.parse(json); + } catch(e) { + json = {}; + } + } + + const q = new URLSearchParams(); + + try { + JSON.stringify(json, (k, v) => { + if (!["object", "function"].includes(typeof(v))) { + q.set(k, v); + } + return v; + }); + } catch (e) {} + + return q.toString(); +} + +function mfdToQueryString(mfd, contentType) { + const params = new URLSearchParams(); + + if (mfd instanceof Uint8Array) { + mfd = new TextDecoder().decode(mfd); + } + + try { + const boundary = contentType.split("boundary=")[1]; + + const parts = mfd.split(new RegExp("-*" + boundary + "-*", "mi")); + + for (let i = 0; i < parts.length; i++) { + const m = parts[i].trim().match(/name="([^"]+)"\r\n\r\n(.*)/mi); + if (m) { + params.set(m[1], m[2]); + } + } + + } catch (e) {} + + return params.toString(); +} + +export { postToGetUrl, getSurt }; diff --git a/test/data/post-test-more.warc b/test/data/post-test-more.warc new file mode 100644 index 0000000..3406895 Binary files /dev/null and b/test/data/post-test-more.warc differ diff --git a/test/data/post-test.warc.gz b/test/data/post-test.warc.gz new file mode 100644 index 0000000..b9cc1f4 Binary files /dev/null and b/test/data/post-test.warc.gz differ diff --git a/test/testIndexer.js b/test/testIndexer.js index dad6948..dcbf9d9 100644 --- a/test/testIndexer.js +++ b/test/testIndexer.js @@ -98,6 +98,25 @@ test('cdx json warc.gz all', index, {"urlkey":"com,example)/","timestamp":"20170306040348","url":"http://example.com/","digest":"3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ","length":609,"offset":3207,"filename":"example.warc.gz"} `); + +test('post append', index, + ['cdx-index', './test/data/post-test.warc.gz'], + `\ +org,httpbin)/post 20140610000859 {"url":"http://httpbin.org/post","mime":"application/json","status":200,"digest":"M532K5WS4GY2H4OVZO6HRPOP47A7KDWU","length":720,"offset":0,"filename":"post-test.warc.gz","method":"POST","requestBody":"?__wb_method=POST&foo=bar&test=abc"} +org,httpbin)/post 20140610001151 {"url":"http://httpbin.org/post","mime":"application/json","status":200,"digest":"M7YCTM7HS3YKYQTAWQVMQSQZBNEOXGU2","length":723,"offset":1196,"filename":"post-test.warc.gz","method":"POST","requestBody":"?__wb_method=POST&A=1&B=[]&C=3"} +org,httpbin)/post?foo=bar 20140610001255 {"url":"http://httpbin.org/post?foo=bar","mime":"application/json","status":200,"digest":"B6E5P6JUZI6UPDTNO4L2BCHMGLTNCUAJ","length":723,"offset":2395,"filename":"post-test.warc.gz","method":"POST","requestBody":"&__wb_method=POST&data=^"} +`); + + +test('post append 2', index, + ['cdx-index', './test/data/post-test-more.warc'], + `\ +org,httpbin)/post 20200809195334 {"url":"https://httpbin.org/post","mime":"application/json","status":200,"digest":"7AWVEIPQMCA4KTCNDXWSZ465FITB7LSK","length":688,"offset":0,"filename":"post-test-more.warc","method":"POST","requestBody":"?__wb_method=POST&test=some+data&another=more%5Edata"} +org,httpbin)/post 20200809195334 {"url":"https://httpbin.org/post","mime":"application/json","status":200,"digest":"BYOQWRSQFW3A5SNUBDSASHFLXGL4FNGB","length":655,"offset":1227,"filename":"post-test-more.warc","method":"POST","requestBody":"?__wb_method=POST&a=json-data"} +org,httpbin)/post 20200810055049 {"url":"https://httpbin.org/post","mime":"application/json","status":200,"digest":"34LEADQD3MOBQ42FCO2WA5TUSEL5QOKP","length":628,"offset":2338,"filename":"post-test-more.warc","method":"POST"} +`); + + test('cdx resource', index, ['cdx-index', './test/data/example-resource.warc.gz'], `\