indexing: add 'noSurt' option for urlkey, simplify request-body appen…

…d logic in indexer
webrecorder · Feb 27, 2021 · 57ea3b0 · 57ea3b0
1 parent 92c5b55
commit 57ea3b0
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 11 deletions.
diff --git a/dist/warcio.js b/dist/warcio.js
@@ -8464,15 +8464,25 @@ function postToGetUrl(request) {
   }
 
   if (query)  {
-    const start = (url.indexOf("?") > 0 ? "&" : "?");
-    request.url += `${start}__wb_method=${method}&${query}`;
+    request.url = appendRequestQuery(request.url, query, request.method);
     request.method = "GET";
+    request.requestBody = query;
     return true;
   }
 
   return false;
 }
 
+function appendRequestQuery(url, query, method) {
+  if (!method) {
+    return url;
+  }
+
+  const start = (url.indexOf("?") > 0 ? "&" : "?");
+
+  return `${url}${start}__wb_method=${method}&${query}`;
+}
+
 function jsonToQueryString(json) {
   if (typeof(json) === "string") {
     try {
@@ -8656,6 +8666,7 @@ class CDXIndexer extends Indexer
     this.includeAll = opts.all;
     this.fields = DEFAULT_CDX_FIELDS;
     this.parseHttp = true;
+    this.noSurt = !!opts.noSurt;
     this._lastRecord = null;
 
     switch (opts.format) {
@@ -8739,10 +8750,11 @@ class CDXIndexer extends Indexer
   indexRecordPair(record, reqRecord, parser, filename) {
     let method;
     let requestBody;
+    let url = record.warcTargetURI;
 
     if (reqRecord && reqRecord.httpHeaders.method !== "GET") {
       const request = {
-        url: record.warcTargetURI,
+        url,
         method: reqRecord.httpHeaders.method,
         headers: reqRecord.httpHeaders.headers,
         postData: reqRecord.payload,
@@ -8751,10 +8763,15 @@ class CDXIndexer extends Indexer
       method = request.method;
 
       if (postToGetUrl(request)) {
-        requestBody = request.url.slice(record.warcTargetURI.length);
+        requestBody = request.requestBody;
+        record.method = method;
+        record.requestBody = requestBody;
+        url = request.url;
       }
     }
 
+    record._urlkey = url;
+
     const res = super.indexRecord(record, parser, filename);
     if (res && record && record._offset !== undefined) {
       res.offset = record._offset;
@@ -8766,6 +8783,7 @@ class CDXIndexer extends Indexer
     if (requestBody) {
       res.requestBody = requestBody;
     }
+
     return res;
   }
 
@@ -8792,7 +8810,8 @@ class CDXIndexer extends Indexer
 
     switch (field) {
       case "urlkey":
-        return getSurt(record.updatedURL ? record.updatedURL : record.warcTargetURI);
+        value = record._urlkey ? record._urlkey : record.warcTargetURI;
+        return this.noSurt ? value : getSurt(value);
 
       case "timestamp":
         value = record.warcDate;
@@ -8828,4 +8847,4 @@ class CDXIndexer extends Indexer
   }
 }
 
-export { AsyncIterReader, BaseAsyncIterReader, CDXIndexer, Indexer, LimitReader, StatusAndHeaders, StatusAndHeadersParser, WARCParser, WARCRecord, WARCSerializer, getSurt, postToGetUrl };
+export { AsyncIterReader, BaseAsyncIterReader, CDXIndexer, Indexer, LimitReader, StatusAndHeaders, StatusAndHeadersParser, WARCParser, WARCRecord, WARCSerializer, appendRequestQuery, getSurt, postToGetUrl };
diff --git a/dist/warcio.min.js b/dist/warcio.min.js
diff --git a/src/cli_main.js b/src/cli_main.js
@@ -44,6 +44,10 @@ function main(args, out) {
       'describe': 'output format',
       'choices': ['json', 'cdxj', 'cdx'],
       'default': 'cdxj'
+    }).
+    option('noSurt', {
+      'describe': 'Use plain urlkey, do not convert to SURT form (Sort-friendly URI Reordering Transform)',
+      'type': 'boolean',
     })
   }, async (args) => {
     /* istanbul ignore next */

diff --git a/src/indexer.js b/src/indexer.js
@@ -2,7 +2,7 @@ import { WARCParser } from './warcparser';
 
 const DEFAULT_FIELDS = 'offset,warc-type,warc-target-uri'.split(',');
 
-import { postToGetUrl, getSurt, appendRequestQuery } from './utils';
+import { postToGetUrl, getSurt } from './utils';
 
 
 // ===========================================================================
@@ -138,6 +138,7 @@ class CDXIndexer extends Indexer
     this.includeAll = opts.all;
     this.fields = DEFAULT_CDX_FIELDS;
     this.parseHttp = true;
+    this.noSurt = !!opts.noSurt;
     this._lastRecord = null;
 
     switch (opts.format) {
@@ -226,10 +227,11 @@ class CDXIndexer extends Indexer
   indexRecordPair(record, reqRecord, parser, filename) {
     let method;
     let requestBody;
+    let url = record.warcTargetURI;
 
     if (reqRecord && reqRecord.httpHeaders.method !== "GET") {
       const request = {
-        url: record.warcTargetURI,
+        url,
         method: reqRecord.httpHeaders.method,
         headers: reqRecord.httpHeaders.headers,
         postData: reqRecord.payload,
@@ -241,9 +243,12 @@ class CDXIndexer extends Indexer
         requestBody = request.requestBody;
         record.method = method;
         record.requestBody = requestBody;
+        url = request.url;
       }
     }
 
+    record._urlkey = url;
+
     const res = super.indexRecord(record, parser, filename);
     if (res && record && record._offset !== undefined) {
       res.offset = record._offset;
@@ -255,6 +260,7 @@ class CDXIndexer extends Indexer
     if (requestBody) {
       res.requestBody = requestBody;
     }
+
     return res;
   }
 
@@ -281,7 +287,8 @@ class CDXIndexer extends Indexer
 
     switch (field) {
       case "urlkey":
-        return getSurt(appendRequestQuery(record.warcTargetURI, record.requestBody, record.method));
+        value = record._urlkey ? record._urlkey : record.warcTargetURI;
+        return this.noSurt ? value : getSurt(value);
 
       case "timestamp":
         value = record.warcDate;