Skip to content

Commit

Permalink
WIP: Make non-GET body canonicalization consistent with pywb
Browse files Browse the repository at this point in the history
  • Loading branch information
@tw4l
tw4l committed Apr 2, 2024
1 parent 0223364 commit 91c67c7
Show file tree
Hide file tree
Showing 2 changed files with 141 additions and 91 deletions.
47 changes: 41 additions & 6 deletions src/lib/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,48 @@ export function jsonToQueryParams(json: string | any, ignoreInvalid = true) {
return key + "." + ++dupes[key] + "_";
};

try {
JSON.stringify(json, (k, v) => {
if (!["object", "function"].includes(typeof v)) {
q.set(getKey(k), v);
const parser = (jsonObj: object, key: string = "") => {

Check failure on line 159 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Type string trivially inferred from a string literal, remove type annotation
// if object with keys, recurse through key/values
let queryValue = "";

if (typeof(jsonObj) === "object" && !(jsonObj instanceof Array)) {
try {
// recurse through key/value pairs

Check failure on line 165 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 8 spaces but found 10
const keys = Object.keys(jsonObj);

Check failure on line 166 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 8 spaces but found 10
for (let i=0; i < keys.length; i++) {

Check failure on line 167 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 8 spaces but found 10
// @ts-ignore

Check failure on line 168 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 10 spaces but found 12

Check failure on line 168 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Do not use "@ts-ignore" because it alters compilation errors
const key = keys[i];

Check failure on line 169 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 10 spaces but found 12
// @ts-ignore

Check failure on line 170 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 10 spaces but found 12

Check failure on line 170 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Do not use "@ts-ignore" because it alters compilation errors
const value = jsonObj[key];

Check failure on line 171 in src/lib/utils.ts

View workflow job for this annotation

GitHub Actions / build (18.x) 

Expected indentation of 10 spaces but found 12
parser(value, key);
}
} catch (e) {
// special case for null
if (jsonObj === null) {
queryValue = "null";
}
}
}

// if array, recurse through values, re-using key
else if (jsonObj instanceof Array) {
for (let i=0; i < jsonObj.length; i++) {
parser(jsonObj[i], key);
}
return v;
});
}

// otherwise, try to cast to string and set query param
if (["string", "number", "boolean"].includes(typeof(jsonObj))) {
queryValue = jsonObj.toString();
}

if (!!queryValue) {
q.set(getKey(key), queryValue);
}
}

try {
parser(json);
} catch (e) {
if (!ignoreInvalid) {
throw e;
Expand Down
185 changes: 100 additions & 85 deletions test/testUtils.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,91 +34,106 @@ describe("utils", () => {
).toBe("abc=def&data=bar&bar=2&a=3&a.2_=4&bar.2_=123&a.3_=5");
});

test("post-to-get empty", () => {
const request = {
postData: "",
headers: new Headers(),
method: "POST",
url: "https://example.com/path/file",
};
const result = postToGetUrl(request);
expect(result).toBe(true);
expect(request.url).toBe(
"https://example.com/path/file?__wb_method=POST&__wb_post_data="
);
});

test("post-to-get binary", () => {
const request = {
postData: new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06]),
headers: new Headers({ "Content-Type": "application/octet-stream" }),
method: "POST",
url: "https://example.com/path/file",
};
const result = postToGetUrl(request);
expect(result).toBe(true);
expect(request.url).toBe(
"https://example.com/path/file?__wb_method=POST&__wb_post_data=AQIDBAUG"
);
});

test("surt with www", () => {
expect(getSurt("https://www23.example.com/some/path")).toBe(
"com,example)/some/path"
);
});

test("surt with www in middle", () => {
expect(getSurt("https://example.com/www2.example/some/value")).toBe(
"com,example)/www2.example/some/value"
);
});

test("surt with www in middle host", () => {
expect(getSurt("https://abc.www.example.com/example")).toBe(
"com,example,www,abc)/example"
);
});

test("surt with default port https", () => {
expect(getSurt("https://www.example.com:443/some/path")).toBe(
"com,example)/some/path"
);
});

test("surt with default port https", () => {
expect(getSurt("https://www.example.com:443/some/path")).toBe(
"com,example)/some/path"
);
});

test("surt with default port http", () => {
expect(getSurt("http://www.example.com:80/some/path")).toBe(
"com,example)/some/path"
);
});

test("surt with default custom port", () => {
expect(getSurt("https://www.example.com:123/some/path")).toBe(
"com,example:123)/some/path"
);
});

test("surt with query args sorted, lowercase", () => {
expect(getSurt("https://www.example.com/some/path?D=1&CC=2&EE=3")).toBe(
"com,example)/some/path?cc=2&d=1&ee=3"
);
});

test("surt with no = param, lowercase", () => {
expect(getSurt("https://www.example.com/some/path?a=b&c&cc=1&d=e")).toBe(
"com,example)/some/path?a=b&c&cc=1&d=e"
);
test("another json with more complicated data", () => {
expect(
toQuery({
"type": "event",
"id": 44.0,
"values": [true, false, null],
"source": {
"type": "component",
"id": "a+b&c= d",
"values": [3, 4]
}
})
).toBe("type=event&id=44.0&values=true&values.2_=false&values.3_=null&type.2_=component&id.2_=a%2Bb%26c%3D+d&values.4_=3&values.5_=4");
});

test("surt with no = param, sort", () => {
expect(getSurt("https://www.example.com/some/path?a=b&c=d&*&z")).toBe(
"com,example)/some/path?*&a=b&c=d&z"
);
});
// test("post-to-get empty", () => {
// const request = {
// postData: "",
// headers: new Headers(),
// method: "POST",
// url: "https://example.com/path/file",
// };
// const result = postToGetUrl(request);
// expect(result).toBe(true);
// expect(request.url).toBe(
// "https://example.com/path/file?__wb_method=POST&__wb_post_data="
// );
// });

// test("post-to-get binary", () => {
// const request = {
// postData: new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06]),
// headers: new Headers({ "Content-Type": "application/octet-stream" }),
// method: "POST",
// url: "https://example.com/path/file",
// };
// const result = postToGetUrl(request);
// expect(result).toBe(true);
// expect(request.url).toBe(
// "https://example.com/path/file?__wb_method=POST&__wb_post_data=AQIDBAUG"
// );
// });

// test("surt with www", () => {
// expect(getSurt("https://www23.example.com/some/path")).toBe(
// "com,example)/some/path"
// );
// });

// test("surt with www in middle", () => {
// expect(getSurt("https://example.com/www2.example/some/value")).toBe(
// "com,example)/www2.example/some/value"
// );
// });

// test("surt with www in middle host", () => {
// expect(getSurt("https://abc.www.example.com/example")).toBe(
// "com,example,www,abc)/example"
// );
// });

// test("surt with default port https", () => {
// expect(getSurt("https://www.example.com:443/some/path")).toBe(
// "com,example)/some/path"
// );
// });

// test("surt with default port https", () => {
// expect(getSurt("https://www.example.com:443/some/path")).toBe(
// "com,example)/some/path"
// );
// });

// test("surt with default port http", () => {
// expect(getSurt("http://www.example.com:80/some/path")).toBe(
// "com,example)/some/path"
// );
// });

// test("surt with default custom port", () => {
// expect(getSurt("https://www.example.com:123/some/path")).toBe(
// "com,example:123)/some/path"
// );
// });

// test("surt with query args sorted, lowercase", () => {
// expect(getSurt("https://www.example.com/some/path?D=1&CC=2&EE=3")).toBe(
// "com,example)/some/path?cc=2&d=1&ee=3"
// );
// });

// test("surt with no = param, lowercase", () => {
// expect(getSurt("https://www.example.com/some/path?a=b&c&cc=1&d=e")).toBe(
// "com,example)/some/path?a=b&c&cc=1&d=e"
// );
// });

// test("surt with no = param, sort", () => {
// expect(getSurt("https://www.example.com/some/path?a=b&c=d&*&z")).toBe(
// "com,example)/some/path?*&a=b&c=d&z"
// );
// });
});

0 comments on commit 91c67c7

Please sign in to comment.