From 2e6fa9ce8554a4a222a0f3e9e4327e221da884cc Mon Sep 17 00:00:00 2001 From: ido Date: Tue, 27 Feb 2024 13:34:38 +0200 Subject: [PATCH] feat: if-range skip --- README.md | 2 +- examples/browser.html | 2 +- .../download-engine/engine/base-download-engine.ts | 8 ++++---- .../base-download-engine-fetch-stream.ts | 14 +++++++++++++- .../download-engine-fetch-stream-fetch.ts | 10 ++++++++-- .../download-engine-fetch-stream-xhr.ts | 10 ++++++++-- .../errors/invalid-content-length-error.ts | 5 +++++ test/download.test.ts | 2 +- 8 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 src/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.ts diff --git a/README.md b/README.md index 408bfe2..985a4b2 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ import {downloadFileBrowser} from "ipull/dist/browser.js"; const downloader = await downloadFileBrowser({ url: 'https://example.com/file.large', - acceptRangeAlwaysTrue: true // cors origin request will not return the range header, but we can force it to be true (multi-connection download) + acceptRangeIsKnown: true // cors origin request will not return the range header, but we can force it to be true (multi-connection download) }); await downloader.download(); diff --git a/examples/browser.html b/examples/browser.html index c269e6f..d0beb0d 100644 --- a/examples/browser.html +++ b/examples/browser.html @@ -26,7 +26,7 @@ const downloader = await downloadFileBrowser({ url: BIG_IMAGE, - acceptRangeAlwaysTrue: true // cors origin request will not return the range header, but we can force it to be true (multipart download) + acceptRangeIsKnown: true // cors origin request will not return the range header, but we can force it to be true (multipart download) }); downloader.on("progress", progress => { diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index 27da8ca..8a8b795 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -9,13 +9,13 @@ import ProgressStatisticsBuilder, {TransferProgressWithStatus} from "../../trans import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; export type InputURLOptions = { partsURL: string[] } | { url: string }; -export type BaseDownloadEngineOptions = DownloadEngineFileOptions & +export type BaseDownloadEngineOptions = + DownloadEngineFileOptions + & BaseDownloadEngineFetchStreamOptions + & { comment?: string; - headers?: Record; - acceptRangeIsKnown?: boolean; fetchStrategy?: FetchStrategy; - defaultFetchDownloadInfo?: BaseDownloadEngineFetchStreamOptions["defaultFetchDownloadInfo"]; }; export interface BaseDownloadEngineEvents extends Omit { diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index 9ade071..1db31b5 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -6,8 +6,9 @@ export type BaseDownloadEngineFetchStreamOptions = { /** * If true, parallel download will be enabled even if the server does not return `accept-range` header, this is good when using cross-origin requests */ - acceptRangeAlwaysTrue?: boolean + acceptRangeIsKnown?: boolean defaultFetchDownloadInfo?: { length: number, acceptRange: boolean } + ignoreIfRangeWithQueryParams?: boolean }; export default abstract class BaseDownloadEngineFetchStream { @@ -32,4 +33,15 @@ export default abstract class BaseDownloadEngineFetchStream { public close(): void | Promise { } + + protected _appendToURL(url: string) { + const parsed = new URL(url); + if (this.options.ignoreIfRangeWithQueryParams) { + const randomText = Math.random() + .toString(36); + parsed.searchParams.set("_ignore", randomText); + } + + return parsed.href; + } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 5569fb1..b3cb134 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -1,8 +1,9 @@ import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js"; +import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream { protected async _fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: (length: number) => void) { - const response = await fetch(url, { + const response = await fetch(this._appendToURL(url), { headers: { accept: "*/*", ...this.options.headers, @@ -10,6 +11,11 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe } }); + const contentLength = parseInt(response.headers.get("content-length")!); + if (contentLength !== end - start) { + throw new InvalidContentLengthError(end - start, contentLength); + } + let receivedLength = 0; const reader = response.body!.getReader(); const chunks = []; @@ -41,7 +47,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe }); const length = parseInt(response.headers.get("content-length")!); - const acceptRange = this.options.acceptRangeAlwaysTrue || response.headers.get("accept-ranges") === "bytes"; + const acceptRange = this.options.acceptRangeIsKnown ?? response.headers.get("accept-ranges") === "bytes"; return { length, diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index 5203878..097e786 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -2,6 +2,7 @@ import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.j import EmptyResponseError from "./errors/empty-response-error.js"; import StatusCodeError from "./errors/status-code-error.js"; import XhrError from "./errors/xhr-error.js"; +import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { @@ -15,12 +16,17 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc const xhr = new XMLHttpRequest(); xhr.responseType = "arraybuffer"; - xhr.open("GET", url, true); + xhr.open("GET", this._appendToURL(url), true); for (const [key, value] of Object.entries(headers)) { xhr.setRequestHeader(key, value); } xhr.onload = function () { + const contentLength = parseInt(xhr.getResponseHeader("content-length")!); + if (contentLength !== end - start) { + throw new InvalidContentLengthError(end - start, contentLength); + } + if (xhr.status >= 200 && xhr.status < 300) { const arrayBuffer = xhr.response; if (arrayBuffer) { @@ -59,7 +65,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { const length = xhr.getResponseHeader("Content-Length") || "-"; - const acceptRange = this.options.acceptRangeAlwaysTrue || xhr.getResponseHeader("Accept-Ranges") === "bytes"; + const acceptRange = this.options.acceptRangeIsKnown ?? xhr.getResponseHeader("Accept-Ranges") === "bytes"; resolve({length: parseInt(length), acceptRange}); } else { reject(new StatusCodeError(url, xhr.status, xhr.statusText, this.options.headers)); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.ts b/src/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.ts new file mode 100644 index 0000000..f690228 --- /dev/null +++ b/src/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.ts @@ -0,0 +1,5 @@ +export default class InvalidContentLengthError extends Error { + constructor(expectedLength: number, gotLength: number) { + super(`Expected ${expectedLength} bytes, but got ${gotLength} bytes. If you on browser try to set "ignoreIfRangeWithQueryParams" to true, this will add a "_ignore" query parameter to the URL to avoid chrome "if-range" header.`); + } +} diff --git a/test/download.test.ts b/test/download.test.ts index fa90d3b..fec30eb 100644 --- a/test/download.test.ts +++ b/test/download.test.ts @@ -11,7 +11,7 @@ describe("File Download", () => { const MIN_PARALLEL_CONNECTIONS = 4; const randomNumber = Math.max(MIN_PARALLEL_CONNECTIONS, Math.floor(Math.random() * 30)); const fetchStream = new DownloadEngineFetchStreamFetch({ - acceptRangeAlwaysTrue: true + acceptRangeIsKnown: true }); const writeStream = new DownloadEngineWriteStreamBrowser(() => { });