fix(length): better defaults for calculating length

src/download/download-engine/download-file/download-engine-file.ts

@@ -239,7 +239,7 @@ export default class DownloadEngineFile extends EventEmitter<DownloadEngineFileE
                 }
 
                 this._progress.chunks[index] = ChunkStatus.COMPLETE;
-                lastChunkSize = this._activeStreamBytes[startChunk];
+                lastChunkSize = chunks.reduce((last, current) => last + current.length, 0);
                 delete this._activeStreamBytes[startChunk];
                 void this._saveProgress();
 

src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts

@@ -4,6 +4,8 @@ import {EventEmitter} from "eventemitter3";
 import {AvailablePrograms} from "../../download-file/download-programs/switch-program.js";
 import HttpError from "./errors/http-error.js";
 
+export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB
+
 export type BaseDownloadEngineFetchStreamOptions = {
     retry?: retry.Options
     headers?: Record<string, string>,

src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts

@@ -1,8 +1,10 @@
-import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, WriteCallback} from "./base-download-engine-fetch-stream.js";
+import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js";
 import InvalidContentLengthError from "./errors/invalid-content-length-error.js";
 import SmartChunkSplit from "./utils/smart-chunk-split.js";
 import {parseContentDisposition} from "./utils/content-disposition.js";
 import StatusCodeError from "./errors/status-code-error.js";
+import {parseHttpContentRange} from "./utils/httpRange.js";
+import {browserCheck} from "./utils/browserCheck.js";
 
 type GetNextChunk = () => Promise<ReadableStreamReadResult<Uint8Array>> | ReadableStreamReadResult<Uint8Array>;
 export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream {
@@ -16,7 +18,6 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
     protected override async fetchWithoutRetryChunks(callback: WriteCallback) {
         const headers: { [key: string]: any } = {
             accept: "*/*",
-            "Accept-Encoding": "identity",
             ...this.options.headers
         };
 
@@ -34,7 +35,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
             throw new StatusCodeError(this.state.url, response.status, response.statusText, headers);
         }
 
-        const contentLength = parseInt(response.headers.get("content-length")!);
+        const contentLength = parseHttpContentRange(response.headers.get("content-range"))?.length ?? parseInt(response.headers.get("content-length")!);
         const expectedContentLength = this._endSize - this._startSize;
         if (this.state.rangeSupport && contentLength !== expectedContentLength) {
             throw new InvalidContentLengthError(expectedContentLength, contentLength);
@@ -61,14 +62,14 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
             throw new StatusCodeError(url, response.status, response.statusText, this.options.headers);
         }
 
-        let length = parseInt(response.headers.get("content-length")!);
-        if (response.headers.get("content-encoding")) {
-            length = 0;
-        }
-
         const acceptRange = this.options.acceptRangeIsKnown ?? response.headers.get("accept-ranges") === "bytes";
         const fileName = parseContentDisposition(response.headers.get("content-disposition"));
 
+        let length = parseInt(response.headers.get("content-length")!);
+        if (response.headers.get("content-encoding") || browserCheck() && MIN_LENGTH_FOR_MORE_INFO_REQUEST < length) {
+            length = acceptRange ? await this.fetchDownloadInfoWithoutRetryContentRange(url) : 0;
+        }
+
         return {
             length,
             acceptRange,
@@ -77,6 +78,20 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe
         };
     }
 
+    protected async fetchDownloadInfoWithoutRetryContentRange(url: string) {
+        const responseGet = await fetch(url, {
+            method: "GET",
+            headers: {
+                accept: "*/*",
+                ...this.options.headers,
+                range: "bytes=0-0"
+            }
+        });
+
+        const contentRange = responseGet.headers.get("content-range");
+        return parseHttpContentRange(contentRange)?.size || 0;
+    }
+
     async chunkGenerator(callback: WriteCallback, getNextChunk: GetNextChunk) {
         const smartSplit = new SmartChunkSplit(callback, this.state);
 

src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts

@@ -1,11 +1,12 @@
-import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, WriteCallback} from "./base-download-engine-fetch-stream.js";
+import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js";
 import EmptyResponseError from "./errors/empty-response-error.js";
 import StatusCodeError from "./errors/status-code-error.js";
 import XhrError from "./errors/xhr-error.js";
 import InvalidContentLengthError from "./errors/invalid-content-length-error.js";
 import retry from "async-retry";
 import {AvailablePrograms} from "../../download-file/download-programs/switch-program.js";
 import {parseContentDisposition} from "./utils/content-disposition.js";
+import {parseHttpContentRange} from "./utils/httpRange.js";
 
 
 export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream {
@@ -43,6 +44,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
 
             xhr.onload = () => {
                 const contentLength = parseInt(xhr.getResponseHeader("content-length")!);
+
                 if (this.state.rangeSupport && contentLength !== end - start) {
                     throw new InvalidContentLengthError(end - start, contentLength);
                 }
@@ -102,7 +104,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
     protected async _fetchChunksWithoutRange(callback: WriteCallback) {
         const relevantContent = await (async (): Promise<Uint8Array> => {
             const result = await this.fetchBytes(this.state.url, 0, this._endSize, this.state.onProgress);
-            return result.slice(this._startSize, this._endSize);
+            return result.slice(this._startSize, this._endSize || result.length);
         })();
 
         let totalReceivedLength = 0;
@@ -132,18 +134,15 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
                 xhr.setRequestHeader(key, value);
             }
 
-            xhr.onload = () => {
+            xhr.onload = async () => {
                 if (xhr.status >= 200 && xhr.status < 300) {
-                    let length = xhr.getResponseHeader("Content-Length") || "-";
+                    const contentLength = parseInt(xhr.getResponseHeader("content-length")!);
+                    const length = MIN_LENGTH_FOR_MORE_INFO_REQUEST < contentLength ? await this.fetchDownloadInfoWithoutRetryContentRange(url) : 0;
                     const fileName = parseContentDisposition(xhr.getResponseHeader("content-disposition"));
-
-                    if (xhr.getResponseHeader("Content-Encoding")) {
-                        length = "0";
-                    }
-
                     const acceptRange = this.options.acceptRangeIsKnown ?? xhr.getResponseHeader("Accept-Ranges") === "bytes";
+
                     resolve({
-                        length: parseInt(length),
+                        length,
                         acceptRange,
                         newURL: xhr.responseURL,
                         fileName
@@ -161,4 +160,31 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc
         });
     }
 
+    protected fetchDownloadInfoWithoutRetryContentRange(url: string) {
+        return new Promise<number>((resolve, reject) => {
+            const xhr = new XMLHttpRequest();
+            xhr.open("GET", url, true);
+
+            const allHeaders = {
+                accept: "*/*",
+                ...this.options.headers,
+                range: "bytes=0-0"
+            };
+            for (const [key, value] of Object.entries(allHeaders)) {
+                xhr.setRequestHeader(key, value);
+            }
+
+            xhr.onload = () => {
+                const contentRange = xhr.getResponseHeader("Content-Range");
+                resolve(parseHttpContentRange(contentRange)?.size || 0);
+            };
+
+            xhr.onerror = () => {
+                reject(new XhrError(`Failed to fetch ${url}`));
+            };
+
+            xhr.send();
+        });
+    }
+
 }

src/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.ts

@@ -1,7 +1,7 @@
 import HttpError from "./http-error.js";
 
 export default class InvalidContentLengthError extends HttpError {
-    constructor(expectedLength: number, gotLength: number) {
+    constructor(expectedLength: number, gotLength: number | string) {
         super(`Expected ${expectedLength} bytes, but got ${gotLength} bytes. If you on browser try to set "ignoreIfRangeWithQueryParams" to true, this will add a "_ignore" query parameter to the URL to avoid chrome "if-range" header.`);
     }
 }

src/download/download-engine/streams/download-engine-fetch-stream/utils/browserCheck.ts

@@ -0,0 +1,3 @@
+export function browserCheck() {
+    return typeof window !== "undefined" && typeof window.document !== "undefined";
+}

src/download/download-engine/streams/download-engine-fetch-stream/utils/httpRange.ts

@@ -0,0 +1,21 @@
+export function parseHttpContentRange(value?: string | null) {
+    try {
+        if (!value) return null;
+        const parts = value.split(" ")[1].split("/");
+        const range = parts[0].split("-");
+        const size = parseInt(parts[1]);
+
+        const start = parseInt(range[0]);
+        const end = parseInt(range[1]);
+        const length = end - start + 1;
+
+        return {
+            start,
+            end,
+            size,
+            length
+        };
+    } catch {
+        return null;
+    }
+}