From 159c5fadc48f4074729f192f2ab568f51f13f43b Mon Sep 17 00:00:00 2001 From: englehardt Date: Wed, 11 Jan 2023 22:56:14 -0500 Subject: [PATCH] Move dns_instrument from onCompleted to onHeadersReceived --- Extension/src/background/dns-instrument.ts | 16 +++---- Extension/src/lib/pending-response.ts | 13 ++++- .../browser-web-request-event-details.ts | 47 +++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/Extension/src/background/dns-instrument.ts b/Extension/src/background/dns-instrument.ts index 2972e7561..d3634416a 100755 --- a/Extension/src/background/dns-instrument.ts +++ b/Extension/src/background/dns-instrument.ts @@ -1,12 +1,12 @@ import { PendingResponse } from "../lib/pending-response"; import { DnsResolved } from "../schema"; -import { WebRequestOnCompletedEventDetails } from "../types/browser-web-request-event-details"; +import { WebRequestOnHeadersReceivedDetails } from "../types/browser-web-request-event-details"; import { allTypes } from "./http-instrument"; import RequestFilter = browser.webRequest.RequestFilter; export class DnsInstrument { private readonly dataReceiver; - private onCompleteListener; + private onHeadersReceivedListener; private pendingResponses: { [requestId: number]: PendingResponse; } = {}; @@ -29,23 +29,23 @@ export class DnsInstrument { /* * Attach handlers to event listeners */ - this.onCompleteListener = (details: WebRequestOnCompletedEventDetails) => { + this.onHeadersReceivedListener = (details: WebRequestOnHeadersReceivedDetails) => { // Ignore requests made by extensions if (requestStemsFromExtension(details)) { return; } const pendingResponse = this.getPendingResponse(details.requestId); - pendingResponse.resolveOnCompletedEventDetails(details); + pendingResponse.resolveOnHeadersReceivedDetails(details); this.onCompleteDnsHandler(details, crawlID); }; - browser.webRequest.onCompleted.addListener(this.onCompleteListener, filter); + browser.webRequest.onHeadersReceived.addListener(this.onHeadersReceivedListener, filter); } public cleanup() { - if (this.onCompleteListener) { - browser.webRequest.onCompleted.removeListener(this.onCompleteListener); + if (this.onHeadersReceivedListener) { + browser.webRequest.onHeadersReceived.removeListener(this.onHeadersReceivedListener); } } @@ -70,7 +70,7 @@ export class DnsInstrument { } private async onCompleteDnsHandler( - details: WebRequestOnCompletedEventDetails, + details: WebRequestOnHeadersReceivedDetails, crawlID, ) { // Create and populate DnsResolve object diff --git a/Extension/src/lib/pending-response.ts b/Extension/src/lib/pending-response.ts index 28bfc20f5..f44f66c83 100644 --- a/Extension/src/lib/pending-response.ts +++ b/Extension/src/lib/pending-response.ts @@ -1,16 +1,20 @@ import { WebRequestOnBeforeRequestEventDetails, WebRequestOnCompletedEventDetails, + WebRequestOnHeadersReceivedDetails } from "../types/browser-web-request-event-details"; import { ResponseBodyListener } from "./response-body-listener"; /** - * Ties together the two separate events that together holds information about both response headers and body + * Ties together three separate events that hold information about both response headers and body */ export class PendingResponse { public readonly onBeforeRequestEventDetails: Promise< WebRequestOnBeforeRequestEventDetails >; + public readonly onHeadersReceivedEventDetails: Promise< + WebRequestOnHeadersReceivedDetails + >; public readonly onCompletedEventDetails: Promise< WebRequestOnCompletedEventDetails >; @@ -18,6 +22,9 @@ export class PendingResponse { public resolveOnBeforeRequestEventDetails: ( details: WebRequestOnBeforeRequestEventDetails, ) => void; + public resolveOnHeadersReceivedDetails: ( + details: WebRequestOnHeadersReceivedDetails, + ) => void; public resolveOnCompletedEventDetails: ( details: WebRequestOnCompletedEventDetails, ) => void; @@ -25,6 +32,9 @@ export class PendingResponse { this.onBeforeRequestEventDetails = new Promise(resolve => { this.resolveOnBeforeRequestEventDetails = resolve; }); + this.onHeadersReceivedEventDetails = new Promise(resolve => { + this.resolveOnHeadersReceivedDetails = resolve; + }); this.onCompletedEventDetails = new Promise(resolve => { this.resolveOnCompletedEventDetails = resolve; }); @@ -37,6 +47,7 @@ export class PendingResponse { public resolved() { return Promise.all([ this.onBeforeRequestEventDetails, + this.onHeadersReceivedEventDetails, this.onCompletedEventDetails, ]); } diff --git a/Extension/src/types/browser-web-request-event-details.ts b/Extension/src/types/browser-web-request-event-details.ts index 20c36173a..716b74f33 100644 --- a/Extension/src/types/browser-web-request-event-details.ts +++ b/Extension/src/types/browser-web-request-event-details.ts @@ -7,6 +7,7 @@ import ResourceType = browser.webRequest.ResourceType; import UploadData = browser.webRequest.UploadData; import HttpHeaders = browser.webRequest.HttpHeaders; +import UrlClassification = browser.webRequest.UrlClassification; export interface FrameAncestor { /** The URL that the document was loaded from. */ @@ -189,3 +190,49 @@ export interface WebRequestOnCompletedEventDetails { */ statusLine: string; } + +export interface WebRequestOnHeadersReceivedDetails { + /** + * The ID of the request. Request IDs are unique within a browser session. As a result, they could be used to relate different events of the same request. + */ + requestId: string; + url: string; + /** Standard HTTP method. */ + method: string; + /** + * The value 0 indicates that the request happens in the main frame; a positive value indicates the ID of a subframe in which the request happens. If the document of a (sub-)frame is loaded (`type` is `main_frame` or `sub_frame`), `frameId` indicates the ID of this frame, not the ID of the outer frame. Frame IDs are unique within a tab. + */ + frameId: number; + /** ID of frame that wraps the frame which sent the request. Set to -1 if no parent frame exists. */ + parentFrameId: number; + /** True for private browsing requests. */ + incognito?: boolean | undefined; + /** The cookie store ID of the contextual identity. */ + cookieStoreId?: string | undefined; + /** URL of the resource that triggered this request. */ + originUrl?: string | undefined; + /** URL of the page into which the requested resource will be loaded. */ + documentUrl?: string | undefined; + /** The ID of the tab in which the request takes place. Set to -1 if the request isn't related to a tab. */ + tabId: number; + /** How the requested resource will be used. */ + type: ResourceType; + /** The time when this signal is triggered, in milliseconds since the epoch. */ + timeStamp: number; + /** + * HTTP status line of the response or the 'HTTP/0.9 200 OK' string for HTTP/0.9 responses (i.e., responses that lack a status line). + */ + statusLine: string; + /** + * The server IP address that the request was actually sent to. Note that it may be a literal IPv6 address. + */ + ip?: string; + /** The HTTP response headers that have been received with this response. */ + responseHeaders?: HttpHeaders | undefined; + /** Standard HTTP status code returned by the server. */ + statusCode: number; + /** Tracking classification if the request has been classified. */ + urlClassification?: UrlClassification | undefined; + /** Indicates if this request and its content window hierarchy is third party. */ + thirdParty: boolean; +} \ No newline at end of file