diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index f3eee9d61..bd1a013a7 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -15,9 +15,9 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union import psutil -import tblib from multiprocess import Queue from selenium.common.exceptions import WebDriverException +from tblib import Traceback, pickling_support from .command_sequence import CommandSequence from .commands.browser_commands import FinalizeCommand @@ -29,10 +29,6 @@ from .errors import BrowserConfigError, BrowserCrashError, ProfileLoadError from .socket_interface import ClientSocket from .storage.storage_providers import TableName - -if TYPE_CHECKING: - from .task_manager import TaskManager - from .types import BrowserId, VisitId from .utilities.multiprocess_utils import ( Process, @@ -40,7 +36,10 @@ parse_traceback_for_sentry, ) -tblib.pickling_support.install() +pickling_support.install() + +if TYPE_CHECKING: + from .task_manager import TaskManager class BrowserManagerHandle: @@ -517,7 +516,7 @@ def _unpack_pickled_error(self, pickled_error: bytes) -> Tuple[str, str]: """Unpacks `pickled_error` into an error `message` and `tb` string.""" exc = pickle.loads(pickled_error) message = traceback.format_exception(*exc)[-1] - tb = json.dumps(tblib.Traceback(exc[2]).to_dict()) + tb = json.dumps(Traceback(exc[2]).to_dict()) return message, tb def kill_browser_manager(self): diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index 2ae04dcf1..cf2c1f19e 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -8,7 +8,7 @@ def query_db( db: Path, query: str, params: Iterable = None, as_tuple: bool = False -) -> List[Union[sqlite3.Row, tuple]]: +) -> List[Union[sqlite3.Row, Tuple[Any, ...]]]: """Run a query against the given db. If params is not None, securely construct a query from the given diff --git a/openwpm/utilities/platform_utils.py b/openwpm/utilities/platform_utils.py index 2ec02debb..e9cb549e2 100644 --- a/openwpm/utilities/platform_utils.py +++ b/openwpm/utilities/platform_utils.py @@ -145,7 +145,7 @@ def get_configuration_string(manager_params, browser_params, versions): config_str += tabulate(table_input, headers=key_dict) config_str += "\n\n========== JS Instrument Settings ==========\n" - config_str += json.dumps(js_config, indent=2, separators=(",", ": ")) + config_str += json.dumps(js_config, indent=None, separators=(",", ":")) config_str += "\n\n========== Input profile tar files ==========\n" if profile_all_none: diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 6768268b5..0e6489a36 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -17,16 +17,16 @@ PAGE_LINKS = { ( - u"http://localtest.me:8000/test_pages/simple_a.html", - u"http://localtest.me:8000/test_pages/simple_c.html", + f"{utilities.BASE_TEST_URL}/simple_a.html", + f"{utilities.BASE_TEST_URL}/simple_c.html", ), ( - u"http://localtest.me:8000/test_pages/simple_a.html", - u"http://localtest.me:8000/test_pages/simple_d.html", + f"{utilities.BASE_TEST_URL}/simple_a.html", + f"{utilities.BASE_TEST_URL}/simple_d.html", ), ( - u"http://localtest.me:8000/test_pages/simple_a.html", - u"http://example.com/test.html?localtest.me", + f"{utilities.BASE_TEST_URL}/simple_a.html", + "http://example.com/test.html?localhost", ), } diff --git a/test/test_dns_instrument.py b/test/test_dns_instrument.py index 48f183292..5c5dab38d 100644 --- a/test/test_dns_instrument.py +++ b/test/test_dns_instrument.py @@ -1,6 +1,14 @@ +import os + +import pytest + from openwpm.utilities import db_utils +@pytest.mark.skipif( + "CI" not in os.environ or os.environ["CI"] == "false", + reason="Makes remote connections", +) def test_name_resolution(default_params, task_manager_creator): manager_params, browser_params = default_params for browser_param in browser_params: diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index ac954d951..ba8ec32a6 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -34,177 +34,169 @@ # resource_type HTTP_REQUESTS = { ( - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"undefined", - u"undefined", - u"undefined", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + "undefined", + "undefined", + "undefined", 0, None, None, - u"main_frame", + "main_frame", ), ( - u"http://localtest.me:8000/test_pages/shared/test_favicon.ico", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_favicon.ico", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), - # ( - # u'http://localtest.me:8000/test_pages/shared/test_favicon.ico', - # u'undefined', - # u'http://localtest.me:8000', - # u'http://localtest.me:8000', - # u'http://localtest.me:8000/test_pages/http_test_page.html', - # 0, None, None, u'image', - # ), ( - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script_2.js", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL}/shared/test_script_2.js", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"script", + "script", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script.js", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_script.js", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"script", + "script", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_image.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/test_pages/http_test_page_2.html", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"sub_frame", + "sub_frame", ), ( - u"http://localtest.me:8000/test_pages/shared/test_style.css", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_style.css", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"stylesheet", + "stylesheet", ), ( - u"http://localtest.me:8000/404.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame1.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame1.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req1.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req1.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), } @@ -212,208 +204,208 @@ # TODO: webext instrumentation doesn't support referrer yet HTTP_RESPONSES = { ( - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", # u'', - u"", + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_favicon.ico", + f"{utilities.BASE_TEST_URL}/shared/test_favicon.ico", # u'', - u"", + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_style.css", - # u'http://localtest.me:8000/test_pages/http_test_page.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_style.css", + # u'http://localhost:8000/test_pages/http_test_page.html', + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script.js", - # u'http://localtest.me:8000/test_pages/http_test_page.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_script.js", + # u'http://localhost:8000/test_pages/http_test_page.html', + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image.png", - # u'http://localtest.me:8000/test_pages/http_test_page.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_image.png", + # u'http://localhost:8000/test_pages/http_test_page.html', + "", ), ( - u"http://localtest.me:8000/test_pages/http_test_page_2.html", - # u'http://localtest.me:8000/test_pages/http_test_page.html', - u"", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", + # u'http://localhost:8000/test_pages/http_test_page.html', + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - # u'http://localtest.me:8000/test_pages/http_test_page_2.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + # u'http://localhost:8000/test_pages/http_test_page_2.html', + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script_2.js", - # u'http://localtest.me:8000/test_pages/http_test_page_2.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_script_2.js", + # u'http://localhost:8000/test_pages/http_test_page_2.html', + "", ), ( - u"http://localtest.me:8000/404.png", - # u'http://localtest.me:8000/test_pages/http_test_page_2.html', - u"", + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", + # u'http://localhost:8000/test_pages/http_test_page_2.html', + "", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - # u'http://localtest.me:8000/test_pages/http_test_page.html', - u"", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + # u'http://localhost:8000/test_pages/http_test_page.html', + "", ), } # format: (source_url, destination_url, location header) HTTP_REDIRECTS = { ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req1.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", - u"req2.png?dst=req3.png&dst=/test_pages/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req1.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", + "req2.png?dst=req3.png&dst=/test_pages/shared/test_image_2.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", - u"req3.png?dst=/test_pages/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", + "req3.png?dst=/test_pages/shared/test_image_2.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - u"/test_pages/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + "/test_pages/shared/test_image_2.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame1.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", - u"frame2.png?dst=/404.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame1.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", + "frame2.png?dst=/404.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", - u"http://localtest.me:8000/404.png", - u"/404.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", + "/404.png", ), } # Data for test_cache_hits_recorded HTTP_CACHED_REQUESTS = { ( - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"undefined", - u"undefined", - u"undefined", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + "undefined", + "undefined", + "undefined", 0, None, None, - u"main_frame", + "main_frame", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script_2.js", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL}/shared/test_script_2.js", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"script", + "script", ), ( - u"http://localtest.me:8000/test_pages/shared/test_script.js", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_script.js", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"script", + "script", ), ( - u"http://localtest.me:8000/test_pages/http_test_page_2.html", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"sub_frame", + "sub_frame", ), ( - u"http://localtest.me:8000/404.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame1.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame1.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page_2.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req1.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req1.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", - u"http://localtest.me:8000/test_pages/http_test_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL}/http_test_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_test_page.html", 0, None, None, - u"image", + "image", ), } @@ -421,174 +413,172 @@ # TODO: referrer isn't recorded by webext instrumentation yet. HTTP_CACHED_RESPONSES = { ( - u"http://localtest.me:8000/test_pages/http_test_page.html", + f"{utilities.BASE_TEST_URL}/http_test_page.html", # u'', 1, ), ( - u"http://localtest.me:8000/test_pages/shared/test_script.js", - # u'http://localtest.me:8000/test_pages/http_test_page.html', + f"{utilities.BASE_TEST_URL}/shared/test_script.js", + # u'http://localhost:8000/test_pages/http_test_page.html', 1, ), ( - u"http://localtest.me:8000/test_pages/http_test_page_2.html", - # u'http://localtest.me:8000/test_pages/http_test_page.html', + f"{utilities.BASE_TEST_URL}/http_test_page_2.html", + # u'http://localhost:8000/test_pages/http_test_page.html', 1, ), ( - u"http://localtest.me:8000/test_pages/shared/test_script_2.js", - # u'http://localtest.me:8000/test_pages/http_test_page_2.html', + f"{utilities.BASE_TEST_URL}/shared/test_script_2.js", + # u'http://localhost:8000/test_pages/http_test_page_2.html', 1, ), ( - u"http://localtest.me:8000/404.png", - # u'http://localtest.me:8000/test_pages/http_test_page_2.html', + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", + # u'http://localhost:8000/test_pages/http_test_page_2.html', 1, ), - (u"http://localtest.me:8000/test_pages/shared/test_image_2.png", 1), + (f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", 1), } # format: (source_url, destination_url) HTTP_CACHED_REDIRECTS = { ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame1.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame1.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/frame2.png", - u"http://localtest.me:8000/404.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/frame2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/404.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req1.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req1.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req2.png", - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", ), ( - u"http://localtest.me:8000/MAGIC_REDIRECT/req3.png", - u"http://localtest.me:8000/test_pages/shared/test_image_2.png", + f"{utilities.BASE_TEST_URL_NOPATH}/MAGIC_REDIRECT/req3.png", + f"{utilities.BASE_TEST_URL}/shared/test_image_2.png", ), } # Test URL attribution for worker script requests HTTP_WORKER_SCRIPT_REQUESTS = { ( - u"http://localtest.me:8000/test_pages/http_worker_page.html", - u"http://localtest.me:8000/test_pages/http_worker_page.html", - u"undefined", - u"undefined", - u"undefined", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", + "undefined", + "undefined", + "undefined", 0, None, None, - u"main_frame", + "main_frame", ), ( - u"http://localtest.me:8000/test_pages/shared/test_favicon.ico", - u"http://localtest.me:8000/test_pages/http_worker_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_worker_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_favicon.ico", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localtest.me:8000/test_pages/shared/worker.js", - u"http://localtest.me:8000/test_pages/http_worker_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_worker_page.html", + f"{utilities.BASE_TEST_URL}/shared/worker.js", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", 0, None, None, - u"script", + "script", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image.png", - u"http://localtest.me:8000/test_pages/http_worker_page.html", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/http_worker_page.html", + f"{utilities.BASE_TEST_URL}/shared/test_image.png", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/http_worker_page.html", 1, None, None, - u"xmlhttprequest", + "xmlhttprequest", ), ( - u"http://localtest.me:8000/test_pages/shared/test_image.png", - u"http://localtest.me:8000/test_pages/shared/worker.js", - u"http://localtest.me:8000", - u"http://localtest.me:8000", - u"http://localtest.me:8000/test_pages/shared/worker.js", + f"{utilities.BASE_TEST_URL}/shared/test_image.png", + f"{utilities.BASE_TEST_URL}/shared/worker.js", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL_NOPATH}", + f"{utilities.BASE_TEST_URL}/shared/worker.js", 1, None, None, - u"xmlhttprequest", + "xmlhttprequest", ), } # Test URL-attribution for Service Worker requests. -# localhost URLs are used instead of localtest.me because service workers -# are only loaded from secure contexts. HTTP_SERVICE_WORKER_REQUESTS = { ( - u"http://localhost:8000/test_pages/http_service_worker_page.html", - u"http://localhost:8000/test_pages/http_service_worker_page.html", - u"undefined", - u"undefined", - u"undefined", + "http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000/test_pages/http_service_worker_page.html", + "undefined", + "undefined", + "undefined", 0, None, None, - u"main_frame", + "main_frame", ), ( - u"http://localhost:8000/test_pages/shared/test_favicon.ico", - u"http://localhost:8000/test_pages/http_service_worker_page.html", - u"http://localhost:8000", - u"http://localhost:8000", - u"http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000/test_pages/shared/test_favicon.ico", + "http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000", + "http://localhost:8000", + "http://localhost:8000/test_pages/http_service_worker_page.html", 0, None, None, - u"image", + "image", ), ( - u"http://localhost:8000/test_pages/shared/service_worker.js", - u"http://localhost:8000/test_pages/http_service_worker_page.html", - u"http://localhost:8000", - u"http://localhost:8000", - u"http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000/test_pages/shared/service_worker.js", + "http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000", + "http://localhost:8000", + "http://localhost:8000/test_pages/http_service_worker_page.html", 0, None, None, - u"script", + "script", ), ( - u"http://localhost:8000/test_pages/shared/test_image.png", - u"http://localhost:8000/test_pages/http_service_worker_page.html", - u"http://localhost:8000", - u"http://localhost:8000", - u"http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000/test_pages/shared/test_image.png", + "http://localhost:8000/test_pages/http_service_worker_page.html", + "http://localhost:8000", + "http://localhost:8000", + "http://localhost:8000/test_pages/http_service_worker_page.html", 1, None, None, - u"xmlhttprequest", + "xmlhttprequest", ), ( - u"http://localhost:8000/test_pages/shared/test_image_2.png", - u"http://localhost:8000/test_pages/shared/service_worker.js", - u"http://localhost:8000", - u"http://localhost:8000", - u"http://localhost:8000/test_pages/shared/service_worker.js", + "http://localhost:8000/test_pages/shared/test_image_2.png", + "http://localhost:8000/test_pages/shared/service_worker.js", + "http://localhost:8000", + "http://localhost:8000", + "http://localhost:8000/test_pages/shared/service_worker.js", 1, None, None, - u"xmlhttprequest", + "xmlhttprequest", ), } @@ -694,7 +684,7 @@ def test_worker_script_requests(self): def test_service_worker_requests(self): """Check correct URL attribution for requests made by service worker""" - test_url = "http://localhost:8000/test_pages" + "/http_service_worker_page.html" + test_url = utilities.BASE_TEST_URL + "/http_service_worker_page.html" db = self.visit(test_url) request_id_to_url = dict() @@ -877,9 +867,9 @@ def test_record_file_upload(self, task_manager_creator): # POST data is stored as JSON in the DB post_body_decoded = json.loads(post_body) expected_body = { - u"username": u"name surname+", - u"upload-css": css_file_content, - u"upload-img": img_file_content, + "username": "name surname+", + "upload-css": css_file_content, + "upload-img": img_file_content, } assert expected_body == post_body_decoded diff --git a/test/test_pages/expected_source.html b/test/test_pages/expected_source.html index 5f31fb5b4..b66dad71e 100644 --- a/test/test_pages/expected_source.html +++ b/test/test_pages/expected_source.html @@ -8,10 +8,10 @@ - Click me! + Click me! Click me also! Click me for a JS alert! Go to google.com - Go to example.com + Go to example.com diff --git a/test/test_pages/recursive_iframes/child1a.html b/test/test_pages/recursive_iframes/child1a.html index d98a31960..12df956c5 100644 --- a/test/test_pages/recursive_iframes/child1a.html +++ b/test/test_pages/recursive_iframes/child1a.html @@ -5,7 +5,7 @@

Depth 1 -- Child A

- - + + diff --git a/test/test_pages/recursive_iframes/child1c.html b/test/test_pages/recursive_iframes/child1c.html index f57a9387e..76758b50e 100644 --- a/test/test_pages/recursive_iframes/child1c.html +++ b/test/test_pages/recursive_iframes/child1c.html @@ -5,6 +5,6 @@

Depth 1 -- Child C

- + diff --git a/test/test_pages/recursive_iframes/child2a.html b/test/test_pages/recursive_iframes/child2a.html index af29c6600..336fa78d3 100644 --- a/test/test_pages/recursive_iframes/child2a.html +++ b/test/test_pages/recursive_iframes/child2a.html @@ -5,6 +5,6 @@

Depth 2 -- Child A

- + diff --git a/test/test_pages/recursive_iframes/child2c.html b/test/test_pages/recursive_iframes/child2c.html index df4061e44..0a02e2a32 100644 --- a/test/test_pages/recursive_iframes/child2c.html +++ b/test/test_pages/recursive_iframes/child2c.html @@ -5,8 +5,8 @@

Depth 2 -- Child C

- - - + + + diff --git a/test/test_pages/recursive_iframes/child3a.html b/test/test_pages/recursive_iframes/child3a.html index e06d37e07..cc1c988bc 100644 --- a/test/test_pages/recursive_iframes/child3a.html +++ b/test/test_pages/recursive_iframes/child3a.html @@ -5,6 +5,6 @@

Depth 3 -- Child A

- + diff --git a/test/test_pages/recursive_iframes/child4a.html b/test/test_pages/recursive_iframes/child4a.html index 41250ff3c..22626bc34 100644 --- a/test/test_pages/recursive_iframes/child4a.html +++ b/test/test_pages/recursive_iframes/child4a.html @@ -5,6 +5,6 @@

Depth 4 -- Child A

- + diff --git a/test/test_pages/recursive_iframes/child5a.html b/test/test_pages/recursive_iframes/child5a.html index 8ffbee39f..73b502548 100644 --- a/test/test_pages/recursive_iframes/child5a.html +++ b/test/test_pages/recursive_iframes/child5a.html @@ -5,6 +5,6 @@

Depth 5 -- Child A

- + diff --git a/test/test_pages/recursive_iframes/parent.html b/test/test_pages/recursive_iframes/parent.html index d244ed886..356e39d13 100644 --- a/test/test_pages/recursive_iframes/parent.html +++ b/test/test_pages/recursive_iframes/parent.html @@ -7,8 +7,8 @@

This is a test page to test the recursive iframe function. The test function grabs the page source of each frame, so any changes to the file will break the tests.

- - - + + + diff --git a/test/test_pages/shared/worker.js b/test/test_pages/shared/worker.js index 9da2b3500..5e1089490 100644 --- a/test/test_pages/shared/worker.js +++ b/test/test_pages/shared/worker.js @@ -1,6 +1,6 @@ const request = new Request( - 'http://localtest.me:8000/test_pages/shared/test_image.png' + 'http://localhost:8000/test_pages/shared/test_image.png' ); fetch(request); diff --git a/test/test_pages/simple_a.html b/test/test_pages/simple_a.html index cf6eb0194..da747162d 100644 --- a/test/test_pages/simple_a.html +++ b/test/test_pages/simple_a.html @@ -10,9 +10,9 @@ - Click me! + Click me! Click me also! Click me for a JS alert! Go to google.com - Go to example.com + Go to example.com diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 46b7787eb..1facc1a02 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -28,13 +28,15 @@ rendered_js_url = utilities.BASE_TEST_URL + "/property_enumeration.html" # Expected nested page source +# This needs to be changed back once #887 is addressed + NESTED_TEST_DIR = "/recursive_iframes/" NESTED_FRAMES_URL = utilities.BASE_TEST_URL + NESTED_TEST_DIR + "parent.html" -D1_BASE = "http://d1." + utilities.BASE_TEST_URL_NOSCHEME + NESTED_TEST_DIR -D2_BASE = "http://d2." + utilities.BASE_TEST_URL_NOSCHEME + NESTED_TEST_DIR -D3_BASE = "http://d3." + utilities.BASE_TEST_URL_NOSCHEME + NESTED_TEST_DIR -D4_BASE = "http://d4." + utilities.BASE_TEST_URL_NOSCHEME + NESTED_TEST_DIR -D5_BASE = "http://d5." + utilities.BASE_TEST_URL_NOSCHEME + NESTED_TEST_DIR +D1_BASE = utilities.BASE_TEST_URL + NESTED_TEST_DIR +D2_BASE = utilities.BASE_TEST_URL + NESTED_TEST_DIR +D3_BASE = utilities.BASE_TEST_URL + NESTED_TEST_DIR +D4_BASE = utilities.BASE_TEST_URL + NESTED_TEST_DIR +D5_BASE = utilities.BASE_TEST_URL + NESTED_TEST_DIR EXPECTED_PARENTS = { NESTED_FRAMES_URL: [], D1_BASE + "child1a.html": [NESTED_FRAMES_URL], @@ -269,7 +271,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode # 2) A relative link to simple_d.html # 3) A javascript: link # 4) A link to www.google.com - # 5) A link to example.com?localtest.me + # 5) A link to example.com?localhost # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( db, @@ -353,7 +355,7 @@ def test_browse_wrapper_http_table_valid( # 2) A relative link to simple_d.html # 3) A javascript: link # 4) A link to www.google.com - # 5) A link to example.com?localtest.me + # 5) A link to example.com?localhost # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( db, diff --git a/test/utilities.py b/test/utilities.py index 621d17ef4..b9e5bfb6a 100644 --- a/test/utilities.py +++ b/test/utilities.py @@ -6,7 +6,7 @@ from urllib.parse import parse_qs, urlparse LOCAL_WEBSERVER_PORT = 8000 -BASE_TEST_URL_DOMAIN = "localtest.me" +BASE_TEST_URL_DOMAIN = "localhost" BASE_TEST_URL_NOPATH = "http://%s:%s" % (BASE_TEST_URL_DOMAIN, LOCAL_WEBSERVER_PORT) BASE_TEST_URL = "%s/test_pages" % BASE_TEST_URL_NOPATH BASE_TEST_URL_NOSCHEME = BASE_TEST_URL.split("//")[1] @@ -93,7 +93,7 @@ def start_server(): We need this since page-mod events in the extension don't fire on `file://*`. Instead, point test code to - `http://localtest.me:8000/test_pages/...` + `http://localhost:8000/test_pages/...` """ print("Starting HTTP Server in a separate thread") # switch to test dir, this is where the test files are