make tds depend on privacy config

DuckDuckGo.xcodeproj/project.pbxproj

@@ -1775,6 +1775,8 @@
 		56BA1E802BAB2E43001CF69F /* ErrorPageTabExtensionTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E7C2BAB290E001CF69F /* ErrorPageTabExtensionTest.swift */; };
 		56BA1E8A2BB1CB5B001CF69F /* CertificateTrustEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */; };
 		56BA1E8B2BB1CB5B001CF69F /* CertificateTrustEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */; };
+		56BC8F012D312B320046059D /* ConfigurationManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */; };
+		56BC8F022D312B320046059D /* ConfigurationManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */; };
 		56CE77612C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */; };
 		56CE77622C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */; };
 		56CEE90E2B7A725B00CF10AA /* InfoPlist.xcstrings in Resources */ = {isa = PBXBuildFile; fileRef = 56CEE90D2B7A6DE100CF10AA /* InfoPlist.xcstrings */; };
@@ -4131,6 +4133,7 @@
 		56BA1E742BAAF70F001CF69F /* SpecialErrorPageTabExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpecialErrorPageTabExtension.swift; sourceTree = "<group>"; };
 		56BA1E7C2BAB290E001CF69F /* ErrorPageTabExtensionTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorPageTabExtensionTest.swift; sourceTree = "<group>"; };
 		56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CertificateTrustEvaluator.swift; sourceTree = "<group>"; };
+		56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConfigurationManagerTests.swift; sourceTree = "<group>"; };
 		56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingSuggestedSearchesProviderTests.swift; sourceTree = "<group>"; };
 		56CEE9092B7A66C500CF10AA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		56CEE90D2B7A6DE100CF10AA /* InfoPlist.xcstrings */ = {isa = PBXFileReference; lastKnownFileType = text.json.xcstrings; path = InfoPlist.xcstrings; sourceTree = "<group>"; };
@@ -7496,6 +7499,7 @@
 			isa = PBXGroup;
 			children = (
 				85AC3B4825DAC9BD00C7D2AA /* ConfigurationStorageTests.swift */,
+				56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */,
 			);
 			path = Configuration;
 			sourceTree = "<group>";
@@ -12454,6 +12458,7 @@
 				3706FE2F293F661700E42796 /* WebViewMock.swift in Sources */,
 				3706FE30293F661700E42796 /* CollectionExtension.swift in Sources */,
 				B630E80129C887ED00363609 /* NSErrorAdditionalInfo.swift in Sources */,
+				56BC8F012D312B320046059D /* ConfigurationManagerTests.swift in Sources */,
 				3706FE31293F661700E42796 /* TabCollectionViewModelDelegateMock.swift in Sources */,
 				3706FE32293F661700E42796 /* BookmarksHTMLReaderTests.swift in Sources */,
 				9F0FFFBC2BCCAEC2007C87DD /* AddEditBookmarkFolderDialogViewModelMock.swift in Sources */,
@@ -14293,6 +14298,7 @@
 				B6CA4824298CDC2E0067ECCE /* AdClickAttributionTabExtensionTests.swift in Sources */,
 				AAEC74B22642C57200C2EFBC /* HistoryCoordinatingMock.swift in Sources */,
 				37D046A12C7DA9A200AEAA50 /* UserBackgroundImagesManagerTests.swift in Sources */,
+				56BC8F022D312B320046059D /* ConfigurationManagerTests.swift in Sources */,
 				56A214AF2CB583BF00E5BC0E /* TrackerMessageProviderTests.swift in Sources */,
 				37CD54B927F1F8AC00F1F7B9 /* AppearancePreferencesTests.swift in Sources */,
 				EEF53E182950CED5002D78F4 /* JSAlertViewModelTests.swift in Sources */,
@@ -15365,8 +15371,8 @@
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
 			requirement = {
-				kind = exactVersion;
-				version = 224.6.0;
+				branch = "sabrina/tds-override";
+				kind = branch;
 			};
 		};
 		9FF521422BAA8FF300B9819B /* XCRemoteSwiftPackageReference "lottie-spm" */ = {

DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -32,17 +32,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/BrowserServicesKit",
       "state" : {
-        "revision" : "20316b105d8874e10cf544a9c97e97237441ae01",
-        "version" : "224.6.0"
+        "branch" : "sabrina/tds-override",
+        "revision" : "70f5a0a29d3a5909def923702ed071778a72040d"
       }
     },
     {
       "identity" : "content-scope-scripts",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "06d244c4e0951ef16217c46f23c4865c5f98dd5c",
-        "version" : "7.4.0"
+        "revision" : "0502ed7de4130bd8705daebaca9aeb20d3e62d15",
+        "version" : "7.5.0"
       }
     },
     {
@@ -104,8 +104,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/privacy-dashboard",
       "state" : {
-        "revision" : "2e2baf7d31c7d8e158a58bc1cb79498c1c727fd2",
-        "version" : "7.5.0"
+        "revision" : "bea4d750913ef82c10cd06e791686501c8e648e4",
+        "version" : "7.6.0"
       }
     },
     {

DuckDuckGo/Application/AppConfigurationURLProvider.swift

@@ -18,12 +18,16 @@
 
 import Configuration
 import Foundation
+import BrowserServicesKit
 
 struct AppConfigurationURLProvider: ConfigurationURLProviding {
 
     // MARK: - Debug
 
-    internal init(customPrivacyConfiguration: URL? = nil) {
+    internal init(privacyConfigurationManager: PrivacyConfigurationManaging = ContentBlocking.shared.privacyConfigurationManager,
+                  featureFlagger: FeatureFlagger = Application.appDelegate.featureFlagger,
+                  customPrivacyConfiguration: URL? = nil) {
+        self.init(privacyConfigurationManager: privacyConfigurationManager, featureFlagger: featureFlagger)
         if let customPrivacyConfiguration {
             // Overwrite custom privacy configuration if provided
             self.customPrivacyConfiguration = customPrivacyConfiguration.absoluteString
@@ -47,6 +51,20 @@ struct AppConfigurationURLProvider: ConfigurationURLProviding {
 
     // MARK: - Main
 
+    var privacyConfigurationManager: PrivacyConfigurationManaging
+    var featureFlagger: FeatureFlagger
+
+    public enum Constants {
+        public static let baseTdsURLString = "https://staticcdn.duckduckgo.com/trackerblocking/"
+        public static let defaultTrackerDataURL = URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/v6/current/macos-tds.json")!
+    }
+
+    init (privacyConfigurationManager: PrivacyConfigurationManaging = ContentBlocking.shared.privacyConfigurationManager,
+          featureFlagger: FeatureFlagger = Application.appDelegate.featureFlagger) {
+        self.privacyConfigurationManager = privacyConfigurationManager
+        self.featureFlagger = featureFlagger
+    }
+
     func url(for configuration: Configuration) -> URL {
         // URLs for privacyConfiguration and trackerDataSet shall match the ones in update_embedded.sh. 
         // Danger checks that the URLs match on every PR. If the code changes, the regex that Danger uses may need an update.
@@ -56,10 +74,101 @@ struct AppConfigurationURLProvider: ConfigurationURLProviding {
         case .bloomFilterExcludedDomains: return URL(string: "https://staticcdn.duckduckgo.com/https/https-mobile-v2-false-positives.json")!
         case .privacyConfiguration: return customPrivacyConfigurationUrl ?? URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/config/v4/macos-config.json")!
         case .surrogates: return URL(string: "https://staticcdn.duckduckgo.com/surrogates.txt")!
-        case .trackerDataSet: return URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/v6/current/macos-tds.json")!
+        case .trackerDataSet:
+            return trackerDataURL()
         // In archived repo, to be refactored shortly (https://staticcdn.duckduckgo.com/useragents/social_ctp_configuration.json)
         case .remoteMessagingConfig: return RemoteMessagingClient.Constants.endpoint
         }
     }
 
+    private func trackerDataURL() -> URL {
+        for experimentType in TdsExperimentType.allCases {
+            if let cohort = featureFlagger.getCohortIfEnabled(for: experimentType.experiment) as? TdsNextExperimentFlag.Cohort,
+               let url = trackerDataURL(for: experimentType.subfeature, cohort: cohort) {
+                return url
+            }
+        }
+        return Constants.defaultTrackerDataURL
+    }
+
+    private func trackerDataURL(for subfeature: any PrivacySubfeature, cohort: TdsNextExperimentFlag.Cohort) -> URL? {
+        guard let settings = privacyConfigurationManager.privacyConfig.settings(for: subfeature),
+              let jsonData = settings.data(using: .utf8) else { return nil }
+        do {
+            if let settingsDict = try JSONSerialization.jsonObject(with: jsonData) as? [String: String],
+               let urlString = cohort == .control ? settingsDict["controlUrl"] : settingsDict["treatmentUrl"] {
+                return URL(string: Constants.baseTdsURLString + urlString)
+            }
+        } catch {
+            print("Failed to parse JSON: \(error)")
+        }
+        return nil
+    }
+
+}
+
+public enum TdsExperimentType: Int, CaseIterable {
+    case baseline
+    case feb25
+    case mar25
+    case apr25
+    case may25
+    case jun25
+    case jul25
+    case aug25
+    case sep25
+    case oct25
+    case nov25
+    case dec25
+
+    var experiment: any FeatureFlagExperimentDescribing {
+        TdsNextExperimentFlag(subfeature: self.subfeature)
+    }
+
+    var subfeature: any PrivacySubfeature {
+        switch self {
+        case .baseline:
+            ContentBlockingSubfeature.tdsNextExperimentBaseline
+        case .feb25:
+            ContentBlockingSubfeature.tdsNextExperimentFeb25
+        case .mar25:
+            ContentBlockingSubfeature.tdsNextExperimentMar25
+        case .apr25:
+            ContentBlockingSubfeature.tdsNextExperimentApr25
+        case .may25:
+            ContentBlockingSubfeature.tdsNextExperimentMay25
+        case .jun25:
+            ContentBlockingSubfeature.tdsNextExperimentJun25
+        case .jul25:
+            ContentBlockingSubfeature.tdsNextExperimentJul25
+        case .aug25:
+            ContentBlockingSubfeature.tdsNextExperimentAug25
+        case .sep25:
+            ContentBlockingSubfeature.tdsNextExperimentSep25
+        case .oct25:
+            ContentBlockingSubfeature.tdsNextExperimentOct25
+        case .nov25:
+            ContentBlockingSubfeature.tdsNextExperimentNov25
+        case .dec25:
+            ContentBlockingSubfeature.tdsNextExperimentDec25
+        }
+    }
+
+}
+
+public struct TdsNextExperimentFlag: FeatureFlagExperimentDescribing {
+    public var rawValue: String
+    public var source: FeatureFlagSource
+
+    init(subfeature: any PrivacySubfeature) {
+        self.source = .remoteReleasable(.subfeature(subfeature))
+        self.rawValue = subfeature.rawValue
+    }
+
+    public typealias CohortType = Cohort
+
+    public enum Cohort: String, FlagCohort {
+        case control
+        case treatment
+    }
 }

DuckDuckGo/Configuration/ConfigurationManager.swift

@@ -28,6 +28,10 @@ import PixelKit
 
 final class ConfigurationManager: DefaultConfigurationManager {
 
+    private lazy var trackerDataManager = ContentBlocking.shared.trackerDataManager
+    private lazy var privacyConfigurationManager = ContentBlocking.shared.privacyConfigurationManager
+    private lazy var contentBlockingManager = ContentBlocking.shared.contentBlockingManager
+
     private enum Constants {
         static let lastConfigurationInstallDateKey = "config.last.installed"
     }
@@ -107,10 +111,30 @@ final class ConfigurationManager: DefaultConfigurationManager {
     private func fetchTrackerBlockingDependencies(isDebug: Bool) async -> Bool {
         var didFetchAnyTrackerBlockingDependencies = false
 
-        var tasks = [Configuration: Task<(), Swift.Error>]()
-        tasks[.trackerDataSet] = Task { try await fetcher.fetch(.trackerDataSet, isDebug: isDebug) }
-        tasks[.surrogates] = Task { try await fetcher.fetch(.surrogates, isDebug: isDebug) }
-        tasks[.privacyConfiguration] = Task { try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug) }
+        // Start surrogates fetch task
+        let surrogatesTask = Task { try await fetcher.fetch(.surrogates, isDebug: isDebug) }
+
+        // Perform privacyConfiguration fetch and update
+        do {
+            try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug)
+            didFetchAnyTrackerBlockingDependencies = true
+            privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
+                                               data: store.loadData(for: .privacyConfiguration))
+        } catch {
+            Logger.config.error(
+                "Failed to complete configuration update to \(Configuration.privacyConfiguration.rawValue, privacy: .public): \(error.localizedDescription, privacy: .public)"
+            )
+            tryAgainSoon()
+        }
+
+        // Start trackerDataSet fetch task after privacyConfiguration completes
+        let trackerDataSetTask = Task { try await fetcher.fetch(.trackerDataSet, isDebug: isDebug) }
+
+        // Wait for surrogates and trackerDataSet tasks
+        let tasks: [(Configuration, Task<(), Swift.Error>)] = [
+            (.surrogates, surrogatesTask),
+            (.trackerDataSet, trackerDataSetTask)
+        ]
 
         for (configuration, task) in tasks {
             do {
@@ -141,11 +165,12 @@ final class ConfigurationManager: DefaultConfigurationManager {
 
     private func updateTrackerBlockingDependencies() {
         lastConfigurationInstallDate = Date()
-        ContentBlocking.shared.trackerDataManager.reload(etag: store.loadEtag(for: .trackerDataSet),
-                                                         data: store.loadData(for: .trackerDataSet))
-        ContentBlocking.shared.privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
-                                                                  data: store.loadData(for: .privacyConfiguration))
-        ContentBlocking.shared.contentBlockingManager.scheduleCompilation()
+
+        trackerDataManager.reload(etag: store.loadEtag(for: .trackerDataSet),
+                                  data: store.loadData(for: .trackerDataSet))
+        privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
+                                           data: store.loadData(for: .privacyConfiguration))
+        contentBlockingManager.scheduleCompilation()
     }
 
     private func updateBloomFilter() async throws {
@@ -199,3 +224,17 @@ extension ConfigurationManager {
         updateTrackerBlockingDependencies()
     }
 }
+
+#if DEBUG
+extension ConfigurationManager {
+    func setContentBlockingManagers(trackerDataManager: TrackerDataManager,
+                                    privacyConfigurationManager: PrivacyConfigurationManager,
+                                    contentBlockingManager: ContentBlockerRulesManagerProtocol) {
+        if Application.runType == .unitTests {
+            self.trackerDataManager = trackerDataManager
+            self.privacyConfigurationManager = privacyConfigurationManager
+            self.contentBlockingManager = contentBlockingManager
+        }
+    }
+}
+#endif

DuckDuckGo/ContentBlocker/Mocks/MockPrivacyConfiguration.swift

@@ -44,6 +44,7 @@ final class MockPrivacyConfiguration: PrivacyConfiguration {
                                                                             state: PrivacyConfigurationData.State.enabled)
     var exceptionsList: (PrivacyFeature) -> [String] = { _ in [] }
     var featureSettings: PrivacyConfigurationData.PrivacyFeature.FeatureSettings = [:]
+    var subfeatureSettings: PrivacyConfigurationData.PrivacyFeature.SubfeatureSettings = ""
 
     func exceptionsList(forFeature featureKey: PrivacyFeature) -> [String] { exceptionsList(featureKey) }
     var isFeatureKeyEnabled: ((PrivacyFeature, AppVersionProvider) -> Bool)?
@@ -76,6 +77,9 @@ final class MockPrivacyConfiguration: PrivacyConfiguration {
     func isInExceptionList(domain: String?, forFeature featureKey: PrivacyFeature) -> Bool { false }
     func settings(for feature: PrivacyFeature) -> PrivacyConfigurationData.PrivacyFeature.FeatureSettings {
         featureSettings }
+    func settings(for subfeature: any BrowserServicesKit.PrivacySubfeature) -> PrivacyConfigurationData.PrivacyFeature.SubfeatureSettings? {
+        subfeatureSettings
+    }
     func userEnabledProtection(forDomain: String) {}
     func userDisabledProtection(forDomain: String) {}
 }

LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift

@@ -171,6 +171,7 @@ final class PrivacyConfigurationManagingMock: PrivacyConfigurationManaging {
 }
 
 final class PrivacyConfigurationMock: PrivacyConfiguration {
+
     var identifier: String = "mock"
     var version: String? = "123456789"
 
@@ -224,6 +225,10 @@ final class PrivacyConfigurationMock: PrivacyConfiguration {
         [String: Any]()
     }
 
+    func settings(for subfeature: any BrowserServicesKit.PrivacySubfeature) -> PrivacyConfigurationData.PrivacyFeature.SubfeatureSettings? {
+        return nil
+    }
+
     func userEnabledProtection(forDomain: String) {
 
     }