Merge remote-tracking branch 'origin/main' into max/remove-cpm-experi…

…ment

Package.resolved

@@ -14,8 +14,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "1733ee59f06f6e725a98cf6cd8322159f59d664b",
-        "version" : "6.31.0"
+        "revision" : "adca39c379b1a124f9990e9d0308c374f32f5018",
+        "version" : "6.32.0"
       }
     },
     {

Package.swift

@@ -53,7 +53,7 @@ let package = Package(
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.3.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "3.0.0"),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.31.0"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.32.0"),
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.1.1"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),

Sources/BrowserServicesKit/InternalUserDecider/UserDefaults+isInternalUser.swift

@@ -0,0 +1,46 @@
+//
+//  UserDefaults+isInternalUser.swift
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Combine
+import Foundation
+
+extension UserDefaults: InternalUserStoring {
+    @objc
+    public dynamic var isInternalUser: Bool {
+        get {
+            bool(forKey: #keyPath(isInternalUser))
+        }
+
+        set {
+            guard newValue != bool(forKey: #keyPath(isInternalUser)) else {
+                return
+            }
+
+            guard newValue else {
+                removeObject(forKey: #keyPath(isInternalUser))
+                return
+            }
+
+            set(newValue, forKey: #keyPath(isInternalUser))
+        }
+    }
+
+    var internalUserPublisher: AnyPublisher<Bool, Never> {
+        publisher(for: \.isInternalUser).eraseToAnyPublisher()
+    }
+}

Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift

@@ -120,6 +120,10 @@ public enum NetworkProtectionSubfeature: String, Equatable, PrivacySubfeature {
     /// Display user tips for Network Protection
     /// https://app.asana.com/0/72649045549333/1208231259093710/f
     case userTips
+
+    /// Enforce routes for the VPN to fix TunnelVision
+    /// https://app.asana.com/0/72649045549333/1208617860225199/f
+    case enforceRoutes
 }
 
 public enum SyncSubfeature: String, PrivacySubfeature {

Sources/DDGSync/DDGSync.swift

@@ -126,7 +126,7 @@ public class DDGSync: DDGSyncing {
         }
         do {
             try await disconnect(deviceId: deviceId)
-            try updateAccount(nil)
+            try removeAccount(reason: .userTurnedOffSync)
         } catch {
             try handleUnauthenticated(error)
         }
@@ -180,7 +180,7 @@ public class DDGSync: DDGSyncing {
 
         do {
             try await dependencies.account.deleteAccount(account)
-            try updateAccount(nil)
+            try removeAccount(reason: .userDeletedAccount)
         } catch {
             try handleUnauthenticated(error)
         }
@@ -194,7 +194,7 @@ public class DDGSync: DDGSyncing {
     }
 
     public func updateServerEnvironment(_ serverEnvironment: ServerEnvironment) {
-        try? updateAccount(nil)
+        try? removeAccount(reason: .serverEnvironmentUpdated)
         dependencies.updateServerEnvironment(serverEnvironment)
         authState = .initializing
         initializeIfNeeded()
@@ -227,11 +227,12 @@ public class DDGSync: DDGSyncing {
         let syncEnabled = dependencies.keyValueStore.object(forKey: Constants.syncEnabledKey) != nil
         guard syncEnabled else {
             try? dependencies.secureStore.removeAccount()
+            dependencies.errorEvents.fire(.accountRemoved(.syncEnabledNotSetOnKeyValueStore))
             authState = .inactive
             return
         }
 
-        var account: SyncAccount?
+        let account: SyncAccount?
         do {
             account = try dependencies.secureStore.account()
         } catch {
@@ -241,29 +242,30 @@ public class DDGSync: DDGSyncing {
 
         authState = account?.state ?? .inactive
 
+        guard let account else {
+            do {
+                try removeAccount(reason: .notFoundInSecureStorage)
+            } catch {
+                dependencies.errorEvents.fire(.failedToRemoveAccount, error: error)
+            }
+            return
+        }
+
         do {
             try updateAccount(account)
         } catch {
             dependencies.errorEvents.fire(.failedToSetupEngine, error: error)
         }
     }
 
-    private func updateAccount(_ account: SyncAccount? = nil) throws {
-        guard account?.state != .initializing else {
+    private func updateAccount(_ account: SyncAccount) throws {
+        guard account.state != .initializing else {
             assertionFailure("Sync has not been initialized properly")
             return
         }
 
-        guard var account, account.state != .inactive else {
-            dependencies.scheduler.isEnabled = false
-            startSyncCancellable?.cancel()
-            syncQueueCancellable?.cancel()
-            isDataSyncingFeatureFlagEnabledCancellable?.cancel()
-            try syncQueue?.dataProviders.forEach { try $0.deregisterFeature() }
-            syncQueue = nil
-            authState = .inactive
-            try dependencies.secureStore.removeAccount()
-            dependencies.keyValueStore.set(nil, forKey: Constants.syncEnabledKey)
+        guard account.state != .inactive else {
+            try removeAccount(reason: .authStateInactive)
             return
         }
 
@@ -274,9 +276,12 @@ public class DDGSync: DDGSyncing {
         try syncQueue.prepareDataModelsForSync(needsRemoteDataFetch: account.state == .addingNewDevice)
 
         if account.state != .active {
-            account = account.updatingState(.active)
+            let activatedAccount = account.updatingState(.active)
+            try dependencies.secureStore.persistAccount(activatedAccount)
+        } else {
+            try dependencies.secureStore.persistAccount(account)
         }
-        try dependencies.secureStore.persistAccount(account)
+
         authState = account.state
         dependencies.keyValueStore.set(true, forKey: Constants.syncEnabledKey)
 
@@ -328,6 +333,19 @@ public class DDGSync: DDGSyncing {
         self.syncQueue = syncQueue
     }
 
+    private func removeAccount(reason: SyncError.AccountRemovedReason) throws {
+        dependencies.scheduler.isEnabled = false
+        startSyncCancellable?.cancel()
+        syncQueueCancellable?.cancel()
+        isDataSyncingFeatureFlagEnabledCancellable?.cancel()
+        try syncQueue?.dataProviders.forEach { try $0.deregisterFeature() }
+        syncQueue = nil
+        authState = .inactive
+        try dependencies.secureStore.removeAccount()
+        dependencies.keyValueStore.set(nil, forKey: Constants.syncEnabledKey)
+        dependencies.errorEvents.fire(.accountRemoved(reason))
+    }
+
     private func handleUnauthenticated(_ error: Error) throws {
         guard let syncError = error as? SyncError,
               case .unexpectedStatusCode(let statusCode) = syncError,
@@ -336,7 +354,7 @@ public class DDGSync: DDGSyncing {
         }
 
         do {
-            try updateAccount(nil)
+            try removeAccount(reason: .unauthenticatedRequest)
             throw SyncError.unauthenticatedWhileLoggedIn
         } catch {
             Logger.sync.error("Failed to delete account upon unauthenticated server response: \(error.localizedDescription, privacy: .public)")

Sources/DDGSync/SyncError.swift

@@ -20,11 +20,22 @@ import Foundation
 
 public enum SyncError: Error, Equatable {
 
+    public enum AccountRemovedReason: String, Equatable {
+        case authStateInactive = "auth-state-inactive"
+        case syncEnabledNotSetOnKeyValueStore = "not-set-on-key-value-store"
+        case notFoundInSecureStorage = "not-found-in-secure-storage"
+        case userTurnedOffSync = "user-turned-off"
+        case userDeletedAccount = "user-deleted-account"
+        case unauthenticatedRequest = "unauthenticated-request"
+        case serverEnvironmentUpdated = "server-environment-updated"
+    }
+
     case noToken
 
     case failedToMigrate
     case failedToLoadAccount
     case failedToSetupEngine
+    case failedToRemoveAccount
 
     case failedToCreateAccountKeys(_ message: String)
     case accountNotFound
@@ -38,7 +49,7 @@ public enum SyncError: Error, Equatable {
     case unableToEncodeRequestBody(_ message: String)
     case unableToDecodeResponse(_ message: String)
     case invalidDataInResponse(_ message: String)
-    case accountRemoved
+    case accountRemoved(_ reason: AccountRemovedReason)
 
     case failedToEncryptValue(_ message: String)
     case failedToDecryptValue(_ message: String)
@@ -49,6 +60,7 @@ public enum SyncError: Error, Equatable {
     case failedToWriteSecureStore(status: OSStatus)
     case failedToReadSecureStore(status: OSStatus)
     case failedToRemoveSecureStore(status: OSStatus)
+    case failedToDecodeSecureStoreData(error: NSError)
 
     case credentialsMetadataMissingBeforeFirstSync
     case receivedCredentialsWithoutUUID
@@ -140,6 +152,10 @@ public enum SyncError: Error, Equatable {
             return [syncErrorString: "unauthenticatedWhileLoggedIn"]
         case .patchPayloadCompressionFailed:
             return [syncErrorString: "patchPayloadCompressionFailed"]
+        case .failedToRemoveAccount:
+            return [syncErrorString: "failedToRemoveAccount"]
+        case .failedToDecodeSecureStoreData:
+            return [syncErrorString: "failedToDecodeSecureStoreData"]
         }
     }
 }
@@ -187,14 +203,20 @@ extension SyncError: CustomNSError {
         case .settingsMetadataNotPresent: return 27
         case .unauthenticatedWhileLoggedIn: return 28
         case .patchPayloadCompressionFailed: return 29
+        case .failedToRemoveAccount: return 30
+        case .failedToDecodeSecureStoreData: return 31
         }
     }
 
     public var errorUserInfo: [String: Any] {
         switch self {
-        case .failedToReadSecureStore(let status), .failedToWriteSecureStore(let status), .failedToRemoveSecureStore(let status):
+        case .failedToReadSecureStore(let status),
+                .failedToWriteSecureStore(let status),
+                .failedToRemoveSecureStore(let status):
             let underlyingError = NSError(domain: "secError", code: Int(status))
             return [NSUnderlyingErrorKey: underlyingError]
+        case .failedToDecodeSecureStoreData(let error):
+            return [NSUnderlyingErrorKey: error]
         default:
             return [:]
         }

Sources/DDGSync/internal/SecureStorage.swift

@@ -65,7 +65,11 @@ struct SecureStorage: SecureStoring {
         }
 
         if let data = item as? Data {
-            return try JSONDecoder.snakeCaseKeys.decode(SyncAccount.self, from: data)
+            do {
+                return try JSONDecoder.snakeCaseKeys.decode(SyncAccount.self, from: data)
+            } catch {
+                throw SyncError.failedToDecodeSecureStoreData(error: error as NSError)
+            }
         }
 
         return nil

Sources/NetworkProtection/Controllers/TunnelController.swift

@@ -33,6 +33,10 @@ public protocol TunnelController {
     ///
     func stop() async
 
+    /// Sends a command to the adapter
+    ///
+    func command(_ command: VPNCommand) async throws
+
     /// Whether the tunnel is connected
     ///
     var isConnected: Bool { get async }

Sources/NetworkProtection/Models/AnyIPAddress.swift

@@ -19,7 +19,7 @@
 import Foundation
 import Network
 
-public enum AnyIPAddress: IPAddress, Hashable, CustomDebugStringConvertible, @unchecked Sendable {
+public enum AnyIPAddress: Hashable, CustomDebugStringConvertible, @unchecked Sendable {
     /// A host specified as an IPv4 address
     case ipv4(IPv4Address)
 
@@ -68,7 +68,7 @@ public enum AnyIPAddress: IPAddress, Hashable, CustomDebugStringConvertible, @un
         }
     }
 
-    private var ipAddress: IPAddress {
+    public var ipAddress: IPAddress {
         switch self {
         case .ipv4(let ip):
             return ip