Merge branch 'dev_1.17.0' into simple_pb_inclusion

Trusted-AI · Dec 19, 2023 · a405f49 · a405f49
2 parents 1f3026b + a281f62
commit a405f49
Show file tree

Hide file tree

Showing 8 changed files with 1,528 additions and 156 deletions.
diff --git a/art/attacks/evasion/__init__.py b/art/attacks/evasion/__init__.py
@@ -18,6 +18,7 @@
     from art.attacks.evasion.brendel_bethge import BrendelBethgeAttack
 
 from art.attacks.evasion.boundary import BoundaryAttack
+from art.attacks.evasion.composite_adversarial_attack import CompositeAdversarialAttackPyTorch
 from art.attacks.evasion.carlini import CarliniL2Method, CarliniLInfMethod, CarliniL0Method
 from art.attacks.evasion.decision_tree_attack import DecisionTreeAttack
 from art.attacks.evasion.deepfool import DeepFool

diff --git a/art/attacks/evasion/composite_adversarial_attack.py b/art/attacks/evasion/composite_adversarial_attack.py
diff --git a/art/attacks/inference/membership_inference/black_box.py b/art/attacks/inference/membership_inference/black_box.py
diff --git a/notebooks/README.md b/notebooks/README.md
@@ -108,6 +108,11 @@ demonstrates a MembershipInferenceBlackBox membership inference attack using sha
 [label_only_membership_inference.ipynb](label_only_membership_inference.ipynb) [[on nbviewer](https://nbviewer.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/label_only_membership_inference.ipynb)]
 demonstrates a LabelOnlyDecisionBoundary membership inference attack on a PyTorch classifier for the MNIST dataset.
 
+[composite-adversarial-attack.ipynb](composite-adversarial-attack.ipynb)[[on nbviewer](https://nbviewer.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/composite-adversarial-attack.ipynb)]
+shows how to launch Composite Adversarial Attack (CAA) on Pytorch-based model ([Hsiung et al., 2023](https://arxiv.org/abs/2202.04235)).
+CAA composites the perturbations in Lp-ball and semantic space (i.e., hue, saturation, rotation, brightness, and contrast),
+and is able to optimize the attack sequence and each attack component, thereby enhancing the efficiency and efficacy of adversarial examples.
+
 ## Metrics
 
 [privacy_metric.ipynb](privacy_metric.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/privacy_metric.ipynb)] 

diff --git a/notebooks/composite-adversarial-attack.ipynb b/notebooks/composite-adversarial-attack.ipynb
diff --git a/tests/attacks/evasion/test_composite_adversarial_attack.py b/tests/attacks/evasion/test_composite_adversarial_attack.py
@@ -0,0 +1,192 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2023
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import logging
+
+import numpy as np
+import pytest
+
+from art.attacks.evasion import CompositeAdversarialAttackPyTorch
+from art.estimators.estimator import BaseEstimator, LossGradientsMixin
+from art.estimators.classification.classifier import ClassifierMixin
+
+from tests.attacks.utils import backend_test_classifier_type_check_fail
+from tests.utils import ARTTestException, get_cifar10_image_classifier_pt
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture()
+def fix_get_cifar10_subset(get_cifar10_dataset):
+    (x_train_cifar10, y_train_cifar10), (x_test_cifar10, y_test_cifar10) = get_cifar10_dataset
+    n_train = 100
+    n_test = 11
+    yield x_train_cifar10[:n_train], y_train_cifar10[:n_train], x_test_cifar10[:n_test], y_test_cifar10[:n_test]
+
+
+@pytest.mark.skip_framework(
+    "tensorflow1", "tensorflow2", "tensorflow2v1", "keras", "non_dl_frameworks", "mxnet", "kerastf", "huggingface"
+)
+def test_generate(art_warning, fix_get_cifar10_subset):
+    try:
+        (x_train, y_train, x_test, y_test) = fix_get_cifar10_subset
+
+        classifier = get_cifar10_image_classifier_pt(from_logits=False, load_init=True)
+        attack = CompositeAdversarialAttackPyTorch(classifier)
+
+        x_train_adv = attack.generate(x=x_train, y=y_train)
+        x_test_adv = attack.generate(x=x_test, y=y_test)
+
+        assert x_train.shape == x_train_adv.shape
+        assert np.min(x_train_adv) >= 0.0
+        assert np.max(x_train_adv) <= 1.0
+        assert x_test.shape == x_test_adv.shape
+        assert np.min(x_test_adv) >= 0.0
+        assert np.max(x_test_adv) <= 1.0
+
+    except ARTTestException as e:
+        art_warning(e)
+
+
+@pytest.mark.skip_framework(
+    "tensorflow1", "tensorflow2", "tensorflow2v1", "keras", "non_dl_frameworks", "mxnet", "kerastf"
+)
+def test_check_params(art_warning):
+    try:
+        classifier = get_cifar10_image_classifier_pt(from_logits=False, load_init=True)
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, enabled_attack=(0, 1, 2, 3, 4, 5, 6, 7))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(-10.0, 0.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(0.0, 10.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(-1, 2.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=3.14)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=("1.0", 2.0))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(-10.0, 0.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(0.0, -10.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(1, 2.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=2.0)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=("1.0", 2.0))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(-450.0, 359.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(10.0, -10.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(1.0, 2))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=10)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=("10", 20.0))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(-10.0, 0.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(0.0, 10.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(-1, 1.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=1.0)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=("1.0", 2.0))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(-10.0, 10.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(0.0, -10.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(1, 2.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=2.0)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=("1.0", 2.0))
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(-0.5, 2.0))
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(8 / 255, -8 / 255))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(-2, 1))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=8 / 255)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(0.0, 10.0, 20.0))
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=("2/255", 3 / 255))
+
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, early_stop="true")
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, early_stop=1)
+
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_iter="max")
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_iter=-5)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_iter=2.5)
+
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter="max")
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter=-5)
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter=2.5)
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, attack_order="schedule")
+
+        with pytest.raises(ValueError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, batch_size=-1)
+
+        with pytest.raises(TypeError):
+            _ = CompositeAdversarialAttackPyTorch(classifier, verbose="true")
+
+    except ARTTestException as e:
+        art_warning(e)
+
+
+@pytest.mark.framework_agnostic
+def test_classifier_type_check_fail(art_warning):
+    try:
+        backend_test_classifier_type_check_fail(
+            CompositeAdversarialAttackPyTorch, [BaseEstimator, LossGradientsMixin, ClassifierMixin]
+        )
+    except ARTTestException as e:
+        art_warning(e)
diff --git a/tests/attacks/inference/attribute_inference/test_true_label_baseline.py b/tests/attacks/inference/attribute_inference/test_true_label_baseline.py
@@ -605,8 +605,8 @@ def transform_other_feature(x):
             baseline_inferred_test
         )
 
-        expected_train_acc = {"nn": 0.81, "rf": 0.95, "gb": 0.95, "lr": 0.81, "dt": 0.94, "knn": 0.87, "svm": 0.81}
-        expected_test_acc = {"nn": 0.88, "rf": 0.79, "gb": 0.8, "lr": 0.88, "dt": 0.74, "knn": 0.86, "svm": 0.88}
+        expected_train_acc = {"nn": 0.81, "rf": 0.93, "gb": 0.95, "lr": 0.81, "dt": 0.94, "knn": 0.87, "svm": 0.81}
+        expected_test_acc = {"nn": 0.88, "rf": 0.78, "gb": 0.8, "lr": 0.88, "dt": 0.74, "knn": 0.86, "svm": 0.88}
 
         assert expected_train_acc[model_type] <= baseline_train_acc
         assert expected_test_acc[model_type] <= baseline_test_acc