diff --git a/src/Task/Base/ParallelExec.php b/src/Task/Base/ParallelExec.php
index 3146976a1..aa9c73e86 100644
--- a/src/Task/Base/ParallelExec.php
+++ b/src/Task/Base/ParallelExec.php
@@ -51,6 +51,11 @@ class ParallelExec extends BaseTask implements CommandInterface, PrintedInterfac
      */
     protected $isPrinted = false;
 
+    /**
+     * @var bool
+     */
+    protected $stopAllOnAnyFailure = false;
+
     /**
      * {@inheritdoc}
      */
@@ -134,6 +139,17 @@ public function waitInterval($waitInterval)
         return $this;
     }
 
+    /**
+     * @param bool $stopAllOnAnyFailure
+     *
+     * @return $this
+     */
+    public function stopAllOnAnyFailure($stopAllOnAnyFailure)
+    {
+        $this->stopAllOnAnyFailure = $stopAllOnAnyFailure;
+        return $this;
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -185,6 +201,12 @@ public function run()
                         }
                     }
                     unset($running[$k]);
+                    if ($this->stopAllOnAnyFailure && $process->getExitCode() !== 0) {
+                        foreach ($running as $otherRunningProcess) {
+                            $otherRunningProcess->stop();
+                        }
+                        $queue = [];
+                    }
                 }
             }
             if (empty($running) && empty($queue)) {
diff --git a/tests/integration/StopAllOnAnyFailureTest.php b/tests/integration/StopAllOnAnyFailureTest.php
new file mode 100644
index 000000000..f4b060c51
--- /dev/null
+++ b/tests/integration/StopAllOnAnyFailureTest.php
@@ -0,0 +1,46 @@
+<?php
+namespace Robo;
+
+use PHPUnit\Framework\TestCase;
+use Robo\Task\Base\Exec;
+use Robo\Task\Base\ParallelExec;
+use Robo\Traits\TestTasksTrait;
+
+class StopAllOnAnyFailureTest extends TestCase
+{
+    use TestTasksTrait;
+    use Task\File\loadTasks;
+
+    protected $fixtures;
+
+    public function setUp()
+    {
+        $this->fixtures = new Fixtures();
+        $this->initTestTasksTrait();
+    }
+
+    public function tearDown()
+    {
+        $this->fixtures->cleanup();
+    }
+
+    public function testParallelProcessesGetStoppedIfStopAllOnAnyFailureIsSet()
+    {
+        // Some tests may left this to true
+        Result::$stopOnFail = false;
+        $this->fixtures->createAndCdToSandbox();
+
+        $filenameOk = uniqid('ok_');
+        $filenameKo = uniqid('ko_');
+
+        /** @var ParallelExec $parallel */
+        $parallel = $this->task(ParallelExec::class);
+        $parallel->stopAllOnAnyFailure(true);
+        $parallel->process($this->task(Exec::class, sprintf('touch %s && false', escapeshellarg($filenameOk))));
+        $parallel->process($this->task(Exec::class, sprintf('sleep 3 && touch %s', escapeshellarg($filenameKo))));
+        $parallel->run();
+
+        $this->assertFileExists($filenameOk);
+        $this->assertFileNotExists($filenameKo);
+    }
+}