Merge pull request #482 from uccross/retry-failed-ceph-setup

Retry the coffea_rados_parquet job 3 times to overcome the rare OSD segfault issue

.github/workflows/ci.yml

@@ -137,13 +137,7 @@ jobs:
         docker build -t coffea-rados-parquet-test \
           --file docker/coffea_rados_parquet/Dockerfile \
           .
-        docker run \
-        -v $(pwd):/w \
-        -w /w \
-        -e IS_CI=true \
-        --privileged \
-        coffea-rados-parquet-test \
-        ./docker/coffea_rados_parquet/script.sh
+        ./docker/coffea_rados_parquet/run.sh
 
   release:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')

docker/coffea_rados_parquet/run.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -ex
+
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=1
+  local max=3
+  local delay=5
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+retry docker run \
+        -v $(pwd):/w \
+        -w /w \
+        -e IS_CI=true \
+        --privileged \
+        coffea-rados-parquet-test \
+        ./docker/coffea_rados_parquet/script.sh
+