diff --git a/trigger/advanced-automated-retries.rego b/trigger/advanced-automated-retries.rego new file mode 100644 index 0000000..d985741 --- /dev/null +++ b/trigger/advanced-automated-retries.rego @@ -0,0 +1,48 @@ +package spacelift + +# set the maximum number of retries +default max_retries := 3 + +# label to use for stack specific retries +default retry_count_key := "spacelift_retry" + +# FN: gets the value of an array of retry labels +obtain_retries_count(arr) := {x | + new_arr := array.concat(arr, [sprintf("%s:0", [retry_count_key])]) + some i + parts := split(new_arr[i], ":") + parts[0] == retry_count_key + x := to_number(parts[1]) +} + +# Get the value of the highest retry stack label +retry_label := max(obtain_retries_count(input.stack.labels)) + +# Set the flag for the next iteration +new_retries := max(obtain_retries_count(input.run.flags)) + 1 + +retry_flag := sprintf("%s:%d", [retry_count_key, new_retries]) + +flag[retry_flag] + +is_failed_and_tracked { + input.run.state == "FAILED" + input.run.type == "TRACKED" +} + +# trigger the stack if the max retry label is 0 +trigger[stack.id] { + stack := input.stack + is_failed_and_tracked + retry_label <= 0 + new_retries <= max_retries +} + +# trigger the stack if the max retry label is defined but only up to the maximum retries +trigger[stack.id] { + stack := input.stack + is_failed_and_tracked + retry_label > 0 + new_retries <= retry_label + new_retries <= max_retries +} diff --git a/trigger/advanced-automated-retries.yml b/trigger/advanced-automated-retries.yml new file mode 100644 index 0000000..ed25a9a --- /dev/null +++ b/trigger/advanced-automated-retries.yml @@ -0,0 +1,13 @@ +name: Advanced Automated Retries +source: advanced-automated-retries.rego +type: trigger +description: | + Sometimes Terraform or Pulumi deployments fail for a reason that has nothing to do with the code + - think eventual consistency between various cloud subsystems, transient API errors etc. + This trigger policy will restart the failed run up to a maximum number of times. This policy can also be configured + on a per stack basis by setting `spacelift_retry:` on the stacks labels. +labels: + - trigger + - retries + - automated + - failed diff --git a/trigger/advanced-automated-retries_test.rego b/trigger/advanced-automated-retries_test.rego new file mode 100644 index 0000000..4fc5e89 --- /dev/null +++ b/trigger/advanced-automated-retries_test.rego @@ -0,0 +1,84 @@ +package spacelift_test + +import data.spacelift +import future.keywords.contains +import future.keywords.if +import future.keywords.in + +# Test Case 1: Failed and Tracked Run without flags +test_failed_and_tracked_run_without_flags if { + result := spacelift.trigger with input as { + "run": { + "state": "FAILED", + "type": "TRACKED", + "flags": [], + }, + "stack": { + "id": "stack-one", + "labels": [], + }, + } + result["stack-one"] +} + +# Test Case 2: Failed and Tracked Run with flags +test_failed_and_tracked_run_with_flags if { + result := spacelift.trigger with input as { + "run": { + "state": "FAILED", + "type": "TRACKED", + "flags": ["spacelift_retry:1"], + }, + "stack": { + "id": "stack-one", + "labels": [], + }, + } + result["stack-one"] +} + +# Test Case 3: Failed and Tracked Run with flags at max +test_failed_and_tracked_run_with_flags_at_max if { + count(spacelift.trigger) == 0 with input as { + "run": { + "state": "FAILED", + "type": "TRACKED", + "flags": ["spacelift_retry:3"], + }, + "stack": { + "id": "stack-one", + "labels": [], + }, + } +} + +# Test Case 4: LABELS: Failed and Tracked Run without flags +test_failed_and_tracked_run_labels_without_flags if { + result := spacelift.trigger with input as { + "run": { + "state": "FAILED", + "type": "TRACKED", + "flags": [], + }, + "stack": { + "id": "stack-one", + "labels": ["spacelift_retry:2"], + }, + } + result["stack-one"] +} + +# Test Case 5: LABELS: Failed and Tracked Run with flags at max +test_failed_and_tracked_run_labels_with_flags_at_max if { + count(spacelift.trigger) == 0 with input as { + "run": { + "state": "FAILED", + "type": "TRACKED", + "flags": ["spacelift_retry:2"], + }, + "stack": { + "id": "stack-one", + "labels": ["spacelift_retry:2"], + }, + } +}