feat: add retry_if support on steps (#490)

* feat: add retry_if support on steps

Signed-off-by: GitHub <[email protected]>

Author: lowlighter

README.md

@@ -302,6 +302,8 @@ testcases:
     method: GET
     url: https://eu.api.ovh.com/1.0/
     retry: 3
+    retry_if: # (optional, lets you early break unrecoverable errors)
+    - result.statuscode ShouldNotEqual 403
     delay: 2
     assertions:
     - result.statuscode ShouldEqual 200

assertion.go

@@ -353,3 +353,22 @@ func findLineNumber(filename, testcase string, stepNumber int, assertion string,
 
 	return countLine
 }
+
+//This evaluates a string of assertions with a given vars scope, and returns a slice of failures (i.e. empty slice = all pass)
+func testConditionalStatement(ctx context.Context, tc *TestCase, assertions []string, vars H, text string) ([]string, error) {
+	var failures []string
+	for _, assertion := range assertions {
+		Debug(ctx, "evaluating %s", assertion)
+		assert, err := parseAssertions(ctx, assertion, vars)
+		if err != nil {
+			Error(ctx, "unable to parse assertion: %v", err)
+			tc.AppendError(err)
+			return failures, err
+		}
+		if err := assert.Func(assert.Actual, assert.Args...); err != nil {
+			s := fmt.Sprintf(text, tc.originalName, err)
+			failures = append(failures, s)
+		}
+	}
+	return failures, nil
+}

process_testcase.go

@@ -139,22 +139,18 @@ func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) {
 }
 
 func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
-	for _, skipAssertion := range tc.Skip {
-		Debug(ctx, "evaluating %s", skipAssertion)
-		assert, err := parseAssertions(ctx, skipAssertion, tc.Vars)
-		if err != nil {
-			Error(ctx, "unable to parse skip assertion: %v", err)
-			tc.AppendError(err)
-			return
-		}
-		if err := assert.Func(assert.Actual, assert.Args...); err != nil {
-			s := fmt.Sprintf("skipping testcase %q: %v", tc.originalName, err)
+
+	results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v")
+	if err != nil {
+		Error(ctx, "unable to evaluate \"skip\" assertions: %v", err)
+		tc.AppendError(err)
+		return
+	}
+	if len(results) > 0 {
+		for _, s := range results {
 			tc.Skipped = append(tc.Skipped, Skipped{Value: s})
 			Warn(ctx, s)
 		}
-	}
-
-	if len(tc.Skipped) > 0 {
 		return
 	}
 

process_teststep.go

@@ -112,6 +112,15 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
 		if assertRes.ok {
 			break
 		}
+		failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tc.computedVars, "")
+		if err != nil {
+			return fmt.Errorf("Error while evaluating retry condition: %v", err)
+		}
+		if len(failures) > 0 {
+			failure := newFailure(*tc, stepNumber, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-retry))
+			tc.Failures = append(tc.Failures, *failure)
+			break
+		}
 	}
 
 	tc.Errors = append(tc.Errors, assertRes.errors...)

tests/failing/retry_if.yml

@@ -0,0 +1,19 @@
+testcases:
+
+- name: test retry
+  steps:
+  - type: exec
+    script: echo pending
+    retry: 2
+    assertions: 
+      - result.systemout ShouldEqual ok
+
+- name: test retry and retry_if
+  steps:
+  - type: exec
+    script: sleep 2 && echo error
+    retry: 5
+    retry_if: 
+      - result.systemout ShouldEqual pending
+    assertions: 
+      - result.systemout ShouldEqual ok

tests/retry_if.yml

@@ -0,0 +1,30 @@
+name: testsuite with retry if
+testcases:
+- name: testsuite with retry if (success)
+  steps:
+  - type: exec
+    # we use a tmp file as "memory" to know whether we're on first attempt or second one
+    script: |
+      test -f /tmp/retry-if-first-attempt
+      RC=$?
+      touch /tmp/retry-if-first-attempt
+      exit $RC
+    retry: 1
+    retry_if: 
+      - result.code ShouldNotEqual 0
+    assertions:
+      - result.code ShouldEqual 0
+
+- name: testsuite with retry if (failing)
+  steps:
+  # spawn a venom sub-process and expect it to fail 
+  - type: exec
+    script: '{{.venom.executable}} run failing/retry_if.yml'
+    assertions:
+      - result.code ShouldEqual 2
+      - result.systemerr ShouldBeEmpty
+      # classic retry
+      - result.systemout ShouldContainSubstring "It's a failure after 3 attempts"
+      # retry with condition (sleep 2 * 5 retries = max 10 seconds)
+      - result.timeseconds ShouldBeLessThan 10
+      - result.systemout ShouldContainSubstring "retry conditions not fulfilled, skipping 5 remaining retries"

types.go

@@ -151,6 +151,10 @@ func (t TestStep) StringSliceValue(name string) ([]string, error) {
 		}
 		return out, nil
 	}
+	//If string is empty, return an empty slice instead
+	if len(out) == 0 {
+		return []string{}, nil
+	}
 	return []string{out}, nil
 }
 

types_executor.go

@@ -25,6 +25,7 @@ type ExecutorRunner interface {
 	ExecutorWithSetup
 	Name() string
 	Retry() int
+	RetryIf() []string
 	Delay() int
 	Timeout() int
 	Info() []string
@@ -39,6 +40,7 @@ type executor struct {
 	Executor
 	name    string
 	retry   int      // nb retry a test case if it is in failure.
+	retryIf []string // retry conditions to check before performing any retries
 	delay   int      // delay between two retries
 	timeout int      // timeout on executor
 	info    []string // info to display after the run and before the assertion
@@ -57,6 +59,10 @@ func (e executor) Retry() int {
 	return e.retry
 }
 
+func (e executor) RetryIf() []string {
+	return e.retryIf
+}
+
 func (e executor) Delay() int {
 	return e.delay
 }
@@ -124,11 +130,12 @@ func (e executor) Run(ctx context.Context, step TestStep) (interface{}, error) {
 	return e.Executor.Run(ctx, step)
 }
 
-func newExecutorRunner(e Executor, name, stype string, retry, delay, timeout int, info []string) ExecutorRunner {
+func newExecutorRunner(e Executor, name, stype string, retry int, retryIf []string, delay, timeout int, info []string) ExecutorRunner {
 	return &executor{
 		Executor: e,
 		name:     name,
 		retry:    retry,
+		retryIf:  retryIf,
 		delay:    delay,
 		timeout:  timeout,
 		info:     info,

venom.go

@@ -108,6 +108,10 @@ func (v *Venom) GetExecutorRunner(ctx context.Context, ts TestStep, h H) (contex
 	if err != nil {
 		return nil, nil, err
 	}
+	retryIf, err := ts.StringSliceValue("retry_if")
+	if err != nil {
+		return nil, nil, err
+	}
 	delay, err := ts.IntValue("delay")
 	if err != nil {
 		return nil, nil, err
@@ -131,19 +135,19 @@ func (v *Venom) GetExecutorRunner(ctx context.Context, ts TestStep, h H) (contex
 	ctx = context.WithValue(ctx, ContextKey("vars"), allKeys)
 
 	if name == "" {
-		return ctx, newExecutorRunner(nil, name, "builtin", retry, delay, timeout, info), nil
+		return ctx, newExecutorRunner(nil, name, "builtin", retry, retryIf, delay, timeout, info), nil
 	}
 
 	if ex, ok := v.executorsBuiltin[name]; ok {
-		return ctx, newExecutorRunner(ex, name, "builtin", retry, delay, timeout, info), nil
+		return ctx, newExecutorRunner(ex, name, "builtin", retry, retryIf, delay, timeout, info), nil
 	}
 
 	if err := v.registerUserExecutors(ctx, name, ts, vars); err != nil {
 		Debug(ctx, "executor %q is not implemented as user executor - err:%v", name, err)
 	}
 
 	if ex, ok := v.executorsUser[name]; ok {
-		return ctx, newExecutorRunner(ex, name, "user", retry, delay, timeout, info), nil
+		return ctx, newExecutorRunner(ex, name, "user", retry, retryIf, delay, timeout, info), nil
 	}
 
 	if err := v.registerPlugin(ctx, name, vars); err != nil {
@@ -152,7 +156,7 @@ func (v *Venom) GetExecutorRunner(ctx context.Context, ts TestStep, h H) (contex
 
 	// then add the executor plugin to the map to not have to load it on each step
 	if ex, ok := v.executorsUser[name]; ok {
-		return ctx, newExecutorRunner(ex, name, "plugin", retry, delay, timeout, info), nil
+		return ctx, newExecutorRunner(ex, name, "plugin", retry, retryIf, delay, timeout, info), nil
 	}
 	return ctx, nil, fmt.Errorf("executor %q is not implemented", name)
 }