Set kubeVersion and added chart-verifier tests (#510)

Set min kubeVersion in Chart.yaml to 1.14. Added a chart-verifier bats
test, and configured to run it in CI. Some verification tests that
haven't been addressed yet are skipped.

Author: tvoran

.circleci/config.yml

@@ -10,6 +10,29 @@ jobs:
     steps:
       - checkout
       - run: bats ./test/unit -t
+
+  chart-verifier:
+    docker:
+      - image: docker.mirror.hashicorp.services/cimg/go:1.16
+    environment:
+      BATS_VERSION: "1.3.0"
+      # Note: the commit SHA is used here since the repo doesn't use release tags
+      CHART_VERIFIER_VERSION: "190d532246a5936dc6a7125e2da917d04e38a672"
+    steps:
+      - checkout
+      - run:
+          name: install chart-verifier
+          command: go get github.com/redhat-certification/chart-verifier@${CHART_VERIFIER_VERSION}
+      - run:
+          name: install bats
+          command: |
+            curl -sSL https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz -o /tmp/bats.tgz
+            tar -zxf /tmp/bats.tgz -C /tmp
+            sudo /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /usr/local
+      - run:
+          name: run chart-verifier tests
+          command: bats ./test/chart -t
+
   acceptance:
     docker:
       # This image is build from test/docker/Test.dockerfile
@@ -66,6 +89,7 @@ workflows:
   build_and_test:
     jobs:
       - bats-unit-test
+      - chart-verifier
       - acceptance:
           requires:
             - bats-unit-test

Chart.yaml

@@ -2,6 +2,7 @@ apiVersion: v2
 name: vault
 version: 0.11.0
 appVersion: 1.7.0
+kubeVersion: ">= 1.14"
 description: Official HashiCorp Vault Chart
 home: https://www.vaultproject.io
 icon: https://github.com/hashicorp/vault/raw/f22d202cde2018f9455dec755118a9b84586e082/Vault_PrimaryLogo_Black.png

README.md

@@ -22,9 +22,9 @@ The versions required are:
 
   * **Helm 3.0+** - This is the earliest version of Helm tested. It is possible
     it works with earlier versions but this chart is untested for those versions.
-  * **Kubernetes 1.9+** - This is the earliest version of Kubernetes tested.
+  * **Kubernetes 1.14+** - This is the earliest version of Kubernetes tested.
     It is possible that this chart works with earlier versions but it is
-    untested. Other versions verified are Kubernetes 1.10, 1.11.
+    untested.
 
 ## Usage
 

test/chart/_helpers.bash

@@ -0,0 +1,18 @@
+# chart_dir returns the directory for the chart
+chart_dir() {
+    echo ${BATS_TEST_DIRNAME}/../..
+}
+
+# check_result checks if the specified test passed
+# results schema example:
+# {
+#   "check": "has-minkubeversion",
+#   "type": "Mandatory",
+#   "outcome": "PASS",
+#   "reason": "Minimum Kubernetes version specified"
+# }
+check_result() {
+  local -r var="$1"
+  local check=$(cat $VERIFY_OUTPUT | jq -r ".results[] | select(.check==\"${var}\").outcome")
+  [ "$check" = "PASS" ]
+}

test/chart/verifier.bats

@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+setup_file() {
+    cd `chart_dir`
+    export VERIFY_OUTPUT="/$BATS_RUN_TMPDIR/verify.json"
+    export CHART_VOLUME=vault-helm-chart-src
+    # Note: currently `latest` is the only tag available in the chart-verifier repo.
+    local IMAGE="quay.io/redhat-certification/chart-verifier:latest"
+
+    local run_cmd="chart-verifier"
+    local chart_src="."
+
+    if [ ! -e $USE_DOCKER ]; then
+        chart_src="/chart"
+        # Create a dummy container which will hold a volume with chart source
+        docker create -v $chart_src --name $CHART_VOLUME alpine:3 /bin/true
+        # Copy the chart source into this volume
+        docker cp . $CHART_VOLUME:$chart_src
+        # Make sure we have the latest version of chart-verifier
+        docker pull $IMAGE
+        # Start chart-verifier using this volume
+        run_cmd="docker run --rm --volumes-from $CHART_VOLUME $IMAGE"
+    fi
+    
+    $run_cmd verify --output json $chart_src 2>&1 | tee $VERIFY_OUTPUT
+}
+
+teardown_file() {
+    if [ ! -e $USE_DOCKER ]; then
+        docker rm $CHART_VOLUME
+    fi
+}
+
+@test "has-minkubeversion" {
+    check_result has-minkubeversion
+}
+
+@test "is-helm-v3" {
+    check_result is-helm-v3
+}
+
+@test "not-contains-crds" {
+    check_result not-contains-crds
+}
+
+@test "helm-lint" {
+    check_result helm-lint
+}
+
+@test "not-contain-csi-objects" {
+    check_result not-contain-csi-objects
+}
+
+@test "has-readme" {
+    check_result has-readme
+}
+
+@test "contains-values" {
+    check_result contains-values
+}
+
+@test "images-are-certified" {
+    skip "Skipping until this has been addressed"
+    check_result images-are-certified
+}
+
+@test "contains-test" {
+    skip "Skipping until this has been addressed"
+    check_result contains-test
+}
+
+@test "contains-values-schema" {
+    skip "Skipping until this has been addressed"
+    check_result contains-values-schema
+}