From af511549ccbe0ff086fa52cc3d0ceacc70fa5c02 Mon Sep 17 00:00:00 2001
From: Wei Fu <fhfuwei@163.com>
Date: Wed, 18 Jul 2018 23:30:48 +0800
Subject: [PATCH] refactor: reorder hack folder

* refactor hack/build, hack/make.sh and Makefile
* refactor .travis.yml to split testing into three jobs
* use gometalinter to do all the linters

Signed-off-by: Wei Fu <fhfuwei@163.com>
---
 .circleci/config.yml                       |  24 +--
 codecov.yml => .codecov.yml                |   0
 .gitignore                                 |   2 +
 .gometalinter.json                         |  18 ++
 .travis.yml                                |  46 +++--
 Makefile                                   | 205 +++++++++++++--------
 client/image_list.go                       |   1 +
 hack/codegen/swagger.sh                    |  34 ++++
 hack/cri-test/test-cri.sh                  | 117 ------------
 hack/cri-test/test-utils.sh                |  93 ----------
 hack/cri-test/versions                     |   2 -
 hack/install/check.sh                      |  24 +++
 hack/install/install_ci_related.sh         |  57 ++++++
 hack/install/install_containerd.sh         |  61 ++++++
 hack/install/install_critest.sh            |  77 ++++++++
 hack/install/install_dumb_init.sh          |  59 ++++++
 hack/install/install_local_persist.sh      |  52 ++++++
 hack/install/install_lxcfs.sh              |  83 +++++++++
 hack/install/install_nsenter.sh            | 101 ++++++++++
 hack/install/install_runc.sh               |  58 ++++++
 hack/testing/run_daemon_cri_integration.sh | 153 +++++++++++++++
 hack/testing/run_daemon_integration.sh     |  68 +++++++
 hack/testing/utils.sh                      |  54 ++++++
 hack/validate_swagger.sh                   |  23 +++
 test/cli_pause_test.go                     |   3 +-
 25 files changed, 1084 insertions(+), 331 deletions(-)
 rename codecov.yml => .codecov.yml (100%)
 create mode 100644 .gometalinter.json
 create mode 100644 hack/codegen/swagger.sh
 delete mode 100755 hack/cri-test/test-cri.sh
 delete mode 100755 hack/cri-test/test-utils.sh
 delete mode 100755 hack/cri-test/versions
 create mode 100644 hack/install/check.sh
 create mode 100755 hack/install/install_ci_related.sh
 create mode 100755 hack/install/install_containerd.sh
 create mode 100755 hack/install/install_critest.sh
 create mode 100755 hack/install/install_dumb_init.sh
 create mode 100755 hack/install/install_local_persist.sh
 create mode 100755 hack/install/install_lxcfs.sh
 create mode 100755 hack/install/install_nsenter.sh
 create mode 100755 hack/install/install_runc.sh
 create mode 100755 hack/testing/run_daemon_cri_integration.sh
 create mode 100755 hack/testing/run_daemon_integration.sh
 create mode 100644 hack/testing/utils.sh
 create mode 100755 hack/validate_swagger.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cfcf03077..36467a76a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -30,29 +30,12 @@ jobs:
     steps:
       - checkout
       - run: 
-          name: install golint
-          command: go get -u golang.org/x/lint/golint
-      - run: 
-          name: install swagger-0.12.0
-          command: |
-            wget --quiet -O /bin/swagger https://github.com/go-swagger/go-swagger/releases/download/0.12.0/swagger_linux_amd64
-            chmod +x /bin/swagger
+          name: install gometalinter linters tools
+          command: hack/install/install_ci_related.sh
       - run: 
           name: use check target in Makefile to check code
           command: make check
-  
-  unit-test:
-    docker:
-      - image: golang:1.9.1
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run:
-          name: unit test
-          command: make unit-test
-      - run:
-          name: upload code coverage report
-          command: bash <(curl -s https://codecov.io/bash)
+
 notify:
   webhooks:
     - url: http://47.96.190.121:6788/circleci_notifications
@@ -63,4 +46,3 @@ workflows:
     jobs:
       - markdownlint-misspell-shellcheck
       - code-check
-      - unit-test
diff --git a/codecov.yml b/.codecov.yml
similarity index 100%
rename from codecov.yml
rename to .codecov.yml
diff --git a/.gitignore b/.gitignore
index 3f6e3d550..a8557882f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,5 @@ coverage.txt
 .bashrc
 .idea
 .DS_Store
+bin/
+coverage/
diff --git a/.gometalinter.json b/.gometalinter.json
new file mode 100644
index 000000000..c983f8459
--- /dev/null
+++ b/.gometalinter.json
@@ -0,0 +1,18 @@
+{
+  "Enable": [
+    "gofmt",
+    "goimports",
+    "golint",
+    "misspell",
+    "vet"
+  ],
+  "Deadline": "2m",
+  "Sort": ["linter", "severity", "path", "line"],
+  "Exclude": [
+    ".*\\.pb\\.go",
+	"pkg/jsonstream/image_progress\\.go:.*::error: unrecognized printf verb 'r'"
+  ],
+  "EnableGC": true,
+  "WarnUnmatchedDirective": true,
+  "Vendor": true
+}
diff --git a/.travis.yml b/.travis.yml
index 01bbe4561..6aac051fa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,22 +14,34 @@ notifications:
     on_failure: always
     on_error: always
 
-before_install:
-  |
-    git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' || {
-      echo travis doesn\'t run for document-only changed
-      exit
-    }
-
-    sudo apt-get update -qq
-    sudo apt-get install -y -qq autoconf automake
-
 env:
-  - TEST_SUITE=integration-test
-  - TEST_SUITE=cri-test
-
-script:
-  - sudo make -e ${TEST_SUITE}
-
+  - TEST_SUITE=unittest
+  - TEST_SUITE=integrationtest
+  - TEST_SUITE=critest
+
+script: |
+  if [[ "${TEST_SUITE}" = "unittest" ]]; then
+    hack/install/install_ci_related.sh
+    make unit-test
+    make coverage
+  elif [[ "${TEST_SUITE}" = "integrationtest" ]]; then
+    make build
+    make build-daemon-integration
+    TEST_FLAGS= make build-integration-test
+    sudo env "PATH=$PATH" make install
+
+    sudo env "PATH=$PATH" make download_dependencies
+    sudo env "PATH=$PATH" make integration-test
+    make coverage
+  else
+    make build
+    TEST_FLAGS= make build-daemon-integration
+    sudo env "PATH=$PATH" make install
+
+    sudo env "PATH=$PATH" make download_dependencies
+    sudo env "PATH=$PATH" make cri-test
+    make coverage
+  fi
+  
 after_success:
-- bash <(curl -s https://codecov.io/bash) || echo Codecov_Did_Not_Collect_Coverage_Reports
+  - bash <(curl -s https://codecov.io/bash) -cF ${TEST_SUITE} -y .codecov.yml
diff --git a/Makefile b/Makefile
index 3b2242cec..a6d2480fa 100644
--- a/Makefile
+++ b/Makefile
@@ -1,115 +1,160 @@
-# Go parameters
-GOBUILD=go build
-GOCLEAN=go clean
-GOTEST=go test
-GOPACKAGES=$(shell go list ./... | grep -v /vendor/ | sed 's/^_//')
-
-# Binary name of CLI and Daemon
-BINARY_NAME=pouchd
+# TEST_FLAGS used as flags of go test.
+TEST_FLAGS ?= -v --race
+
+# DAEMON_BINARY_NAME is the name of binary of daemon.
+DAEMON_BINARY_NAME=pouchd
+
+# CLI_BINARY_NAME is the name of binary of pouch client.
 CLI_BINARY_NAME=pouch
 
-# Base path used to install pouch & pouchd
-DESTDIR=/usr/local
+# DAEMON_INTEGRATION_BINARY_NAME is the name of test binary of daemon.
+DAEMON_INTEGRATION_BINARY_NAME=pouchd-integration
 
-.PHONY: build
-build: server client
+# INTEGRATION_TESTCASE_BINARY_NAME is the name of binary of integration cases.
+INTEGRATION_TESTCASE_BINARY_NAME=pouchd-integration-test
 
-.PHONY: pre
-pre:
-	@./hack/build pre
+# DEST_DIR is base path used to install pouch & pouchd
+DEST_DIR=/usr/local
 
-.PHONY: server
-server: pre modules
-	@./hack/build server
+# the following variables used for the daemon build
 
-.PHONY: client
-client: pre
-	@./hack/build client
+# API_VERSION is used for Daemon API Version in go build.
+API_VERSION="1.24"
 
-.PHONY: testserver
-testserver: pre modules
-	@./hack/build testserver
+# VERSION is used for Daemon Release Version in go build.
+VERSION ?= "1.0.0-rc1"
 
-.PHONY: clean
-clean:
-	$(GOCLEAN)
-	rm -f $(BINARY_NAME)
-	rm -f $(CLI_BINARY_NAME)
-	./hack/build clean
-	./hack/module --clean
+# GIT_COMMIT is used for Daemon GitCommit in go build.
+GIT_COMMIT=$(shell git describe --dirty --always --tags 2> /dev/null || true)
 
-.PHONY: check
-check: pre fmt lint vet validate-swagger
+# BUILD_TIME is used for Daemon BuildTime in go build.
+BUILD_TIME=$(shell date --rfc-3339 s 2> /dev/null | sed -e 's/ /T/')
 
-.PHONY: fmt
-fmt: ## run go fmt
-	@echo $@
-	@which gofmt
-	@test -z "$$(gofmt -s -l . 2>/dev/null | grep -Fv 'vendor/' | grep -v ".pb.go$$" | tee /dev/stderr)" || \
-		(echo "please format Go code with 'gofmt -s -w'" && false)
-	@test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
-		(echo "please indent proto files with tabs only" && false)
-	@test -z "$$(find . -path ./vendor -prune -o -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
-		(echo "meta fields in proto files must have option (gogoproto.nullable) = false" && false)
-
-.PHONY: lint
-lint: ## run go lint
+VERSION_PKG=github.com/alibaba/pouch
+DEFAULT_LDFLAGS="-X ${VERSION_PKG}/version.GitCommit=${GIT_COMMIT} \
+		  -X ${VERSION_PKG}/version.Version=${VERSION} \
+		  -X ${VERSION_PKG}/version.ApiVersion=${API_VERSION} \
+		  -X ${VERSION_PKG}/version.BuildTime=${BUILD_TIME}"
+
+# COVERAGE_PACKAGES is the coverage we care about.
+COVERAGE_PACKAGES=$(shell go list ./... | \
+				  grep -v github.com/alibaba/pouch$$ | \
+				  grep -v github.com/alibaba/pouch/storage/volume/examples/demo | \
+				  grep -v github.com/alibaba/pouch/test | \
+				  grep -v github.com/alibaba/pouch/cli | \
+				  grep -v github.com/alibaba/pouch/cri/apis | \
+				  grep -v github.com/alibaba/pouch/apis/types )
+
+COVERAGE_PACKAGES_LIST=$(shell echo $(COVERAGE_PACKAGES) | tr " " ",")
+
+build: build-daemon build-cli ## build PouchContainer both daemon and cli binaries
+
+build-daemon: modules ## build PouchContainer daemon binary
 	@echo $@
-	@which golint
-	@test -z "$$(golint ./... | grep -Fv 'vendor/' | grep -v ".pb.go:" | tee /dev/stderr)"
+	@mkdir -p bin
+	GOOS=linux go build -ldflags ${DEFAULT_LDFLAGS} -o bin/${DAEMON_BINARY_NAME} -tags 'selinux' 
 
-.PHONY: vet
-vet: ## run go vet
+build-cli: ## build PouchContainer cli binary
 	@echo $@
-	@test -z "$$(./hack/build vet)"
+	@mkdir -p bin
+	go build -o bin/${CLI_BINARY_NAME} github.com/alibaba/pouch/cli
 
-.PHONY: unit-test
-unit-test: pre modules ## run go test
+build-daemon-integration: modules ## build PouchContainer Daemon Integration Testing binary
 	@echo $@
-	@./hack/build unit-test
+	@mkdir -p bin
+	go test -c ${TEST_FLAGS} \
+		-cover -covermode=atomic -coverpkg ${COVERAGE_PACKAGES_LIST} \
+		-o bin/${DAEMON_INTEGRATION_BINARY_NAME}
 
-.PHONY: validate-swagger
-validate-swagger: ## run swagger validate
+build-integration-test: modules ## build PouchContainer Integration test-case binary
 	@echo $@
-	@swagger validate apis/swagger.yml
+	@mkdir -p bin
+	go test -c \
+		-o bin/${INTEGRATION_TESTCASE_BINARY_NAME} github.com/alibaba/pouch/test
 
-.PHONY: modules
-modules:
+modules: ## run modules to generate volume related code
+	@echo $@
 	@./hack/module --clean
 	@./hack/module --add-volume=github.com/alibaba/pouch/storage/volume/modules/tmpfs
 	@./hack/module --add-volume=github.com/alibaba/pouch/storage/volume/modules/local
 
-# build binaries
-# install them to /usr/local/bin/
-# remove binaries
-.PHONY: install
-install: build ## build and install binary into /usr/local/bin
+install: ## install pouch and pouchd binary into /usr/local/bin
 	@echo $@
-	@echo "installing $(BINARY_NAME) and $(CLI_BINARY_NAME) to $(DESTDIR)/bin"
-	@mkdir -p $(DESTDIR)/bin
-	@install $(BINARY_NAME) $(DESTDIR)/bin
-	@install $(CLI_BINARY_NAME) $(DESTDIR)/bin
+	@mkdir -p $(DEST_DIR)/bin
+	install bin/$(CLI_BINARY_NAME) $(DEST_DIR)/bin
+	install bin/$(DAEMON_BINARY_NAME) $(DEST_DIR)/bin
 
-.PHONY: uninstall
 uninstall: ## uninstall pouchd and pouch binary
 	@echo $@
-	@rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARY_NAME)))
-	@rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(CLI_BINARY_NAME)))
+	@rm -f $(addprefix $(DEST_DIR)/bin/,$(notdir $(DAEMON_BINARY_NAME)))
+	@rm -f $(addprefix $(DEST_DIR)/bin/,$(notdir $(CLI_BINARY_NAME)))
+
+.PHONY: download_dependencies
+download_dependencies:
+	@echo $@
+	hack/install/install_ci_related.sh
+	hack/install/install_containerd.sh
+	hack/install/install_dumb_init.sh
+	hack/install/install_local_persist.sh
+	hack/install/install_lxcfs.sh
+	hack/install/install_nsenter.sh
+	hack/install/install_runc.sh
+
+.PHONY: clean
+clean: ## clean to remove bin/* and files created by module
+	@go clean
+	@rm -f bin/*
+	@rm -rf coverage/*
+	@./hack/module --clean
+
+
+.PHONY: check
+check: gometalinter validate-swagger ## run all linters
+
+.PHONY: validate-swagger
+validate-swagger: ## run swagger validate
+	@echo $@
+	./hack/validate_swagger.sh
+
+# gometalinter consumes .gometalinter.json as config.
+.PHONY: gometalinter
+gometalinter: ## run gometalinter for go source code
+	@echo $@
+	gometalinter --config .gometalinter.json ./...
+
+
+.PHONY: unit-test
+unit-test: modules ## run go unit-test
+	@echo $@
+	@mkdir -p coverage
+	@( for pkg in ${COVERAGE_PACKAGES}; do \
+		go test ${TEST_FLAGS} \
+			-cover -covermode=atomic \
+			-coverprofile=coverage/unit-test-`echo $$pkg | tr "/" "_"`.out \
+			$$pkg || exit; \
+	done )
 
-# For integration-test and test, PATH is not set under sudo, then we set up path mannually.
-# Ref https://unix.stackexchange.com/questions/83191/how-to-make-sudo-preserve-path
 .PHONY: integration-test
-integration-test: ## build binary and run integration-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build integration-test"
+integration-test: ## run daemon integration-test
+	@echo $@
+	@mkdir -p coverage
+	./hack/testing/run_daemon_integration.sh
 
 .PHONY: cri-test
-cri-test: ## build binary and run cri-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build cri-test"
+cri-test: ## run v1 alpha2 cri-test
+	@echo $@
+	@mkdir -p coverage
+	./hack/testing/run_daemon_cri_integration.sh
 
 .PHONY: test
-test: ## run the build integration-test cri-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build integration-test cri-test"
+test: unit-test integration-test cri-test ## run the unit-test, integration-test and cri-test
+
+.PHONY: coverage
+coverage: ## combine coverage after test
+	@echo $@
+	@gocovmerge coverage/* > coverage.txt
+
 
 .PHONY: help
 help: ## this help
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-28s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
diff --git a/client/image_list.go b/client/image_list.go
index bbf34516e..87bddf78c 100644
--- a/client/image_list.go
+++ b/client/image_list.go
@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+
 	"github.com/alibaba/pouch/apis/types"
 )
 
diff --git a/hack/codegen/swagger.sh b/hack/codegen/swagger.sh
new file mode 100644
index 000000000..a1620d9f7
--- /dev/null
+++ b/hack/codegen/swagger.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly SWAGGER_VERSION=0.12.0
+
+# swagger::check_version checks the command and the version.
+swagger::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v swagger || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    return
+  fi
+
+  version="$(swagger version | head -n 1 | cut -d " " -f 2)"
+  if [[ "${SWAGGER_VERSION}" != "${version}" ]]; then
+    echo false
+    return
+  fi
+  echo true
+}
+
+# swagger::install installs the swagger binary.
+swagger::install() {
+  echo ">>>> install swagger-${SWAGGER_VERSION} <<<<"
+  local url
+  url="https://github.com/go-swagger/go-swagger/releases/download"
+  url="${url}/${SWAGGER_VERSION}/swagger_linux_amd64"
+
+  wget --quiet -O /usr/local/bin/swagger "${url}"
+  chmod +x /usr/local/bin/swagger
+}
diff --git a/hack/cri-test/test-cri.sh b/hack/cri-test/test-cri.sh
deleted file mode 100755
index 51b9f574f..000000000
--- a/hack/cri-test/test-cri.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o nounset
-set -o pipefail
-
-source "$(dirname "${BASH_SOURCE[0]}")/test-utils.sh"
-
-POUCH_SOCK="/var/run/pouchcri.sock"
-
-# CRI_FOCUS focuses the test to run.
-# With the CRI manager completes its function, we may need to expand this field.
-CRI_FOCUS=${CRI_FOCUS:-}
-
-# CRI_SKIP skips the test to skip.
-CRI_SKIP=${CRI_SKIP:-"RunAsUserName|seccomp localhost|should error on create with wrong options|runtime should support reopening container log"}
-# REPORT_DIR is the the directory to store test logs.
-REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"}
-
-# Check GOPATH
-if [[ -z "${GOPATH}" ]]; then
-  echo "GOPATH is not set"
-  exit 1
-fi
-
-# For multiple GOPATHs, keep the first one only
-GOPATH=${GOPATH%%:*}
-
-# Install CNI first
-mkdir -p /etc/cni/net.d /opt/cni/bin
-
-git clone https://github.com/containernetworking/plugins "$GOPATH/src/github.com/containernetworking/plugins"
-cd "$GOPATH/src/github.com/containernetworking/plugins"
-
-./build.sh
-cp bin/* /opt/cni/bin
-
-# Create CNI configuration file
-sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
-{
-    "cniVersion": "0.3.1",
-    "name": "mynet",
-    "plugins": [
-        {
-            "type": "bridge",
-            "bridge": "cni0",
-            "isGateway": true,
-            "ipMasq": true,
-            "ipam": {
-                "type": "host-local",
-                "subnet": "10.30.0.0/16",
-                "routes": [
-                    { "dst": "0.0.0.0/0"   }
-                ]
-            }
-        },
-        {
-            "type": "portmap",
-            "capabilities": {"portMappings": true},
-            "snat": true
-        }
-    ]
-}
-EOF'
-
-sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
-{
-    "cniVersion": "0.3.1",
-    "type": "loopback"
-}
-EOF'
-
-CRITEST=${GOPATH}/bin/critest
-CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools
-GINKGO=${GOPATH}/bin/ginkgo
-GINKGO_PKG=github.com/onsi/ginkgo/ginkgo
-
-# Install critest
-if [ ! -x "$(command -v "${CRITEST}")" ]; then
-  go get -d ${CRITOOL_PKG}/...
-  cd "${GOPATH}/src/${CRITOOL_PKG}"
-  git fetch --all
-  git checkout "${CRITOOL_VERSION}"
-  make
-fi
-which "${CRITEST}"
-
-# Install ginkgo
-if [ ! -x "$(command -v "${GINKGO}")" ]; then
-  go get -u ${GINKGO_PKG}
-fi
-which "${GINKGO}"
-
-mkdir -p "${REPORT_DIR}"
-test_setup "${REPORT_DIR}"
-
-# Run cri validation test
-sudo env PATH="${PATH}" GOPATH="${GOPATH}" "${CRITEST}" --runtime-endpoint=${POUCH_SOCK} --ginkgo.focus="${CRI_FOCUS}" --ginkgo.skip="${CRI_SKIP}"
-test_exit_code=$?
-
-test_teardown
-
-exit ${test_exit_code}
-
diff --git a/hack/cri-test/test-utils.sh b/hack/cri-test/test-utils.sh
deleted file mode 100755
index 8b5b21c22..000000000
--- a/hack/cri-test/test-utils.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-. "${ROOT}/versions"
-# POUCH_FLAGS are the extra flags to use when start pouchd.
-POUCH_FLAGS=${POUCH_FLAGS:-""}
-# RESTART_WAIT_PERIOD is the period to wait before restarting pouchd/containerd.
-RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
-
-# POUCH_SOCK=/var/run/pouchcri.sock
-
-pouch_pid=
-containerd_pid=
-
-# test_setup starts containerd and pouchd.
-test_setup() {
-  local report_dir=$1 
-
-  # Start containerd
-  if [ ! -x "$(command -v containerd)" ]; then
-    echo "containerd is not installed, please run hack/make.sh"
-    exit 1
-  fi
-  containerd_pid_command=$(pgrep containerd)
-  containerd_pid=${containerd_pid_command}
-  if [ ! -n "${containerd_pid}" ]; then
-    keepalive "/usr/local/bin/containerd" "${RESTART_WAIT_PERIOD}" &> "${report_dir}/containerd.log" &
-    containerd_pid=$!
-  fi
-  # Wait for containerd to be running by using the containerd client ctr to check the version
-  # of the containerd server. Wait an increasing amount of time after each of five attempts
-  readiness_check "ctr version"
-
-  # Start pouchd
-  pouch_pid_command=$(pgrep pouchd)
-  pouch_pid=${pouch_pid_command}
-  if [ ! -n "${pouch_pid}" ]; then
-    keepalive "pouchd --enable-cri --sandbox-image=gcr.io/google_containers/pause-amd64:3.0 ${POUCH_FLAGS}" \
-	  "${RESTART_WAIT_PERIOD}" &> "${report_dir}/pouch.log" &
-    pouch_pid=$!
-  fi
-  readiness_check "pouch version"
-}
-
-# test_teardown kills containerd and cri-containerd.
-test_teardown() {
-  if [ -n "${containerd_pid}" ]; then
-    kill ${containerd_pid}
-  fi
-  if [ -n "${pouch_pid}" ]; then
-    kill ${pouch_pid}
-  fi
-  sudo pkill containerd
-}
-
-# keepalive runs a command and keeps it alive.
-# keepalive process is eventually killed in test_teardown.
-keepalive() {
-  local command=$1
-  echo "${command}"
-  local wait_period=$2
-  while true; do
-    ${command}
-    sleep "${wait_period}"
-  done
-}
-
-# readiness_check checks readiness of a daemon with specified command.
-readiness_check() {
-  local command=$1
-  local MAX_ATTEMPTS=5
-  local attempt_num=1
-  until ${command} &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
-  do
-      echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..."
-      sleep $(( attempt_num++ ))
-  done
-}
-
diff --git a/hack/cri-test/versions b/hack/cri-test/versions
deleted file mode 100755
index cef334d7e..000000000
--- a/hack/cri-test/versions
+++ /dev/null
@@ -1,2 +0,0 @@
-CRITOOL_VERSION=v1.0.0-beta.0
-
diff --git a/hack/install/check.sh b/hack/install/check.sh
new file mode 100644
index 000000000..190c1b7e1
--- /dev/null
+++ b/hack/install/check.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# detect_os return linux OS distribution.
+#
+# NOTE: only support Ubuntu/Debian/Centos/Redhat
+detect_os() {
+  local os dist
+
+  os="$(uname -s | tr "[:upper:]" "[:lower:]")"
+
+  if [[ "${os}" = "linux" ]]; then
+    if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then
+      dist="$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//)"
+    elif [ -f /etc/redhat-release -o -f /etc/centos-release ]; then
+      dist="RedHat"
+    fi
+  fi
+
+  if [ -z "${dist}" ]; then
+    echo >&2 "failed to detect os distribution"
+    exit 1
+  fi
+  echo "${dist}"
+}
diff --git a/hack/install/install_ci_related.sh b/hack/install/install_ci_related.sh
new file mode 100755
index 000000000..daa3bb8f2
--- /dev/null
+++ b/hack/install/install_ci_related.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+source "../codegen/swagger.sh"
+
+# ci_related::install_gometalinter installs gometalinter for linters.
+ci_related::install_gometalinter() {
+  local has_installed
+
+  has_installed="$(command -v gometalinter || echo false)"
+  if [[ "${has_installed}" != "false" ]]; then
+    echo "gometalinter has been installed."
+    return
+  fi
+
+  go get -u github.com/alecthomas/gometalinter
+  gometalinter --install > /dev/null
+}
+
+# ci_related::install_gocovmerge installs gocovmerge for coverage combine.
+ci_related::install_gocovmerge() {
+  local has_installed
+
+  has_installed="$(command -v gocovmerge || echo false)"
+  if [[ "${has_installed}" != "false" ]]; then
+    echo "gocovmerge has been installed."
+    return
+  fi
+
+  go get -u github.com/wadey/gocovmerge
+}
+
+# ci_related::install_swagger installs swagger binary.
+ci_related::install_swagger() {
+  local has_installed
+
+  has_installed="$(swagger::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "swagger-${SWAGGER_VERSION} has been installed."
+    exit 0
+  fi
+
+  swagger::install
+  command -v swagger
+}
+
+main() {
+  echo "instsall CI related tools..."
+  ci_related::install_gometalinter
+  ci_related::install_gocovmerge
+  ci_related::install_swagger
+  echo
+}
+
+main
diff --git a/hack/install/install_containerd.sh b/hack/install/install_containerd.sh
new file mode 100755
index 000000000..6e9b4a139
--- /dev/null
+++ b/hack/install/install_containerd.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly CONTAINERD_VERSION="1.0.3"
+
+# containerd::check_version checks the command and the version.
+containerd::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v containerd || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+
+  version="$(containerd -v | cut -d " " -f 3 | cut -c 2-)"
+  if [[ "${CONTAINERD_VERSION}" != "${version}" ]]; then
+    echo false
+    exit 0
+  fi
+
+  echo true
+}
+
+# containerd::install downloads the binary from release url.
+containerd::install() {
+  local url target tmpdir
+
+  target="containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz"
+  url="https://github.com/containerd/containerd/releases/download"
+  url="${url}/v${CONTAINERD_VERSION}/${target}"
+
+  tmpdir="$(mktemp -d /tmp/containerd-install-XXXXXX)"
+  trap 'rm -rf /tmp/containerd-install-*' EXIT
+
+  wget --quiet "${url}" -P "${tmpdir}"
+  tar xf "${tmpdir}/${target}" -C "${tmpdir}"
+  cp -f "${tmpdir}"/bin/* /usr/local/bin/
+}
+
+main() {
+  local has_installed
+
+  has_installed="$(containerd::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "containerd-${CONTAINERD_VERSION} has been installed."
+    exit 0
+  fi
+
+  echo ">>>> install containerd-${CONTAINERD_VERSION} <<<<"
+
+  containerd::install
+
+  # final check
+  command -v containerd > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/install/install_critest.sh b/hack/install/install_critest.sh
new file mode 100755
index 000000000..5988f4f56
--- /dev/null
+++ b/hack/install/install_critest.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly CRITEST_VERSION=1.0.0-beta.0
+
+# keep the first one only
+GOPATH="${GOPATH%%:*}"
+
+# add bin folder into PATH.
+export PATH="${GOPATH}/bin:${PATH}"
+
+# critest::check_version checks the command and the version.
+critest::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v critest || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+
+  version="$(critest -version | head -n 1 | cut -d " " -f 3)"
+  if [[ "${CRITEST_VERSION}" != "${version}" ]]; then
+    echo false
+    exit 0
+  fi
+
+  echo true
+}
+
+# critest::install downloads the package and build.
+critest::install() {
+  local workdir pkg
+
+  pkg="github.com/kubernetes-incubator/cri-tools"
+  workdir="${GOPATH}/src/${pkg}"
+
+  go get -d "${pkg}"/...
+  cd "${workdir}"
+  git fetch --all
+  git checkout "v${CRITEST_VERSION}"
+  make
+  cd -
+}
+
+# critest::install_ginkgo installs ginkgo if missing.
+critest::install_ginkgo() {
+  local has_installed pkg
+
+  pkg="github.com/onsi/ginkgo/ginkgo"
+  has_installed="$(command -v ginkgo || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    go get -u "${pkg}"
+  fi
+
+  command -v ginkgo > /dev/null
+}
+
+main() {
+  critest::install_ginkgo
+
+  local has_installed
+  has_installed="$(critest::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "critest-${CRITEST_VERSION} has been installed."
+    exit 0
+  fi
+
+  echo ">>>> install critest-${CRITEST_VERSION} <<<<"
+  critest::install
+
+  command -v critest > /dev/null
+  command -v ginkgo > /dev/null
+}
+
+main
diff --git a/hack/install/install_dumb_init.sh b/hack/install/install_dumb_init.sh
new file mode 100755
index 000000000..69776af3a
--- /dev/null
+++ b/hack/install/install_dumb_init.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly DUMB_INIT_VERSION="1.2.1"
+
+# dumb_init::check_version checks the command and the version.
+dumb_init::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v dumb-init || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+
+  version="$(dumb-init -V 2>&1 | cut -d " " -f 2 | cut -c 2-)"
+  if [[ "${DUMB_INIT_VERSION}" != "${version}" ]]; then
+    echo false
+    exit 0
+  fi
+
+  echo true
+}
+
+# dumb_init::install downloads the binary from release url.
+dumb_init::install() {
+  local url target
+
+  target="/tmp/dumb-init"
+
+  url="https://github.com/Yelp/dumb-init/releases/download"
+  url="${url}/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_amd64"
+
+  wget --quiet -O "${target}" "${url}"
+  mv "${target}" /usr/bin/
+  chmod +x /usr/bin/dumb-init
+}
+
+main() {
+  local has_installed
+
+  has_installed="$(dumb_init::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "dumb-init-${DUMB_INIT_VERSION} has been installed."
+    exit 0
+  fi
+
+  echo ">>>> install dumb-init-${DUMB_INIT_VERSION} <<<<"
+
+  dumb_init::install
+
+  # final check
+  command -v dumb-init > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/install/install_local_persist.sh b/hack/install/install_local_persist.sh
new file mode 100755
index 000000000..459000e8d
--- /dev/null
+++ b/hack/install/install_local_persist.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly LOCAL_PERSIST_VERSION="1.3.0"
+readonly cmd="local-persist"
+
+# local_persist::check_version checks the command and the version.
+local_persist::check_version() {
+  local has_installed
+
+  has_installed="$(command -v local-persist || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+  echo true
+}
+
+# local_persist::install downloads the binary from release url.
+local_persist::install() {
+  local url target
+
+  target="/tmp/${cmd}"
+  url="https://github.com/CWSpear/local-persist/releases/download"
+  url="${url}/v${LOCAL_PERSIST_VERSION}/local-persist-linux-amd64"
+
+  wget --quiet -O "${target}" "${url}"
+  chmod +x "${target}"
+  mv "${target}" /usr/bin/
+}
+
+main() {
+  local has_installed
+
+  has_installed="$(local_persist::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "${cmd}-${LOCAL_PERSIST_VERSION} has been installed."
+    exit 0
+  fi
+
+  echo ">>>> install ${cmd}-${LOCAL_PERSIST_VERSION} <<<<"
+
+  local_persist::install
+
+  # final check
+  command -v "${cmd}" > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/install/install_lxcfs.sh b/hack/install/install_lxcfs.sh
new file mode 100755
index 000000000..876f5c6b5
--- /dev/null
+++ b/hack/install/install_lxcfs.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+source "./check.sh"
+
+# lxcfs::centos::install_dependencies will use yum to install dependencies.
+lxcfs::centos::install_dependencies() {
+  local arch
+  arch="$(uname -p)"
+
+  yum install -y -q \
+    autobook \
+    autoconf-doc \
+    autoconf2.13 \
+    autotools-dev \
+    autoconf-archive \
+    git \
+    gnu-standards \
+    libtool \
+    make \
+    m4 \
+    "fuse-devel.${arch}" \
+    "pam-devel.${arch}" \
+    "fuse.${arch}"
+}
+
+# lxcfs::centos::pull_build will pull stable-2.0 to build lxcfs.
+lxcfs::centos::pull_build() {
+  local tmpdir branch
+
+  tmpdir="$(mktemp -d /tmp/lxcfs-build-XXXXXX)"
+  branch="stable-2.0"
+
+  trap 'rm -rf /tmp/lxcfs-build-*' EXIT
+
+  git clone -b "${branch}" https://github.com/lxc/lxcfs.git "${tmpdir}/lxcfs"
+  cd "${tmpdir}/lxcfs"
+
+  ./bootstrap.sh
+  ./configure
+  make install
+
+  ln -s /usr/local/bin/lxcfs /usr/bin/lxcfs
+  mkdir -p /var/lib/lxcfs
+}
+
+# lxcfs::ubuntu::install will use repository manager to install lxcfs.
+#
+# FIXME: It is 3.0.0 version which different with stable-2.0 in centos!!!!!!!
+lxcfs::ubuntu::install() {
+  add-apt-repository -y ppa:ubuntu-lxc/lxcfs-stable
+  apt-get update -q -y
+  apt-get install -y -q lxcfs
+}
+
+main() {
+  local os_dist has_installed
+
+  has_installed="$(command -v lxcfs || echo false)"
+  if [[ "${has_installed}" != "false" ]]; then
+    echo "lxcfs has been installed!"
+    exit 0
+  fi
+
+  echo ">>>> install lxcfs <<<<"
+
+  os_dist="$(detect_os)"
+  if [[ "${os_dist}" = "Ubuntu" ]]; then
+    lxcfs::ubuntu::install > /dev/null
+  else
+    lxcfs::centos::install_dependencies > /dev/null
+    lxcfs::centos::pull_build > /dev/null
+  fi
+
+  # final check
+  command -v lxcfs > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/install/install_nsenter.sh b/hack/install/install_nsenter.sh
new file mode 100755
index 000000000..0afb2074b
--- /dev/null
+++ b/hack/install/install_nsenter.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+source "./check.sh"
+
+readonly NSENTER_VERSION="2.24.1"
+readonly cmd="nsenter"
+
+# nsenter::check_version checks the command and the version.
+nsenter::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v nsenter1 || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+
+  version="$(nsenter -V | cut -d " " -f 4)"
+  if [[ "${NSENTER_VERSION}" != "${version}" ]]; then
+    echo false
+    exit 0
+  fi
+
+  echo true
+}
+
+# nsenter::ubuntu::install_dependencies will use apt-get to install dependencies.
+nsenter::ubuntu::install_dependencies() {
+  apt-get install -y -q \
+    libncurses5-dev \
+    libslang2-dev \
+    gettext \
+    zlib1g-dev \
+    libselinux1-dev \
+    debhelper \
+    lsb-release \
+    pkg-config \
+    po-debconf \
+    autoconf \
+    automake \
+    autopoint \
+    libtool
+}
+
+# nsenter::ubuntu::install will install nsenter.
+nsenter::ubuntu::install() {
+  local url target tmpdir
+
+  target="util-linux-${NSENTER_VERSION}.tar.gz"
+  url="https://www.kernel.org/pub/linux/utils/util-linux/v2.24"
+  url="${url}/${target}"
+
+  tmpdir="$(mktemp -d /tmp/nsenter-install-XXXXXX)"
+  trap 'rm -rf /tmp/nsenter-install-*' EXIT
+
+  wget --quiet "${url}" -P "${tmpdir}"
+  tar xf "${tmpdir}/${target}" -C "${tmpdir}"
+
+  cd "${tmpdir}/util-linux-${NSENTER_VERSION}"
+  ./autogen.sh
+  autoreconf -vfi
+  ./configure
+  make
+
+  cp "${cmd}" /usr/local/bin
+}
+
+# nsenter::centos::install will install nsenter.
+nsenter::centos::install() {
+  yum install -y -q util-linux
+}
+
+main() {
+  local os_dist has_installed
+
+  has_installed="$(nsenter::check_version)"
+  if [[ "${has_installed}" == "true" ]]; then
+    echo "${cmd}-${NSENTER_VERSION} has been installed!"
+    exit 0
+  fi
+
+  echo ">>>> install ${cmd}-${NSENTER_VERSION} <<<<"
+
+  os_dist="$(detect_os)"
+  if [[ "${os_dist}" = "Ubuntu" ]]; then
+    nsenter::ubuntu::install_dependencies > /dev/null
+    nsenter::ubuntu::install > /dev/null
+  else
+    nsenter::centos::install > /dev/null
+  fi
+
+  # final check
+  command -v "${cmd}" > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/install/install_runc.sh b/hack/install/install_runc.sh
new file mode 100755
index 000000000..22631aed4
--- /dev/null
+++ b/hack/install/install_runc.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly RUNC_VERSION="1.0.0-rc4-1"
+
+# runc::check_version checks the command and the version.
+runc::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v runc || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    exit 0
+  fi
+
+  version="$(runc -v | head -1 | cut -d " " -f 3)"
+  if [[ "${RUNC_VERSION}" != "${version}" ]]; then
+
+    echo false
+    exit 0
+  fi
+
+  echo true
+}
+
+# runc::install downloads the binary from release url.
+runc::install() {
+  local url
+
+  url="https://github.com/alibaba/runc/releases/download"
+  url="${url}/v${RUNC_VERSION}/runc.amd64"
+
+  wget --quiet "${url}" -P /usr/local/bin
+  chmod +x /usr/local/bin/runc.amd64
+  mv /usr/local/bin/runc.amd64 /usr/local/bin/runc
+}
+
+main() {
+  local has_installed
+
+  has_installed="$(runc::check_version)"
+  if [[ "${has_installed}" = "true" ]]; then
+    echo "runc-${RUNC_VERSION} has been installed."
+    exit 0
+  fi
+
+  echo ">>>> install runc-${RUNC_VERSION} <<<<"
+
+  runc::install
+
+  # final check
+  command -v runc > /dev/null
+
+  echo
+}
+
+main
diff --git a/hack/testing/run_daemon_cri_integration.sh b/hack/testing/run_daemon_cri_integration.sh
new file mode 100755
index 000000000..0fa9ec368
--- /dev/null
+++ b/hack/testing/run_daemon_cri_integration.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+source utils.sh
+
+cd ../../
+readonly REPO_BASE="$(pwd -P)"
+
+# keep the first one only
+GOPATH="${GOPATH%%:*}"
+
+# add bin folder into PATH so that pouch-integration is available.
+export PATH="${REPO_BASE}/bin:${PATH}"
+
+# add bin folder into PATH.
+export PATH="${GOPATH}/bin:${PATH}"
+
+# CRI_SKIP skips the test to skip.
+DEFAULT_CRI_SKIP="RunAsUserName|seccomp localhost"
+DEFAULT_CRI_SKIP="${DEFAULT_CRI_SKIP}|should error on create with wrong options"
+DEFAULT_CRI_SKIP="${DEFAULT_CRI_SKIP}|runtime should support reopening container log"
+CRI_SKIP="${CRI_SKIP:-"${DEFAULT_CRI_SKIP}"}"
+
+# CRI_FOCUS focuses the test to run.
+# With the CRI manager completes its function, we may need to expand this field.
+CRI_FOCUS=${CRI_FOCUS:-}
+
+POUCH_SOCK="/var/run/pouchcri.sock"
+
+# tmplog_dir stores the background job log data
+tmplog_dir="$(mktemp -d /tmp/integration-daemon-cri-testing-XXXXX)"
+pouchd_log="${tmplog_dir}/pouchd.log"
+local_persist_log="${tmplog_dir}/local_persist.log"
+trap 'rm -rf /tmp/integration-daemon-cri-testing-*' EXIT
+
+# daemon cri integration coverage profile
+coverage_profile="${REPO_BASE}/coverage/integration_daemon_cri_profile.out"
+rm -rf "${coverage_profile}"
+
+
+# integration::install_critest installs test case.
+integration::install_critest() {
+  hack/install/install_critest.sh
+}
+
+# integration::install_cni installs cni plugins.
+integration::install_cni() {
+  echo "install cni..."
+
+  local workdir pkg
+
+  # for multiple GOPATHs, keep the first one only
+  pkg="github.com/containernetworking/plugins"
+  workdir="${GOPATH}/src/${pkg}"
+
+  # downloads github.com/containernetworking/plugins
+  go get -u -d "${pkg}"/...
+
+  # build and copy into /opt/cni/bin
+  "${workdir}"/build.sh
+  mkdir -p /etc/cni/net.d /opt/cni/bin
+  cp "${workdir}"/bin/* /opt/cni/bin
+
+  # setup the config
+  sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
+{
+    "cniVersion": "0.3.1",
+    "name": "mynet",
+    "plugins": [
+        {
+            "type": "bridge",
+            "bridge": "cni0",
+            "isGateway": true,
+            "ipMasq": true,
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.30.0.0/16",
+                "routes": [
+                    { "dst": "0.0.0.0/0"   }
+                ]
+            }
+        },
+        {
+            "type": "portmap",
+            "capabilities": {"portMappings": true},
+            "snat": true
+        }
+    ]
+}
+EOF'
+
+  sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
+{
+    "cniVersion": "0.3.1",
+    "type": "loopback"
+}
+EOF'
+
+  echo
+}
+
+
+# integration::run_daemon_cri_test_cases runs CRI test cases.
+integration::run_daemon_cri_test_cases() {
+  echo "start pouch daemon cri integration test..."
+  local code
+
+  set +e
+  critest --runtime-endpoint=${POUCH_SOCK} \
+    --ginkgo.focus="${CRI_FOCUS}" --ginkgo.skip="${CRI_SKIP}"
+  code=$?
+
+  integration::stop_local_persist
+  integration::stop_pouchd
+  set -e
+
+  if [[ "${code}" != "0" ]]; then
+    echo "failed to pass integration cases!"
+    echo "there is daemon logs..."
+    cat "${pouchd_log}"
+    exit ${code}
+  fi
+
+  # sleep for pouchd stop and got the coverage
+  sleep 5
+}
+
+main() {
+  local cmd flags
+  cmd="pouchd-integration"
+  flags=" -test.coverprofile=${coverage_profile} DEVEL"
+  flags="${flags} --enable-cri --sandbox-image=gcr.io/google_containers/pause-amd64:3.0"
+
+  integration::install_cni
+  integration::install_critest
+
+  integration::stop_local_persist
+  integration::run_local_persist_background "${local_persist_log}"
+  integration::stop_pouchd
+  integration::run_pouchd_background "${cmd}" "${flags}" "${pouchd_log}"
+
+  set +e; integration::ping_pouchd; code=$?; set -e
+  if [[ "${code}" != "0" ]]; then
+    echo "there is daemon logs..."
+    cat "${pouchd_log}"
+    exit ${code}
+  fi
+  integration::run_daemon_cri_test_cases
+}
+
+main "$@"
diff --git a/hack/testing/run_daemon_integration.sh b/hack/testing/run_daemon_integration.sh
new file mode 100755
index 000000000..d5dcc85c9
--- /dev/null
+++ b/hack/testing/run_daemon_integration.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+source utils.sh
+
+readonly REPO_BASE="$(cd ../../ && pwd -P)"
+
+# add bin folder into PATH so that pouch-integration is available.
+export PATH="${REPO_BASE}/bin:${PATH}"
+
+# tmplog_dir stores the background job log data
+tmplog_dir="$(mktemp -d /tmp/integration-testing-XXXXX)"
+pouchd_log="${tmplog_dir}/pouchd.log"
+local_persist_log="${tmplog_dir}/local_persist.log"
+trap 'rm -rf /tmp/integration-testing-*' EXIT
+
+# daemon integration coverage profile
+coverage_profile="${REPO_BASE}/coverage/integration_daemon_profile.out"
+rm -rf "${coverage_profile}"
+
+# integration::run_daemon_test_cases starts cases.
+integration::run_daemon_test_cases() {
+  echo "start pouch daemon integration test..."
+  local code
+
+  cp -rf "${REPO_BASE}/test/tls" /tmp/
+
+  set +e
+  "${REPO_BASE}/bin/pouchd-integration-test" -test.v -check.v
+  code=$?
+  integration::stop_local_persist
+  integration::stop_pouchd
+  set -e
+
+  if [[ "${code}" != "0" ]]; then
+    echo "failed to pass integration cases!"
+    echo "there is daemon logs...."
+    cat "${pouchd_log}"
+    exit ${code}
+  fi
+
+  # sleep for pouchd stop and got the coverage
+  sleep 5
+}
+
+main() {
+  local cmd flags
+  cmd="pouchd-integration"
+  flags=" -test.coverprofile=${coverage_profile} DEVEL"
+  flags="${flags} --debug --enable-lxcfs --add-runtime runv=runv"
+
+  integration::stop_local_persist
+  integration::run_local_persist_background "${local_persist_log}"
+  integration::stop_pouchd
+  integration::run_pouchd_background "${cmd}" "${flags}" "${pouchd_log}"
+
+  set +e; integration::ping_pouchd; code=$?; set -e
+  if [[ "${code}" != "0" ]]; then
+    echo "there is daemon logs..."
+    cat "${pouchd_log}"
+    exit ${code}
+  fi
+  integration::run_daemon_test_cases
+}
+
+main "$@"
diff --git a/hack/testing/utils.sh b/hack/testing/utils.sh
new file mode 100644
index 000000000..b7a0d559a
--- /dev/null
+++ b/hack/testing/utils.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# integration::run_local_persist_background runs local-persist in background.
+integration::run_local_persist_background() {
+  local log_file
+  log_file=$1
+
+  echo "start local-persist volume plugin..."
+  local-persist > "${log_file}" 2 >&1 &
+}
+
+# integration::stop_local_persist stop local-persist.
+integration::stop_local_persist() {
+  echo "stop local-persist volume plugin..."
+  set +e; pkill local-persist; set -e
+}
+
+# integration::run_pouchd_background runs pouchd in background.
+integration::run_pouchd_background() {
+  echo "start pouch daemon..."
+
+  local cmd flags log_file
+
+  cmd="$1"
+  flags="$2"
+  log_file="$3"
+
+  cmd="${cmd} ${flags}"
+  ${cmd} > "${log_file}" 2>&1 &
+}
+
+# integration::stop_pouchd stops pouchd.
+integration::stop_pouchd() {
+  echo "stop pouch daemon..."
+  set +e; pkill -3 pouchd; set -e
+}
+
+# integration::ping_pouchd makes sure that pouchd started.
+integration::ping_pouchd() {
+  local timeout
+
+  # make sure that it's up.
+  timeout=30
+  while true;
+  do
+    if pouch version 2> /dev/null; then
+      break
+    elif (( $((timeout--)) == 0 ));then
+      echo "failed to start pouch daemon in background!"
+      exit 1
+    fi
+    sleep 1
+  done
+}
diff --git a/hack/validate_swagger.sh b/hack/validate_swagger.sh
new file mode 100755
index 000000000..c6ca2d9e5
--- /dev/null
+++ b/hack/validate_swagger.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+readonly REPO_BASE="$(cd .. && pwd -P)"
+
+source "${REPO_BASE}/hack/codegen/swagger.sh"
+
+main() {
+  local has_installed
+
+  has_installed="$(swagger::check_version)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo >&2 "swagger-${SWAGGER_VERSION} should be installed."
+    exit 1
+  fi
+
+  swagger validate "${REPO_BASE}/apis/swagger.yml"
+}
+
+main
diff --git a/test/cli_pause_test.go b/test/cli_pause_test.go
index c98f255aa..4756c5147 100644
--- a/test/cli_pause_test.go
+++ b/test/cli_pause_test.go
@@ -1,12 +1,13 @@
 package main
 
 import (
+	"strings"
+
 	"github.com/alibaba/pouch/test/command"
 	"github.com/alibaba/pouch/test/environment"
 
 	"github.com/go-check/check"
 	"github.com/gotestyourself/gotestyourself/icmd"
-	"strings"
 )
 
 // PouchPauseSuite is the test suite for pause CLI.