run-functional-tests

#!/bin/bash
DOCKER_IMAGE=${DOCKER_IMAGE:-"nyang23/haleap:15.5"}
PROJECT_PATH=$(dirname $(dirname `realpath $0`))
PROJECT_INSIDE="/opt/crmsh"
DOCKER_SERVICE="docker.service"
COROSYNC_CONF="/etc/corosync/corosync.conf"
COROSYNC_AUTH="/etc/corosync/authkey"
HA_NETWORK_FIRST="ha_network_first"
HA_NETWORK_SECOND="ha_network_second"
declare -a HA_NETWORK_ARRAY
declare -a HA_NETWORK_V6_ARRAY
HA_NETWORK_ARRAY[0]=$HA_NETWORK_FIRST
HA_NETWORK_ARRAY[1]=$HA_NETWORK_SECOND
HA_NETWORK_V6_ARRAY[0]="2001:db8:10::/64"
HA_NETWORK_V6_ARRAY[1]="2001:db8:20::/64"
BEHAVE_CASE_DIR="$(dirname $0)/features/"
BEHAVE_CASE_EXCLUDE="sbd|ocfs2"

read -r -d '' SSHD_CONFIG_AZURE << EOM
PermitRootLogin no
AuthorizedKeysFile .ssh/authorized_keys
ChallengeResponseAuthentication no
UsePAM yes
X11Forwarding yes
ClientAliveInterval 180
Subsystem sftp	/usr/lib/ssh/sftp-server
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL
PasswordAuthentication no
EOM

read -r -d '' COROSYNC_CONF_TEMPLATE << EOM
totem {
        version: 2
        cluster_name: hacluster
        clear_node_high_bit: yes
        interface {
                ringnumber: 0
                mcastport: 5405
                ttl: 1
        }

        transport: udpu
        crypto_hash: sha1
        crypto_cipher: aes256
        token: 5000
        join: 60
        max_messages: 20
        token_retransmits_before_loss_const: 10
}

logging {
        fileline: off
        to_stderr: no
        to_logfile: no
        logfile: /var/log/cluster/corosync.log
        to_syslog: yes
        debug: off
        timestamp: on
        logger_subsys {
                subsys: QUORUM
                debug: off
        }

}

nodelist {
}

quorum {

        # Enable and configure quorum subsystem (default: off)
        # see also corosync.conf.5 and votequorum.5
        provider: corosync_votequorum
}
EOM


fatal() {
	error $*
	exit 1
}


error() {
	echo "ERROR: $*"
}


warning() {
	echo "WARNING: $*"
}


info() {
	echo "INFO: $*"
}


is_number() {
	num=$1
	test ! -z "$num" && test "$num" -eq "$num" 2> /dev/null && test "$num" -gt 0 2> /dev/null
}


check_docker_env() {
	# check if docker available
	systemctl list-unit-files $DOCKER_SERVICE &> /dev/null
	if [ "$?" -ne 0 ];then
		fatal "$DOCKER_SERVICE is not available"
	fi
	# check if docker.service started
	systemctl is-active $DOCKER_SERVICE &> /dev/null
	if [ "$?" -ne 0 ];then
		fatal "$DOCKER_SERVICE is not active"
	fi
	# check if docker cgroup driver is systemd
	docker info 2> /dev/null|grep -q "Cgroup Driver: systemd"
	if [ "$?" -ne 0 ];then
		warning "docker cgroup driver suggest to be \"systemd\""
	fi

	[ "$1" == "cleanup" ] && return
	# check if ha network already exists
	for network in ${HA_NETWORK_ARRAY[@]};do
		docker network ls|grep -q "$network"
		if [ "$?" -eq 0 ];then
			fatal "HA specific network \"$network\" already exists"
		fi
	done
}


get_test_case_array() {
	test -d $BEHAVE_CASE_DIR || fatal "Cannot find '$BEHAVE_CASE_DIR'"
	ls $BEHAVE_CASE_DIR|grep "\.feature"|grep -Ev "$BEHAVE_CASE_EXCLUDE"
}


echo_test_cases() {
	case_arry=`get_test_case_array`
	echo "Index|File Name|Description"
	index=1
	 for f in ${case_arry[@]};do
		desc=`awk -F: '/Feature/{print $2}' $BEHAVE_CASE_DIR/$f`
		printf "%3s    %-40s %-60s\n" $index $f "$desc"
		index=$(($index+1))
	done
	printf "%3s    %-40s  %-60s\n" $index "regression test" "Original regression test"
}


usage_and_exit() {
	prog=`basename $0`
	cat <<END
Usage: $prog [OPTIONS]|[TESTCASE INDEX]
$prog is a tool for developers to setup the cluster in containers to run functional tests.
The container image is based on Tumbleweed with preinstalled packages of the cluster stack include pacemaker/corosync/crmsh and many others.
Users can make the code change under crmsh.git including test cases. This tool will pick up the code change and "make install" to all running containers.

OPTIONS:
  -h, --help		Show this help message and exit
  -l            	List existing functional test cases and exit
  -n NUM        	Only setup a cluster with NUM nodes(containers)
  -x			Don't config corosync on containers(with -n option)
  -d			Cleanup the cluster containers
  -u			Create normal users, and Azure like ssh environment
  -q			Create a qnetd node(with -n and -x option)

EXAMPLES:
To launch 2 nodes with the running cluster with the very basic corosync.conf
# crmsh.git/test/run-functional-tests -n 2

To launch 2 nodes without the cluster stack running to play with "crm cluster init/join"
# crmsh.git/run-functional-tests -n 2 -x

To launch 2 nodes without the cluster stack running, and a qnetd node(named 'qnetd-node')
# crmsh.git/run-functional-tests -n 2 -x -q

To list the existing test cases. Users could add his own new test cases.
# crmsh.git/test/run-functional-tests -l

To run a single or a number of functional test cases
# crmsh.git/test/run-functional-tests 1
# crmsh.git/test/run-functional-tests 1 2 3

To clean up the all containers which are generated by this tool
# crmsh.git/test/run-functional-tests -d
END
	exit 1
}


docker_exec() {
	name=$1
	cmd=$2
	docker exec -t $name /bin/sh -c "$cmd"
}

set_sshd_config_like_in_azure() {
	node_name=$1
	docker_exec $node_name "echo \"$SSHD_CONFIG_AZURE\" > /etc/ssh/sshd_config"
	docker_exec $node_name "systemctl restart sshd.service"
}

create_custom_user() {
	user_name=$1
	user_id=$2
	docker_exec $node_name "useradd -m -s /bin/bash ${user_name} 2>/dev/null"
	docker_exec $node_name "chmod u+w /etc/sudoers"
	docker_exec $node_name "echo \"${user_name} ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers"
	docker_exec $node_name "chmod u-w /etc/sudoers"
	docker_exec $node_name "echo 'export PATH=\$PATH:/usr/sbin/' >> ~${user_name}/.bashrc"
	docker_exec $node_name "echo -e \"linux\\nlinux\" | passwd ${user_name} 2>/dev/null"
	docker_exec $node_name "cp -r /root/.ssh ~${user_name}/ && chown ${user_name}:haclient -R ~${user_name}/.ssh"
	info "Create user '$user_name' on $node_name"
}

create_alice_bob_carol() {
	# Custom users are alice, bob and carol and they are as important as the root
	# and eventually they should be already in the docker image
	# However now, let's create them here
	create_custom_user "alice" "1000"
	create_custom_user "bob" "1001"
	create_custom_user "carol" "1002"
}

deploy_ha_node() {
	node_name=$1
	docker_options="-d --name $node_name -h $node_name --privileged --shm-size 1g"
	make_cmd="cd $PROJECT_INSIDE;./autogen.sh && ./configure --prefix /usr && make install && make install-crmconfDATA prefix= && cp /usr/bin/crm /usr/sbin"

	info "Deploying \"$node_name\"..."
	docker run --restart always $docker_options $DOCKER_IMAGE &> /dev/null
	for network in ${HA_NETWORK_ARRAY[@]};do
		docker network connect $network $node_name
	done

	if [ "$node_name" != "qnetd-node" ];then
		rm_qnetd_cmd="rpm -q corosync-qnetd && rpm -e corosync-qnetd"
		docker_exec $node_name "$rm_qnetd_cmd" &> /dev/null
	fi
	docker_exec $node_name "rm -rf /run/nologin"
	docker_exec $node_name "echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config"

	if [ "$node_name" != "qnetd-node" ];then
	  docker cp $PROJECT_PATH $node_name:/opt/crmsh
	  info "Building crmsh on \"$node_name\"..."
	  docker_exec $node_name "$make_cmd" 1> /dev/null || \
		fatal "Building failed on $node_name!"
	  docker_exec $node_name "chown hacluster:haclient -R /var/log/crmsh"
	  docker_exec $node_name "chmod g+w -R /var/log/crmsh"
	  create_alice_bob_carol
	  if [ "$NORMAL_USER_FLAG" -eq 1 ];then
		set_sshd_config_like_in_azure $node_name
          fi
	else
	  docker_exec $node_name "useradd -m -s /bin/bash alice 2>/dev/null"
	  docker_exec $node_name "echo \"alice ALL=(ALL) NOPASSWD:ALL\" > /etc/sudoers.d/alice"
	  docker_exec $node_name "cp -r /root/.ssh ~alice/ && chown alice:users -R ~alice/.ssh"
	  info "Create user 'alice' on $node_name"
	  [ "$NORMAL_USER_FLAG" -eq 1 ] && set_sshd_config_like_in_azure $node_name
	fi
}


create_node() {
	info "Loading docker image $DOCKER_IMAGE..."
        docker pull $DOCKER_IMAGE &> /dev/null

	for index in ${!HA_NETWORK_ARRAY[@]};do
		network=${HA_NETWORK_ARRAY[$index]}
		info "Create ha specific docker network \"$network\"..."
		docker network create --ipv6 --subnet ${HA_NETWORK_V6_ARRAY[$index]} $network &> /dev/null
	done

	for node in $*;do
		deploy_ha_node $node &
	done
	wait
}


config_cluster() {
	node_num=$#
	insert_str=""
	container_ip_array=(`docker network inspect $HA_NETWORK_ARRAY -f '{{range .Containers}}{{printf "%s " .IPv4Address}}{{end}}'`)

	for i in $(seq $node_num -1 1);do
		ip=`echo ${container_ip_array[$((i-1))]}|awk -F/ '{print $1}'`
		insert_str+="\\n\\tnode {\n\t\tring0_addr: $ip\n\t\tnodeid: $i\n\t}"
	done
	corosync_conf_str=$(sed "/nodelist/a \\${insert_str}" <(echo "$COROSYNC_CONF_TEMPLATE"))
	if [ $node_num -eq 2 ];then
		corosync_conf_str=$(sed "/corosync_votequorum/a \\\\ttwo_node: 1" <(echo "$corosync_conf_str"))
	fi

	info "Copy corosync.conf to $*"
	for node in $*;do
		if [ $node == $1 ];then
			docker_exec $1 "echo \"$corosync_conf_str\" >> $COROSYNC_CONF"
			docker_exec $1 "corosync-keygen -l -k $COROSYNC_AUTH &> /dev/null"
		else
			while :
			do
				docker_exec $1 "ssh -T -o Batchmode=yes $node true &> /dev/null" && break
				sleep 1
			done
			docker_exec $1 "scp -p $COROSYNC_CONF $COROSYNC_AUTH $node:/etc/corosync &> /dev/null"
		fi
	done
}


start_cluster() {
	for node in $*;do
		docker_exec $node "crm cluster enable && crm cluster start" 1> /dev/null
		if [ "$?" -eq 0 ];then
			info "Cluster service started on \"$node\""
		else
			fatal "Failed to start cluster service on \"$node\""
		fi
	done
}


container_already_exists() {
	docker ps -a|grep -q "$1"
	if [ "$?" -eq 0 ];then
		fatal "Container \"$1\" already running"
	fi
}


setup_cluster() {
	hanodes_arry=()
	is_number $1
	if [ "$?" -eq 0 ];then
		for i in $(seq 1 $1);do
			hanodes_arry+=("hanode$i")
		done
	else
		hanodes_arry=($*)
	fi

	if [ $WITH_QNETD_NODE -eq 1 ];then
	  create_node ${hanodes_arry[@]} "qnetd-node"
	else
	  create_node ${hanodes_arry[@]}
	fi

	[ "$CONFIG_COROSYNC_FLAG" -eq 0 ] && return
	config_cluster ${hanodes_arry[@]}
	start_cluster ${hanodes_arry[@]}
	docker_exec "hanode1" "crm configure property stonith-enabled=false"
}


cleanup_container() {
	node=$1
	info "Cleanup container \"$node\"..."
	docker container stop $node &> /dev/null
	docker container rm $node &> /dev/null
}


cleanup_cluster() {
	exist_network_array=()
	for network in ${HA_NETWORK_ARRAY[@]};do
		docker network ls|grep -q $network && exist_network_array+=($network)
	done
	if [ ${#exist_network_array[@]} -eq 0 ];then
		info "Already cleaned up"
		return 0
	fi

	container_array=(`docker network inspect $exist_network_array -f '{{range .Containers}}{{printf "%s " .Name}}{{end}}'`)
	for node in ${container_array[@]};do
		cleanup_container $node &
	done
	wait

	for network in ${exist_network_array[@]};do
		info "Cleanup ha specific docker network \"$network\"..."
		docker network rm $network &> /dev/null
	done
}


adjust_test_case() {
	node_name=$1
	replace_arry=(`docker_exec $node_name "grep -o -E '@(hanode[0-9]+|qnetd-node)\.ip[6]?\.(default|[0-9])' $2|sort -u|dos2unix"`)
	for item in ${replace_arry[@]};do
		item_str=${item##@}
		node=`echo $item_str|cut -d "." -f 1`
		ip_version=`echo $item_str|cut -d "." -f 2|tr -d "\r"`
		ip_search_str="IPAddress"
		if [ "$ip_version" == "ip6" ];then
			ip_search_str="GlobalIPv6Address"
		fi
		index=`echo $item_str|cut -d "." -f 3|tr -d "\r"`
		if [ "$index" == "default" ];then
			ip=`docker container inspect $node -f "{{range .NetworkSettings.Networks}}{{printf \"%s \" .$ip_search_str}}{{end}}"|awk '{print $1}'|tr -d "\r"`
		else
			ip=`docker container inspect $node -f "{{.NetworkSettings.Networks.${HA_NETWORK_ARRAY[$index]}.$ip_search_str}}"|tr -d "\r"`
		fi
		item=`echo $item|tr -d "\r"`
		docker_exec $node_name "sed -i s/$item/$ip/g $2"
	done

	vip_replace_array=(`docker_exec $node_name "grep -o -E '@vip\.[0-9]' $2|sort -u|dos2unix"`)
	for item in ${vip_replace_array[@]};do
		index=`echo $item|cut -d "." -f 2|tr -d "\r"`
		suffix=$((123+index))
		ip=`docker container inspect $node_name -f "{{.NetworkSettings.Networks.${HA_NETWORK_ARRAY[0]}.IPAddress}}"|tr -d "\r"`
		vip=`echo $ip|sed "s/\.[0-9][0-9]*$/\.$suffix/g"|tr -d "\r"`
		item=`echo $item|tr -d "\r"`
		docker_exec $node_name "sed -i s/$item/$vip/g $2"
	done
}


run_origin_regression_test() {
	CONFIG_COROSYNC_FLAG=0
	setup_cluster "hanode1"
	docker_exec "hanode1" "echo -e '[core]\nhas_fa_advised_op = yes' > /etc/crm/crm.conf"
	docker_exec "hanode1" "sh /usr/share/crmsh/tests/regression.sh"
	return $?
}


prepare_coverage_env() {
	for node in $*; do
            docker exec -t $node /bin/sh -c 'sed -i '\''1a\import coverage\nimport atexit\ncov=coverage.Coverage(config_file="/opt/crmsh/test/features/coveragerc")\natexit.register(lambda:(cov.stop(),cov.save()))\ncov.start()'\'' /usr/sbin/crm'
	done
}


fetch_coverage_report() {
	local unique_id=$(dd if=/dev/urandom count=1 bs=6 | base64 | tr '+/' '-_')
	for node in $*; do
		docker_exec "$node" "coverage combine; coverage xml -o /opt/coverage.xml"
		# see https://github.com/codecov/codecov-cli/blob/master/codecov_cli/services/upload/coverage_file_finder.py
		docker cp "$node":/opt/coverage.xml coverage."$unique_id"."$node".xml
	done
}


WITH_QNETD_NODE=0
NORMAL_USER_FLAG=0
CONFIG_COROSYNC_FLAG=1
SETUP_N_NODES_CLUSTER=0
options=$(getopt -l "help" -o "hldxuqn:" -- "$@")
eval set -- "$options"
while true;do
case $1 in
-h|--help) usage_and_exit;;
-l)
	echo_test_cases
	exit 0
	;;
-d)
	check_docker_env cleanup
	cleanup_cluster
	exit $?
	;;
-x)
	CONFIG_COROSYNC_FLAG=0
	shift
	;;
-u)
	NORMAL_USER_FLAG=1
	shift
	;;
-q)
	WITH_QNETD_NODE=1
	shift
	;;
-n)
	check_docker_env
	shift
	is_number $1 || fatal "-n option need a number larger than 0"
	SETUP_N_NODES_CLUSTER=$1
	shift
	;;
--)
	shift
	break
	;;
esac
done

if [ $SETUP_N_NODES_CLUSTER -ge 1 ];then
	setup_cluster $SETUP_N_NODES_CLUSTER
	exit $?
fi

if [ "$#" -eq 0 ];then
	usage_and_exit
fi

# used by github action
if [ "$1" == "_get_index_of" ];then
	shift
	pattern=""
	for item in $*;do
		pattern+="${item}|"
	done
	echo_test_cases|grep -E "(${pattern%%|})(\.feature|\s+Original regression test)"|awk '{print $1}'|tr -s '\n' ' '
	exit 0
fi

for case_num in $*;do
	echo_test_cases|grep -E "\s+$case_num\s" &> /dev/null
	if [ "$?" -ne 0 ];then
		error "\"$case_num\" is an invalid index"
		echo_test_cases
		exit 1
	fi
done

for case_num in $*;do
	if [ "$case_num" -ne $1 ];then
		check_docker_env cleanup
		cleanup_cluster
		echo
	fi
	check_docker_env
	test_case_array=(`get_test_case_array`)
	if [ $case_num -gt ${#test_case_array[*]} ];then
		run_origin_regression_test || exit 1
		continue
	fi
	case_file=$BEHAVE_CASE_DIR/${test_case_array[$((case_num-1))]}
	case_file_in_container="$PROJECT_INSIDE/test/features/`basename $case_file`"
	node_arry=(`awk -F: '/Need nodes/{print $2}' $case_file`)
	CONFIG_COROSYNC_FLAG=0
	setup_cluster ${node_arry[@]}
	adjust_test_case ${node_arry[0]} $case_file_in_container
	echo
        prepare_coverage_env "${node_arry[@]}"
	if [ "$NORMAL_USER_FLAG" -eq 0 ];then
		info "Running \"$case_file_in_container\" under 'root'..."
		docker_exec ${node_arry[0]} "behave --no-logcapture $case_file_in_container || exit 1" || exit 1
	else
		info "Running \"$case_file_in_container\" under normal user 'alice'..."
                docker_exec ${node_arry[0]} "su - alice -c 'sudo behave --no-logcapture $case_file_in_container || exit 1'" || exit 1
	fi
	fetch_coverage_report "${node_arry[@]}"
	echo
done