From 1bcb6e93162da351c5efae15d59e3f773449eeff Mon Sep 17 00:00:00 2001 From: Sandino Araico Sanchez Date: Wed, 25 Jan 2017 22:30:16 -0600 Subject: [PATCH] high availability bonding tests --- scripts/scan-dmidecode | 18 +++++----- scripts/scan-netdev | 5 ++- scripts/screen-netdev | 4 ++- scripts/setup-netdev | 76 ++++++++++++++++++++++++++++++++++++++++++ scripts/stat-bonding | 46 +++++++++++++++++++++++++ scripts/test-batch | 23 +++++++++++++ scripts/test-bonding | 54 ++++++++++++++++++++++++++++++ scripts/test-ping | 21 ++++++++++++ 8 files changed, 236 insertions(+), 11 deletions(-) create mode 100755 scripts/setup-netdev create mode 100755 scripts/stat-bonding create mode 100755 scripts/test-batch create mode 100755 scripts/test-bonding create mode 100755 scripts/test-ping diff --git a/scripts/scan-dmidecode b/scripts/scan-dmidecode index 85b8bec..7deb43b 100755 --- a/scripts/scan-dmidecode +++ b/scripts/scan-dmidecode @@ -5,16 +5,16 @@ SERVER_SERIAL=`dmidecode -s system-serial-number | sed 's/\s.*$//'` MY_DATETIME=`date +%Y-%m-%d_%H-%M-%S` OUTPUT_HOME=/output/hwtests OUTPUT_DIR=$OUTPUT_HOME/$SERVER_SERIAL -OUTPUT_BAD="$OUTPUT_HOME/ $OUTPUT_DIR/netdev.lst for DEV in \ - `cat /proc/net/dev | grep ':' | cut -d ':' -f 1` + `cat /proc/net/dev \ + | grep -v bond \ + | grep ':' \ + | cut -d ':' -f 1` do MAC=`ethtool -P $DEV | cut -d ':' -f 2-9 | sed 's/\s\+//g'` #MAC=`cat /sys/class/net/$DEV/address` diff --git a/scripts/screen-netdev b/scripts/screen-netdev index 89f81ce..967146b 100755 --- a/scripts/screen-netdev +++ b/scripts/screen-netdev @@ -2,7 +2,7 @@ touch /run/test.pid cat > /usr/local/etc/screenrc << EOF -screen -t tests stat-netdev +screen -t tests stat-bonding split -v focus bottom screen -t status journalctl -f @@ -23,6 +23,8 @@ echo #exec < /dev/tty3 > /dev/tty3 #chvt 3 +echo "=====> Begin setup <=====" +setup-netdev echo "=====> Begin tests <=====" screen -c /usr/local/etc/screenrc diff --git a/scripts/setup-netdev b/scripts/setup-netdev new file mode 100755 index 0000000..11cb37b --- /dev/null +++ b/scripts/setup-netdev @@ -0,0 +1,76 @@ +#!/bin/bash + +source scan-netdev + +modprobe bonding + +# All vlans are using netmask /24 +MASK=24 + +for VLAN in 401 402 403 404 405 406 407 408 ; do + if [[ -f /etc/netdev/vlan.$VLAN.csv ]] ; then + VLAN_CONF=`grep $SERVER_SERIAL \ + /etc/netdev/vlan.$VLAN.csv` + fi + if [[ -z $VLAN_CONF ]] ; then + continue + fi + MY_HOSTNAME=`hostname` + #HOSTNAME,SERIAL,IP_ADDRESS,BOND,MODE,SLAVE0,SLAVE1 + VLAN_HOSTNAME=`echo $VLAN_CONF \ + | cut -d ',' -f 1` + if [[ -n $VLAN_HOSTNAME ]] ; then + if [[ "$MY_HOSTNAME" != "$VLAN_HOSTNAME" ]] ; then + hostname $VLAN_HOSTNAME + fi + fi + VLAN_IP=`echo $VLAN_CONF \ + | cut -d ',' -f 3` + VLAN_BOND=`echo $VLAN_CONF \ + | cut -d ',' -f 4` + BOND_MODE=`echo $VLAN_CONF \ + | cut -d ',' -f 5` + SLAVE_0=`echo $VLAN_CONF \ + | cut -d ',' -f 6` + SLAVE_1=`echo $VLAN_CONF \ + | cut -d ',' -f 7` + if [[ -z $VLAN_BOND ]] ; then + continue + fi + if [[ ! -d /sys/class/net/$VLAN_BOND ]] ; then + if [[ ! -f /sys/class/net/bonding_masters ]] ; then + echo "Missing bonding_masters special file" + exit 22 + fi + echo "+$VLAN_BOND" \ + > /sys/class/net/bonding_masters + sleep 3 + if [[ ! -d /sys/class/net/$VLAN_BOND ]] ; then + echo "$VLAN_BOND not available" + exit 23 + fi + fi + MY_MODE=`cat /sys/class/net/$VLAN_BOND/bonding/mode \ + | cut -d ' ' -f 2` + if [[ "$MY_MODE" != "$BOND_MODE" ]] ; then + echo $BOND_MODE \ + > /sys/class/net/$VLAN_BOND/bonding/mode + sleep 2 + fi + ip link set $VLAN_BOND up + sleep 5 + ifenslave $VLAN_BOND $SLAVE_0 $SLAVE_1 + sleep 5 + if [[ $BOND_MODE == 1 ]] ; then + ip addr add $VLAN_IP/$MASK dev $VLAN_BOND + else + ip link add link $VLAN_BOND \ + name $VLAN_BOND.$VLAN \ + type vlan id $VLAN + ip link set $VLAN_BOND.$VLAN up + ip addr add $VLAN_IP/$MASK dev $VLAN_BOND.$VLAN + fi +done + +systemctl restart sshd + diff --git a/scripts/stat-bonding b/scripts/stat-bonding new file mode 100755 index 0000000..c8b03d9 --- /dev/null +++ b/scripts/stat-bonding @@ -0,0 +1,46 @@ +#!/bin/bash + +source scan-netdev +if [[ "$1" == "out" ]] ; then + OUT=1 +else + source test-variables +fi +#setup-netdev +#sleep 5 +#systemctl start sshd +#sleep 1 + +while [[ 1 ]] ; do + if [[ -z $OUT ]] ; then + clear + fi + date + for BOND_N in {0..4} ; do + BOND=bond$BOND_N + if [[ -d /sys/class/net/$BOND ]] ; then + echo -n "$BOND:" + SLAVES=`cat /sys/class/net/$BOND/bonding/slaves` + MODE=`cat /sys/class/net/$BOND/bonding/mode \ + | cut -d ' ' -f 2` + for DEV in $SLAVES ; do + LINKING=`cat /sys/class/net/$DEV/operstate` + if [[ "$LINKING" == "up" ]] ; then + echo -n " $GRN$DEV$OFF up" + else + echo -n " $RED$DEV$OFF down" + fi + done + echo -n " Mode: $BLD$MODE$OFF" + echo + if [[ $MODE == 1 ]] ; then + ACTIVE=`cat /sys/class/net/$BOND/bonding/active_slave` + echo "Active: $ACTIVE" + fi + fi + done + if [[ -n $OUT ]] ; then + exit 0 + fi + sleep 2 +done diff --git a/scripts/test-batch b/scripts/test-batch new file mode 100755 index 0000000..72ef1b1 --- /dev/null +++ b/scripts/test-batch @@ -0,0 +1,23 @@ +#!/bin/bash + +source scan-dmidecode +source test-variables +#rm -f /output/hwtests/*/netdev/*.ethtool +source scan-netdev + +echo $$ > /run/batch.pid + +echo -n "setup-netdev... " +setup-netdev >/dev/null & +sleep 1 +jobs +disown +#stat-bonding out > $OUTPUT_DIR/stat-bonding.pre.1.out +#test-bonding +#stat-bonding out > $OUTPUT_DIR/stat-bonding.post.1.out +# Take snapshot +#mkdir -p $SNAPSHOT_D +#cp -af $OUTPUT_DIR/* $SNAPSHOT_D/ + +rm -f /run/batch.pid +#echo '--' diff --git a/scripts/test-bonding b/scripts/test-bonding new file mode 100755 index 0000000..dde8311 --- /dev/null +++ b/scripts/test-bonding @@ -0,0 +1,54 @@ +#!/bin/bash + +source scan-netdev +sleep_5 () { + for WAIT in 1 2 3 4 5 ; do + echo -n '.' + sleep 1 + done + echo +} + +echo "First ping test:" +test-ping | tee $OUTPUT_DIR/test-ping.out \ + | grep error + +for BOND_N in {0..4} ; do + BOND=bond$BOND_N + if [[ -d /sys/class/net/$BOND ]] ; then + cp /sys/class/net/$BOND/bonding/slaves \ + $OUTPUT_DIR/netdev/$BOND.slaves + fi +done + +SLAVE_NO=1 +#for SLAVE_NO in 1 ; do + echo -n "Removing slave $SLAVE_NO..." + for BOND_N in {0..4} ; do + BOND=bond$BOND_N + [[ -d /sys/class/net/$BOND ]] || continue + SLAVE=`cut -d ' ' -f $SLAVE_NO \ + $OUTPUT_DIR/netdev/$BOND.slaves` + ifenslave -d $BOND $SLAVE + done + sleep_5 + echo -n "Dummy ping..." + test-ping > /dev/null + sleep_5 + echo "Real ping test:" + test-ping | tee $OUTPUT_DIR/slave$SLAVE_NO-ping.out \ + | grep error + echo -n "Reinserting slave $SLAVE_NO..." + for BOND_N in {0..4} ; do + [[ -d /sys/class/net/$BOND ]] || continue + BOND=bond$BOND_N + SLAVE=`cut -d ' ' -f $SLAVE_NO \ + $OUTPUT_DIR/netdev/$BOND.slaves` + ifenslave $BOND $SLAVE + done + sleep_5 + echo -n "Dummy ping..." + test-ping > /dev/null + sleep_5 +#done + diff --git a/scripts/test-ping b/scripts/test-ping new file mode 100755 index 0000000..4940f2b --- /dev/null +++ b/scripts/test-ping @@ -0,0 +1,21 @@ +#!/bin/bash + +source scan-netdev + +for VLAN in {401..408} ; do + if [[ -f /etc/netdev/vlan.$VLAN.csv ]] ; then + VLAN_CONF=`grep $SERVER_SERIAL \ + /etc/netdev/vlan.$VLAN.csv` + fi + if [[ -z $VLAN_CONF ]] ; then + continue + fi + IP_LIST=`cat /etc/netdev/vlan.$VLAN.csv \ + | cut -d ',' -f 3 \ + | grep '^[0-9]' | sort` + for IP in $IP_LIST; do + echo -n "$IP - " + ping -c5 -f $IP | grep 'received' + done +done +