high availability bonding tests

This commit is contained in:
Sandino Araico Sanchez 2017-01-25 22:30:16 -06:00
parent 3f75af7dda
commit 1bcb6e9316
8 changed files with 236 additions and 11 deletions

View file

@ -5,16 +5,16 @@ SERVER_SERIAL=`dmidecode -s system-serial-number | sed 's/\s.*$//'`
MY_DATETIME=`date +%Y-%m-%d_%H-%M-%S`
OUTPUT_HOME=/output/hwtests
OUTPUT_DIR=$OUTPUT_HOME/$SERVER_SERIAL
OUTPUT_BAD="$OUTPUT_HOME/<BAD"
#OUTPUT_BAD="$OUTPUT_HOME/<BAD"
SNAPSHOT_D=$OUTPUT_HOME/snapshot/$SERVER_SERIAL/$MY_DATETIME
if [[ -d $OUTPUT_BAD ]] ; then
if [[ -f $OUTPUT_BAD/dmidecode ]] ; then
IS_MY_SERIAL=`cat $OUTPUT_BAD/dmidecode | grep $SERVER_SERIAL`
if [[ -n $IS_MY_SERIAL ]] ; then
mv -vf $OUTPUT_BAD $OUTPUT_DIR
fi
fi
fi
#if [[ -d $OUTPUT_BAD ]] ; then
# if [[ -f $OUTPUT_BAD/dmidecode ]] ; then
# IS_MY_SERIAL=`cat $OUTPUT_BAD/dmidecode | grep $SERVER_SERIAL`
# if [[ -n $IS_MY_SERIAL ]] ; then
# mv -vf $OUTPUT_BAD $OUTPUT_DIR
# fi
# fi
#fi
export SERVER_SERIAL
export OUTPUT_DIR
export SNAPSHOT_D

View file

@ -6,7 +6,10 @@ fi
echo "#DEV MA:CA:DD:RE:SS" > $OUTPUT_DIR/netdev.lst
for DEV in \
`cat /proc/net/dev | grep ':' | cut -d ':' -f 1`
`cat /proc/net/dev \
| grep -v bond \
| grep ':' \
| cut -d ':' -f 1`
do
MAC=`ethtool -P $DEV | cut -d ':' -f 2-9 | sed 's/\s\+//g'`
#MAC=`cat /sys/class/net/$DEV/address`

View file

@ -2,7 +2,7 @@
touch /run/test.pid
cat > /usr/local/etc/screenrc << EOF
screen -t tests stat-netdev
screen -t tests stat-bonding
split -v
focus bottom
screen -t status journalctl -f
@ -23,6 +23,8 @@ echo
#exec < /dev/tty3 > /dev/tty3
#chvt 3
echo "=====> Begin setup <====="
setup-netdev
echo "=====> Begin tests <====="
screen -c /usr/local/etc/screenrc

76
scripts/setup-netdev Executable file
View file

@ -0,0 +1,76 @@
#!/bin/bash
source scan-netdev
modprobe bonding
# All vlans are using netmask /24
MASK=24
for VLAN in 401 402 403 404 405 406 407 408 ; do
if [[ -f /etc/netdev/vlan.$VLAN.csv ]] ; then
VLAN_CONF=`grep $SERVER_SERIAL \
/etc/netdev/vlan.$VLAN.csv`
fi
if [[ -z $VLAN_CONF ]] ; then
continue
fi
MY_HOSTNAME=`hostname`
#HOSTNAME,SERIAL,IP_ADDRESS,BOND,MODE,SLAVE0,SLAVE1
VLAN_HOSTNAME=`echo $VLAN_CONF \
| cut -d ',' -f 1`
if [[ -n $VLAN_HOSTNAME ]] ; then
if [[ "$MY_HOSTNAME" != "$VLAN_HOSTNAME" ]] ; then
hostname $VLAN_HOSTNAME
fi
fi
VLAN_IP=`echo $VLAN_CONF \
| cut -d ',' -f 3`
VLAN_BOND=`echo $VLAN_CONF \
| cut -d ',' -f 4`
BOND_MODE=`echo $VLAN_CONF \
| cut -d ',' -f 5`
SLAVE_0=`echo $VLAN_CONF \
| cut -d ',' -f 6`
SLAVE_1=`echo $VLAN_CONF \
| cut -d ',' -f 7`
if [[ -z $VLAN_BOND ]] ; then
continue
fi
if [[ ! -d /sys/class/net/$VLAN_BOND ]] ; then
if [[ ! -f /sys/class/net/bonding_masters ]] ; then
echo "Missing bonding_masters special file"
exit 22
fi
echo "+$VLAN_BOND" \
> /sys/class/net/bonding_masters
sleep 3
if [[ ! -d /sys/class/net/$VLAN_BOND ]] ; then
echo "$VLAN_BOND not available"
exit 23
fi
fi
MY_MODE=`cat /sys/class/net/$VLAN_BOND/bonding/mode \
| cut -d ' ' -f 2`
if [[ "$MY_MODE" != "$BOND_MODE" ]] ; then
echo $BOND_MODE \
> /sys/class/net/$VLAN_BOND/bonding/mode
sleep 2
fi
ip link set $VLAN_BOND up
sleep 5
ifenslave $VLAN_BOND $SLAVE_0 $SLAVE_1
sleep 5
if [[ $BOND_MODE == 1 ]] ; then
ip addr add $VLAN_IP/$MASK dev $VLAN_BOND
else
ip link add link $VLAN_BOND \
name $VLAN_BOND.$VLAN \
type vlan id $VLAN
ip link set $VLAN_BOND.$VLAN up
ip addr add $VLAN_IP/$MASK dev $VLAN_BOND.$VLAN
fi
done
systemctl restart sshd

46
scripts/stat-bonding Executable file
View file

@ -0,0 +1,46 @@
#!/bin/bash
source scan-netdev
if [[ "$1" == "out" ]] ; then
OUT=1
else
source test-variables
fi
#setup-netdev
#sleep 5
#systemctl start sshd
#sleep 1
while [[ 1 ]] ; do
if [[ -z $OUT ]] ; then
clear
fi
date
for BOND_N in {0..4} ; do
BOND=bond$BOND_N
if [[ -d /sys/class/net/$BOND ]] ; then
echo -n "$BOND:"
SLAVES=`cat /sys/class/net/$BOND/bonding/slaves`
MODE=`cat /sys/class/net/$BOND/bonding/mode \
| cut -d ' ' -f 2`
for DEV in $SLAVES ; do
LINKING=`cat /sys/class/net/$DEV/operstate`
if [[ "$LINKING" == "up" ]] ; then
echo -n " $GRN$DEV$OFF up"
else
echo -n " $RED$DEV$OFF down"
fi
done
echo -n " Mode: $BLD$MODE$OFF"
echo
if [[ $MODE == 1 ]] ; then
ACTIVE=`cat /sys/class/net/$BOND/bonding/active_slave`
echo "Active: $ACTIVE"
fi
fi
done
if [[ -n $OUT ]] ; then
exit 0
fi
sleep 2
done

23
scripts/test-batch Executable file
View file

@ -0,0 +1,23 @@
#!/bin/bash
source scan-dmidecode
source test-variables
#rm -f /output/hwtests/*/netdev/*.ethtool
source scan-netdev
echo $$ > /run/batch.pid
echo -n "setup-netdev... "
setup-netdev >/dev/null &
sleep 1
jobs
disown
#stat-bonding out > $OUTPUT_DIR/stat-bonding.pre.1.out
#test-bonding
#stat-bonding out > $OUTPUT_DIR/stat-bonding.post.1.out
# Take snapshot
#mkdir -p $SNAPSHOT_D
#cp -af $OUTPUT_DIR/* $SNAPSHOT_D/
rm -f /run/batch.pid
#echo '--'

54
scripts/test-bonding Executable file
View file

@ -0,0 +1,54 @@
#!/bin/bash
source scan-netdev
sleep_5 () {
for WAIT in 1 2 3 4 5 ; do
echo -n '.'
sleep 1
done
echo
}
echo "First ping test:"
test-ping | tee $OUTPUT_DIR/test-ping.out \
| grep error
for BOND_N in {0..4} ; do
BOND=bond$BOND_N
if [[ -d /sys/class/net/$BOND ]] ; then
cp /sys/class/net/$BOND/bonding/slaves \
$OUTPUT_DIR/netdev/$BOND.slaves
fi
done
SLAVE_NO=1
#for SLAVE_NO in 1 ; do
echo -n "Removing slave $SLAVE_NO..."
for BOND_N in {0..4} ; do
BOND=bond$BOND_N
[[ -d /sys/class/net/$BOND ]] || continue
SLAVE=`cut -d ' ' -f $SLAVE_NO \
$OUTPUT_DIR/netdev/$BOND.slaves`
ifenslave -d $BOND $SLAVE
done
sleep_5
echo -n "Dummy ping..."
test-ping > /dev/null
sleep_5
echo "Real ping test:"
test-ping | tee $OUTPUT_DIR/slave$SLAVE_NO-ping.out \
| grep error
echo -n "Reinserting slave $SLAVE_NO..."
for BOND_N in {0..4} ; do
[[ -d /sys/class/net/$BOND ]] || continue
BOND=bond$BOND_N
SLAVE=`cut -d ' ' -f $SLAVE_NO \
$OUTPUT_DIR/netdev/$BOND.slaves`
ifenslave $BOND $SLAVE
done
sleep_5
echo -n "Dummy ping..."
test-ping > /dev/null
sleep_5
#done

21
scripts/test-ping Executable file
View file

@ -0,0 +1,21 @@
#!/bin/bash
source scan-netdev
for VLAN in {401..408} ; do
if [[ -f /etc/netdev/vlan.$VLAN.csv ]] ; then
VLAN_CONF=`grep $SERVER_SERIAL \
/etc/netdev/vlan.$VLAN.csv`
fi
if [[ -z $VLAN_CONF ]] ; then
continue
fi
IP_LIST=`cat /etc/netdev/vlan.$VLAN.csv \
| cut -d ',' -f 3 \
| grep '^[0-9]' | sort`
for IP in $IP_LIST; do
echo -n "$IP - "
ping -c5 -f $IP | grep 'received'
done
done