tests/tester.sh

   1 #!/bin/bash
   2 #
   3 # tester.sh - reads testcases from file and tests busybox applets vs GNU
   4 # counterparts
   5 #
   6 # This should be run from within the tests/ directory. Before you run it, you
   7 # should compile up a busybox that has all applets and all features turned on.
   8
   9 # set up defaults (can be changed with cmd-line options)
  10 BUSYBOX=../busybox
  11 TESTCASES=testcases
  12 LOGFILE=tester.log
  13 CONFIG_OUT=bb.out
  14 GNU_OUT=gnu.out
  15 SETUP=""
  16 CLEANUP=""
  17 KEEPTMPFILES="no"
  18 DEBUG=2
  19
  20
  21 #while getopts 'p:t:l:b:g:s:c:kd:' opt
  22 while getopts 'p:t:l:s:c:kd:' opt
  23 do
  24         case $opt in
  25                 p) BUSYBOX=$OPTARG; ;;
  26                 t) TESTCASES=$OPTARG; ;;
  27                 l) LOGFILE=$OPTARG; ;;
  28 #               b) CONFIG_OUT=$OPTARG; ;;
  29 #               g) GNU_OUT=$OPTARG; ;;
  30                 s) SETUP=$OPTARG; ;;
  31                 c) CLEANUP=$OPTARG; ;;
  32                 k) KEEPTMPFILES="yes"; ;;
  33                 d) DEBUG=$OPTARG; ;;
  34                 *)
  35                         echo "usage: $0 [-ptlbgsc]"
  36                         echo "  -p PATH  path to busybox executable (default=$BUSYBOX)"
  37                         echo "  -t FILE  run testcases in FILE (default=$TESTCASES)"
  38                         echo "  -l FILE  log test results in FILE (default=$LOGFILE)"
  39 #                       echo "  -b FILE  store temporary busybox output in FILE"
  40 #                       echo "  -g FILE  store temporary GNU output in FILE"
  41                         echo "  -s FILE  (setup) run commands in FILE before testcases"
  42                         echo "  -c FILE  (cleanup) run commands in FILE after testcases"
  43                         echo "  -k       keep temporary output files (don't delete them)"
  44                         echo "  -d NUM   set level of debugging output"
  45                         echo "           0 = no output"
  46                         echo "           1 = output failures / whoops lines only"
  47                         echo "           2 = (default) output setup / cleanup msgs and testcase lines"
  48                         echo "           3+= other debug noise (internal stuff)"
  49                         exit 1
  50                         ;;
  51         esac
  52 done
  53 #shift `expr $OPTIND - 1`
  54
  55
  56 # maybe print some debug output
  57 if [ $DEBUG -ge 3 ]
  58 then
  59         echo "BUSYBOX=$BUSYBOX"
  60         echo "TESTCASES=$TESTCASES"
  61         echo "LOGFILE=$LOGFILE"
  62         echo "CONFIG_OUT=$CONFIG_OUT"
  63         echo "GNU_OUT=$GNU_OUT"
  64         echo "SETUP=$SETUP"
  65         echo "CLEANUP=$CLEANUP"
  66         echo "DEBUG=$DEBUG"
  67 fi
  68
  69
  70 # do sanity checks
  71 if [ ! -e $BUSYBOX ]
  72 then
  73         echo "Busybox executable: $BUSYBOX not found!"
  74         exit 1
  75 fi
  76
  77 if [ ! -e $TESTCASES ]
  78 then
  79         echo "Testcases file: $TESTCASES not found!"
  80         exit 1
  81 fi
  82
  83
  84 # do normal setup
  85 [ -e $LOGFILE ] && rm $LOGFILE
  86 unalias -a      # gets rid of aliases that might create different output
  87
  88
  89 # do extra setup (if any)
  90 if [ ! -z "$SETUP" ]
  91 then
  92         [ $DEBUG -ge 2 ] && echo "running setup commands in $SETUP"
  93         source $SETUP
  94 fi
  95
  96
  97 # go through each line in the testcase file
  98 cat $TESTCASES | while read line
  99 do
 100         #echo $line
 101         # only process non-blank lines and non-comment lines
 102         if [ "$line" ]
 103         then
 104                 if [ `echo "$line" | cut -c1` != "#" ]
 105                 then
 106
 107                         # test if the applet was compiled into busybox
 108                         # (this only tests the applet at the beginning of the line)
 109                         #applet=`echo $line | cut -d' ' -f1`
 110                         applet=`echo $line | sed 's/\(^[^ ;]*\)[ ;].*/\1/'`
 111                         $BUSYBOX 2>&1 | grep -qw $applet
 112                         if [ $? -eq 1 ]
 113                         then
 114                                 echo "WHOOPS: $applet not compiled into busybox" | tee -a $LOGFILE
 115                         else
 116
 117                                 # execute line using gnu / system programs
 118                                 [ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
 119                                 sh -c "$line" > $GNU_OUT
 120
 121                                 # change line to include "busybox" before every statement
 122                                 line="$BUSYBOX $line"
 123                                 # is this a bash-2-ism?
 124                                 # line=${line//;/; $BUSYBOX }
 125                                 # line=${line//|/| $BUSYBOX }
 126                                 # assume $BUSYBOX has no commas
 127                                 line=`echo "$line" | sed -e 's,;,; '$BUSYBOX, \
 128                                                        -e 's, |, | '$BUSYBOX,`
 129
 130                                 # execute line using busybox programs
 131                                 [ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
 132                                 sh -c "$line" > $CONFIG_OUT
 133
 134                                 # see if they match
 135                                 diff -q $CONFIG_OUT $GNU_OUT > /dev/null
 136                                 if [ $? -eq 1 ]
 137                                 then
 138                                         [ $DEBUG -ge 1 ] && echo "FAILED: $line" | tee -a $LOGFILE
 139                                         diff -u $CONFIG_OUT $GNU_OUT >> $LOGFILE
 140                                 fi
 141                         fi
 142                 fi
 143         fi
 144 done
 145
 146 [ $DEBUG -gt 0 ] && echo "Finished. Results are in $LOGFILE"
 147
 148
 149 # do normal cleanup
 150 [ "$KEEPTMPFILES" = "no" ] && rm -f $CONFIG_OUT $GNU_OUT
 151
 152
 153 # do extra cleanup (if any)
 154 if [ ! -z "$CLEANUP" ]
 155 then
 156         [ $DEBUG -ge 2 ] && echo "running cleanup commands in $CLEANUP"
 157         source $CLEANUP
 158 fi