diff options
Diffstat (limited to 'security/nss/automation/buildbot-slave/build.sh')
| -rwxr-xr-x | security/nss/automation/buildbot-slave/build.sh | 548 |
1 files changed, 548 insertions, 0 deletions
diff --git a/security/nss/automation/buildbot-slave/build.sh b/security/nss/automation/buildbot-slave/build.sh new file mode 100755 index 000000000..00e749672 --- /dev/null +++ b/security/nss/automation/buildbot-slave/build.sh @@ -0,0 +1,548 @@ +#! /bin/bash + +# Ensure a failure of the first command inside a pipe +# won't be hidden by commands later in the pipe. +# (e.g. as in ./dosomething | grep) + +set -o pipefail + +proc_args() +{ + while [ -n "$1" ]; do + OPT=$(echo $1 | cut -d= -f1) + VAL=$(echo $1 | cut -d= -f2) + + case $OPT in + "--build-nss") + BUILD_NSS=1 + ;; + "--test-nss") + TEST_NSS=1 + ;; + "--check-abi") + CHECK_ABI=1 + ;; + "--build-jss") + BUILD_JSS=1 + ;; + "--test-jss") + TEST_JSS=1 + ;; + "--memtest") + NSS_TESTS="memleak" + export NSS_TESTS + ;; + "--nojsssign") + NO_JSS_SIGN=1 + ;; + *) + echo "Usage: $0 ..." + echo " --memtest - run the memory leak tests" + echo " --nojsssign - try to sign jss" + echo " --build-nss" + echo " --build-jss" + echo " --test-nss" + echo " --test-jss" + echo " --check-abi" + exit 1 + ;; + esac + + shift + done +} + +set_env() +{ + TOPDIR=$(pwd) + HGDIR=$(pwd)$(echo "/hg") + OUTPUTDIR=$(pwd)$(echo "/output") + LOG_ALL="${OUTPUTDIR}/all.log" + LOG_TMP="${OUTPUTDIR}/tmp.log" + + echo "hello" |grep --line-buffered hello >/dev/null 2>&1 + [ $? -eq 0 ] && GREP_BUFFER="--line-buffered" +} + +print_log() +{ + DATE=$(date "+TB [%Y-%m-%d %H:%M:%S]") + echo "${DATE} $*" + echo "${DATE} $*" >> ${LOG_ALL} +} + +print_result() +{ + TESTNAME=$1 + RET=$2 + EXP=$3 + + if [ ${RET} -eq ${EXP} ]; then + print_log "${TESTNAME} PASSED" + else + print_log "${TESTNAME} FAILED" + fi +} + +print_env() +{ + print_log "######## Environment variables ########" + + uname -a | tee -a ${LOG_ALL} + if [ -e "/etc/redhat-release" ]; then + cat "/etc/redhat-release" | tee -a ${LOG_ALL} + fi + # don't print the MAIL command, it might contain a password + env | grep -v "^MAIL=" | tee -a ${LOG_ALL} +} + +set_cycle() +{ + BITS=$1 + OPT=$2 + + if [ "${BITS}" = "64" ]; then + USE_64=1 + JAVA_HOME=${JAVA_HOME_64} + PORT_DBG=${PORT_64_DBG} + PORT_OPT=${PORT_64_OPT} + else + USE_64= + JAVA_HOME=${JAVA_HOME_32} + PORT_DBG=${PORT_32_DBG} + PORT_OPT=${PORT_32_OPT} + fi + export USE_64 + export JAVA_HOME + + BUILD_OPT= + if [ "${OPT}" = "OPT" ]; then + BUILD_OPT=1 + XPCLASS=xpclass.jar + PORT=${PORT_OPT} + else + BUILD_OPT= + XPCLASS=xpclass_dbg.jar + PORT=${PORT_DBG} + fi + export BUILD_OPT + + PORT_JSS_SERVER=$(expr ${PORT} + 20) + PORT_JSSE_SERVER=$(expr ${PORT} + 40) + + export PORT + export PORT_JSS_SERVER + export PORT_JSSE_SERVER +} + +build_nss() +{ + print_log "######## NSS - build - ${BITS} bits - ${OPT} ########" + + print_log "$ cd ${HGDIR}/nss" + cd ${HGDIR}/nss + + print_log "$ ${MAKE} ${NSS_BUILD_TARGET}" + #${MAKE} ${NSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} | grep ${GREP_BUFFER} "^${MAKE}" + ${MAKE} ${NSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} + RET=$? + print_result "NSS - build - ${BITS} bits - ${OPT}" ${RET} 0 + + if [ ${RET} -eq 0 ]; then + return 0 + else + tail -100 ${LOG_ALL} + return ${RET} + fi +} + +build_jss() +{ + print_log "######## JSS - build - ${BITS} bits - ${OPT} ########" + + print_log "$ cd ${HGDIR}/jss" + cd ${HGDIR}/jss + + print_log "$ ${MAKE} ${JSS_BUILD_TARGET}" + #${MAKE} ${JSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} | grep ${GREP_BUFFER} "^${MAKE}" + ${MAKE} ${JSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} + RET=$? + print_result "JSS build - ${BITS} bits - ${OPT}" ${RET} 0 + [ ${RET} -eq 0 ] || return ${RET} + + print_log "$ cd ${HGDIR}/dist" + cd ${HGDIR}/dist + + if [ -z "${NO_JSS_SIGN}" ]; then + print_log "cat ${TOPDIR}/keystore.pw | ${JAVA_HOME}/bin/jarsigner -keystore ${TOPDIR}/keystore -internalsf ${XPCLASS} jssdsa" + cat ${TOPDIR}/keystore.pw | ${JAVA_HOME}/bin/jarsigner -keystore ${TOPDIR}/keystore -internalsf ${XPCLASS} jssdsa >> ${LOG_ALL} 2>&1 + RET=$? + print_result "JSS - sign JAR files - ${BITS} bits - ${OPT}" ${RET} 0 + [ ${RET} -eq 0 ] || return ${RET} + fi + print_log "${JAVA_HOME}/bin/jarsigner -verify -certs ${XPCLASS}" + ${JAVA_HOME}/bin/jarsigner -verify -certs ${XPCLASS} >> ${LOG_ALL} 2>&1 + RET=$? + print_result "JSS - verify JAR files - ${BITS} bits - ${OPT}" ${RET} 0 + [ ${RET} -eq 0 ] || return ${RET} + + return 0 +} + +test_nss() +{ + print_log "######## NSS - tests - ${BITS} bits - ${OPT} ########" + + if [ "${OS_TARGET}" = "Android" ]; then + print_log "$ cd ${HGDIR}/nss/tests/remote" + cd ${HGDIR}/nss/tests/remote + print_log "$ make test_android" + make test_android 2>&1 | tee ${LOG_TMP} | grep ${GREP_BUFFER} ": #" + OUTPUTFILE=${HGDIR}/tests_results/security/*.1/output.log + else + print_log "$ cd ${HGDIR}/nss/tests" + cd ${HGDIR}/nss/tests + print_log "$ ./all.sh" + ./all.sh 2>&1 | tee ${LOG_TMP} | egrep ${GREP_BUFFER} ": #|^\[.{10}\] " + OUTPUTFILE=${LOG_TMP} + fi + + cat ${LOG_TMP} >> ${LOG_ALL} + tail -n2 ${HGDIR}/tests_results/security/*.1/results.html | grep END_OF_TEST >> ${LOG_ALL} + RET=$? + + print_log "######## details of detected failures (if any) ########" + grep -B50 -w FAILED ${OUTPUTFILE} + [ $? -eq 1 ] || RET=1 + + print_result "NSS - tests - ${BITS} bits - ${OPT}" ${RET} 0 + return ${RET} +} + +check_abi() +{ + print_log "######## NSS ABI CHECK - ${BITS} bits - ${OPT} ########" + print_log "######## creating temporary HG clones ########" + + rm -rf ${HGDIR}/baseline + mkdir ${HGDIR}/baseline + BASE_NSS=`cat ${HGDIR}/nss/automation/abi-check/previous-nss-release` + hg clone -u "${BASE_NSS}" "${HGDIR}/nss" "${HGDIR}/baseline/nss" + if [ $? -ne 0 ]; then + echo "invalid tag in automation/abi-check/previous-nss-release" + return 1 + fi + + BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH + hg clone -u "${BASE_NSPR}" "${HGDIR}/nspr" "${HGDIR}/baseline/nspr" + if [ $? -ne 0 ]; then + echo "nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt" + # Assume that version hasn't been released yet, fall back to trunk + pushd "${HGDIR}/baseline/nspr" + hg update default + popd + fi + + print_log "######## building baseline NSPR/NSS ########" + pushd ${HGDIR}/baseline/nss + + print_log "$ ${MAKE} ${NSS_BUILD_TARGET}" + ${MAKE} ${NSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} + RET=$? + print_result "NSS - build - ${BITS} bits - ${OPT}" ${RET} 0 + if [ ${RET} -ne 0 ]; then + tail -100 ${LOG_ALL} + return ${RET} + fi + popd + + ABI_PROBLEM_FOUND=0 + ABI_REPORT=${OUTPUTDIR}/abi-diff.txt + rm -f ${ABI_REPORT} + PREVDIST=${HGDIR}/baseline/dist + NEWDIST=${HGDIR}/dist + ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnssdbm3.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so" + for SO in ${ALL_SOs}; do + if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then + touch ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt + fi + abidiff --hd1 $PREVDIST/public/ --hd2 $NEWDIST/public \ + $PREVDIST/*/lib/$SO $NEWDIST/*/lib/$SO \ + > ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt + RET=$? + cat ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt \ + | grep -v "^Functions changes summary:" \ + | grep -v "^Variables changes summary:" \ + > ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt + rm -f ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt + ABIDIFF_ERROR=$((($RET & 0x01) != 0)) + ABIDIFF_USAGE_ERROR=$((($RET & 0x02) != 0)) + ABIDIFF_ABI_CHANGE=$((($RET & 0x04) != 0)) + ABIDIFF_ABI_INCOMPATIBLE_CHANGE=$((($RET & 0x08) != 0)) + ABIDIFF_UNKNOWN_BIT_SET=$((($RET & 0xf0) != 0)) + + # If abidiff reports an error, or a usage error, or if it sets a result + # bit value this script doesn't know yet about, we'll report failure. + # For ABI changes, we don't yet report an error. We'll compare the + # result report with our whitelist. This allows us to silence changes + # that we're already aware of and have been declared acceptable. + + REPORT_RET_AS_FAILURE=0 + if [ $ABIDIFF_ERROR -ne 0 ]; then + print_log "abidiff reported ABIDIFF_ERROR." + REPORT_RET_AS_FAILURE=1 + fi + if [ $ABIDIFF_USAGE_ERROR -ne 0 ]; then + print_log "abidiff reported ABIDIFF_USAGE_ERROR." + REPORT_RET_AS_FAILURE=1 + fi + if [ $ABIDIFF_UNKNOWN_BIT_SET -ne 0 ]; then + print_log "abidiff reported ABIDIFF_UNKNOWN_BIT_SET." + REPORT_RET_AS_FAILURE=1 + fi + + if [ $ABIDIFF_ABI_CHANGE -ne 0 ]; then + print_log "Ignoring abidiff result ABI_CHANGE, instead we'll check for non-whitelisted differences." + fi + if [ $ABIDIFF_ABI_INCOMPATIBLE_CHANGE -ne 0 ]; then + print_log "Ignoring abidiff result ABIDIFF_ABI_INCOMPATIBLE_CHANGE, instead we'll check for non-whitelisted differences." + fi + + if [ $REPORT_RET_AS_FAILURE -ne 0 ]; then + ABI_PROBLEM_FOUND=1 + print_log "abidiff {$PREVDIST , $NEWDIST} for $SO FAILED with result $RET, or failed writing to ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt" + fi + if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then + ABI_PROBLEM_FOUND=1 + print_log "FAILED to access report file: ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt" + fi + + diff -wB -u ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt \ + ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT} + if [ ! -f ${ABI_REPORT} ]; then + ABI_PROBLEM_FOUND=1 + print_log "FAILED to compare exepcted and new report: ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt" + fi + done + + if [ -s ${ABI_REPORT} ]; then + print_log "FAILED: there are new unexpected ABI changes" + cat ${ABI_REPORT} + return 1 + elif [ $ABI_PROBLEM_FOUND -ne 0 ]; then + print_log "FAILED: failure executing the ABI checks" + cat ${ABI_REPORT} + return 1 + fi + + return 0 +} + +test_jss() +{ + print_log "######## JSS - tests - ${BITS} bits - ${OPT} ########" + + print_log "$ cd ${HGDIR}/jss" + cd ${HGDIR}/jss + + print_log "$ ${MAKE} platform" + PLATFORM=$(${MAKE} platform) + print_log "PLATFORM=${PLATFORM}" + + print_log "$ cd ${HGDIR}/jss/org/mozilla/jss/tests" + cd ${HGDIR}/jss/org/mozilla/jss/tests + + print_log "$ perl all.pl dist ${HGDIR}/dist/${PLATFORM}" + perl all.pl dist ${HGDIR}/dist/${PLATFORM} 2>&1 | tee ${LOG_TMP} + cat ${LOG_TMP} >> ${LOG_ALL} + + tail -n2 ${LOG_TMP} | grep JSSTEST_RATE > /dev/null + RET=$? + + grep FAIL ${LOG_TMP} + [ $? -eq 1 ] || RET=1 + + print_result "JSS - tests - ${BITS} bits - ${OPT}" ${RET} 0 + return ${RET} +} + +create_objdir_dist_link() +{ + # compute relevant 'dist' OBJDIR_NAME subdirectory names for JSS and NSS + OS_TARGET=`uname -s` + OS_RELEASE=`uname -r | sed 's/-.*//' | sed 's/-.*//' | cut -d . -f1,2` + CPU_TAG=_`uname -m` + # OBJDIR_NAME_COMPILER appears to be defined for NSS but not JSS + OBJDIR_NAME_COMPILER=_cc + LIBC_TAG=_glibc + IMPL_STRATEGY=_PTH + if [ "${RUN_BITS}" = "64" ]; then + OBJDIR_TAG=_${RUN_BITS}_${RUN_OPT}.OBJ + else + OBJDIR_TAG=_${RUN_OPT}.OBJ + fi + + # define NSS_OBJDIR_NAME + NSS_OBJDIR_NAME=${OS_TARGET}${OS_RELEASE}${CPU_TAG}${OBJDIR_NAME_COMPILER} + NSS_OBJDIR_NAME=${NSS_OBJDIR_NAME}${LIBC_TAG}${IMPL_STRATEGY}${OBJDIR_TAG} + print_log "create_objdir_dist_link(): NSS_OBJDIR_NAME='${NSS_OBJDIR_NAME}'" + + # define JSS_OBJDIR_NAME + JSS_OBJDIR_NAME=${OS_TARGET}${OS_RELEASE}${CPU_TAG} + JSS_OBJDIR_NAME=${JSS_OBJDIR_NAME}${LIBC_TAG}${IMPL_STRATEGY}${OBJDIR_TAG} + print_log "create_objdir_dist_link(): JSS_OBJDIR_NAME='${JSS_OBJDIR_NAME}'" + + if [ -e "${HGDIR}/dist/${NSS_OBJDIR_NAME}" ]; then + SOURCE=${HGDIR}/dist/${NSS_OBJDIR_NAME} + TARGET=${HGDIR}/dist/${JSS_OBJDIR_NAME} + ln -s ${SOURCE} ${TARGET} >/dev/null 2>&1 + fi +} + +build_and_test() +{ + if [ -n "${BUILD_NSS}" ]; then + build_nss + [ $? -eq 0 ] || return 1 + fi + + if [ -n "${TEST_NSS}" ]; then + test_nss + [ $? -eq 0 ] || return 1 + fi + + if [ -n "${CHECK_ABI}" ]; then + check_abi + [ $? -eq 0 ] || return 1 + fi + + if [ -n "${BUILD_JSS}" ]; then + create_objdir_dist_link + build_jss + [ $? -eq 0 ] || return 1 + fi + + if [ -n "${TEST_JSS}" ]; then + test_jss + [ $? -eq 0 ] || return 1 + fi + + return 0 +} + +run_cycle() +{ + print_env + build_and_test + RET=$? + + grep ^TinderboxPrint ${LOG_ALL} + + return ${RET} +} + +prepare() +{ + rm -rf ${OUTPUTDIR}.oldest >/dev/null 2>&1 + mv ${OUTPUTDIR}.older ${OUTPUTDIR}.oldest >/dev/null 2>&1 + mv ${OUTPUTDIR}.old ${OUTPUTDIR}.older >/dev/null 2>&1 + mv ${OUTPUTDIR}.last ${OUTPUTDIR}.old >/dev/null 2>&1 + mv ${OUTPUTDIR} ${OUTPUTDIR}.last >/dev/null 2>&1 + mkdir -p ${OUTPUTDIR} + + # Remove temporary test files from previous jobs, that weren't cleaned up + # by move_results(), e.g. caused by unexpected interruptions. + rm -rf ${HGDIR}/tests_results/ + + cd ${HGDIR}/nss + + if [ -n "${FEWER_STRESS_ITERATIONS}" ]; then + sed -i 's/-c_1000_/-c_500_/g' tests/ssl/sslstress.txt + fi + + return 0 +} + +move_results() +{ + cd ${HGDIR} + if [ -n "${TEST_NSS}" ]; then + mv -f tests_results ${OUTPUTDIR} + fi + tar -c -z --dereference -f ${OUTPUTDIR}/dist.tgz dist + rm -rf dist +} + +run_all() +{ + set_cycle ${BITS} ${OPT} + prepare + run_cycle + RESULT=$? + print_log "### result of run_cycle is ${RESULT}" + move_results + return ${RESULT} +} + +main() +{ + VALID=0 + RET=1 + FAIL=0 + + for BITS in 32 64; do + echo ${RUN_BITS} | grep ${BITS} > /dev/null + [ $? -eq 0 ] || continue + for OPT in DBG OPT; do + echo ${RUN_OPT} | grep ${OPT} > /dev/null + [ $? -eq 0 ] || continue + + VALID=1 + set_env + run_all + RET=$? + print_log "### result of run_all is ${RET}" + if [ ${RET} -ne 0 ]; then + FAIL=${RET} + fi + done + done + + if [ ${VALID} -ne 1 ]; then + echo "Need to set valid bits/opt values." + return 1 + fi + + return ${FAIL} +} + +#function killallsub() +#{ +# FINAL_RET=$? +# for proc in `jobs -p` +# do +# kill -9 $proc +# done +# return ${FINAL_RET} +#} +#trap killallsub EXIT + +#IS_RUNNING_FILE="./build-is-running" + +#if [ -a $IS_RUNNING_FILE ]; then +# echo "exiting, because old job is still running" +# exit 1 +#fi + +#touch $IS_RUNNING_FILE + +echo "tinderbox args: $0 $@" +. ${ENVVARS} +proc_args "$@" +main + +RET=$? +print_log "### result of main is ${RET}" + +#rm $IS_RUNNING_FILE +exit ${RET} |