add compare-results.sh, call from start-testsuite.sh
Compare current test results to the expected results, and exit in error on discrepancies. Add compare-result.sh: (trivially) grep junit xml output to determine which tests passed and which didn't, and compare against an expected-result.log, another junit file from a previous run. Summarize and determine success. Include an "xfail" feature: tests that are expected to fail are marked as "xfail", unexpected failures as "FAIL". In various subdirs, copy the current jenkins jobs' junit xml outputs as expected-results.log, so that we will start getting useful output in both jenkins runs and manual local runs. In start-testsuite.sh, after running the tests, invoke the results comparison. Due to the single-line parsing nature, the script so far does not distinguish between error and failure. I doubt that we actually need to do that though. Related: OS#3136 Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
This commit is contained in:
Neels Hofmeyr
parent
def4ad4bce
commit
3cf797d7a5
|
|
@ -1,6 +1,7 @@
|
|||
deps/*/
|
||||
*.o
|
||||
*.log
|
||||
!expected-results.log
|
||||
*.so
|
||||
compile
|
||||
*/.gitignore
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='BSC_Tests' tests='68' failures='4' errors='2' skipped='0' inconc='0' time='932.00'>
|
||||
<testcase classname='BSC_Tests' name='TC_ctrl_msc_connection_status' time='2.267915'/>
|
||||
<testcase classname='BSC_Tests' name='TC_ctrl_msc0_connection_status' time='2.225696'/>
|
||||
<testcase classname='BSC_Tests' name='TC_ctrl' time='4.242800'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_noreply' time='2.229171'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_counter' time='2.226685'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_ack_noest' time='5.225675'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_noreply' time='24.248343'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_refused' time='4.231838'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_act_nack' time='2.728319'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_exhaustion' time='2.239509'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_rel_rll_rel_ind' time='8.276291'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_rel_conn_fail' time='8.290978'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_rel_hard_clear' time='7.431206'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_rel_hard_rlsd' time='4.451855'/>
|
||||
<testcase classname='BSC_Tests' name='TC_chan_rel_a_reset' time='4.437509'/>
|
||||
<testcase classname='BSC_Tests' name='TC_outbound_connect' time='2.230366'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_cic_only' time='2.432230'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_csd' time='2.435962'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_ctm' time='2.431946'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_sign' time='3.139859'>
|
||||
<error type='DTE'></error>
|
||||
</testcase>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_fr_a5_0' time='5.362074'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1' time='5.360999'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1_codec_missing' time='3.371507'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_fr_a5_3' time='5.373823'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_fr_a5_4' time='5.385205'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_codec_fr' time='5.370247'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_codec_hr' time='5.370154'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_codec_efr' time='5.373357'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_codec_amr_f' time='5.369318'/>
|
||||
<testcase classname='BSC_Tests' name='TC_assignment_codec_amr_h' time='5.362888'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rll_est_ind_inact_lchan' time='2.224803'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi1' time='4.226237'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi3' time='4.231547'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sacch' time='4.246674'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan' time='8.753319'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_tmsi_nochan' time='8.759194'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_tmsi_any' time='8.750134'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_tmsi_sdcch' time='8.754159'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_f' time='8.758687'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_hf' time='9.289936'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi' time='8.956272'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_ci' time='8.956649'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_ci' time='8.969455'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lai' time='8.763181'>
|
||||
<failure type='fail-verdict'>Timeout expecting { msg_disc := { msg_group := RSL_MDISC_CCHAN (6), transparent := false }, msg_type := RSL_MT_PAGING_CMD (21), ies := { { iei := ?, body := { chan_nr := { u := { ch0 := RSL_CHAN_NR_PCH_AGCH (18) }, tn := ? } } }, { iei := ?, body := { paging_group := ? } }, { iei := ?, body := { ms_identity := { len := ?, payload := ? } } }, * } }
|
||||
BSC_Tests.ttcn:2203 BSC_Tests control part
|
||||
BSC_Tests.ttcn:1131 TC_paging_imsi_nochan_lai testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac' time='8.752368'>
|
||||
<error type='DTE'></error>
|
||||
</testcase>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_all' time='8.751176'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_plmn_lac_rnc' time='8.563604'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_rnc' time='8.551772'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_rnc' time='8.568040'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs' time='8.762731'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs_empty' time='8.548303'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi_unknown_cid' time='8.557896'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_a_reset' time='8.759365'>
|
||||
<failure type='fail-verdict'>Received PAGING after A-RESET
|
||||
BSC_Tests.ttcn:2212 BSC_Tests control part
|
||||
BSC_Tests.ttcn:1306 TC_paging_imsi_a_reset testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_imsi_load' time='12.961436'/>
|
||||
<testcase classname='BSC_Tests' name='TC_paging_counter' time='20.788631'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rsl_drop_counter' time='2.227479'/>
|
||||
<testcase classname='BSC_Tests' name='TC_rsl_unknown_unit_id' time='0.010705'/>
|
||||
<testcase classname='BSC_Tests' name='TC_oml_unknown_unit_id' time='0.010646'/>
|
||||
<testcase classname='BSC_Tests' name='TC_classmark' time='3.185732'/>
|
||||
<testcase classname='BSC_Tests' name='TC_unsol_ass_fail' time='8.149850'/>
|
||||
<testcase classname='BSC_Tests' name='TC_unsol_ass_compl' time='8.155615'/>
|
||||
<testcase classname='BSC_Tests' name='TC_unsol_ho_fail' time='8.147592'/>
|
||||
<testcase classname='BSC_Tests' name='TC_err_82_short_msg' time='8.161174'/>
|
||||
<testcase classname='BSC_Tests' name='TC_err_84_unknown_msg' time='3.174456'/>
|
||||
<testcase classname='BSC_Tests' name='TC_ho_int' time='7.423814'/>
|
||||
<testcase classname='BSC_Tests' name='TC_bssap_rlsd_does_not_cause_bssmap_reset' time='14.310727'/>
|
||||
<testcase classname='BSC_Tests' name='TC_bssmap_clear_does_not_cause_bssmap_reset' time='45.459198'>
|
||||
<failure type='fail-verdict'>Timeout of T_guard
|
||||
BSC_Tests.ttcn:2230 BSC_Tests control part
|
||||
BSC_Tests.ttcn:2056 TC_bssmap_clear_does_not_cause_bssmap_reset testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='BSC_Tests' name='TC_ms_rel_ind_does_not_cause_bssmap_reset' time='45.320202'>
|
||||
<failure type='fail-verdict'>Timeout of T_guard
|
||||
BSC_Tests.ttcn:2231 BSC_Tests control part
|
||||
BSC_Tests.ttcn:2115 TC_ms_rel_ind_does_not_cause_bssmap_reset testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
#!/usr/bin/env bash
|
||||
expected_file="$1"
|
||||
results_file="$2"
|
||||
|
||||
usage() {
|
||||
echo "
|
||||
Usage:
|
||||
|
||||
$(basename "$0") expected_results.junit-log current_results.junit-log [--allow-* [...]]
|
||||
|
||||
Return 0 if the expected results match the current results exactly.
|
||||
|
||||
--allow-skip Allow runnning less tests than are listed in the expected file.
|
||||
Default is to return failure on any skipped tests.
|
||||
--allow-new Allow more test results than found in the expected file.
|
||||
Default is to return failure on any unknown tests.
|
||||
--allow-xpass If a test was expected to fail but passed, return success.
|
||||
Default is to return failure on any mismatch.
|
||||
"
|
||||
}
|
||||
|
||||
if [ ! -f "$expected_file" ]; then
|
||||
usage
|
||||
echo "Expected file not found: '$expected_file'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$results_file" ]; then
|
||||
usage
|
||||
echo "Current results file not found: '$results_file'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
shift
|
||||
shift
|
||||
|
||||
allow_xpass=0
|
||||
allow_skip=0
|
||||
allow_new=0
|
||||
|
||||
while test -n "$1"; do
|
||||
arg="$1"
|
||||
if [ "x$arg" = "x--allow-xpass" ]; then
|
||||
allow_xpass=1
|
||||
elif [ "x$arg" = "x--allow-skip" ]; then
|
||||
allow_skip=1
|
||||
elif [ "x$arg" = "x--allow-new" ]; then
|
||||
allow_new=1
|
||||
else
|
||||
usage
|
||||
echo "Unknown argument: '$arg'"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
echo "Comparing expected results $expected_file against results in $results_file
|
||||
--------------------"
|
||||
|
||||
parse_testcase() {
|
||||
line="$1"
|
||||
suite_name="$(echo "$line" | sed 's,.*classname='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
|
||||
test_name="$(echo "$line" | sed 's,.*\<name='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
|
||||
if [ -n "$(echo "$line" | grep '/>$')" ]; then
|
||||
test_result="pass"
|
||||
else
|
||||
test_result="FAIL"
|
||||
fi
|
||||
}
|
||||
|
||||
pass=0
|
||||
xfail=0
|
||||
more_failures=0
|
||||
more_successes=0
|
||||
skipped=0
|
||||
new=0
|
||||
|
||||
while read line; do
|
||||
parse_testcase "$line"
|
||||
exp_suite_name="$suite_name"
|
||||
exp_test_name="$test_name"
|
||||
exp_test_result="$test_result"
|
||||
matched="0"
|
||||
|
||||
while read line; do
|
||||
parse_testcase "$line"
|
||||
if [ "x$exp_suite_name" != "x$suite_name" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ "x$exp_test_name" != "x$test_name" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ "x$exp_test_result" = "x$test_result" ]; then
|
||||
if [ "x$exp_test_result" = "xFAIL" ]; then
|
||||
exp_test_result="xfail"
|
||||
(( xfail += 1 ))
|
||||
else
|
||||
(( pass += 1 ))
|
||||
fi
|
||||
echo "$exp_test_result $suite_name.$test_name"
|
||||
else
|
||||
if [ "x$exp_test_result" = "xFAIL" ]; then
|
||||
exp_test_result="xfail"
|
||||
fi
|
||||
echo "$exp_test_result->$test_result $suite_name.$test_name"
|
||||
if [ "x$test_result" = "xFAIL" ]; then
|
||||
(( more_failures += 1 ))
|
||||
else
|
||||
(( more_successes += 1 ))
|
||||
fi
|
||||
fi
|
||||
matched="1"
|
||||
break
|
||||
done <<< "$(grep "<testcase" "$results_file")"
|
||||
|
||||
if [ "x$matched" = "x0" ]; then
|
||||
echo "skipped $exp_suite_name.$exp_test_name"
|
||||
(( skipped += 1 ))
|
||||
fi
|
||||
|
||||
done <<< "$(grep "<testcase" "$expected_file")"
|
||||
|
||||
# Also catch all new tests that aren't covered in the expected results
|
||||
while read line; do
|
||||
parse_testcase "$line"
|
||||
got_suite_name="$suite_name"
|
||||
got_test_name="$test_name"
|
||||
got_test_result="$test_result"
|
||||
matched="0"
|
||||
|
||||
while read line; do
|
||||
parse_testcase "$line"
|
||||
if [ "x$got_suite_name" != "x$suite_name" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ "x$got_test_name" != "x$test_name" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
matched="1"
|
||||
break
|
||||
done <<< "$(grep "<testcase" "$expected_file")"
|
||||
|
||||
if [ "x$matched" = "x0" ]; then
|
||||
echo "NEW-$got_test_result $got_suite_name.$got_test_name"
|
||||
(( new += 1 ))
|
||||
fi
|
||||
|
||||
done <<< "$(grep "<testcase" "$results_file")"
|
||||
|
||||
echo "--------------------"
|
||||
overall_verdict=0
|
||||
|
||||
if [ "x$pass" != x0 ]; then
|
||||
echo "$pass pass"
|
||||
fi
|
||||
|
||||
if [ "x$xfail" != x0 ]; then
|
||||
echo "$xfail xfail"
|
||||
fi
|
||||
|
||||
if [ "x$skipped" != x0 ]; then
|
||||
echo "$skipped skipped"
|
||||
if [ "x$allow_skip" = x0 ]; then
|
||||
overall_verdict=4
|
||||
echo " Failing due to skipped tests. If tests were removed, update the expected results!"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "x$new" != x0 ]; then
|
||||
echo "$new new"
|
||||
if [ "x$allow_new" = x0 ]; then
|
||||
overall_verdict=3
|
||||
echo " Failing due to new tests. Update the expected results!"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "x$more_successes" != x0 ]; then
|
||||
echo "$more_successes pass unexpectedly"
|
||||
if [ "x$allow_xpass" = x0 ]; then
|
||||
overall_verdict=2
|
||||
echo " Update the expected results!"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "x$more_failures" != x0 ]; then
|
||||
echo "$more_failures FAIL"
|
||||
overall_verdict=1
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "x$overall_verdict" != x0 ]; then
|
||||
echo "FAILURE"
|
||||
else
|
||||
echo "ok"
|
||||
fi
|
||||
exit $overall_verdict
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='GGSN_Tests' tests='11' failures='0' errors='0' skipped='0' inconc='0' time='205.00'>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp4_act_deact' time='0.006364'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_ipcp' time='0.003752'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_pcodns' time='0.003510'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_gtpu_access' time='6.011141'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp4_clients_interact' time='0.006517'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp6_act_deact' time='0.003312'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_pcodns' time='0.003403'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_icmp6' time='0.005114'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_gtpu_access' time='9.018065'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_pdp6_clients_interact' time='6.014695'/>
|
||||
<testcase classname='GGSN_Tests' name='TC_echo_req_resp' time='0.001674'/>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='Titan' tests='10' failures='2' errors='0' skipped='0' inconc='0' time='118.00'>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_sai_err_invalid_imsi' time='0.008962'>
|
||||
<failure type='fail-verdict'>Unexpected SAI ERROR Cause
|
||||
HLR_Tests.ttcn:558 HLR_Tests control part
|
||||
HLR_Tests.ttcn:364 TC_gsup_sai_err_invalid_imsi testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_sai' time='0.431623'/>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_ul_unknown_imsi' time='0.009124'/>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_sai_err_unknown_imsi' time='0.008056'/>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_ul' time='0.402349'/>
|
||||
<testcase classname='HLR_Tests' name='TC_vty' time='0.151754'/>
|
||||
<testcase classname='HLR_Tests' name='TC_vty_msisdn_isd' time='5.076033'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ISD.req
|
||||
HLR_Tests.ttcn:564 HLR_Tests control part
|
||||
HLR_Tests.ttcn:485 TC_vty_msisdn_isd testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_purge_cs' time='0.459958'/>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_purge_ps' time='0.467724'/>
|
||||
<testcase classname='HLR_Tests' name='TC_gsup_purge_unknown' time='0.008034'/>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='Titan' tests='26' failures='1' errors='0' skipped='1' inconc='0' time='177.00'>
|
||||
<testcase classname='MGCP_Test' name='TC_selftest' time='0.002990'>
|
||||
<skipped>no verdict</skipped>
|
||||
</testcase>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx' time='0.008097'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_noprefix' time='0.007103'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_unsupp_mode' time='0.006485'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_early_bidir_mode' time='0.023678'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_unsupp_param' time='0.005827'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_missing_callid' time='0.006864'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_missing_mode' time='0.006328'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_unsupp_packet_intv' time='0.005716'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_illegal_double_lco' time='0.014705'>
|
||||
<failure type='fail-verdict'>
|
||||
MGCP_Test.ttcn:887 MGCP_Test control part
|
||||
MGCP_Test.ttcn:436 TC_crcx_illegal_double_lco testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_sdp' time='0.008481'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_wildcarded' time='0.006528'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_wildcarded_exhaust' time='0.033504'/>
|
||||
<testcase classname='MGCP_Test' name='TC_mdcx_without_crcx' time='0.023214'/>
|
||||
<testcase classname='MGCP_Test' name='TC_dlcx_without_crcx' time='0.006506'/>
|
||||
<testcase classname='MGCP_Test' name='TC_mdcx_wildcarded' time='0.006675'/>
|
||||
<testcase classname='MGCP_Test' name='TC_dlcx_wildcarded' time='0.022936'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_connid' time='0.006310'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid' time='0.006509'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep' time='0.006822'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_inval' time='0.007519'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_connid_inval' time='0.006912'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_retrans' time='0.006681'/>
|
||||
<testcase classname='MGCP_Test' name='TC_crcx_dlcx_30ep' time='0.032986'/>
|
||||
<testcase classname='MGCP_Test' name='TC_rtpem_selftest' time='5.514538'/>
|
||||
<testcase classname='MGCP_Test' name='TC_two_crcx_and_rtp' time='1.111828'/>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='Titan' tests='45' failures='12' errors='1' skipped='0' inconc='0' time='497.00'>
|
||||
<testcase classname='MSC_Tests' name='TC_cr_before_reset' time='4.009127'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_tmsi' time='1.035697'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_notmsi' time='1.045502'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_reject' time='6.041282'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1829 MSC_Tests control part
|
||||
MSC_Tests.ttcn:502 TC_lu_imsi_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_timeout_gsup' time='11.044559'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1830 MSC_Tests control part
|
||||
MSC_Tests.ttcn:531 TC_lu_imsi_timeout_gsup testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi' time='1.036704'/>
|
||||
<testcase classname='MSC_Tests' name='TC_cmserv_imsi_unknown' time='1.021692'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_and_mo_call' time='4.291881'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_auth_sai_timeout' time='6.032964'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_auth_sai_err' time='1.028503'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_clear_request' time='2.029726'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_disconnect' time='2.028908'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_by_imei' time='6.030599'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_by_tmsi_noauth_unknown' time='1.246818'/>
|
||||
<testcase classname='MSC_Tests' name='TC_imsi_detach_by_imsi' time='1.238427'/>
|
||||
<testcase classname='MSC_Tests' name='TC_imsi_detach_by_tmsi' time='1.228194'/>
|
||||
<testcase classname='MSC_Tests' name='TC_imsi_detach_by_imei' time='1.233577'/>
|
||||
<testcase classname='MSC_Tests' name='TC_emerg_call_imei_reject' time='6.025962'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1843 MSC_Tests control part
|
||||
MSC_Tests.ttcn:903 TC_emerg_call_imei_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_emerg_call_imsi' time='4.295068'/>
|
||||
<testcase classname='MSC_Tests' name='TC_cm_serv_req_vgcs_reject' time='6.251082'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1845 MSC_Tests control part
|
||||
MSC_Tests.ttcn:940 TC_cm_serv_req_vgcs_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_cm_serv_req_vbs_reject' time='6.249901'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1846 MSC_Tests control part
|
||||
MSC_Tests.ttcn:961 TC_cm_serv_req_vbs_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_cm_serv_req_lcs_reject' time='6.244740'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1847 MSC_Tests control part
|
||||
MSC_Tests.ttcn:982 TC_cm_serv_req_lcs_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_cm_reest_req_reject' time='6.250134'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1848 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1003 TC_cm_reest_req_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_auth_2G_fail' time='1.042594'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_13' time='1.052763'/>
|
||||
<testcase classname='MSC_Tests' name='TC_cl3_no_payload' time='1.021188'/>
|
||||
<testcase classname='MSC_Tests' name='TC_cl3_rnd_payload' time='5.996049'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand or SCCP Release
|
||||
MSC_Tests.ttcn:1852 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1125 TC_cl3_rnd_payload testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_establish_and_nothing' time='6.263255'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1853 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1142 TC_establish_and_nothing testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_mo_setup_and_nothing' time='31.481429'>
|
||||
<failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
|
||||
MSC_Tests.ttcn:1854 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1166 TC_mo_setup_and_nothing testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_mo_crcx_ran_timeout' time='5.267724'/>
|
||||
<testcase classname='MSC_Tests' name='TC_mo_crcx_ran_reject' time='31.279836'>
|
||||
<failure type='fail-verdict'>Timeout waiting for channel release
|
||||
MSC_Tests.ttcn:1856 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1283 TC_mo_crcx_ran_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_mt_crcx_ran_reject' time='31.062826'>
|
||||
<failure type='fail-verdict'>Timeout waiting for channel release
|
||||
MSC_Tests.ttcn:1857 MSC_Tests control part
|
||||
MSC_Tests.ttcn:1380 TC_mt_crcx_ran_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_mo_setup_and_dtmf_dup' time='2.468132'/>
|
||||
<testcase classname='MSC_Tests' name='TC_gsup_cancel' time='1.259044'>
|
||||
<error type='DTE'></error>
|
||||
</testcase>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_1_13' time='1.049565'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_13' time='1.054199'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1' time='1.034815'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_no_cm' time='1.038335'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_2' time='1.042416'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_013_2' time='1.050732'/>
|
||||
<testcase classname='MSC_Tests' name='TC_mo_release_timeout' time='25.264303'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_and_mt_call_no_dlcx_resp' time='10.273413'/>
|
||||
<testcase classname='MSC_Tests' name='TC_reset_two' time='2.999986'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_and_mt_call' time='4.284560'/>
|
||||
<testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_log_msc_debug' time='1.040196'/>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='Titan' tests='22' failures='5' errors='2' skipped='0' inconc='0' time='491.00'>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach' time='7.164349'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_mnc3' time='7.007076'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_auth_id_timeout' time='35.991976'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_auth_sai_timeout' time='36.007949'>
|
||||
<failure type='fail-verdict'>Tguard timeout
|
||||
SGSN_Tests.ttcn:1157 SGSN_Tests control part
|
||||
SGSN_Tests.ttcn:480 TC_attach_auth_sai_timeout testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_auth_sai_reject' time='36.020705'>
|
||||
<failure type='fail-verdict'>Tguard timeout
|
||||
SGSN_Tests.ttcn:1158 SGSN_Tests control part
|
||||
SGSN_Tests.ttcn:501 TC_attach_auth_sai_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_timeout' time='37.025962'>
|
||||
<failure type='fail-verdict'>Tguard timeout
|
||||
SGSN_Tests.ttcn:1159 SGSN_Tests control part
|
||||
SGSN_Tests.ttcn:529 TC_attach_gsup_lu_timeout testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_reject' time='37.024621'>
|
||||
<failure type='fail-verdict'>Tguard timeout
|
||||
SGSN_Tests.ttcn:1160 SGSN_Tests control part
|
||||
SGSN_Tests.ttcn:558 TC_attach_gsup_lu_reject testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_combined' time='7.007181'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_accept_all' time='7.995753'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_closed' time='8.017589'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_rau_unknown' time='6.998580'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_rau' time='7.008848'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_detach_unknown_nopoweroff' time='6.993647'>
|
||||
<error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 77, which has already terminated.</error>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_detach_unknown_poweroff' time='6.982069'>
|
||||
<error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 83, which has already terminated.</error>
|
||||
</testcase>
|
||||
<testcase classname='SGSN_Tests' name='TC_detach_nopoweroff' time='7.010464'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_detach_poweroff' time='12.011997'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_pdp_act' time='6.050101'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_pdp_act_unattached' time='5.999444'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user' time='6.053814'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_pdp_act_ggsn_reject' time='6.038840'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mo' time='6.041516'/>
|
||||
<testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mt' time='36.021021'>
|
||||
<failure type='fail-verdict'>Tguard timeout
|
||||
SGSN_Tests.ttcn:1175 SGSN_Tests control part
|
||||
SGSN_Tests.ttcn:1148 TC_attach_pdp_act_user_deact_mt testcase
|
||||
</failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='Titan' tests='4' failures='0' errors='0' skipped='0' inconc='0' time='106.00'>
|
||||
<testcase classname='SIP_Tests' name='TC_mt_success_rel_gsm' time='5.032948'/>
|
||||
<testcase classname='SIP_Tests' name='TC_mt_success_rel_sip' time='5.032518'/>
|
||||
<testcase classname='SIP_Tests' name='TC_mo_success_rel_gsm' time='5.032548'/>
|
||||
<testcase classname='SIP_Tests' name='TC_mo_success_rel_sip' time='5.059944'/>
|
||||
</testsuite>
|
||||
|
|
@ -10,7 +10,9 @@ if [ $# -lt 1 ]; then
|
|||
fi
|
||||
|
||||
SUITE=$1
|
||||
CFG=$(basename "$SUITE").cfg
|
||||
SUITE_DIR="$(dirname "$SUITE")"
|
||||
SUITE_NAME="$(basename "$SUITE")"
|
||||
CFG="$SUITE_NAME.cfg"
|
||||
if [ $# -gt 1 ]; then
|
||||
CFG=$2
|
||||
fi
|
||||
|
|
@ -19,4 +21,26 @@ if [ $# -gt 2 ]; then
|
|||
TEST=$3
|
||||
fi
|
||||
|
||||
LD_LIBRARY_PATH=$(dirname "$SUITE"):/usr/lib/titan:/usr/ttcn3/lib ttcn3_start $SUITE $CFG $TEST
|
||||
LD_LIBRARY_PATH="$SUITE_DIR:/usr/lib/titan:/usr/ttcn3/lib" ttcn3_start $SUITE $CFG $TEST
|
||||
|
||||
expected="$SUITE_DIR/expected-results.log"
|
||||
if [ ! -f "$expected" ]; then
|
||||
echo "No expected results found, not comparing outcome. ($expected)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# find the most recent junit output log here
|
||||
last_log="$(ls -1tr junit*.log | tail -n 1)"
|
||||
if [ ! -f "$last_log" ]; then
|
||||
echo "No junit log found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compare="$SUITE_DIR/../compare-results.sh"
|
||||
if [ ! -x "$compare" ]; then
|
||||
echo "ERROR: cannot find $compare"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
"$compare" "$expected" "$last_log" $OSMO_TTCN3_COMPARE_ARGS
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
<?xml version="1.0"?>
|
||||
<testsuite name='osmo-ttcn3-hacks/sysinfo' tests='22' failures='3' errors='16' skipped='0' inconc='0' time='200.00'>
|
||||
<testcase classname='Test' name='TC_si1' time='0.001430'/>
|
||||
<testcase classname='Test' name='TC_telnet' time='0.004703'/>
|
||||
<testcase classname='Test' name='TC_si_default' time='8.009958'>
|
||||
<failure type='fail-verdict'>No SI messages for TC=0!
|
||||
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_si_sched_2bis' time='10.034880'>
|
||||
<failure type='fail-verdict'>No SI messages for TC=0!
|
||||
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_si_sched_2ter' time='10.035191'>
|
||||
<failure type='fail-verdict'>No SI messages for TC=0!
|
||||
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_si_sched_2ter_2bis' time='9.020499'>
|
||||
<error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_si_sched_2quater' time='2.027051'/>
|
||||
<testcase classname='Test' name='TC_si_sched_13' time='9.024905'>
|
||||
<error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_si_sched_13_2bis_2ter_2quater' time='9.024221'>
|
||||
<error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_neci' time='11.005209'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_cell_resel_hyst' time='11.020545'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_rxlev_acc_min' time='11.016459'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_cellid' time='11.014834'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_lac' time='11.014391'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_rach_tx_int' time='11.016352'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_rach_max_tx' time='11.014782'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_attach' time='11.005593'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_dtx_ul' time='10.013151'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_emerg_allowed' time='11.014455'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_bs_pa_mfrms' time='11.005825'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_bs_ag_blks_res' time='11.014585'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
<testcase classname='Test' name='TC_radio_link_timeout' time='11.025433'>
|
||||
<error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
Loading…
Reference in New Issue