From: Ondrej Lichtner <olichtne(a)redhat.com>
Setting the max deviation value to 10%, meaning that if the measured
packet rate interval is larger than +-10% interval, we automatically
mark this as a failure.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
recipes/regression_tests/phase3/ovs-dpdk-pvp.py | 41 ++++++++++++++++++++----
recipes/regression_tests/phase3/ovs-dpdk-pvp.xml | 1 +
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git a/recipes/regression_tests/phase3/ovs-dpdk-pvp.py
b/recipes/regression_tests/phase3/ovs-dpdk-pvp.py
index 027dae2..c476f7f 100644
--- a/recipes/regression_tests/phase3/ovs-dpdk-pvp.py
+++ b/recipes/regression_tests/phase3/ovs-dpdk-pvp.py
@@ -28,7 +28,7 @@ def run_ssh_command_on_bg_channel(command, channel, host,
guest_name=""):
desc = "%s running in bg: %s" % (guest_name, command)
host.run(custom_mod, desc=desc)
-def compare_results(host, result, baseline):
+def compare_results(host, result, baseline, max_dev):
avg1 = result.get_value("rx_rate").get_result()
dev1 = result.get_value("rx_rate_deviation").get_result()
interval1 = (avg1 - dev1, avg1 + dev1)
@@ -38,16 +38,43 @@ def compare_results(host, result, baseline):
interval2 = (avg2 - dev2, avg2 + dev2)
if interval1[1] < interval2[0]:
- desc = "Measured rate %.2f +-%.2f pps is lower "\
- "than threshold %.2f +-%.2f pps"% (avg1, dev1, avg2, dev2)
+ desc = ("Measured rate {:.2f} +-{:.2f} pps is lower "
+ "than threshold {:.2f} +-{:.2f} pps".
+ format(avg1, dev1, avg2, dev2))
custom_mod = ctl.get_module("Custom", options = {"fail":
True})
else:
- desc = "Measured rate %.2f +-%.2f pps is higher "\
- "than threshold %.2f +-%.2f pps"% (avg1, dev1, avg2, dev2)
+ desc = ("Measured rate {:.2f} +-{:.2f} pps is higher "
+ "than threshold {:.2f} +-{:.2f} pps".
+ format(avg1, dev1, avg2, dev2))
custom_mod = ctl.get_module("Custom")
host.run(custom_mod, desc=desc)
+def report_result(host, result, max_dev):
+ avg = result.get_value("rx_rate").get_result()
+ dev = result.get_value("rx_rate_deviation").get_result()
+
+ desc = "Measured rate is {:.2f} +-{:.2f} pps".format(avg, dev)
+ fail = False
+ if re.match("\d+%", max_dev):
+ if ((float(dev) / avg) * 100) > int(max_dev[:-1]):
+ desc = ("Measured rate {:.2f} +-{:.2f} pps has bigger "
+ "deviation than allowed (+-{})".
+ format(avg, dev, max_dev))
+ fail = True
+ elif re.match("\d+", max_dev):
+ if dev > int(max_dev):
+ desc = ("Measured rate {:.2f} +-{:.2f} pps has bigger "
+ "deviation than allowed (+-{} pps)".
+ format(avg, dev, max_dev))
+ fail = True
+ elif max_dev == None:
+ pass
+ else:
+ raise Exception("Unsupported format for max_dev argument.")
+
+ custom_mod = ctl.get_module("Custom", options={"fail": fail})
+ host.run(custom_mod, desc=desc)
# ------
# SETUP
@@ -93,6 +120,7 @@ trex_dir = ctl.get_alias("trex_dir")
pkt_size = int(ctl.get_alias("pkt_size"))
test_duration = int(ctl.get_alias("test_duration"))
test_runs = int(ctl.get_alias("test_runs"))
+max_dev = ctl.get_alias("max_dev")
pr_comment = generate_perfrepo_comment([h1, h2], pr_user_comment)
pr_comment += "\n<BR>DPDK version: {}".format(dpdk_version)
@@ -430,8 +458,9 @@ baseline = perf_api.get_baseline_of_result(pr_result)
pr_result.set_comment(pr_comment)
perf_api.save_result(pr_result, official_result)
+report_result(h1, pr_result.get_testExecution(), max_dev)
if not isinstance(baseline, Noop):
- compare_results(h1, pr_result.get_testExecution(), baseline.get_texec())
+ compare_results(h1, pr_result.get_testExecution(), baseline.get_texec(), max_dev)
#============================================
# Configuration cleanup
diff --git a/recipes/regression_tests/phase3/ovs-dpdk-pvp.xml
b/recipes/regression_tests/phase3/ovs-dpdk-pvp.xml
index 35241ad..e9edf74 100644
--- a/recipes/regression_tests/phase3/ovs-dpdk-pvp.xml
+++ b/recipes/regression_tests/phase3/ovs-dpdk-pvp.xml
@@ -18,6 +18,7 @@
<alias name="pkt_size" value="64" />
<alias name="test_duration" value="60" />
<alias name="test_runs" value="5" />
+ <alias name="max_dev" value="10%"/>
</define>
<network>
<host id="host1">
--
2.15.1