5:12 a.m.
From: Ondrej Lichtner <olichtne(a)redhat.com>
This module defines classes for storing Result data related to a test
run. Most are generated automatically by LNST during test execution and
a tester also has a Recipe interface available to create Result objects
for custom entries.
These will later be integrated with the rest of the LNST Controller so
that they're created in the right places and accessible as Recipe data
after the test run is finished for optional post-processing.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/RecipeResults.py | 98 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
create mode 100644 lnst/Controller/RecipeResults.py
diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py
new file mode 100644
index 0000000..b79516d
--- /dev/null
+++ b/lnst/Controller/RecipeResults.py
@@ -0,0 +1,98 @@
+"""
+This module defines classes for storing Result data related to a test run.
+Most are generated automatically by LNST during test execution and a tester
+also has a Recipe interface available to create Result objects for custom
+entries.
+
+Copyright 2018 Red Hat, Inc.
+Licensed under the GNU General Public License, version 2 as
+published by the Free Software Foundation; see COPYING for details.
+"""
+
+__author__ = """
+olichtne(a)redhat.com (Ondrej Lichtner)
+"""
+
+import time
+
+class BaseResult(object):
+ """Base class for storing result data
+
+ should not be instantiated directly, only defines the interface"""
+ def __init__(self, success=True):
+ self._timestamp = time.time()
+ self._success = success
+
+ @property
+ def timestamp(self):
+ return self._timestamp
+
+ @property
+ def success(self):
+ return self._success
+
+ @property
+ def short_desc(self):
+ return "Short description of result if relevant"
+
+ @property
+ def data(self):
+ return None
+
+class JobResult(BaseResult):
+ """Base class for storing result data of Jobs
+
+ should not be instantiated directly, just stores the Job instance"""
+ def __init__(self, job, success):
+ super(JobResult, self).__init__(success)
+
+ self._job = job
+
+ @property
+ def job(self):
+ return self._job
+
+class JobStartResult(JobResult):
+ """Generated automatically when a Job is succesfully started on a
slave"""
+ @BaseResult.short_desc.getter
+ def short_desc(self):
+ return "Job started: {}".format(str(self.job))
+
+class JobFinishResult(JobResult):
+ """Generated automatically when a Job is finished on a slave
+
+ success depends on the Job passed value and returns the data returned as
+ a result of the Job."""
+ def __init__(self, job):
+ super(JobFinishResult, self).__init__(job, True)
+
+ @BaseResult.success.getter
+ def success(self):
+ return self._job.passed
+
+ @BaseResult.short_desc.getter
+ def short_desc(self):
+ return "Job finished: {}".format(str(self.job))
+
+ @BaseResult.data.getter
+ def data(self):
+ return self.job.result
+
+class Result(BaseResult):
+ """Class intended to store aribitrary tester supplied data
+
+ Will be created when the tester calls the Recipe interface for adding
+ results."""
+ def __init__(self, success, short_desc="", data=None):
+ super(Result, self).__init__(success)
+
+ self._short_desc = short_desc
+ self._data = data
+
+ @BaseResult.short_desc.getter
+ def short_desc(self):
+ return self._short_desc
+
+ @BaseResult.data.getter
+ def data(self):
+ return self._data
--
2.16.1
5:12 a.m.
New subject: [PATCH 2/7] Controller: move print_match_description to MachineMapper
From: Ondrej Lichtner <olichtne(a)redhat.com>
Renamed the method to format_match_description and made it return the
formatted string describing a match that gets returned by a
MachineMapper.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/Controller.py | 15 +++------------
lnst/Controller/MachineMapper.py | 13 +++++++++++++
2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 0d30026..a41887e 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -27,6 +27,7 @@ from lnst.Controller.Config import CtlConfig
from lnst.Controller.MessageDispatcher import MessageDispatcher
from lnst.Controller.SlavePoolManager import SlavePoolManager
from lnst.Controller.MachineMapper import MachineMapper
+from lnst.Controller.MachineMapper import format_match_description
from lnst.Controller.Host import Hosts, Host
from lnst.Controller.Recipe import BaseRecipe
@@ -118,7 +119,8 @@ class Controller(object):
expand="match_%d" % i)
i += 1
- self._print_match_description(match)
+ for line in format_match_description(match).split():
+ logging.info(line)
try:
self._map_match(match, req)
recipe.test()
@@ -218,14 +220,3 @@ class Controller(object):
config.load_config(gitcfg)
return config
-
- def _print_match_description(self, match):
- logging.info("Pool match description:")
- if match["virtual"]:
- logging.info(" Setup is using virtual machines.")
- for m_id, m in sorted(match["machines"].iteritems()):
- logging.info(" host \"%s\" uses \"%s\"" %
(m_id, m["target"]))
- for if_id, match in m["interfaces"].iteritems():
- pool_id = match["target"]
- logging.info(" interface \"%s\" matched to
\"%s\"" %\
- (if_id, pool_id))
diff --git a/lnst/Controller/MachineMapper.py b/lnst/Controller/MachineMapper.py
index 39b2acc..13488eb 100644
--- a/lnst/Controller/MachineMapper.py
+++ b/lnst/Controller/MachineMapper.py
@@ -16,6 +16,19 @@ from lnst.Controller.Common import ControllerError
class MapperError(ControllerError):
pass
+def format_match_description(match):
+ output = []
+ output.append("Pool match description:")
+ if match["virtual"]:
+ output.append(" Setup is using virtual machines.")
+ for m_id, m in sorted(match["machines"].iteritems()):
+ output.append(" host \"{}\" uses
\"{}\"".format(m_id, m["target"]))
+ for if_id, match in m["interfaces"].iteritems():
+ pool_id = match["target"]
+ output.append(" interface \"{}\" matched to
\"{}\"".
+ format(if_id, pool_id))
+ return "\n".join(output)
+
class MachineMapper(object):
"""Implements a matching algorithm that maps requirements to available
hosts
--
2.16.1
5:12 a.m.
New subject: [PATCH 3/7] Recipe: add class RecipeRun
From: Ondrej Lichtner <olichtne(a)redhat.com>
Instances of the RecipeRun class represent a single test run of a
recipe. They store information about Results (executed Jobs or tester
defined) and about the matchine match that was used.
A recipe instance stores a list of these RecipeRun objects and all of
them can be accessed after testing for optional post-processing.
The Recipe class now also has an interface method for the tester which
can be used to add Result objects to the current run, this is intended
to be used from within the tester defined "test" method.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/Controller.py | 3 ++-
lnst/Controller/Recipe.py | 39 +++++++++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index a41887e..5b06a04 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -29,7 +29,7 @@ from lnst.Controller.SlavePoolManager import SlavePoolManager
from lnst.Controller.MachineMapper import MachineMapper
from lnst.Controller.MachineMapper import format_match_description
from lnst.Controller.Host import Hosts, Host
-from lnst.Controller.Recipe import BaseRecipe
+from lnst.Controller.Recipe import BaseRecipe, RecipeRun
class Controller(object):
"""The LNST Controller class
@@ -123,6 +123,7 @@ class Controller(object):
logging.info(line)
try:
self._map_match(match, req)
+ recipe._init_run(RecipeRun(match))
recipe.test()
except Exception as exc:
logging.error("Recipe execution terminated by unexpected
exception")
diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py
index 9380a35..a10f081 100644
--- a/lnst/Controller/Recipe.py
+++ b/lnst/Controller/Recipe.py
@@ -14,6 +14,7 @@ import copy
from lnst.Common.Parameters import Parameters, Param
from lnst.Controller.Requirements import _Requirements, HostReq
from lnst.Controller.Common import ControllerError
+from lnst.Controller.RecipeResults import BaseResult, Result
class RecipeError(ControllerError):
"""Exception thrown by the BaseRecipe class"""
@@ -81,6 +82,7 @@ class BaseRecipe(object):
and checked if mandatory Parameters have values.
"""
self._ctl = None
+ self.runs = []
self.req = _Requirements()
self.params = Parameters()
for attr in dir(self):
@@ -121,3 +123,40 @@ class BaseRecipe(object):
def test(self):
"""Method to be implemented by the Tester"""
raise NotImplementedError("Method test must be defined by a child
class.")
+
+ def _init_run(self, run):
+ self.runs.append(run)
+
+ @property
+ def current_run(self):
+ if len(self.runs) > 0:
+ return self.runs[-1]
+ else:
+ return None
+
+ def add_result(self, success, description="", data=None):
+ self.current_run.add_result(Result(success, description, data))
+
+class RecipeRun(object):
+ def __init__(self, match, desc=None):
+ self._match = match
+ self._desc = desc
+ self._results = []
+
+ def add_result(self, result):
+ if not isinstance(result, BaseResult):
+ raise RecipeError("result must be a BaseActionResult instance.")
+
+ self._results.append(result)
+
+ @property
+ def match(self):
+ return self._match
+
+ @property
+ def description(self):
+ return self._desc
+
+ @property
+ def results(self):
+ return self._results
--
2.16.1
5:12 a.m.
New subject: [PATCH 4/7] lnst.Controller.Controller: use the Recipe instance for test init
From: Ondrej Lichtner <olichtne(a)redhat.com>
When a recipe is succesfully matched and the Controller starts setting
up the Machine objects we should pass the Recipe instance. This is
important since the Machine object needs to be able to generate
JobResult objects.
Another minor improvement is that a correct recipe name will be used
(the name of the Recipe class) instead of the filename of the currently
running executable which was incorrectly used until now.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/Controller.py | 11 +++++------
lnst/Controller/Machine.py | 6 +++++-
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py
index 5b06a04..5f58f09 100644
--- a/lnst/Controller/Controller.py
+++ b/lnst/Controller/Controller.py
@@ -122,7 +122,7 @@ class Controller(object):
for line in format_match_description(match).split():
logging.info(line)
try:
- self._map_match(match, req)
+ self._map_match(match, req, recipe)
recipe._init_run(RecipeRun(match))
recipe.test()
except Exception as exc:
@@ -143,7 +143,7 @@ class Controller(object):
self._msg_dispatcher.wait_for_condition(condition)
- def _map_match(self, match, requested):
+ def _map_match(self, match, requested, recipe):
self._machines = {}
self._hosts = Hosts()
pool = self._pools.get_machine_pool(match["pool_name"])
@@ -154,7 +154,7 @@ class Controller(object):
host = getattr(self._hosts, m_id)
machine.set_id(m_id)
- self._prepare_machine(machine)
+ self._prepare_machine(machine, recipe)
for if_id, i in m["interfaces"].items():
host._map_device(if_id, i)
@@ -168,13 +168,12 @@ class Controller(object):
setattr(host, name, new_virt_dev)
new_virt_dev._enable()
- def _prepare_machine(self, machine):
+ def _prepare_machine(self, machine, recipe):
self._log_ctl.add_slave(machine.get_id())
machine.set_mac_pool(self._mac_pool)
machine.set_network_bridges(self._network_bridges)
- recipe_name = os.path.basename(sys.argv[0])
- machine.set_recipe(recipe_name)
+ machine.set_recipe(recipe)
def _cleanup_slaves(self):
if self._machines == None:
diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py
index cd14f78..b429721 100644
--- a/lnst/Controller/Machine.py
+++ b/lnst/Controller/Machine.py
@@ -70,6 +70,8 @@ class Machine(object):
self._port = ctl_config.get_option('environment', 'rpcport')
self._msg_dispatcher = msg_dispatcher
+
+ self._recipe = None
self._mac_pool = None
self._interfaces = []
@@ -231,12 +233,14 @@ class Machine(object):
self._slave_desc = slave_desc
- def set_recipe(self, recipe_name):
+ def set_recipe(self, recipe):
""" Reserves the machine for the specified recipe
Also sends Device classes from the controller and initializes the
InterfaceManager on the Slave and builds the device database.
"""
+ self._recipe = recipe
+ recipe_name = recipe.__class__.__name__
self.rpc_call("set_recipe", recipe_name)
self._send_device_classes()
self.rpc_call("init_if_manager")
--
2.16.1
5:12 a.m.
New subject: [PATCH 5/7] lnst.Controller.Machine: generate and store JobResult objects
From: Ondrej Lichtner <olichtne(a)redhat.com>
Generates a JobStartResult object when the "run_job" rpc call returns
and stores it in the current RecipeRun.
Generates a JobFinishResult object when a job_finished message is
received and stores it in the current RecipeRun.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/Machine.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py
index b429721..09e7540 100644
--- a/lnst/Controller/Machine.py
+++ b/lnst/Controller/Machine.py
@@ -21,6 +21,7 @@ from lnst.Common.TestModule import BaseTestModule
from lnst.Common.Version import lnst_version
from lnst.Controller.Common import ControllerError
from lnst.Controller.CtlSecSocket import CtlSecSocket
+from lnst.Controller.RecipeResults import JobStartResult, JobFinishResult
from lnst.Devices import device_classes
from lnst.Devices.Device import Device
from lnst.Devices.RemoteDevice import RemoteDevice
@@ -365,7 +366,10 @@ class Machine(object):
if job._desc is not None:
logging.info("Job description: %s" % job._desc)
- return self.rpc_call("run_job", job._to_dict(), netns=job.netns)
+ res = self.rpc_call("run_job", job._to_dict(), netns=job.netns)
+
+ self._recipe.current_run.add_result(JobStartResult(job, res))
+ return res
def wait_for_job(self, job, timeout):
res = True
@@ -426,6 +430,7 @@ class Machine(object):
job_id = msg["job_id"]
job = self._jobs[job_id]
job._res = msg["result"]
+ self._recipe.current_run.add_result(JobFinishResult(job))
def kill(self, job, signal):
if job.id not in self._jobs:
--
2.16.1
5:12 a.m.
New subject: [PATCH 6/7] lnst.Controller.Job: passed should reflect what was expected
From: Ondrej Lichtner <olichtne(a)redhat.com>
In case the Job was expected to fail, the passed attribute should
reflect this.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/Job.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py
index 96cf045..1026938 100644
--- a/lnst/Controller/Job.py
+++ b/lnst/Controller/Job.py
@@ -88,10 +88,13 @@ class Job(object):
def passed(self):
"""Indicates whether or not the Job passed
+ The return value is True or False based on if the Job was expected to
+ pass or fail.
+
Type: Boolean
"""
try:
- return self._res["passed"]
+ return self._res["passed"] == self._expect
except:
return False
--
2.16.1
5:12 a.m.
New subject: [PATCH 7/7] add lnst.Controller.RunSummaryFormatter
From: Ondrej Lichtner <olichtne(a)redhat.com>
Instances of this class can process RecipeRun objects and output a
formatted run summary string.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
---
lnst/Controller/RunSummaryFormatter.py | 102 +++++++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
create mode 100644 lnst/Controller/RunSummaryFormatter.py
diff --git a/lnst/Controller/RunSummaryFormatter.py
b/lnst/Controller/RunSummaryFormatter.py
new file mode 100644
index 0000000..efc5c15
--- /dev/null
+++ b/lnst/Controller/RunSummaryFormatter.py
@@ -0,0 +1,102 @@
+"""
+Defines the RunSummaryFormatter class which is can be used to process a
+RecipeRun object to return a formatted run summary string.
+
+Copyright 2018 Red Hat, Inc.
+Licensed under the GNU General Public License, version 2 as
+published by the Free Software Foundation; see COPYING for details.
+"""
+
+__author__ = """
+olichtne(a)redhat.com (Ondrej Lichtner)
+"""
+
+from lnst.Common.Colours import decorate_with_preset
+from lnst.Controller.Common import ControllerError
+from lnst.Controller.MachineMapper import format_match_description
+from lnst.Controller.Recipe import BaseRecipe, RecipeRun
+from lnst.Controller.RecipeResults import BaseResult, JobResult, Result
+from lnst.Controller.RecipeResults import JobStartResult, JobFinishResult
+
+class RunFormatterException(ControllerError):
+ pass
+
+class RunSummaryFormatter(object):
+ def __init__(self):
+ #TODO changeable format?
+ self._format = ""
+
+ def _format_success(self, success):
+ if success:
+ return decorate_with_preset("PASS", "pass")
+ else:
+ return decorate_with_preset("FAIL", "fail")
+
+ def _format_source(self, res):
+ if isinstance(res, JobResult):
+ return "Host {} job {}".format(res.job.host.hostid, res.job.id)
+ elif isinstance(res, Result):
+ return "TestResult:"
+ else:
+ return ""
+
+ def _format_data(self, data, prefix=" ", level=1):
+ output = []
+ if data is not None:
+ if isinstance(data, dict):
+ for key, value in data.items():
+ output.append("{pref}{key}:".format(pref=level*prefix,
+ key=key))
+ nest_res = self._format_data(value, level=level+1)
+ if len(nest_res) == 1:
+ output[-1] += " " + nest_res[0].lstrip()
+ else:
+ output.extend(nest_res)
+ elif isinstance(data, list):
+ for i, v in enumerate(data):
+ output.append("{pref}item {i}:".format(pref=level*prefix,
+ i=i))
+ output.extend(self._format_data(v, level=level+1))
+ else:
+ for line in str(data).split('\n'):
+ output.append("{pref}{val}".format(pref=level*prefix,
+ val=line))
+ return output
+
+ def format_run(self, run):
+ if not isinstance(run, RecipeRun):
+ raise RunFormatterException("run must be a RecipeRun instance.")
+
+ output_lines = []
+ output_lines.append("RUN SUMMARY")
+ output_lines.append("Description:")
+ if run.description:
+ output_lines.extend(str(run.description).split("\n"))
+
+ output_lines.extend(format_match_description(run.match))
+
+ overall_result = True
+ start_finish = False
+ for i, res in enumerate(run.results):
+ overall_result = overall_result and res.success
+
+ try:
+ if (isinstance(run.results[i+1], JobFinishResult) and
+ isinstance(res, JobStartResult) and res.success):
+ start_finish = True
+ continue
+ start_finish = False
+ except IndexError:
+ pass
+
+ output_lines.append("{res}\t{src}\t{desc}".format(
+ res = self._format_success(res.success),
+ src = self._format_source(res),
+ desc = res.short_desc))
+
+ output_lines.extend(self._format_data(res.data))
+
+ output_lines.append("Overall result of this Run: {}".
+ format(self._format_success(overall_result)))
+
+ return "\n".join(output_lines)
--
2.16.1
2263
days inactive
2263
days old
lnst-developers@lists.fedorahosted.org
6 comments
1 participants
participants (1)
-
olichtne＠redhat.com