[lnst] lnst-ctl: add -r, --reduce-sync option
by Jiří Pírko
commit b23bbf4282a279e223901df194b9ca1d295380db
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:39 2014 +0100
lnst-ctl: add -r, --reduce-sync option
This option disables the full resource synchronization for python tasks.
Instead it relies on the user to use the sync_resources HostAPI method
to manually synchronize required modules and tools.
The option doesn't have any effect on recipes that use xml tasks, these
will always be automatically synchronized in reduced mode when only
required resources are copied to slaves.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst-ctl | 29 +++++++++++++++++++----------
lnst/Controller/NetTestController.py | 11 ++++++++---
2 files changed, 27 insertions(+), 13 deletions(-)
---
diff --git a/lnst-ctl b/lnst-ctl
index 4ce280e..2dad6aa 100755
--- a/lnst-ctl
+++ b/lnst-ctl
@@ -48,6 +48,8 @@ def usage(retval=0):
print " -p, --packet-capture capture and log all ongoing " \
"network communication during the test"
print " -x, --result=FILE file to write xml_result"
+ print " -r, --reduce-sync reduces resource synchronization "\
+ "for python tasks, see documentation"
sys.exit(retval)
def store_alias(alias_def, aliases_dict):
@@ -66,12 +68,13 @@ def store_alias(alias_def, aliases_dict):
def process_recipe(action, file_path, res_serializer,
packet_capture, log_ctl, pool_checks,
- defined_aliases, overriden_aliases):
+ defined_aliases, overriden_aliases, reduce_sync):
nettestctl = NetTestController(file_path, log_ctl,
res_serializer=res_serializer,
pool_checks=pool_checks,
defined_aliases=defined_aliases,
- overriden_aliases=overriden_aliases)
+ overriden_aliases=overriden_aliases,
+ reduce_sync=reduce_sync)
if action == "run":
return nettestctl.run_recipe(packet_capture)
elif action == "config_only":
@@ -79,8 +82,9 @@ def process_recipe(action, file_path, res_serializer,
elif action == "match_setup":
return nettestctl.match_setup()
-def get_recipe_result(args, file_path, res_serializer, packet_capture,
- log_ctl, pool_checks, defined_aliases, overriden_aliases):
+def get_recipe_result(args, file_path, res_serializer, packet_capture, log_ctl,
+ pool_checks, defined_aliases, overriden_aliases,
+ reduce_sync):
res_serializer.add_recipe(file_path)
log_ctl.set_recipe(file_path)
@@ -90,7 +94,7 @@ def get_recipe_result(args, file_path, res_serializer, packet_capture,
try:
res = process_recipe(args, file_path, res_serializer,
packet_capture, log_ctl, pool_checks,
- defined_aliases, overriden_aliases)
+ defined_aliases, overriden_aliases, reduce_sync)
except Exception as err:
log_exc_traceback()
logging.error(err)
@@ -119,10 +123,10 @@ def main():
try:
opts, args = getopt.getopt(
sys.argv[1:],
- "dhr:c:x:poma:A:",
- ["debug", "help", "recipe=", "config", "result=",
+ "dhrc:x:poma:A:",
+ ["debug", "help", "config", "result=",
"packet-capture", "disable-pool-checks", "no-colours",
- "define_alias", "override_alias"]
+ "define_alias", "override_alias", "reduce-sync"]
)
except getopt.GetoptError as err:
print str(err)
@@ -156,6 +160,7 @@ def main():
coloured_output = True
defined_aliases = {}
overriden_aliases = {}
+ reduce_sync = False
for opt, arg in opts:
if opt in ("-d", "--debug"):
debug += 1
@@ -175,6 +180,8 @@ def main():
store_alias(arg, defined_aliases)
elif opt in ("-A", "--override-alias"):
store_alias(arg, overriden_aliases)
+ elif opt in ("-r", "--reduce-sync"):
+ reduce_sync = True
if config_path is not None:
if not os.path.isfile(config_path):
@@ -228,14 +235,16 @@ def main():
packet_capture,
log_ctl, pool_checks,
defined_aliases,
- overriden_aliases)
+ overriden_aliases,
+ reduce_sync)
else:
recipe_head_log_entry(recipe_path)
fp, res, rv = get_recipe_result(action, recipe_path,
res_serializer,
packet_capture,
log_ctl, pool_checks,
- defined_aliases, overriden_aliases)
+ defined_aliases, overriden_aliases,
+ reduce_sync)
summary.append((fp, res))
if rv > retval:
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index d931a47..5adc8e9 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -49,12 +49,14 @@ def ignore_event(**kwarg):
class NetTestController:
def __init__(self, recipe_path, log_ctl,
res_serializer=None, pool_checks=True,
- defined_aliases=None, overriden_aliases=None):
+ defined_aliases=None, overriden_aliases=None,
+ reduce_sync=False):
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = recipe_path
self._msg_dispatcher = MessageDispatcher(log_ctl)
+ self._reduce_sync = reduce_sync
self.remove_saved_machine_config()
@@ -220,8 +222,11 @@ class NetTestController:
if resource_sync:
for task in self._recipe['tasks']:
if 'commands' not in task:
- sync_table = res_table
- break
+ if not self._reduce_sync:
+ sync_table = res_table
+ break
+ else:
+ continue
for cmd in task['commands']:
if 'machine' not in cmd or cmd['machine'] != m_id:
continue
10 years
[lnst] Task: add HostAPI method sync_resources
by Jiří Pírko
commit 23880569317db35a28d37e680821eafa1c856766
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:38 2014 +0100
Task: add HostAPI method sync_resources
This method provides new functionality to the python tasks. With this
the user can specify exactly which resources should be synchronized to
a single slave machine.
Since in case of python tasks all resources are automatically
synchronized this method doesn't have any practical meaning yet. This
will be changed in the following commits.
Example of use:
m1 = ctl.get_host("1")
m1.sync_resources(modules=["IcmpPing"], tools=["multicast"])
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/Task.py | 21 +++++++++++++++++++++
1 files changed, 21 insertions(+), 0 deletions(-)
---
diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py
index c933cfb..b028231 100644
--- a/lnst/Controller/Task.py
+++ b/lnst/Controller/Task.py
@@ -229,6 +229,27 @@ class HostAPI(object):
iface = self._m.get_interface(interface_id)
return Prefix(iface, addr_number)
+ def sync_resources(self, modules=[], tools=[]):
+ res_table = self._ctl._ctl._resource_table
+ sync_table = {'module': {}, 'tools': {}}
+ for mod in modules:
+ if mod in res_table['module']:
+ sync_table['module'][mod] = res_table['module'][mod]
+ else:
+ msg = "Module '%s' not found on the controller"\
+ % mod
+ raise TaskError(msg, cmd)
+
+ for tool in tools:
+ if tool in res_table['tools']:
+ sync_table['tools'][tool] = res_table['tools'][tool]
+ else:
+ msg = "Tool '%s' not found on the controller"\
+ % tool
+ raise TaskError(msg, cmd)
+
+ self._m.sync_resources(sync_table)
+
class ModuleAPI(object):
""" An API class representing a module. """
10 years
[lnst] NetTestSlave: resource_table init
by Jiří Pírko
commit 0ea40c4049f9dc8daba783a13a08d584660a7091
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:37 2014 +0100
NetTestSlave: resource_table init
When the resource_table is initialized or cleared it should have at
least empty 'module' and 'tools' subdictionaries created. This is so
that proper exceptions are raised in case nothing was synchronized to
the slave - before this commit an exception KeyError 'module not
defined' would be raised, instead an exception "module/tool xyz not
found" is more appropriate.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Slave/NetTestSlave.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
---
diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py
index 28c595e..dca0a59 100644
--- a/lnst/Slave/NetTestSlave.py
+++ b/lnst/Slave/NetTestSlave.py
@@ -58,7 +58,7 @@ class SlaveMethods:
self._cache = ResourceCache(lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
- self._resource_table = {}
+ self._resource_table = {'module': {}, 'tools': {}}
def hello(self, recipe_path):
self.machine_cleanup()
@@ -261,7 +261,7 @@ class SlaveMethods:
return True
def clear_resource_table(self):
- self._resource_table = {}
+ self._resource_table = {'module': {}, 'tools': {}}
return True
def has_resource(self, res_hash):
10 years
[lnst] NetTestController: sync only required resources
by Jiří Pírko
commit 2be991e91458843b539bf3f36e960c7dbedd572a
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:36 2014 +0100
NetTestController: sync only required resources
From now the controller will only synchronize resources required by the
slave machine for recipe execution. This is done by traversing the
commands that will be executed and searching for those that require any
kind of resource.
This commit also removes a piece of code from the function
_prepare_command that handled raising exceptions when a tool was not
found, this is now handled in the code that traverses the recipe
commands.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/NetTestController.py | 34 +++++++++++++++++++++++++---------
1 files changed, 25 insertions(+), 9 deletions(-)
---
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index 342149e..d931a47 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -215,9 +215,31 @@ class NetTestController:
recipe_name = os.path.basename(self._recipe_path)
machine.configure(recipe_name)
- sync_table = {}
+ res_table = self._resource_table
+ sync_table = {'module': {}, 'tools': {}}
if resource_sync:
- sync_table = self._resource_table
+ for task in self._recipe['tasks']:
+ if 'commands' not in task:
+ sync_table = res_table
+ break
+ for cmd in task['commands']:
+ if 'machine' not in cmd or cmd['machine'] != m_id:
+ continue
+ if cmd['type'] == 'test':
+ mod = cmd['module']
+ if mod in res_table['module']:
+ sync_table['module'][mod] = res_table['module'][mod]
+ else:
+ msg = "Module '%s' not found on the controller"\
+ % mod
+ raise RecipeError(msg, cmd)
+ if cmd['type'] == 'exec' and 'from' in cmd:
+ tool = cmd['from']
+ if tool in res_table['tools']:
+ sync_table['tools'][tool] = res_table['tools'][tool]
+ else:
+ msg = "Tool '%s' not found on the controller" % tool
+ raise RecipeError(msg, cmd)
machine.sync_resources(sync_table)
def _prepare_interface(self, m_id, iface_xml_data):
@@ -326,13 +348,7 @@ class NetTestController:
cmd["command"] = cmd_data["command"]
if "from" in cmd_data:
- tool = cmd_data["from"]
- if tool in self._resource_table["tools"]:
- cmd["command"] = cmd_data["command"]
- cmd["from"] = cmd_data["from"]
- else:
- msg = "Tool '%s' not found on the controller" % tool
- raise RecipeError(msg, cmd_data)
+ cmd["from"] = cmd_data["from"]
elif cmd["type"] in ["wait", "intr", "kill"]:
# XXX The internal name (proc_id) is different, because
# bg_id is already used by LNST in a different context
10 years
[lnst] NetTestController: skip resource sync for config_only
by Jiří Pírko
commit 5a524de715cc62e64822cad86d434239b313f554
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:35 2014 +0100
NetTestController: skip resource sync for config_only
This commit makes it so that config_only runs don't synchronize
resources (test modules and test tools). The code will later be extended
so that when synchronizing resources only the ones needed by the recipe
are sent to machines.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/NetTestController.py | 14 +++++++++-----
1 files changed, 9 insertions(+), 5 deletions(-)
---
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index 2a18b1b..342149e 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -166,12 +166,12 @@ class NetTestController:
return mreq
- def _prepare_network(self):
+ def _prepare_network(self, resource_sync=True):
recipe = self._recipe
machines = self._machines
for m_id in machines.keys():
- self._prepare_machine(m_id)
+ self._prepare_machine(m_id, resource_sync)
for machine_xml_data in recipe["machines"]:
m_id = machine_xml_data["id"]
@@ -203,7 +203,7 @@ class NetTestController:
provisioner = sp.get_provisioner_id(m_id)
logging.info(" machine %s uses %s" % (m_id, provisioner))
- def _prepare_machine(self, m_id):
+ def _prepare_machine(self, m_id, resource_sync=True):
machine = self._machines[m_id]
address = socket.gethostbyname(machine.get_hostname())
@@ -214,7 +214,11 @@ class NetTestController:
recipe_name = os.path.basename(self._recipe_path)
machine.configure(recipe_name)
- machine.sync_resources(self._resource_table)
+
+ sync_table = {}
+ if resource_sync:
+ sync_table = self._resource_table
+ machine.sync_resources(sync_table)
def _prepare_interface(self, m_id, iface_xml_data):
machine = self._machines[m_id]
@@ -505,7 +509,7 @@ class NetTestController:
def config_only_recipe(self):
try:
self._prepare_provisioning()
- self._prepare_network()
+ self._prepare_network(resource_sync=False)
except (KeyboardInterrupt, Exception) as exc:
msg = "Exception raised during configuration."
logging.error(msg)
10 years
[lnst] schema-recipe: fix python tasks
by Jiří Pírko
commit 7910023e8cbaf9fa734409672cd5cf31120233ab
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Mon Mar 3 16:00:34 2014 +0100
schema-recipe: fix python tasks
The schema didn't accept python tasks. This commit fixes that, python
tasks can now be defined by an element:
<task python="path_to_python_task"/>
task element with the python attribute defined doesn't have any content
allowed.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
schema-recipe.rng | 24 ++++++++++++++++++++----
1 files changed, 20 insertions(+), 4 deletions(-)
---
diff --git a/schema-recipe.rng b/schema-recipe.rng
index bd0a8df..7803849 100644
--- a/schema-recipe.rng
+++ b/schema-recipe.rng
@@ -45,16 +45,32 @@
</define>
<define name="task">
+ <choice>
+ <element name="task">
+ <attribute name="python"/>
+
+ <optional>
+ <attribute name="quit_on_fail">
+ <choice>
+ <value>yes</value>
+ <value>no</value>
+ <value>true</value>
+ <value>false</value>
+ </choice>
+ </attribute>
+ </optional>
+ </element>
+ <ref name="xml_task"/>
+ </choice>
+ </define>
+
+ <define name="xml_task">
<element name="task">
<optional>
<attribute name="label"/>
</optional>
<optional>
- <attribute name="python"/>
- </optional>
-
- <optional>
<attribute name="quit_on_fail">
<choice>
<value>yes</value>
10 years
[PATCH 0/9] resource synchronization enhancements
by Ondrej Lichtner
From: Ondrej Lichtner <olichtne(a)redhat.com>
The following patch series improves the resource synchronization.
The first part makes it so that only resources required by a slave machine, to
succesfully run a task, will be copied.
The last 3 patches add two new attributes to the <task> recipe element -
module_dir and tools_dir that can specify an additional directory that should be
searched for resources.
Ondrej Lichtner (9):
schema-recipe: fix python tasks
NetTestController: skip resource sync for config_only
NetTestController: sync only required resources
NetTestSlave: resource_table init
Task: add HostAPI method sync_resources
lnst-ctl: add -r, --reduce-sync option
XmlParser: save full filename in element objects
recipe xml format: add module_dir and tools_dir attributes
NetTestController: add support for {module,tools}_dir
lnst-ctl | 29 +++++++++-----
lnst/Controller/NetTestController.py | 78 ++++++++++++++++++++++++++++++------
lnst/Controller/RecipeParser.py | 16 ++++++++
lnst/Controller/Task.py | 21 ++++++++++
lnst/Controller/XmlParser.py | 2 +-
lnst/Slave/NetTestSlave.py | 4 +-
schema-recipe.rng | 40 ++++++++++++++++--
7 files changed, 160 insertions(+), 30 deletions(-)
--
1.8.5.3
10 years