Hey, I feel like I've rewritten half of the autoqa harness. Oh, I did!
Damn. :)
So, here's the patch that adds control.autoqa files for our tests. This
means:
1. multihook support
2. autotest labels support
3. tests itself choosing when to run instead of hooks forcing them to run
And some other goodies:
4. 'noarch' arch support
5. --autotest_server option fix
6. improved documentation, templates put into doc/ directory
I've tried to put all those things into separate commits (as much as it
was possible), but I've not always succeeded, so bear with me. The patch
is too large for ML, so please do:
$ git log origin/master..origin/control.autoqa
and inspect the commits, or
$ git diff origin/master..origin/control.autoqa
and see all the changes.
Below I attach just the diff of the autoqa harness itself.
I've tried to document the changes in commit logs, but I'm sure there
will be many questions. Shoot.
======
diff --git a/autoqa b/autoqa
index d0473f8..4144f31 100755
--- a/autoqa
+++ b/autoqa
@@ -28,6 +28,8 @@ import tempfile
import StringIO
import urlgrabber
import socket
+import copy
+import fnmatch
from ConfigParser import *
from subprocess import call
@@ -89,6 +91,30 @@ def prep_controlfile(controlfile, extradata):
os.close(fd)
return name
+def eval_test_vars(test, test_vars):
+ '''Take a test's control.autoqa file, have the test_vars argument as the
+ input data, evaluate the file, and return the modified test_vars.
+ Arguments:
+ * test - name of the test for which control.autoqa should be executed
+ * test_vars - dictionary with test variables that will be used for input
+ Returns: dictionary with test variables that have been evaluated (and
+ probably modified) by test's control.autoqa file
+ '''
+ cfile = open(os.path.join(conf['testdir'], test, 'control.autoqa'))
+ # we need to deepcopy test_vars, so we don't change the input argument at all
+ # (there are nested structures)
+ vars = copy.deepcopy(test_vars)
+ # execute the file
+ exec cfile in vars
+ cfile.close()
+ # leave only those keys there were defined before, delete all other keys
+ # (there could have appeared some new ones we don't need, like builtins
+ # or temporary variables)
+ for key in vars.keys():
+ if not key in test_vars.keys():
+ del vars[key]
+ return vars
+
def maybe_call(cmdlist, dryrun=False, verbose=False):
if dryrun or verbose:
print ' '.join(cmdlist)
@@ -100,22 +126,24 @@ def maybe_call(cmdlist, dryrun=False, verbose=False):
print "Command failed: %s" % str(e)
return None
-# TODO add info about required distro
-def schedule_job(controlfile, required_arch=None, email=None, name=None, dryrun=False):
+def schedule_job(controlfile, required_arch=None, email=None, name=None, dryrun=False, labels=[]):
cmd = ['/usr/bin/atest', 'job', 'create']
if email:
cmd += ['-e', email]
# some hooks/tests may require special machines
- if required_arch:
- cmd += ['-m', '*%s' % required_arch]
- # NOTE this doesn't work without -m, so we need to pick something..
- # currently we require an --arch flag though, so this will never happen
- else:
- # FIXME schedule against distro label with e.g. '-b fc11'
- cmd += ['-m', '*x86_64']
+ # autotest currently doesn't support 'noarch' tests, so execute them on x86_64
+ if not required_arch or required_arch == 'noarch':
+ required_arch = 'x86_64'
+ # for 'i[3-6]86' arch we have 'i386' autotest label, let's convert it
+ if fnmatch.fnmatch(required_arch, 'i?86'):
+ required_arch = 'i386'
+ cmd += ['-m', '*%s' % required_arch]
+ # schedule against additional labels, like distro label ('fc13')
+ if labels:
+ cmd += ['-d', ','.join(labels)]
+ cmd += ['-f', controlfile]
cmd.append(name) # job name
- thiscmd = cmd + ['-f', controlfile]
- return maybe_call(thiscmd, dryrun)
+ return maybe_call(cmd, dryrun)
def run_test_locally(controlfile, name=None, dryrun=False):
cmd = ['/usr/share/autotest/client/bin/autotest', '--verbose']
@@ -124,13 +152,31 @@ def run_test_locally(controlfile, name=None, dryrun=False):
cmd.append(controlfile)
return maybe_call(cmd, dryrun)
+def prep_test_vars(hook, archs, autoqa_args):
+ '''Prepare variables that should be redirected to a test's control.autoqa
+ file, so the test can then decide whether to run and how.
+ Arguments:
+ hook - name of the hook that calls this test
+ archs - list of architectures to be executed on
+ autoqa_args - dictionary of autoqa arguments that will be given to
+ test's control file
+ Returns a dictionary of all the test variables, including autoqa_args
+ as one of its items.
+ '''
+ test_vars = {}
+ test_vars['autoqa_args'] = autoqa_args
+ test_vars['hook'] = hook
+ test_vars['archs'] = archs
+ test_vars['labels'] = []
+ test_vars['execute'] = True
+ return test_vars
+
# Sanity check our installation
if not os.path.isdir(conf['hookdir']):
print "Can't find hooks in %s. Check your installation." % conf['hookdir']
sys.exit(1)
-# known hooks = dirs in hookdir that have a 'testlist' file
-known_hooks = [d for d in os.listdir(conf['hookdir']) \
- if os.path.exists(os.path.join(conf['hookdir'],d,'testlist'))]
+# known hooks = dirs in hookdir
+known_hooks = [d for d in os.listdir(conf['hookdir'])]
# Set up the option parser
parser = optparse.OptionParser(usage="%prog HOOKNAME [options] ...",
@@ -139,23 +185,25 @@ parser.add_option('-h', '--help', action='help',
help='show this help message (or hook help message if HOOKNAME given) and \
exit')
parser.add_option('-a', '--arch', action='append', default=[],
- help='arch to run the test(s) on. can be used multiple times')
-# XXX TODO '-d', '--distro', help='distro label to schedule this test against'
+ help="arch to run the test(s) on; can be used multiple times; by default \
+'noarch' arch is used")
parser.add_option('-t', '--test', action='append',
- help='run only the given test(s). can be used multiple times')
+ help="run only the given test(s) instead of all relevant ones; can be used \
+multiple times; if you specify a test that wouldn't be run by default it will \
+be forced to run")
# XXX --skiptest/--exclude?
parser.add_option('--keep-control-file', action='store_true',
- help='Do not delete generated control files')
+ help='do not delete generated control files')
parser.add_option('--dryrun', '--dry-run', action='store_true', dest='dryrun',
- help='Do not actually execute commands, just show what would be done \
+ help='do not actually execute commands, just show what would be done \
(implies --keep-control-file)')
parser.add_option('--local', action='store_true', dest='local',
- help='Do not schedule jobs - run test(s) directly on the local machine')
+ help='do not schedule jobs - run test(s) directly on the local machine')
parser.add_option('-l', '--list-tests', action='store_true', dest='listtests',
help='list the tests for the given hookname - do not run any tests')
parser.add_option('--autotest-server', action='store', default=None,
- help='Sets the autotest-server hostname. Used for creating URLs to results.\
-Hostname of the local machine is used by default.')
+ help='sets the autotest-server hostname used for creating URLs to results;\
+hostname of the local machine is used by default')
# Read and validate the hookname
# Check for no args, or just -h/--help
if len(sys.argv) == 1 or sys.argv[1] in ('-h', '--help'):
@@ -175,61 +223,66 @@ hook.extend_parser(parser)
(opts, args) = parser.parse_args()
args.pop(0) # dump hookname
+# Bail out if we didn't get at least one argument
+if not args:
+ parser.error('No test argument given - nothing to test!')
+
# Run the tests locally, or schedule them through autotest?
run_local = (opts.local or (conf['local'].lower() == 'true'))
-# Determine list of architectures
-if run_local:
- opts.arch = [os.uname()[4]] # not really important that we get this right
-
-# Get the initial testlist
-# TODO try/except
-testlist = []
-testlist_file = open(os.path.join(hookdir,'testlist'))
-testlist = [t for t in shlex.shlex(testlist_file)]
-testlist_file.close()
-controlfiles = [os.path.join(conf['testdir'], t, 'control') for t in testlist]
-# Use the hook-specific code to filter the testlist
-testlist = hook.process_testlist(opts, args, testlist)
-# Allow user overrides
+if not opts.arch or run_local:
+ opts.arch = ['noarch']
+# it doesn't make sense to have 'noarch' and some other arch specified, but
+# some watcher may still provide us with such combination
+# if this is the case, just delete the 'noarch' item
+while 'noarch' in opts.arch and len(opts.arch) > 1:
+ opts.arch.remove('noarch')
+
+# Override autotest_server if required
+if opts.autotest_server:
+ conf['autotest_server'] = opts.autotest_server
+
+# Ask hook to determine all arguments needed for tests. These variables will
+# be then written into the control file as autoqa_args dictionary.
+autoqa_args = hook.process_testdata(opts, args)
+
+# Evaluate control.autoqa file for every test to get a list of tests to execute
+tests = [test for test in os.listdir(conf['testdir'])]
+default_test_vars = prep_test_vars(hookname, opts.arch, autoqa_args)
+test_vars = {} # dict of test->its test vars
+for test in tests[:]:
+ try:
+ test_vars[test] = eval_test_vars(test, default_test_vars)
+ except IOError as e:
+ print "Error: Can't evaluate test '%s': %s" % (test, e)
+ tests.remove(test)
+testlist = [test for test,vars in test_vars.iteritems() if vars['execute'] == True]
+
+# Allow testlist user override
+# User may force some test to run even though it wouldn't be run by default.
+# This is useful for example for helloworld test.
if opts.test:
for t in opts.test:
- if t not in testlist:
- parser.error('Unknown test %s' % t)
+ if t not in tests:
+ parser.error('Unknown test: %s' % t)
testlist = opts.test
+
# Print testlist, if requested
if opts.listtests:
print ' '.join(testlist)
sys.exit(0)
-# Bail out if we didn't get at least one argument
-if not args:
- parser.error('No test argument given - nothing to test!')
-# XXX TODO allow no arch if we have a specified distro?
-if not (opts.arch or run_local):
- parser.error('No arch specified')
-
# We're ready to run/queue tests now.
-for arch in opts.arch:
- # N.B. process_testdata may grow new keyword arguments if we add new autoqa
- # args that add another loop here..
- testdata = hook.process_testdata(opts, args, arch=arch)
- if not 'autotest_server' in testdata.keys():
- if opts.autotest_server is not None:
- testdata['autotest_server'] = opts.autotest_server
- else:
- testdata['autotest_server'] = conf['autotest_server']
- # XXX FIXME: tests need to be able to indicate that they do not require
- # any specific arch (e.g. rpmlint can run on any arch)
- for test in testlist:
- try:
- template = os.path.join(conf['testdir'], test, 'control')
- control = prep_controlfile(template, testdata)
- except IOError, e:
- print "WARNING: could not process control file for %s: %s" % (test,
- str(e))
- continue
+for test in testlist:
+ try:
+ template = os.path.join(conf['testdir'], test, 'control')
+ control = prep_controlfile(template, test_vars[test]['autoqa_args'])
+ except IOError, e:
+ print "WARNING: could not process control file for %s: %s" % (test,
+ str(e))
+ continue
+ for arch in test_vars[test]['archs']:
testname='%s:%s.%s' % (hookname, test, arch)
email = conf['notification_email']
@@ -237,14 +290,13 @@ for arch in opts.arch:
retval = run_test_locally(control, name=testname,
dryrun=opts.dryrun)
else:
- # XXX FIXME add required_distro, set arch=None for noarch tests
retval = schedule_job(control, email=email, name=testname,
- required_arch=arch,
- dryrun=opts.dryrun)
+ required_arch=arch, dryrun=opts.dryrun,
+ labels=test_vars[test]['labels'])
if retval != 0:
print "ERROR: failed to schedule job %s" % testname
- if opts.keep_control_file or opts.dryrun:
- print "keeping %s at user request" % control
- else:
- os.remove(control)
+ if opts.keep_control_file or opts.dryrun:
+ print "keeping %s at user request" % control
+ else:
+ os.remove(control)
