From: Matt Keeler <mkeeler(a)tresys.com>
Check against output files not just rc,stdout and stderr.
Some fixes to test files for what the actual output should be.
---
testing/harness/secstate_harness/testcase.py | 82 ++++++++++++++------
.../tests/import_xccdf_tgz/commands/import.stdout | 5 +-
.../output/var/lib/secstate/configs/PassComp.cfg | 2 +-
testing/tests/import_xccdf_tgz/test.manifest | 1 -
4 files changed, 61 insertions(+), 29 deletions(-)
diff --git a/testing/harness/secstate_harness/testcase.py
b/testing/harness/secstate_harness/testcase.py
index 4cae8a9..f994622 100644
--- a/testing/harness/secstate_harness/testcase.py
+++ b/testing/harness/secstate_harness/testcase.py
@@ -31,7 +31,7 @@ def join(lst):
return ''.join(lst)
class Command:
- results_template = 'Command %(command_name)s
Results:\n%(rc_results)s%(stdout_results)s%(stderr_results)s\n\tCommand Completed
Successfully: %(success)s'
+ results_template = 'Command %(command_name)s
Results:\n%(rc_results)s%(stdout_results)s%(stderr_results)sCommand Completed
Successfully: %(success)s\n'
rc_results = 'Expected RC : %(expected)s, Actual RC : %(actual)s'
stdout_results = 'Expected STDOUT :\n%(expected)s\nActual STDOUT
:\n%(actual)s\n'
stderr_results = 'Expected STDERR :\n%(expected)s\nActual STDERR
:\n%(actual)s\n'
@@ -72,20 +72,24 @@ class Command:
def cache_rc(self):
rc_file = open(self.rc_path, 'r')
- self.expected_rc = rc_file.readline().strip('\n')
+ self.expected_rc = int(rc_file.readline().strip('\n'))
rc_file.close()
def cache_stdout(self):
if self.stdout_exists:
stdout = open(self.stdout_path)
- self.expected_stdout = stdout.readlines()
+ self.expected_stdout = [line.rstrip('\n') for line in
stdout.readlines()]
stdout.close()
+ if not self.expected_stdout:
+ self.expected_stdout = ['']
def cache_stderr(self):
if self.stderr_exists:
stderr = open(self.stderr_path)
- self.expected_stderr = stderr.readlines()
+ self.expected_stderr = [line.rstrip('\n') for line in
stderr.readlines()]
stderr.close()
+ if not self.expected_stderr:
+ self.expected_stderr = ['']
def execute(self, chroot):
if not self.cmd_args:
@@ -116,12 +120,12 @@ class Command:
return False
elif not self.expected_stdout:
self.cache_stdout()
-
- return self.stdout == self.expected_stdout
+
+ return self.stdout != self.expected_stdout
def stdout_differ_string(self):
if self.does_stdout_differ():
- return self.stdout_results % {'expected' :
join(tab(self.expected_stdout)), 'actual' : join(tab(self.stdout))}
+ return self.stdout_results % {'expected' :
'\n'.join(tab(self.expected_stdout)), 'actual' :
'\n'.join(tab(self.stdout))}
else:
return ''
@@ -132,12 +136,11 @@ class Command:
return False
elif not self.expected_stderr:
self.cache_stderr()
- return self.stderr == self.expected_stderr
+ return self.stderr != self.expected_stderr
-
def stderr_differ_string(self):
if self.does_stderr_differ():
- return self.stderr_results % { 'expected' : tab(self.expected_stderr),
'actual' : tab(self.stderr)}
+ return self.stderr_results % { 'expected' :
'\n'.join(tab(self.expected_stderr)), 'actual' :
'\n'.join(tab(self.stderr))}
else:
return ''
@@ -148,7 +151,7 @@ class Command:
return False
elif not self.expected_rc:
self.cache_rc()
- return self.rc == self.expected_rc
+ return self.rc != self.expected_rc
def rc_differ_string(self):
if self.does_rc_differ():
@@ -158,9 +161,9 @@ class Command:
def get_results_string(self):
rep = { 'command_name' : self.command_name,
- 'rc_results' : self.rc_differ_string() + '\n' if
self.rc_differ_string() else '',
- 'stdout_results' :
join(tab(self.stdout_differ_string().split('\n'))) + '\n' if
self.rc_differ_string() else '',
- 'stderr_results' : '%s\n' %
join(tab(self.stderr_differ_string().split('\n'))) + '\n' if
self.rc_differ_string() else '',
+ 'rc_results' : (join(tab([self.rc_differ_string()])) +
'\n') if self.does_rc_differ() else join(tab(['Expected RC matches Actual
RC\n'])),
+ 'stdout_results' :
('\n'.join(tab(self.stdout_differ_string().split('\n'))) + '\n')
if self.does_stdout_differ() else join(tab(['Expect STDOUT matches Actual
STDOUT\n'])),
+ 'stderr_results' :
('\n'.join(tab(self.stderr_differ_string().split('\n'))) + '\n')
if self.does_stderr_differ() else join(tab(['Expect STDERR matches Actual
STDERR\n'])),
'success' : not (self.does_rc_differ() or self.does_stdout_differ()
or self.does_stderr_differ())}
return self.results_template % rep
@@ -233,7 +236,34 @@ class TestCase:
shutil.copy2(fpath, os.path.join(chroot_dir, basename))
def check_output_files(self):
- pass
+ rc = True
+ expected = 'Expected File Contents:\n%s'
+ actual = 'Actual File Contents:\n%s'
+ fdiffer = 'Output File %(oname)s differs:\n%(expected)s\n%(actual)s\n'
+ output_path = os.path.join(self.files_dir, 'output')
+ for root,dirs,files in os.walk(output_path):
+ for file in files:
+ opath = os.path.join(root, file)
+ dirname = re.sub(r'^%s(.*)' % re.escape(output_path), r'\1',
root).lstrip(os.path.sep)
+ cpath = os.path.join(self.chroot, dirname, file)
+
+ ofile = open(opath)
+ otext = ofile.readlines()
+ ofile.close()
+
+ cfile = open(cpath)
+ ctext = cfile.readlines()
+ cfile.close()
+
+ if otext != ctext:
+ rc = False
+ etext = '\n'.join(tab((expected %
join(tab(otext))).split('\n')))
+ atext = '\n'.join(tab((actual %
join(tab(ctext))).split('\n')))
+ dtext = fdiffer % {'oname' : os.path.join(dirname,file),
'expected' : etext, 'actual' : atext}
+
sys.stdout.write('\n'.join(tab(dtext.rstrip('\n').split('\n'))) +
'\n')
+ if rc:
+ sys.stdout.write('\n'.join(tab(['All Output Files matched expected
output'])) + '\n\n')
+ return rc
def run_test(self):
sys.stdout.write('Test %s:\n' % self.test_name)
@@ -243,9 +273,11 @@ class TestCase:
if rc:
rc = self.__run_verification()
if rc:
- sys.stdout.write('Test %s Completed Successfully: True\n' %
self.test_name)
- return True
- sys.stdout.write('Test %s Completed Successfully: False\n' %
self.tests_name)
+ rc = self.check_output_files()
+ if rc:
+ sys.stdout.write('Test %s Completed Successfully: True\n' %
self.test_name)
+ return True
+ sys.stdout.write('Test %s Completed Successfully: False\n' %
self.test_name)
return False
def __run_requires(self):
@@ -259,12 +291,12 @@ class TestCase:
if cmd.does_rc_differ() or cmd.does_stdout_differ() or
cmd.does_stderr_differ():
cmds_ok = False
- sys.stdout.write('%s\n\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
+ sys.stdout.write('%s\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
if not cmds_ok:
break
- sys.stdout.write(join(tab(['Requires Section Completed Successfully: %s\n'
% str(cmds_ok)])))
+ sys.stdout.write(join(tab(['Requires Section Completed Successfully:
%s\n\n' % str(cmds_ok)])))
return cmds_ok
def __run_tests(self):
@@ -275,12 +307,12 @@ class TestCase:
if cmd.does_rc_differ() or cmd.does_stdout_differ() or
cmd.does_stderr_differ():
cmds_ok = False
- sys.stdout.write('%s\n\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
+ sys.stdout.write('%s\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
if not cmds_ok:
break
- sys.stdout.write(join(tab(['Tests Section Completed Successfully: %s\n' %
str(cmds_ok)])))
+ sys.stdout.write(join(tab(['Tests Section Completed Successfully: %s\n\n' %
str(cmds_ok)])))
return cmds_ok
def __run_verification(self):
@@ -293,11 +325,11 @@ class TestCase:
if cmd.does_rc_differ() or cmd.does_stdout_differ() or
cmd.does_stderr_differ():
cmds_ok = False
- sys.stdout.write('%s\n\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
+ sys.stdout.write('%s\n' %
'\n'.join(tab(cmd.get_results_string().split('\n'), num_tabs=2)))
if not cmds_ok:
break
- sys.stdout.write(join(tab(['Verification Section Completed Successfully:
%s\n' % str(cmds_ok)])))
+ sys.stdout.write(join(tab(['Verification Section Completed Successfully:
%s\n\n' % str(cmds_ok)])))
return cmds_ok
-
\ No newline at end of file
+
diff --git a/testing/tests/import_xccdf_tgz/commands/import.stdout
b/testing/tests/import_xccdf_tgz/commands/import.stdout
index 177b335..8151aeb 100644
--- a/testing/tests/import_xccdf_tgz/commands/import.stdout
+++ b/testing/tests/import_xccdf_tgz/commands/import.stdout
@@ -1,6 +1,7 @@
2-19PasswordComplexity_Lowercase.xml
-2-22PasswordComplexity_Special.xml
2-20PasswordComplexity_MinLen.xml
2-21PasswordComplexity_Numeric.xml
-PassComp.xccdf.xml
2-23PasswordComplexity_Uppercase.xml
+2-22PasswordComplexity_Special.xml
+PassComp.xccdf.xml
+
diff --git
a/testing/tests/import_xccdf_tgz/files/output/var/lib/secstate/configs/PassComp.cfg
b/testing/tests/import_xccdf_tgz/files/output/var/lib/secstate/configs/PassComp.cfg
index 46852ed..e589a45 100644
--- a/testing/tests/import_xccdf_tgz/files/output/var/lib/secstate/configs/PassComp.cfg
+++ b/testing/tests/import_xccdf_tgz/files/output/var/lib/secstate/configs/PassComp.cfg
@@ -1,4 +1,4 @@
[PassComp]
selected = True
-file = PassComp.xccdf.xml
+file = /var/lib/secstate/benchmarks/PassComp/PassComp.xccdf.xml
diff --git a/testing/tests/import_xccdf_tgz/test.manifest
b/testing/tests/import_xccdf_tgz/test.manifest
index e6ed303..566cd6f 100644
--- a/testing/tests/import_xccdf_tgz/test.manifest
+++ b/testing/tests/import_xccdf_tgz/test.manifest
@@ -1,5 +1,4 @@
[requires]
-usage
[test]
import
--
1.6.5.2