autoqa-devel May 2010

autoqa-devel@lists.fedorahosted.org

8 participants
66 discussions

[AutoQA] #154: pst: Abort when old and new package versions match
by fedora-badges 14 May '10

14 May '10

#154: pst: Abort when old and new package versions match -------------------------+-------------------------------------------------- Reporter: kparal | Owner: psss Type: enhancement | Status: new Priority: major | Milestone: Package Sanity Tests Component: tests | Version: 1.0 Keywords: | -------------------------+-------------------------------------------------- Currently this command works: {{{ pst -y -o tzdata-2009o-2.fc12 -n tzdata-2009o-2.fc12 }}} but it prints many FAILs in the process. I think we could just detect if any of the old packages is equal to any of the new packages and abort the test in this case. There is no sense in testing the same package anyway, is it? -- Ticket URL: <https://fedorahosted.org/autoqa/ticket/154> AutoQA <http://autoqa.fedorahosted.org> Automated QA project

1 0

[AutoQA] #151: RFE: opt-in email from autoqa's post-koji-build tests
by fedora-badges 13 May '10

13 May '10

#151: RFE: opt-in email from autoqa's post-koji-build tests -----------------------+---------------------------------------------------- Reporter: skvidal | Owner: Type: task | Status: new Priority: minor | Milestone: Component: reporting | Version: 1.0 Keywords: | -----------------------+---------------------------------------------------- Right now all koji builds are being tested with rpmguard and rpmlint on the backend. The output is going to: https://fedorahosted.org/mailman/listinfo/autoqa-results It'd be great if there were a way for a maintainer to opt-in to this kind of info for their package. Not sure where the info would be stored - but the results from each build, instead of emailing to autoqa-results would send that to $srpmname- owner(a)fedoraproject.org and then anyone who is on that email alias gets the notices. so what we need is: 1. a place to say which packages have opted in 2. maybe a partial solution to ticket 148 3. that sounds like it. -- Ticket URL: <https://fedorahosted.org/autoqa/ticket/151> AutoQA <http://autoqa.fedorahosted.org> Automated QA project

1 11

[PATCH 1/2] make rpmguard and rpmlint tests have option of sending email to pkg owners if they optin - using mechanism in ticket 151
by Seth Vidal 13 May '10

13 May '10

From: Seth Vidal <skvidal(a)fedoraproject.org> --- lib/python/repoinfo.py | 3 ++- lib/python/util.py | 18 ++++++++++++++++++ repoinfo.conf | 14 ++++++++++++++ tests/rpmguard/rpmguard.py | 18 ++++++++++++++---- tests/rpmlint/rpmlint.py | 18 ++++++++++++++---- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/lib/python/repoinfo.py b/lib/python/repoinfo.py index f5d1044..de7332c 100644 --- a/lib/python/repoinfo.py +++ b/lib/python/repoinfo.py @@ -47,7 +47,8 @@ class RepoinfoConfig(object): 'tag': self.config.get(name,'tag'), 'url': self.config.get(name,'url'), 'path': self.config.get(name,'path'), - 'name': name # kind of unnecessary + 'name': name, # kind of unnecessary + 'collection_name': self.config.get(name, 'collection_name'), } return repo def getrepo_by_tag(self, tag): diff --git a/lib/python/util.py b/lib/python/util.py index 4a533ec..67ae699 100644 --- a/lib/python/util.py +++ b/lib/python/util.py @@ -27,6 +27,8 @@ import datetime import urlgrabber.grabber import urlgrabber.progress import config +import urllib +import socket def timestamp_to_compose_id(timestamp=None, serial=1): if not timestamp: @@ -91,6 +93,22 @@ def checkhash(filename, algo, hexdigest): hasher.hexdigest() == hexdigest) return r +def check_opt_in(pkgname, release): + """take a package name and release and check for an autoqa-optin file""" + + url = 'http://fedorapeople.org/packages/' + fullurl = url + '%s/%s/%s/autoqa-optin' % (name[0].lower(), name, release) + tmout = socket.getdefaulttimeout() + socket.setdefaulttimeout(5) # if it takes more than + try: + h = urllib.urlopen(fullurl) + except IOError, e: + return False + if h.getcode() == 200: + return True + return False + + grabber = urlgrabber.grabber.URLGrabber() if hasattr(sys.stdout, "fileno") and os.isatty(sys.stdout.fileno()): grabber.opts.progress_obj = urlgrabber.progress.TextMeter() diff --git a/repoinfo.conf b/repoinfo.conf index d1dc535..a3ebed4 100644 --- a/repoinfo.conf +++ b/repoinfo.conf @@ -12,66 +12,80 @@ rawhideurl = %(baseurl)s/%(path)s/%(arch)s/os arches = i386, x86_64 path = development/rawhide url = %(rawhideurl)s +collection_name = devel [branched] arches = i386, x86_64 # path will change when Fedora 13 is released path = development/13 url = %(rawhideurl)s +collection_name = F-13 [f13] # path will change when Fedora 13 is released path = development/13 url = %(rawhideurl)s +collection_name = F-13 [f13-updates] path = 13 url = %(updatesurl)s parents = f13 +collection_name = F-13 [f13-updates-testing] path = testing/13 url = %(updatesurl)s parents = f13-updates, f13 +collection_name = F-13 [f12] path = 12 url = %(goldurl)s +collection_name = F-12 [f12-updates] path = 12 url = %(updatesurl)s parents = f12 +collection_name = F-12 [f12-updates-testing] path = testing/12 url = %(updatesurl)s parents = f12-updates, f12 +collection_name = F-12 [f11] path = 11 url = %(goldurl)s +collection_name = F-11 [f11-updates] path = 11 url = %(updatesurl)s parents = f11 +collection_name = F-11 [f11-updates-testing] path = testing/11 url = %(updatesurl)s parents = f11-updates, f11 +collection_name = F-11 [f10] path = 10 url = %(goldurl)s +collection_name = F-10 [f10-updates] path = 10 url = %(updatesurl)s parents = f10 +collection_name = F-10 [f10-updates-testing] path = testing/10 url = %(updatesurl)s parents = f10-updates, f10 +collection_name = F-10 diff --git a/tests/rpmguard/rpmguard.py b/tests/rpmguard/rpmguard.py index 61eec64..e91b1bf 100644 --- a/tests/rpmguard/rpmguard.py +++ b/tests/rpmguard/rpmguard.py @@ -23,6 +23,7 @@ from autotest_lib.client.bin.test_config import config_loader from autotest_lib.client.common_lib import error import autoqa.koji_utils import autoqa.util +from autoqa.repoinfo import repoinfo import os import rpmUtils.miscutils @@ -175,13 +176,22 @@ rpmlint | cut -d ' ' -f 1", retain_output=True) out.write(log) out.close() - # email results to mailing list + # email results to mailing list and to pkg owner if they optin + send_to = [] result_email = self.config.get('test','result_email') - if result_email: + if result_email and isinstance(result_email, str): + send_to = mail_to.split(',') + + collection_name = repoinfo.config.getrepo_by_tag(kojitag)['collection_name'] + if collection_name and autoqa.util.check_opt_in(name, collection_name): + #FIXME - hardcoded partial email address here - obviously sub-par + send_to.append('%s-owner(a)fedoraproject.org') % name + + if send_to: warning_count = len([line for line in log.splitlines() \ if line.startswith('W: ')]) - print 'Sending email with results to %s ...' % result_email - utils.send_email(mail_to=result_email, + print 'Sending email with results to %s ...' % ', '.join(send_to) + utils.send_email(mail_to=send_to, mail_from=self.config.get('test','mail_from'), mail_server=self.config.get('test','smtpserver'), subject='rpmguard: %u warnings for package %s' % \ diff --git a/tests/rpmlint/rpmlint.py b/tests/rpmlint/rpmlint.py index 7ad6f90..304d42c 100644 --- a/tests/rpmlint/rpmlint.py +++ b/tests/rpmlint/rpmlint.py @@ -22,6 +22,7 @@ from autotest_lib.client.bin.test_config import config_loader from autotest_lib.client.common_lib import error import autoqa.koji_utils, autoqa.util import os +from autoqa.repoinfo import repoinfo class rpmlint(test.test): version = 1 # increment if setup() changes @@ -60,13 +61,22 @@ class rpmlint(test.test): out.write(self.results) out.close() - # email results to mailing list + # email results to mailing list and to pkg owner if they optin + send_to = [] result_email = self.config.get('test','result_email') - if result_email: + if result_email and isinstance(result_email, str): + send_to = mail_to.split(',') + + collection_name = repoinfo.config.getrepo_by_tag(kojitag)['collection_name'] + if collection_name and autoqa.util.check_opt_in(name, collection_name): + #FIXME - hardcoded partial email address here - obviously sub-par + send_to.append('%s-owner(a)fedoraproject.org') % name + + if send_to: # parse last line for summary ('N errors, N warnings') summary = ((self.results.splitlines()[-1]).split(';')[-1])[1:-1] - print 'Sending email with results to %s ...' % result_email - utils.send_email(mail_to=result_email, + print 'Sending email with results to %s ...' % ', '.join(send_to) + utils.send_email(mail_to=send_to, mail_from=self.config.get('test','mail_from'), mail_server=self.config.get('test','smtpserver'), subject='rpmlint: %s for package %s' % \ -- 1.7.0.1

3 7

Re: rpmguard -- what's next?
by Kamil Paral 13 May '10

13 May '10

Hello, I have similar ideas long time in my head, created a ticket now: https://fedorahosted.org/autoqa/ticket/153 Of course rpmguard should be modular, at least from source code perspective. That would allow easier addition of new checks. Completely separate plugins in a yum way are maybe too much, but that's up to consideration. I really like the modular approach used in rpmlint. I created rpmguard as a quick hack and also as my first longer Python script. If time allows, I would like to convert it into a tool with a more decent architecture (modular design amongst other things), which would also offer me to get my hands dirty with more complex Python programming - looking forward to it. Now the only question is about time frame. I believe our top priority is now to have AutoQA with ResultDB up-and-running, which will allow us to finally inform people about glitches in our/their repositories and packages. Having nicely structured test scripts comes only after that, I assume. Right, wrong? So it can take some time until I'm given the opportunity to work on it. But hey, any contribution is welcome, Seth (or anyone else)! :)

2 1

rpmguard -- what's next?
by James Laska 12 May '10

12 May '10

Greetings folks, Please don't take this post to mean, "drop what you are doing and let's focus on rpmguard". I believe our current focus is correctly centered around automating the package update acceptance test plan [1]. Kamil and I have discussed some long-term plans for rpmguard, but I also wanted to bring the discussion to the list for group consumption/review. Quick background, rpmguard does a fair amount of comparative tests against old and new set of packages. Heck, the tests are event documented [2] (nicely done Kamil!). Today, Seth Vidal directed me to bug#564018 and asked whether rpmguard had support for testing whether a package has a *large* change in it's %provides. In the bug, the erlang package went from 50 provides to 22157 provides. This package did go through rpmguard and warnings were generated (for a good laugh, see [3]). So kudos are in order for capturing the change in an automated manner. What interested me about this exercise was how a new test might be added to rpmguard. The power of rpmguard is in abstracting the details of locating the previous packages and providing a common framework for comparisons against the two package sets. Presently, the tests exist in the main driver. How do we want to extend this in the future make it easier/faster to support new comparison tests? Should tests exist in stand-alone python scripts, where they all accept a common set of arguments? It seems like rpmlint is structured this way [4], is this good/bad ... does this make user-contributed comparative tests easier? Something Kamil proposed a while back, should rpmguard be a stand-alone tool (much like rpmlint)? Instead of being bundled inside autoqa, anyone could do: `yum install rpmguard` Apologies for the ranty nature of the email, I'm still thinking this through. My main objective is to get a sense where rpmguard should go. Hopefully these thoughts can lead to a wiki or TRAC roadmap, and something we could implement once our immediate objects are behind us. Thanks, James [1] https://fedoraproject.org/wiki/QA:Package_Update_Acceptance_Test_Plan [2] https://fedorahosted.org/autoqa/wiki/RpmguardChecks [3] https://fedorahosted.org/pipermail/autoqa-results/2010-April/017795.html [4] http://rpmlint.zarb.org/cgi-bin/trac.cgi/browser/trunk

2 4

[PATCH] 65,130 - autotest_server and job.tag in autoqa_conf variable
by Josef Skladanka 07 May '10

07 May '10

Hello gang, take a look at the patch attached - both autotest_server and job.tag are stored in the autoqa_conf variable. To get the job.tag there, I had to do a little 'hack' - the autoqa_conf is stored in the control file as a string, so when creating this string, I add a parameter jobtag = %s to the [general] section of the config, and append % (job.tag, ) behind the string, so it gets evaluated, when the control file is imported (i.e. when the job.tag value is known). The respective part of the control file then looks like this: autoqa_conf = ''' [test] smtpserver = localhost result_email = mail_from = autoqa(a)fedoraproject.org [general] autotest_server = dhcp-30-103.brq.redhat.com jobtag = %s notification_email = local = false hookdir = /usr/share/autoqa testdir = /usr/share/autotest/client/site_tests ''' % (job.tag, ) so nothing needs to change in control files (no new parameter is required), only the tests will need minor adjustments in the initialize method: def initialize(self, envr, name, kojitag, config): self.config = config_loader(config, self.tmpdir) autotest_server = self.config.get('general', 'autotest_server') jobtag = self.config.get('general', 'jobtag') self.autotest_url = "http://%s/results/%s/" % (autotest_server, jobtag) And then of course the test needs to take advantage of the self.autotest_url variable (i.e. add the URL to the e-mail). What do you think about this approach - I belive it's quite clean (apart from the minor job.tag 'hack') and hope you'll like it. joza ------- diff --git a/autoqa b/autoqa index efb94e3..cf8b6ee 100755 --- a/autoqa +++ b/autoqa @@ -27,6 +27,7 @@ import optparse import tempfile import StringIO import urlgrabber +import socket from ConfigParser import * from subprocess import call @@ -37,6 +38,7 @@ conf = { 'testdir': '/usr/share/autotest/client/site_tests', 'hookdir': '/usr/share/autoqa', 'notification_email': '', + 'autotest_server': socket.gethostname(), } cfg_parser = SafeConfigParser() # used by prep_controlfile try: @@ -58,6 +60,10 @@ def prep_controlfile(controlfile, extradata): prepended with key='value' lines for each item in extradata and an 'autoqa_conf' variable which holds the contents of the autoqa configfile. ''' + # the jobtag must be evaluated on every single testrun (since it changes :-)) + # so line 'jobtag = %s' is added to the [general] part of the autoqa_conf + # string, and then is replaced with job.tag when the code is executed + cfg_parser.set('general', 'jobtag', '%s') controldata = open(controlfile) (fd, name) = tempfile.mkstemp(prefix='autoqa-control.') os.write(fd, '# -*- coding: utf-8 -*-\n\n') @@ -65,7 +71,7 @@ def prep_controlfile(controlfile, extradata): cfgdata = StringIO.StringIO() cfg_parser.write(cfgdata) cfgdata.seek(0) - os.write(fd, "autoqa_conf = '''\n%s'''\n\n" % cfgdata.read()) + os.write(fd, "autoqa_conf = '''\n%s''' %% (job.tag, )\n\n" % cfgdata.read()) except IOError: pass for k,v in extradata.iteritems(): @@ -139,6 +145,9 @@ parser.add_option('--local', action='store_true', dest='local', help='Do not schedule jobs - run test(s) directly on the local machine') parser.add_option('-l', '--list-tests', action='store_true', dest='listtests', help='list the tests for the given hookname - do not run any tests') +parser.add_option('--autotest-server', action='store', default=None, + help='Sets the autotest-server hostname. Used for creating URLs to results.\ +Hostname of the local machine is used by default.') # Read and validate the hookname # Check for no args, or just -h/--help if len(sys.argv) == 1 or sys.argv[1] in ('-h', '--help'): @@ -197,6 +206,11 @@ for arch in opts.arch: # N.B. process_testdata may grow new keyword arguments if we add new autoqa # args that add another loop here.. testdata = hook.process_testdata(opts, args, arch=arch) + if not 'autotest_server' in testdata.keys(): + if opts.autotest_server is not None: + testdata['autotest_server'] = opts.autotest_server + else: + testdata['autotest_server'] = conf['autotest_server'] # XXX FIXME: tests need to be able to indicate that they do not require # any specific arch (e.g. rpmlint can run on any arch) for test in testlist:

2 4

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

autoqa-devel May 2010