May 2010 - cas - Fedora Mailing-Lists

Branch 'devel' - cas-admin caslib/util.py

by Adam Stokes

cas-admin | 10 ++++++---- caslib/util.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) New commits: commit cae222a2b239319753fcd2cdae5408ccebbbc66e Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 15:53:37 2010 -0400 mimeinfo update diff --git a/cas-admin b/cas-admin index dd7b9e2..d5f6cc6 100755 --- a/cas-admin +++ b/cas-admin @@ -81,7 +81,7 @@ def queryWorker(input): if not debug_IN_DB: # add RPM to database session = Session() - debug_record = Debug(debug) + debug_record = Debug(debug, mime="application/x-rpm") session.add(debug_record) session.commit() if util.testMimetype(debug): @@ -100,13 +100,15 @@ def queryWorker(input): session.add(timestamp_record) # commit to database session.commit() - elif util.testMimetype(debug, "application/x-executable"): + elif util.testMimetype(debug, mime="application/x-executable"): timestamp = coreTimestamp(debug, BUFFERSIZE) timestamp_record = Timestamp(timestamp, - debug, + debug[1:], debug_record.debugId) session.add(timestamp_record) session.commit() + else: + genprint(_e[6]) session.close() def queryLocalRpms(): @@ -126,7 +128,7 @@ def queryLocalRpms(): cmd = cmd + "\( -iregex %s" % (rpmfilter[0],) if rpmfilter_len > 1: concat_filter = ''.join([" -o -iregex %s " % (i,) for i in rpmfilter[1:]]) - cmd = cmd + + concat_filter + " \) " + cmd = cmd + concat_filter + " \) " # append any file/dir exclusions if "REGEXEXCLUDEDIR" in settings: for i in settings["REGEXEXCLUDEDIR"].split(','): diff --git a/caslib/util.py b/caslib/util.py index c609519..b79e131 100644 --- a/caslib/util.py +++ b/caslib/util.py @@ -158,7 +158,7 @@ class Utility(object): self.make_exe(crashExe) return - def testMimetype(self, fname, mime="application/x-rpm"): + def testMimetype(self, fname, mime): """ tests file for mime info """ from caslib.core import casexecute cmd = "file -i %s" % (fname,)

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin

by Adam Stokes

cas-admin | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) New commits: commit bedb0c8620be6d77efc03cbcc5f274ceab220c9d Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 15:39:56 2010 -0400 cas-admin concat find expression properly depending on amount of syntaxes diff --git a/cas-admin b/cas-admin index a2eddc9..dd7b9e2 100755 --- a/cas-admin +++ b/cas-admin @@ -117,17 +117,16 @@ def queryLocalRpms(): """ # create Queues task_queue = Queue() - cmd = "find -L %s -regextype %s" % (settings["KERNELS"], - settings["REGEXTYPE"]) + cmd = "find -L %s -regextype %s " % (settings["KERNELS"], + settings["REGEXTYPE"]) + concat_filter = "" if "RPMFILTER" in settings: rpmfilter = settings["RPMFILTER"].split(',') rpmfilter_len = len(rpmfilter) + cmd = cmd + "\( -iregex %s" % (rpmfilter[0],) if rpmfilter_len > 1: - cmd = "\( -iregex %s" % (rpmfilter[0],) - cmd = cmd + [" -o %s " % (i,) for i in rpmfilter[1:]] - cmd = cmd + "\)" - else: - cmd = "\( -iregex %s \)" % (rpmfilter[0],) + concat_filter = ''.join([" -o -iregex %s " % (i,) for i in rpmfilter[1:]]) + cmd = cmd + + concat_filter + " \) " # append any file/dir exclusions if "REGEXEXCLUDEDIR" in settings: for i in settings["REGEXEXCLUDEDIR"].split(','):

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin caslib/util.py

by Adam Stokes

cas-admin | 10 ++++++++-- caslib/util.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) New commits: commit 86b2f79912ff2ec0b6eb82fa3dadf22afaa45e93 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 15:24:00 2010 -0400 fix cmd concat diff --git a/cas-admin b/cas-admin index d1d8a26..a2eddc9 100755 --- a/cas-admin +++ b/cas-admin @@ -120,8 +120,14 @@ def queryLocalRpms(): cmd = "find -L %s -regextype %s" % (settings["KERNELS"], settings["REGEXTYPE"]) if "RPMFILTER" in settings: - for i in settings["RPMFILTER"].split(','): - cmd = cmd + " \( -iregex %s \) , " % (i,) + rpmfilter = settings["RPMFILTER"].split(',') + rpmfilter_len = len(rpmfilter) + if rpmfilter_len > 1: + cmd = "\( -iregex %s" % (rpmfilter[0],) + cmd = cmd + [" -o %s " % (i,) for i in rpmfilter[1:]] + cmd = cmd + "\)" + else: + cmd = "\( -iregex %s \)" % (rpmfilter[0],) # append any file/dir exclusions if "REGEXEXCLUDEDIR" in settings: for i in settings["REGEXEXCLUDEDIR"].split(','): diff --git a/caslib/util.py b/caslib/util.py index 0be1d82..c609519 100644 --- a/caslib/util.py +++ b/caslib/util.py @@ -15,7 +15,8 @@ import os import re import sys import logging -from caslib.core import casexecute + +from caslib import error_messages as _e if sys.version_info[:2] < (2,6): from caslib.cas_subprocess import Popen, PIPE, call @@ -159,6 +160,7 @@ class Utility(object): def testMimetype(self, fname, mime="application/x-rpm"): """ tests file for mime info """ + from caslib.core import casexecute cmd = "file -i %s" % (fname,) results, err = casexecute(cmd, True) if mime in results.strip():

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin caslib/util.py

by Adam Stokes

cas-admin | 4 ++-- caslib/util.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) New commits: commit 8511a4fa98ae89f5c99ea1ce80a6bd36e960a9b6 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 15:01:44 2010 -0400 testMimetype method added diff --git a/cas-admin b/cas-admin index 36b2136..d1d8a26 100755 --- a/cas-admin +++ b/cas-admin @@ -84,7 +84,7 @@ def queryWorker(input): debug_record = Debug(debug) session.add(debug_record) session.commit() - if not coreIsCorefile(debug): + if util.testMimetype(debug): genprint("(extracting) %-50s" % (os.path.basename(debug),)) tmpkernel = extractDebug(debug, dst) # Sort through extracted debug for each type @@ -100,7 +100,7 @@ def queryWorker(input): session.add(timestamp_record) # commit to database session.commit() - elif coreIsCorefile(debug): + elif util.testMimetype(debug, "application/x-executable"): timestamp = coreTimestamp(debug, BUFFERSIZE) timestamp_record = Timestamp(timestamp, debug, diff --git a/caslib/util.py b/caslib/util.py index ab865e5..0be1d82 100644 --- a/caslib/util.py +++ b/caslib/util.py @@ -15,6 +15,7 @@ import os import re import sys import logging +from caslib.core import casexecute if sys.version_info[:2] < (2,6): from caslib.cas_subprocess import Popen, PIPE, call @@ -156,3 +157,11 @@ class Utility(object): self.make_exe(crashExe) return + def testMimetype(self, fname, mime="application/x-rpm"): + """ tests file for mime info """ + cmd = "file -i %s" % (fname,) + results, err = casexecute(cmd, True) + if mime in results.strip(): + return True + return _e[6] +

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin caslib/db.py cas.spec

by Adam Stokes

cas-admin | 66 ++++++++++++++++++++++++++++++++++------------------------- cas.spec | 2 - caslib/db.py | 52 ++++++++++------------------------------------ 3 files changed, 52 insertions(+), 68 deletions(-) New commits: commit 5aa2c4583d1cfd8986b01c3a7d10ee72debe7d43 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 14:50:11 2010 -0400 support both RPM and extracted debugs diff --git a/cas-admin b/cas-admin index fcaac9d..36b2136 100755 --- a/cas-admin +++ b/cas-admin @@ -19,8 +19,13 @@ """ cas-admin - fingerprinting utility for cas """ import sys -if sys.version_info[:2] < (2,6): - raise SystemExit("Python >= 2.6 required") + +try: + from multiprocessing import Process, Queue, current_process, cpu_count, Lock +except ImportError: + raise SystemExit('Unable to import multiprocessing module. This is shipped by' \ + 'default in Python 2.6 and above. If using Python 2.4/2.5 a' \ + 'backport is available at http://pypi.python.org/pypi/multiprocessing') try: from caslib.db import * # Setup session for db interaction @@ -35,9 +40,8 @@ import optparse import datetime import paramiko import tempfile -from multiprocessing import Process, Queue, current_process, cpu_count, Lock -from caslib.core import casexecute, coreTimestamp +from caslib.core import casexecute, coreTimestamp, coreIsCorefile from caslib.util import Utility, genprint from caslib.rpmutils import extractDebug from caslib import error_messages as _e @@ -70,30 +74,38 @@ util = Utility() PROCESSES = cpu_count() def queryWorker(input): """ worker function to process rpm debugs """ - for rpm, dst in iter(input.get, 'STOP'): + for debug, dst in iter(input.get, 'STOP'): session = Session() - rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first() + debug_IN_DB = session.query(Debug).filter_by(debugPath=debug).first() session.close() - if not rpm_IN_DB: + if not debug_IN_DB: # add RPM to database session = Session() - rpm_record = RPM(rpm) - session.add(rpm_record) + debug_record = Debug(debug) + session.add(debug_record) session.commit() - genprint("(extracting) %-50s" % (os.path.basename(rpm),)) - tmpkernel = extractDebug(rpm, dst) - # Sort through extracted debug for each type - # e.g. hugemem, PAE, smp, largesmp - for item in tmpkernel: - vmlinux = item.strip() - debugKernel = os.path.normpath(vmlinux) - timestamp = coreTimestamp(debugKernel, BUFFERSIZE) - # add rpm id, debug, timestamp to database + if not coreIsCorefile(debug): + genprint("(extracting) %-50s" % (os.path.basename(debug),)) + tmpkernel = extractDebug(debug, dst) + # Sort through extracted debug for each type + # e.g. hugemem, PAE, smp, largesmp + for item in tmpkernel: + vmlinux = item.strip() + debugKernel = os.path.normpath(vmlinux) + timestamp = coreTimestamp(debugKernel, BUFFERSIZE) + # add id, debug, timestamp to database + timestamp_record = Timestamp(timestamp, + debugKernel, + debug_record.debugId) + session.add(timestamp_record) + # commit to database + session.commit() + elif coreIsCorefile(debug): + timestamp = coreTimestamp(debug, BUFFERSIZE) timestamp_record = Timestamp(timestamp, - debugKernel, - rpm_record.rpmId) + debug, + debug_record.debugId) session.add(timestamp_record) - # commit to database session.commit() session.close() @@ -108,15 +120,15 @@ def queryLocalRpms(): cmd = "find -L %s -regextype %s" % (settings["KERNELS"], settings["REGEXTYPE"]) if "RPMFILTER" in settings: - for i in settings["RPMFILTER"].split(): - cmd = cmd + " \( -iregex %s \) " % (i,) + for i in settings["RPMFILTER"].split(','): + cmd = cmd + " \( -iregex %s \) , " % (i,) # append any file/dir exclusions if "REGEXEXCLUDEDIR" in settings: - for i in settings["REGEXEXCLUDEDIR"].split(): - cmd = cmd + " \( -not -path %i \) " % (i,) + for i in settings["REGEXEXCLUDEDIR"].split(','): + cmd = cmd + " \( -not -path %s \) " % (i,) if "REGEXEXCLUDEFILE" in settings: - for i in settings["REGEXEXCLUDEFILE"].split(): - cmd = cmd + " \( -not -name %i \) " % (i,) + for i in settings["REGEXEXCLUDEFILE"].split(','): + cmd = cmd + " \( -not -name %s \) " % (i,) results, err = casexecute(cmd, True) dst = tempfile.mkdtemp(prefix='cas-') for rpm in results.split(): diff --git a/cas.spec b/cas.spec index 7deb8b0..dc3abe3 100644 --- a/cas.spec +++ b/cas.spec @@ -11,7 +11,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch Url: http://fedorahosted.org/cas BuildRequires: python-devel -%if 0%{?rhel} +%if 0%{?rhel} <= 5 Requires: python-sqlite %endif Requires: python-paramiko diff --git a/caslib/db.py b/caslib/db.py index a3c5f2b..e3d490a 100644 --- a/caslib/db.py +++ b/caslib/db.py @@ -38,16 +38,16 @@ class Core(Base): def __repr__(self): return "<Core('%s','%s')>" % (self.coreFile, self.coreHash) -class RPM(Base): - __tablename__ = 'rpms' - rpmId = Column(Integer, primary_key=True) - rpmPath = Column(String(255)) +class Debug(Base): + __tablename__ = 'debugs' + debugId = Column(Integer, primary_key=True) + debugPath = Column(String(255)) - def __init__(self, rpmPath): - self.rpmPath = rpmPath + def __init__(self, debugPath): + self.debugPath = debugPath def __repr__(self): - return "<RPM('%s')>" % (self.rpmPath,) + return "<Debug('%s')>" % (self.debugPath,) class Job(Base): __tablename__ = 'jobs' @@ -72,52 +72,24 @@ class Job(Base): self.vmcore, self.path, self.msg) -class JobArchive(Base): - """ Primarily for tracking """ - __tablename__ = 'jobarchive' - jobId = Column(Integer, primary_key=True) - identifier = Column(String(100)) - email = Column(String(40)) - vmcore = Column(String(255)) - path = Column(String(255)) - msg = Column(String(255)) - created = Column(DateTime) - ipaddr = Column(Integer) - - def __init__(self, identifier, email, vmcore, ipaddr, path=None, msg=None): - self.identifier = identifier - self.email = email - self.vmcore = vmcore - self.path = path - self.msg = msg - self.ipaddr = ipaddr - - def __repr__(self): - return "<Job('%s','%s','%s','%s','%s')>" % (self.identifier, - self.email, - self.vmcore, - self.path, - self.msg, - self.ipaddr) - class Timestamp(Base): __tablename__ = 'timestamps' stampId = Column(Integer, primary_key=True) stampKey = Column(String(255)) debugPath = Column(String(255)) - rpmId = Column(Integer, ForeignKey("rpms.rpmId")) + debugId = Column(Integer, ForeignKey("debugs.debugId")) - rpm = relationship(RPM, backref=backref('timestamps')) + debug = relationship(Debug, backref=backref('timestamps')) - def __init__(self, stampKey, debugPath, rpmId): + def __init__(self, stampKey, debugPath, debugId): self.stampKey = stampKey self.debugPath = debugPath - self.rpmId = rpmId + self.debugId = debugId def __repr__(self): return "<Timestamp('%s','%s','%s')>" % (self.stampKey, self.debugPath, - self.rpmId) + self.debugId) class Server(Base): __tablename__ = 'servers'

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin cas.conf

by Adam Stokes

cas-admin | 20 ++++++++++++++------ cas.conf | 11 ++++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) New commits: commit 785ae78fbe551a9437f8bc9e50bdeb800de8a6c6 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 12:31:48 2010 -0400 added feature for defining multiple searches, exclusions (directories, files) diff --git a/cas-admin b/cas-admin index cdc23a9..fcaac9d 100755 --- a/cas-admin +++ b/cas-admin @@ -68,7 +68,7 @@ util = Utility() # Gather number of cpu's PROCESSES = cpu_count() -def queryWorker(input, mutex): +def queryWorker(input): """ worker function to process rpm debugs """ for rpm, dst in iter(input.get, 'STOP'): session = Session() @@ -105,17 +105,25 @@ def queryLocalRpms(): """ # create Queues task_queue = Queue() - cmd = "find -L %s -regextype %s -iregex %s" % \ - (settings["KERNELS"], - settings["REGEXTYPE"], - settings["RPMFILTER"]) + cmd = "find -L %s -regextype %s" % (settings["KERNELS"], + settings["REGEXTYPE"]) + if "RPMFILTER" in settings: + for i in settings["RPMFILTER"].split(): + cmd = cmd + " \( -iregex %s \) " % (i,) + # append any file/dir exclusions + if "REGEXEXCLUDEDIR" in settings: + for i in settings["REGEXEXCLUDEDIR"].split(): + cmd = cmd + " \( -not -path %i \) " % (i,) + if "REGEXEXCLUDEFILE" in settings: + for i in settings["REGEXEXCLUDEFILE"].split(): + cmd = cmd + " \( -not -name %i \) " % (i,) results, err = casexecute(cmd, True) dst = tempfile.mkdtemp(prefix='cas-') for rpm in results.split(): task_queue.put((rpm, dst)) for i in range(PROCESSES): - Process(target=queryWorker, args=(task_queue, mutex)).start() + Process(target=queryWorker, args=(task_queue,)).start() for i in range(PROCESSES): task_queue.put('STOP') diff --git a/cas.conf b/cas.conf index 72a585f..0c76527 100644 --- a/cas.conf +++ b/cas.conf @@ -25,9 +25,17 @@ kernels=/mnt/kernels # set -regextype see `man find` regexType=posix-extended +# set directories to exclude separated by comma +# uses shell globbing +regexExcludeDir=*data*,*src* + +# set filenames to exclude separated by comma +# uses shell globbing +# regexExcludeFile=*tmp*,*\.*rcfiles* + # Compose -regextype regular expression for determining what # kernel debug rpms you wish to search for -rpmFilter=[^data].*kerne.+-debuginfo-[0-9].*\.rpm +rpmFilter=.*debuginfo.* # debug level (DEBUG, INFO) debugLevel=DEBUG @@ -65,3 +73,4 @@ autoPurge=no # Valid options are {number, None, or blank} # If set to None or blank default will be ~515M # buffersize=None +

13 years, 11 months

1
0
0 / 0

Branch 'devel' - cas-admin cas.conf doc/index.rst

by Adam Stokes

cas-admin | 19 ++++++++++--------- cas.conf | 7 +++++-- doc/index.rst | 4 ++++ 3 files changed, 19 insertions(+), 11 deletions(-) New commits: commit 3ac5745a0e08c3b9d7bb0b22c250352ae6e0c006 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 11:58:39 2010 -0400 add flexibilty to posix regular expression syntax diff --git a/cas-admin b/cas-admin index 75c071d..cdc23a9 100755 --- a/cas-admin +++ b/cas-admin @@ -35,7 +35,7 @@ import optparse import datetime import paramiko import tempfile -from multiprocessing import Process, Queue, current_process, cpu_count +from multiprocessing import Process, Queue, current_process, cpu_count, Lock from caslib.core import casexecute, coreTimestamp from caslib.util import Utility, genprint @@ -68,11 +68,12 @@ util = Utility() # Gather number of cpu's PROCESSES = cpu_count() -def queryWorker(input): +def queryWorker(input, mutex): """ worker function to process rpm debugs """ for rpm, dst in iter(input.get, 'STOP'): session = Session() rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first() + session.close() if not rpm_IN_DB: # add RPM to database session = Session() @@ -94,7 +95,7 @@ def queryWorker(input): session.add(timestamp_record) # commit to database session.commit() - session.close() + session.close() def queryLocalRpms(): """ query debug kernels located on the filesystem @@ -104,22 +105,22 @@ def queryLocalRpms(): """ # create Queues task_queue = Queue() - # Uses emacs regex -- see `man find` - cmd = "find -L %s -iregex %s" % (settings["KERNELS"], - settings["RPMFILTER"]) + cmd = "find -L %s -regextype %s -iregex %s" % \ + (settings["KERNELS"], + settings["REGEXTYPE"], + settings["RPMFILTER"]) results, err = casexecute(cmd, True) dst = tempfile.mkdtemp(prefix='cas-') for rpm in results.split(): - task_queue.put([rpm, dst]) + task_queue.put((rpm, dst)) for i in range(PROCESSES): - Process(target=queryWorker, args(task_queue).start() + Process(target=queryWorker, args=(task_queue, mutex)).start() for i in range(PROCESSES): task_queue.put('STOP') # Cleanup extracted debugs shutil.rmtree(dst) - genprint("CAS Database generation complete.") return def queryServers(): diff --git a/cas.conf b/cas.conf index ba5efd2..72a585f 100644 --- a/cas.conf +++ b/cas.conf @@ -22,9 +22,12 @@ sshkey=dss # indefinately recursive kernels=/mnt/kernels -# Compose emacs regular expression for determining what +# set -regextype see `man find` +regexType=posix-extended + +# Compose -regextype regular expression for determining what # kernel debug rpms you wish to search for -rpmFilter=.*kerne.+-debuginfo-[0-9].*\.rpm +rpmFilter=[^data].*kerne.+-debuginfo-[0-9].*\.rpm # debug level (DEBUG, INFO) debugLevel=DEBUG diff --git a/doc/index.rst b/doc/index.rst index 3b41ede..e906a15 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -60,6 +60,7 @@ each section and describe its meaning:: casuser=root sshkey=dss kernels=/mnt/kernels + regexType=posix-extended rpmFilter=.*kerne.+-debuginfo-[0-9].*\.rpm debugLevel=DEBUG workDirectory=/cores/processed @@ -80,6 +81,9 @@ each section and describe its meaning:: stored. This can range anywhere from an nfs mount, samba share, local disk or any other type of media the cas server can access. +``regexType``: (**Required**) Defines the regex syntax to use within ``find``. Doing a +``man find`` and searching for regextype will give you the allowable syntaxess. + ``rpmFilters``: (**Required**) This is a emacs based regular expression which is essentially passed to a find command to locate the various kernel-debuginfo packages defined in ``kernels`` directive.

13 years, 11 months

1
0
0 / 0

Branch 'devel' - 3 commits - cas-admin caslib/db.py overseer/command.py

by Adam Stokes

cas-admin | 44 ++++++++++++++++++++++++++++++-------------- caslib/db.py | 7 +++++-- overseer/command.py | 42 +++++++++++++++++++++++++++++++++++------- 3 files changed, 70 insertions(+), 23 deletions(-) New commits: commit 20ecd7bddc742b69fc884f52e681b494f6be3ddb Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 10:20:10 2010 -0400 process out db query diff --git a/cas-admin b/cas-admin index 35abdc4..75c071d 100755 --- a/cas-admin +++ b/cas-admin @@ -19,8 +19,8 @@ """ cas-admin - fingerprinting utility for cas """ import sys -if sys.version_info[:2] < (2,4): - raise SystemExit("Python >= 2.4 required") +if sys.version_info[:2] < (2,6): + raise SystemExit("Python >= 2.6 required") try: from caslib.db import * # Setup session for db interaction @@ -35,6 +35,7 @@ import optparse import datetime import paramiko import tempfile +from multiprocessing import Process, Queue, current_process, cpu_count from caslib.core import casexecute, coreTimestamp from caslib.util import Utility, genprint @@ -65,18 +66,11 @@ if config.has_option("advanced", "buffersize"): util = Utility() -def queryLocalRpms(): - """ query debug kernels located on the filesystem - - Arguments: - - `path`: filesystem location of debug kernels - """ - # Uses emacs regex -- see `man find` - cmd = "find -L %s -iregex %s" % (settings["KERNELS"], - settings["RPMFILTER"]) - results, err = casexecute(cmd, True) - dst = tempfile.mkdtemp(prefix='cas-') - for rpm in results.split(): +# Gather number of cpu's +PROCESSES = cpu_count() +def queryWorker(input): + """ worker function to process rpm debugs """ + for rpm, dst in iter(input.get, 'STOP'): session = Session() rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first() if not rpm_IN_DB: @@ -101,6 +95,28 @@ def queryLocalRpms(): # commit to database session.commit() session.close() + +def queryLocalRpms(): + """ query debug kernels located on the filesystem + + Arguments: + - `path`: filesystem location of debug kernels + """ + # create Queues + task_queue = Queue() + # Uses emacs regex -- see `man find` + cmd = "find -L %s -iregex %s" % (settings["KERNELS"], + settings["RPMFILTER"]) + results, err = casexecute(cmd, True) + dst = tempfile.mkdtemp(prefix='cas-') + for rpm in results.split(): + task_queue.put([rpm, dst]) + + for i in range(PROCESSES): + Process(target=queryWorker, args(task_queue).start() + for i in range(PROCESSES): + task_queue.put('STOP') + # Cleanup extracted debugs shutil.rmtree(dst) genprint("CAS Database generation complete.") commit 47092a53e807e2afa7c316456e0ff703018622de Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 09:45:38 2010 -0400 added remote-ip-addr tracking in jobs diff --git a/caslib/db.py b/caslib/db.py index ccbf840..a3c5f2b 100644 --- a/caslib/db.py +++ b/caslib/db.py @@ -82,20 +82,23 @@ class JobArchive(Base): path = Column(String(255)) msg = Column(String(255)) created = Column(DateTime) + ipaddr = Column(Integer) - def __init__(self, identifier, email, vmcore, path=None, msg=None): + def __init__(self, identifier, email, vmcore, ipaddr, path=None, msg=None): self.identifier = identifier self.email = email self.vmcore = vmcore self.path = path self.msg = msg + self.ipaddr = ipaddr def __repr__(self): return "<Job('%s','%s','%s','%s','%s')>" % (self.identifier, self.email, self.vmcore, self.path, - self.msg) + self.msg, + self.ipaddr) class Timestamp(Base): __tablename__ = 'timestamps' diff --git a/overseer/command.py b/overseer/command.py index 5d3a3eb..02cc040 100755 --- a/overseer/command.py +++ b/overseer/command.py @@ -73,7 +73,8 @@ class Root: if not session.query(Job).filter_by(identifier=data['identifier']).first(): jobRecord = Job(data['identifier'], data['email'], - data['vmcore']) + data['vmcore'], + cherrypy.request.headers['Remote-Addr']) session.add(jobRecord) session.commit() record = jobRecord commit e79c2485fa79e6aeaa09fc948c520571bc888087 Author: adam stokes <astokes(a)fedoraproject.org> Date: Thu May 13 09:28:58 2010 -0400 added sa to dict function for json exporting diff --git a/overseer/command.py b/overseer/command.py index 5f5f364..5d3a3eb 100755 --- a/overseer/command.py +++ b/overseer/command.py @@ -24,7 +24,7 @@ if sys.version_info[:2] < (2,6): import os import cherrypy -import simplejson as json +import simplejson from caslib.db import * from mako.template import Template from mako.lookup import TemplateLookup @@ -38,6 +38,24 @@ def run(cmd): p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) return p +def gmap(obj): + for item in obj.__dict__.items(): + if item[0][0] is '_': + continue + if isinstance(item[1], unicode): + yield [item[0], str(item[1])] + elif isinstance(item[1], datetime): + # datetime objects aren't json iterable so we convert to time string + yield [item[0], item[1].strftime("%I:%M%p %Y/%d/%m")] + else: + yield item + +def json(obj): + if isinstance(obj, list): + return simplejson.dumps( map(lambda x: dict(x), map(lambda x: gmap(x), obj)) ) + else: + return simplejson.dumps( dict(gmap(obj)) ) + class Root: @cherrypy.expose def index(self): @@ -68,8 +86,11 @@ class Root: p = Process(target=run, args=(cmd,)) p.start() p.join() - mytemplate = mylookup.get_template("create.html") - return mytemplate.render(record=record) + if 'json' in data: + return json(record) + else: + mytemplate = mylookup.get_template("create.html") + return mytemplate.render(record=record) else: raise cherrypy.HTTPRedirect('/') @@ -81,19 +102,25 @@ class Root: mytemplate = mylookup.get_template("status.html") session.close() + # check for json output first + if 'json' in data: + return json(status_query) if status_query: return mytemplate.render(status=status_query) else: return mytemplate.render(errors={'error' : 'Job Not Found'}) @cherrypy.expose - def list(self): + def list(self, **data): """ list all jobs """ session = Session() job_query = session.query(Job).order_by(Job.jobId.desc()).all() session.close() - mytemplate = mylookup.get_template("list.html") - return mytemplate.render(job_query=job_query) + if 'json' in data: + return json(job_query) + else: + mytemplate = mylookup.get_template("list.html") + return mytemplate.render(job_query=job_query) def main(): root = Root()

13 years, 11 months

1
0
0 / 0

Branch 'devel' - caslib/cas.py caslib/db.py

by Adam Stokes

caslib/cas.py | 2 +- caslib/db.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) New commits: commit cca149e3a3a4a0c4d95ca0c480b45f6583b549a2 Author: adam stokes <astokes(a)fedoraproject.org> Date: Wed May 12 16:06:47 2010 -0400 Attempting to fix out of bounds session handling diff --git a/caslib/cas.py b/caslib/cas.py index 4b9470c..e46e017 100755 --- a/caslib/cas.py +++ b/caslib/cas.py @@ -207,7 +207,7 @@ class CasApplication(object): debugKernel = os.path.abspath("vmlinux") else: filterString = "*/%s" % (timestamp.debugPath,) - extractDebug(timestamp.rpmPath, self.storagePath, + extractDebug(timestamp.rpm.rpmPath, self.storagePath, filter=filterString, return_results=False) diff --git a/caslib/db.py b/caslib/db.py index 01aeb77..ccbf840 100644 --- a/caslib/db.py +++ b/caslib/db.py @@ -20,7 +20,9 @@ from datetime import datetime from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, scoped_session, relationship, backref +from sqlalchemy.orm import sessionmaker, scoped_session, backref +from sqlalchemy.orm import relation as relationship + Base = declarative_base() class Core(Base): """ Corefile information """

13 years, 11 months

1
0
0 / 0

Branch 'devel' - caslib/db.py

by Adam Stokes

caslib/db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) New commits: commit 426b5d3dd858d1ba1913832c89bdeddb37267086 Author: adam stokes <astokes(a)fedoraproject.org> Date: Wed May 12 15:54:18 2010 -0400 Setup many-to-one relationship diff --git a/caslib/db.py b/caslib/db.py index 3e74b37..01aeb77 100644 --- a/caslib/db.py +++ b/caslib/db.py @@ -19,6 +19,8 @@ from datetime import datetime # Mapper Object from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, scoped_session, relationship, backref Base = declarative_base() class Core(Base): """ Corefile information """ @@ -100,6 +102,8 @@ class Timestamp(Base): debugPath = Column(String(255)) rpmId = Column(Integer, ForeignKey("rpms.rpmId")) + rpm = relationship(RPM, backref=backref('timestamps')) + def __init__(self, stampKey, debugPath, rpmId): self.stampKey = stampKey self.debugPath = debugPath @@ -127,8 +131,6 @@ class Server(Base): self.port, self.arch) -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, scoped_session # Read cas.conf for database path import ConfigParser config = ConfigParser.RawConfigParser()

13 years, 11 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

cas May 2010