cas lib/cas version
by Adam Stokes
cas | 11 ++++++++---
lib/cas/utilities.py | 26 +++++++++++++++-----------
version | 2 +-
3 files changed, 24 insertions(+), 15 deletions(-)
New commits:
commit 7bb15d20e43a4f54280321940b608c0431e52b39
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Thu Oct 30 12:02:04 2008 -0400
finalized on the downloader, altered cas to accept both localfiles and urlbased files.
support for http and ftp only at this time
diff --git a/cas b/cas
index 95f1e11..1590628 100755
--- a/cas
+++ b/cas
@@ -7,7 +7,7 @@ import os
import ConfigParser
import shutil
-from cas.utilities import Utilities, CoreTool, dprint, sprint
+from cas.utilities import Utilities, CoreTool, dprint, sprint, Download
from cas.rpmutils import Tools
if sys.version_info[:2] < (2,4):
@@ -27,6 +27,8 @@ class CoreHandler(object):
self.tool = CoreTool()
def run(self):
+ if (self.filename.startswith("http") or self.filename.startswith("ftp")):
+ self.filename = Download(self.filename, self.dst).get()
if not os.path.isfile(self.filename):
print("Unable to find file %s" % (self.filename,))
sys.exit(1)
@@ -50,6 +52,7 @@ class TimestampHandler(object):
def run(self):
rpmDB = self.util.load(RPMS)
coreTimestamp = self.tool.timestamp(self.corefile)
+ dprint(coreTimestamp, DPRINT)
if coreTimestamp:
for k,v in rpmDB.iteritems():
for coreObj in rpmDB[k]:
@@ -59,7 +62,8 @@ class TimestampHandler(object):
else:
dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT)
sys.exit(1)
- return False
+ dprint("Unable to match (%s,%s) with debugKernel" % (self.corefile, coreTimestamp), DPRINT)
+ sys.exit(1)
class CasApplication(object):
def __init__(self, args):
@@ -90,13 +94,14 @@ class CasApplication(object):
if not os.path.isdir(self.storagePath):
os.makedirs(self.storagePath)
corefile = CoreHandler(self.filename, self.storagePath).run()
+ dprint(corefile, DPRINT)
debuginfo, debugKernel = TimestampHandler(corefile).run()
filterString = "*/%s" % (debugKernel,)
self.rpmTool.extract(debuginfo, self.storagePath,
filter=filterString,
return_results=False)
- self.tool.buildCrashFile(self.storagePath, vmcore, debugKernel)
+ self.tool.buildCrashFile(self.storagePath, corefile, debugKernel)
print("Job on %s complete and located in %s." % (self.filename,self.storagePath))
return
diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py
index 074b3c5..937d8c9 100755
--- a/lib/cas/utilities.py
+++ b/lib/cas/utilities.py
@@ -8,6 +8,7 @@ import sys
import shutil
import tarfile
import logging
+import urlparse
import urlgrabber.grabber as grabber
from subprocess import Popen, PIPE
@@ -181,8 +182,8 @@ class Utilities(object):
# Build crash input file
crashInputPath = os.path.join(dst, file_in)
crashInputFH = open(crashInputPath, "w")
- crashInFH.write(",".join(crashInputCmds))
- crashInFH.close()
+ crashInputFH.write("".join(crashInputCmds))
+ crashInputFH.close()
vmcorePath = os.path.join(dst, vmcore)
crashCmd = "#!/bin/sh\ncrash %s %s -i %s -s\n" % (vmcorePath,debug,
@@ -210,8 +211,11 @@ class CoreTool(object):
raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,))
else:
tar = tarfile.open(self.filepath, "r")
- tar.extractall(self.dst)
- for root, dirs, files in self.util.directoryList(dst):
+ # python 2.4 doesn't have extractall method
+ for tarinfo in tar:
+ tar.extract(tarfinfo.name, self.dst)
+ #tar.extractall(self.dst)
+ for root, dirs, files in self.util.directoryList(self.dst):
for file in files:
if self.isCorefile(file):
return os.path.join(root,file)
@@ -245,8 +249,8 @@ class CoreTool(object):
raise CoreException("Unable to retrieve timestamp from: %s" % (path,))
class Download(object):
-""" borrowed from anaconda's urlinstall method
-"""
+ """ borrowed from anaconda's urlinstall method
+ """
def __init__(self, url, dst):
self.url = url
self.dst = dst
@@ -255,14 +259,14 @@ class Download(object):
self.file = os.path.basename(path)
self.output = os.path.join(self.dst, self.file)
- def status(self, cur_bytes, tot_bytes):
- sys.stdout.write("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r")
+ def status(self, cur_percent):
+ sys.stdout.write("Downloading %3d%%" % (cur_percent) + "\r")
sys.stdout.flush()
def get(self):
try:
url = grabber.urlopen(self.url)
- except: grabber.URLGrabError, e:
+ except grabber.URLGrabError, e:
raise IOError (e.errno, e.strerror)
# check size
@@ -280,9 +284,9 @@ class Download(object):
tot = len(buf)
while len(buf) > 0:
if filesize is not None:
- self.status((100*tot)/filesize, filesize)
+ self.status((100*tot)/filesize)
else:
- self.status(tot/1024, filesize)
+ self.status(tot/1024)
f.write(buf)
buf = url.read(65535)
tot += len(buf)
diff --git a/version b/version
index d5fc183..dd9b839 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 70
+0.13 71
15 years, 6 months
lib/cas
by Adam Stokes
lib/cas/utilities.py | 43 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 40 insertions(+), 3 deletions(-)
New commits:
commit a8da19b400f740cc2e66541fbd3f0bc75b4bfb5e
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Thu Oct 30 10:50:54 2008 -0400
download support added
diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py
index fea79d6..074b3c5 100755
--- a/lib/cas/utilities.py
+++ b/lib/cas/utilities.py
@@ -8,6 +8,7 @@ import sys
import shutil
import tarfile
import logging
+import urlgrabber.grabber as grabber
from subprocess import Popen, PIPE
@@ -180,7 +181,7 @@ class Utilities(object):
# Build crash input file
crashInputPath = os.path.join(dst, file_in)
crashInputFH = open(crashInputPath, "w")
- crashInFH.write(crashInputCmds)
+ crashInFH.write(",".join(crashInputCmds))
crashInFH.close()
vmcorePath = os.path.join(dst, vmcore)
@@ -244,12 +245,48 @@ class CoreTool(object):
raise CoreException("Unable to retrieve timestamp from: %s" % (path,))
class Download(object):
+""" borrowed from anaconda's urlinstall method
+"""
def __init__(self, url, dst):
self.url = url
self.dst = dst
+ (scheme, netloc, path, query, fragid) = urlparse.urlsplit(self.url)
+ self.file = os.path.basename(path)
+ self.output = os.path.join(self.dst, self.file)
+
def status(self, cur_bytes, tot_bytes):
- print("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r")
+ sys.stdout.write("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r")
+ sys.stdout.flush()
def get(self):
- pass
+ try:
+ url = grabber.urlopen(self.url)
+ except: grabber.URLGrabError, e:
+ raise IOError (e.errno, e.strerror)
+
+ # check size
+ try:
+ filesize = int(url.info()["Content-Length"])
+ if filesize == 0:
+ filesize = None
+ except:
+ filesize = None
+
+ #write output
+ f = open(self.output, "w+")
+
+ buf = url.read(65535)
+ tot = len(buf)
+ while len(buf) > 0:
+ if filesize is not None:
+ self.status((100*tot)/filesize, filesize)
+ else:
+ self.status(tot/1024, filesize)
+ f.write(buf)
+ buf = url.read(65535)
+ tot += len(buf)
+
+ f.close()
+ url.close()
+ return self.output
15 years, 6 months
cas lib/cas
by Adam Stokes
cas | 9 +--------
lib/cas/utilities.py | 35 +++++++++++++++++++++++++++++++++++
2 files changed, 36 insertions(+), 8 deletions(-)
New commits:
commit 78162295c6852f938b03bef73bba02133cb9aa70
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Wed Oct 29 21:10:51 2008 -0400
build out crash file to use the -i switch for a more automated approach.
this gets rid of the need for expect, however, i may need to use some sort of
distributed control system like func in order to process the cores depending
on the architecture type.
diff --git a/cas b/cas
index 620238c..95f1e11 100755
--- a/cas
+++ b/cas
@@ -96,14 +96,7 @@ class CasApplication(object):
filter=filterString,
return_results=False)
- crash_cmd = "#!/bin/sh\ncrash %s %s\n" % (os.path.join(self.storagePath,
- debugKernel),
- corefile)
- crash_exe = os.path.join(self.storagePath,"crash")
- fh = open(crash_exe,"w")
- fh.write(crash_cmd)
- fh.close()
- self.tool.make_exe(crash_exe)
+ self.tool.buildCrashFile(self.storagePath, vmcore, debugKernel)
print("Job on %s complete and located in %s." % (self.filename,self.storagePath))
return
diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py
index a9bea61..fea79d6 100755
--- a/lib/cas/utilities.py
+++ b/lib/cas/utilities.py
@@ -169,6 +169,30 @@ class Utilities(object):
return supportArch[k]
return False
+ def buildCrashFile(self, dst, vmcore, debug, file_in="crash.in"):
+ """ build crash file with predefined debug
+ commands
+ """
+ crashInputCmds = ['bt -a > bt.out\n','sys > sys.out\n','sys -c > sys-c.out\n',
+ 'kmem -f > kmem-f.out\n', 'log > log.out\n',
+ 'mod > mod.out\n', 'exit']
+
+ # Build crash input file
+ crashInputPath = os.path.join(dst, file_in)
+ crashInputFH = open(crashInputPath, "w")
+ crashInFH.write(crashInputCmds)
+ crashInFH.close()
+
+ vmcorePath = os.path.join(dst, vmcore)
+ crashCmd = "#!/bin/sh\ncrash %s %s -i %s -s\n" % (vmcorePath,debug,
+ crashInputPath)
+ crashExe = os.path.join(dst,"crash")
+ fh = open(crashExe,"w")
+ fh.write(crashCmd)
+ fh.close()
+ self.make_exe(crashExe)
+ return
+
class CoreException(Exception):
pass
@@ -218,3 +242,14 @@ class CoreTool(object):
if out:
return out
raise CoreException("Unable to retrieve timestamp from: %s" % (path,))
+
+class Download(object):
+ def __init__(self, url, dst):
+ self.url = url
+ self.dst = dst
+
+ def status(self, cur_bytes, tot_bytes):
+ print("Downloaded %s of %s" % (cur_bytes, tot_bytes) + "\r")
+
+ def get(self):
+ pass
15 years, 6 months
cas version
by Adam Stokes
cas | 3 ---
version | 2 +-
2 files changed, 1 insertion(+), 4 deletions(-)
New commits:
commit 7ad38167081db03b39adc4f8a39c6833fdc7d2fc
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Fri Oct 24 18:46:24 2008 -0400
working copy of cas complete.
next step, add support for processing queues for that automated touch
diff --git a/cas b/cas
index 5d5d16f..620238c 100755
--- a/cas
+++ b/cas
@@ -56,9 +56,6 @@ class TimestampHandler(object):
debugKernel, timestamp = coreObj
if timestamp and coreTimestamp in timestamp:
return (k, debugKernel)
- else:
- dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT)
- sys.exit(1)
else:
dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT)
sys.exit(1)
diff --git a/version b/version
index 2e9ff79..d5fc183 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 69
+0.13 70
15 years, 6 months
cas cas.conf casprint lib/cas version
by Adam Stokes
cas | 7 ++++---
cas.conf | 14 ++++++++++++--
casprint | 37 +++++++++++++++++++++++--------------
lib/cas/rpmutils.py | 3 +--
version | 2 +-
5 files changed, 41 insertions(+), 22 deletions(-)
New commits:
commit 588f80a3f10bdaeb5bd3b220d9ca1c294035d62e
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Thu Oct 23 19:08:33 2008 -0400
casprint seems to capture all extracted debugs and timestamp.
db tree is sorted in the form of
RPM
'-> [
( debugKernel, timestamp )
( debugKernel, timestamp )
]
added support for find's emacs regex for fine tuning
kernel-debuginfo searches.
cas is about 85% complete with has the ability to detect
timestamps within cores, test for proper corefile and extraction.
builds out a crash wrapper for ease of use when loading the necessary
files for crash to proceed.
diff --git a/cas b/cas
index 007878c..5d5d16f 100755
--- a/cas
+++ b/cas
@@ -51,10 +51,11 @@ class TimestampHandler(object):
rpmDB = self.util.load(RPMS)
coreTimestamp = self.tool.timestamp(self.corefile)
if coreTimestamp:
- dprint(coreTimestamp, DPRINT)
for k,v in rpmDB.iteritems():
- if rpmDB[k].timestamp and coreTimestamp in rpmDB[k].timestamp:
- return (k, rpmDB[k].debugKernel)
+ for coreObj in rpmDB[k]:
+ debugKernel, timestamp = coreObj
+ if timestamp and coreTimestamp in timestamp:
+ return (k, debugKernel)
else:
dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT)
sys.exit(1)
diff --git a/cas.conf b/cas.conf
index 6de30b4..3d995a5 100644
--- a/cas.conf
+++ b/cas.conf
@@ -1,19 +1,29 @@
[settings]
# Where kernels are stored
-# NOTE: this _can_ include symlinked directories, just be careful they are indefinately
-# recursive
+# NOTE: this _can_ include symlinked directories, just be careful they are
+# indefinately recursive
kernels=/mnt/kernels
+
# Database which houses all kernel found in $kernels
rpms=/var/db/cas/rpms.db
+
+# Compose emacs regular expression for determing what
+# kernel debug rpms you wish to search for
+rpmFilter=.*kernel-debuginfo-[0-9].*\.rpm
+
# Define where to store necessary debug information
# NOTE: This is more than likely going to be temporary storage
# so you could essentially set this to /tmp if space permitted
debugs=/cores/debugs
+
# debug print on? (True/False)
dprint=True
+
# define work directory
workDirectory=/cores/processed
+
# database to house cas servers
servers=/var/db/cas/servers.db
+
# define processing jobs db
jobs=/var/db/cas/jobs.db
diff --git a/casprint b/casprint
index 4fcff4e..adf568d 100755
--- a/casprint
+++ b/casprint
@@ -6,7 +6,6 @@ import ConfigParser
import optparse
import sys
-from cas.db import Core
from cas.utilities import Utilities, CoreTool, dprint, sprint
from cas.rpmutils import Analyze, Tools
from subprocess import Popen, PIPE
@@ -20,6 +19,7 @@ config = ConfigParser.ConfigParser()
config.read("/etc/cas.conf")
KERNELS = config.get("settings","kernels")
RPMS = config.get("settings","rpms")
+RPMFILTER = config.get("settings","rpmFilter")
DEBUGS = config.get("settings","debugs")
DPRINT = config.get("settings","dprint")
@@ -27,43 +27,52 @@ class CasDatabaseHandler(object):
def __init__(self):
self.util = Utilities()
if os.path.isfile(RPMS):
- self.debugList = self.util.load(RPMS)
+ self.rpmDB = self.util.load(RPMS)
else:
- self.debugList = {}
+ self.rpmDB = {}
def rpmExist(self, rpm):
""" checks existence of rpm in db
"""
- if self.debugList.has_key(rpm):
+ if self.rpmDB.has_key(rpm):
return True
return False
def run(self):
rpms = []
- cmd = ["find", "-L", KERNELS, "-iname", "kernel-debuginfo*rpm"]
+ # Uses emacs regex -- see `man find`
+ cmd = ["find", "-L", KERNELS, "-iregex", RPMFILTER]
pipe = Popen(cmd, stdout=PIPE, stderr=PIPE)
for line in pipe.stdout:
rpms.append(line.strip())
sprint("(found) %-100s" % (os.path.basename(line.strip()),))
totalRpms = len(rpms)
count = 0
+ """ Build database out in the form of
+ RPM - [( DebugKernel, Timestamp),
+ ( DebugKernel2, Timestamp2)]
+ """
for x in rpms:
if not self.rpmExist(x):
+ self.rpmDB[x] = []
count = count + 1
dst = os.path.join(DEBUGS, str(count))
rpmTool = Tools()
- coreObj = Core()
sprint("(extracting) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x)))
results = rpmTool.extract(x, dst)
+ # Sort through extracted debug for each type
+ # e.g. hugemem, PAE, smp, largesmp
for item in results:
vmlinux = item.strip()
stamper = CoreTool()
- coreObj.debugKernel = os.path.normpath(vmlinux)
- coreObj.timestamp = stamper.timestamp(coreObj.debugKernel)
- self.debugList[x] = coreObj
- sprint("(timestamp) %-100s" % (coreObj.debugKernel,))
- rmtree(dst)
- self.util.save(self.debugList, RPMS)
+ debugKernel = os.path.normpath(vmlinux)
+ timestamp = stamper.timestamp(debugKernel)
+ # Build tuple of each debug type
+ self.rpmDB[x].append((debugKernel, timestamp))
+ sprint("(timestamp) %-100s" % (debugKernel,))
+ self.util.save(self.rpmDB, RPMS)
+ # Cleanup extracted debugs
+ rmtree(dst)
return
class CasprintApplication(object):
@@ -89,10 +98,10 @@ class CasprintApplication(object):
util.mkdir(DEBUGS)
if self.opts.buildDB:
- print("::: Building CAS DB instance. :::")
+ print("::: Starting CAS DB instance. :::")
dbHandler = CasDatabaseHandler().run()
else:
- print("Please define -b switch to update the CAS database.")
+ print("Please define -b switch to initialize/update the CAS database.")
raise sys.exit(1)
if __name__=="__main__":
diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py
index 44c8f9b..7c6d03e 100644
--- a/lib/cas/rpmutils.py
+++ b/lib/cas/rpmutils.py
@@ -1,4 +1,3 @@
-# rpm.py
# rpm utilities
import rpm
@@ -67,7 +66,7 @@ class Tools(object):
stdout=PIPE,stderr=PIPE)
out, err = p2.communicate()
if return_results:
- tmp = err.split("\n")[:1]
+ tmp = err.splitlines()[:-1]
for item in tmp:
self.filter_results.append(item)
return self.filter_results
diff --git a/version b/version
index d4d75d0..2e9ff79 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 65
+0.13 69
15 years, 6 months
cas lib/cas version
by Adam Stokes
cas | 42 +++++++++++++++++++++++-------------------
lib/cas/utilities.py | 8 ++++----
version | 2 +-
3 files changed, 28 insertions(+), 24 deletions(-)
New commits:
commit bf425bd75095b2d12ea91f16990d2f74a2997437
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Thu Oct 23 11:41:30 2008 -0400
rpmutils is failing to capture all extracted kernels from stdout, needs fixing
diff --git a/cas b/cas
index 88a9074..007878c 100755
--- a/cas
+++ b/cas
@@ -8,6 +8,7 @@ import ConfigParser
import shutil
from cas.utilities import Utilities, CoreTool, dprint, sprint
+from cas.rpmutils import Tools
if sys.version_info[:2] < (2,4):
raise SystemExit("Python >= 2.4 required")
@@ -36,25 +37,29 @@ class CoreHandler(object):
corepath = self.tool.extractCore(self.filename, self.dst)
if corepath:
return os.path.realpath(corepath)
- return False
+ else:
+ dprint("Unable to determine corefile", DPRINT)
+ raise sys.exit(1)
class TimestampHandler(object):
- def __init__(self, corefile, db):
+ def __init__(self, corefile):
self.corefile = corefile
- self.db = db
self.util = Utilities()
self.tool = CoreTool()
def run(self):
- rpmDB = self.util.load(self.db)
+ rpmDB = self.util.load(RPMS)
coreTimestamp = self.tool.timestamp(self.corefile)
if coreTimestamp:
dprint(coreTimestamp, DPRINT)
- for k,v in rpmDB:
- if coreTimestamp and coreTimestamp in rpmDB[k].coreObj.timestamp:
- return rpmDB[k]
+ for k,v in rpmDB.iteritems():
+ if rpmDB[k].timestamp and coreTimestamp in rpmDB[k].timestamp:
+ return (k, rpmDB[k].debugKernel)
+ else:
+ dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT)
+ sys.exit(1)
else:
- dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT)
+ dprint("Cannot process a timestamp from: %s" % (self.corefile,), DPRINT)
sys.exit(1)
return False
@@ -62,6 +67,7 @@ class CasApplication(object):
def __init__(self, args):
self.parse_options(args)
self.tool = Utilities()
+ self.rpmTool = Tools()
def parse_options(self, args):
parser = optparse.OptionParser(usage="cas -i ID "
@@ -86,23 +92,21 @@ class CasApplication(object):
if not os.path.isdir(self.storagePath):
os.makedirs(self.storagePath)
corefile = CoreHandler(self.filename, self.storagePath).run()
- if not corefile:
- dprint("Unable to determine corefile : %s" % (corefile,), DPRINT)
- sys.exit(1)
- coreObj = TimestampHandler(corefile, RPMS).run()
- try:
- # extract debug kernel to workDirectory
- except:
- dprint("Unable to perform symlink %s" % (os.path.basename(debug)), DPRINT)
- sys.exit(1)
+ debuginfo, debugKernel = TimestampHandler(corefile).run()
+ filterString = "*/%s" % (debugKernel,)
+ self.rpmTool.extract(debuginfo, self.storagePath,
+ filter=filterString,
+ return_results=False)
- crash_cmd = "/bin/sh\ncrash %s %s" % (symlink_dst, corefile)
+ crash_cmd = "#!/bin/sh\ncrash %s %s\n" % (os.path.join(self.storagePath,
+ debugKernel),
+ corefile)
crash_exe = os.path.join(self.storagePath,"crash")
fh = open(crash_exe,"w")
fh.write(crash_cmd)
fh.close()
self.tool.make_exe(crash_exe)
- dprint("Job on %s complete and located in %s." % (self.filename,self.storagePath), DPRINT)
+ print("Job on %s complete and located in %s." % (self.filename,self.storagePath))
return
if __name__=="__main__":
diff --git a/lib/cas/utilities.py b/lib/cas/utilities.py
old mode 100644
new mode 100755
index 9b40d59..a9bea61
--- a/lib/cas/utilities.py
+++ b/lib/cas/utilities.py
@@ -20,7 +20,7 @@ logging.basicConfig(level=logging.DEBUG,
def dprint(msg, debug=True):
if debug:
sys.stderr.write("(debug) %s" % (msg,))
- logging.error(msg)
+ logging.debug(msg)
def sprint(msg):
""" function to print status messages
@@ -182,7 +182,7 @@ class CoreTool(object):
self.dst = dst
self.filepath = filepath
if not tarfile.is_tarfile(self.filepath):
- raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,))
+ raise CoreException("%s : is not a properly compressed tarfile." % (self.filepath,))
else:
tar = tarfile.open(self.filepath, "r")
tar.extractall(self.dst)
@@ -207,10 +207,10 @@ class CoreTool(object):
def timestamp(self, path):
""" captures fingerprint from core
"""
- match='Linux\sversion.*\d{1,4}|#1\s.*\d{1,2}'
+ match='Linux\sversion.*20\d{1,2}|#1\s.*20\d{1,2}'
try:
fd=open('%s' % (path))
- except IOError:
+ except IOError:
return False
fd.seek(0)
b = os.read(fd.fileno(),54000000)
diff --git a/version b/version
index 1312e50..d4d75d0 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 64
+0.13 65
15 years, 6 months
casprint lib/cas version
by Adam Stokes
casprint | 2 +-
lib/cas/rpmutils.py | 2 +-
version | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
New commits:
commit d554231bd2a365d4f29eb2a7ad9fa132f49475af
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Wed Oct 22 22:55:31 2008 -0400
casprint stores proper path to vmlinux within cpio archive
incremental saves to rpmdb to help with speed when updating database.
diff --git a/casprint b/casprint
index bcfb0aa..4fcff4e 100755
--- a/casprint
+++ b/casprint
@@ -63,7 +63,7 @@ class CasDatabaseHandler(object):
self.debugList[x] = coreObj
sprint("(timestamp) %-100s" % (coreObj.debugKernel,))
rmtree(dst)
- self.util.save(self.debugList, RPMS)
+ self.util.save(self.debugList, RPMS)
return
class CasprintApplication(object):
diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py
index abd7306..44c8f9b 100644
--- a/lib/cas/rpmutils.py
+++ b/lib/cas/rpmutils.py
@@ -69,7 +69,7 @@ class Tools(object):
if return_results:
tmp = err.split("\n")[:1]
for item in tmp:
- self.filter_results.append(os.path.join(self.dst,os.path.normpath(item)))
+ self.filter_results.append(item)
return self.filter_results
return
diff --git a/version b/version
index 5362fd9..1312e50 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 63
+0.13 64
15 years, 6 months
cas
by Adam Stokes
cas | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
New commits:
commit b2c15170ee3602ce580e685575dff87c6416ec8f
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Wed Oct 22 22:54:13 2008 -0400
*** empty log message ***
diff --git a/cas b/cas
index 2172479..88a9074 100755
--- a/cas
+++ b/cas
@@ -46,14 +46,13 @@ class TimestampHandler(object):
self.tool = CoreTool()
def run(self):
- debugList = self.util.load(self.db)
+ rpmDB = self.util.load(self.db)
coreTimestamp = self.tool.timestamp(self.corefile)
if coreTimestamp:
dprint(coreTimestamp, DPRINT)
- for item in debugList:
- debug, timestamp = item
- if coreTimestamp and coreTimestamp in timestamp:
- return debug
+ for k,v in rpmDB:
+ if coreTimestamp and coreTimestamp in rpmDB[k].coreObj.timestamp:
+ return rpmDB[k]
else:
dprint("Unable to match fingerprint : %s" % (coreTimestamp,), DPRINT)
sys.exit(1)
@@ -90,11 +89,9 @@ class CasApplication(object):
if not corefile:
dprint("Unable to determine corefile : %s" % (corefile,), DPRINT)
sys.exit(1)
- debug = TimestampHandler(corefile, RPMS).run()
+ coreObj = TimestampHandler(corefile, RPMS).run()
try:
- symlink_dst = os.path.join(self.storagePath,
- os.path.basename(debug))
- os.symlink(debug,symlink_dst)
+ # extract debug kernel to workDirectory
except:
dprint("Unable to perform symlink %s" % (os.path.basename(debug)), DPRINT)
sys.exit(1)
15 years, 6 months
cas cas.conf casprint lib/cas version
by Adam Stokes
cas | 4 ++--
cas.conf | 7 ++++---
casprint | 16 ++++++++++------
lib/cas/rpmutils.py | 12 +++++++-----
version | 2 +-
5 files changed, 24 insertions(+), 17 deletions(-)
New commits:
commit 0f8ae2fc50a228fbc562c0dba38a394699184314
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Wed Oct 22 12:19:23 2008 -0400
Reworked rpmutils library to return debug kernel results if asked Altered casprint to cleanup after itself, we are going to continue storing all related information in the debuginfo packages for now
diff --git a/cas b/cas
index 099ea6e..2172479 100755
--- a/cas
+++ b/cas
@@ -27,11 +27,11 @@ class CoreHandler(object):
def run(self):
if not os.path.isfile(self.filename):
- dprint("Unable to find file specified by -f FILENAME.", DPRINT)
+ print("Unable to find file %s" % (self.filename,))
sys.exit(1)
if self.tool.isCorefile(self.filename):
shutil.move(self.filename,
- os.path.join(self.dst,self.filename))
+ os.path.join(self.dst, self.filename))
return os.path.realpath(self.filename)
corepath = self.tool.extractCore(self.filename, self.dst)
if corepath:
diff --git a/cas.conf b/cas.conf
index 2e558a2..6de30b4 100644
--- a/cas.conf
+++ b/cas.conf
@@ -5,14 +5,15 @@
kernels=/mnt/kernels
# Database which houses all kernel found in $kernels
rpms=/var/db/cas/rpms.db
-# Database to house all kernel fingperint
# Define where to store necessary debug information
+# NOTE: This is more than likely going to be temporary storage
+# so you could essentially set this to /tmp if space permitted
debugs=/cores/debugs
# debug print on? (True/False)
dprint=True
# define work directory
workDirectory=/cores/processed
# database to house cas servers
-servers=/var/db/cas/casservers.db
+servers=/var/db/cas/servers.db
# define processing jobs db
-jobs=/var/db/cas/casjobs.db
+jobs=/var/db/cas/jobs.db
diff --git a/casprint b/casprint
index eca970b..bcfb0aa 100755
--- a/casprint
+++ b/casprint
@@ -10,6 +10,7 @@ from cas.db import Core
from cas.utilities import Utilities, CoreTool, dprint, sprint
from cas.rpmutils import Analyze, Tools
from subprocess import Popen, PIPE
+from shutil import rmtree
if sys.version_info[:2] < (2,4):
raise SystemExit("Python >= 2.4 required")
@@ -51,14 +52,17 @@ class CasDatabaseHandler(object):
count = count + 1
dst = os.path.join(DEBUGS, str(count))
rpmTool = Tools()
+ coreObj = Core()
+ sprint("(extracting) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x)))
results = rpmTool.extract(x, dst)
- sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x)))
for item in results:
vmlinux = item.strip()
stamper = CoreTool()
- timestamp = stamper.timestamp(vmlinux)
- self.debugList[x] = (vmlinux, timestamp)
- sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),))
+ coreObj.debugKernel = os.path.normpath(vmlinux)
+ coreObj.timestamp = stamper.timestamp(coreObj.debugKernel)
+ self.debugList[x] = coreObj
+ sprint("(timestamp) %-100s" % (coreObj.debugKernel,))
+ rmtree(dst)
self.util.save(self.debugList, RPMS)
return
@@ -85,10 +89,10 @@ class CasprintApplication(object):
util.mkdir(DEBUGS)
if self.opts.buildDB:
- dprint("Building CAS DB instance.", DPRINT)
+ print("::: Building CAS DB instance. :::")
dbHandler = CasDatabaseHandler().run()
else:
- dprint("Please define -b switch to update the CAS database\n", DPRINT)
+ print("Please define -b switch to update the CAS database.")
raise sys.exit(1)
if __name__=="__main__":
diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py
index 5cecce4..abd7306 100644
--- a/lib/cas/rpmutils.py
+++ b/lib/cas/rpmutils.py
@@ -53,7 +53,7 @@ class Tools(object):
self.filter = filter
self.rpm2cpio = tool
self.cpio = "/bin/cpio"
- self.cpio_args = "-imud"
+ self.cpio_args = "-imudv"
self.filter_results = []
# change to destination directory
@@ -63,11 +63,13 @@ class Tools(object):
# pipe to handle extraction, e.g. rpm2cpio kernel.rpm | cpio -imud */vmlinux
p1 = Popen([self.rpm2cpio, self.rpm], stdout=PIPE)
- (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout,
- stdout=open(os.devnull, "w"),stderr=PIPE).communicate()
+ p2 = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout,
+ stdout=PIPE,stderr=PIPE)
+ out, err = p2.communicate()
if return_results:
- for item in out.stderr.readlines():
- self.filter_results.append(os.path.join(self.dst,item))
+ tmp = err.split("\n")[:1]
+ for item in tmp:
+ self.filter_results.append(os.path.join(self.dst,os.path.normpath(item)))
return self.filter_results
return
diff --git a/version b/version
index 6b6bf85..5362fd9 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.13 61
+0.13 63
15 years, 6 months
2 commits - casprint lib/cas
by Adam Stokes
casprint | 68 ++++++++++++++++++++++------------------------------
lib/cas/rpmutils.py | 16 ++++++++----
2 files changed, 41 insertions(+), 43 deletions(-)
New commits:
commit ba98f0cfab10a3291f70298ae02d889059e7fa57
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Tue Oct 21 21:35:34 2008 -0400
re-worked casprint
diff --git a/lib/cas/rpmutils.py b/lib/cas/rpmutils.py
index 23b024b..5cecce4 100644
--- a/lib/cas/rpmutils.py
+++ b/lib/cas/rpmutils.py
@@ -37,14 +37,15 @@ class Analyze(object):
return ".%s" % (i,)
else:
return hdr["FILENAMES"]
-
+
class Tools(object):
""" provide file alteration tools
"""
def __init__(self):
pass
- def extract(self, rpm, dst, tool="/usr/bin/rpm2cpio", filter="*/vmlinux"):
+ def extract(self, rpm, dst, tool="/usr/bin/rpm2cpio",
+ filter="*/vmlinux", return_results=True):
""" extract file(s) from rpm
"""
self.rpm = rpm
@@ -53,15 +54,20 @@ class Tools(object):
self.rpm2cpio = tool
self.cpio = "/bin/cpio"
self.cpio_args = "-imud"
+ self.filter_results = []
# change to destination directory
if not os.path.isdir(self.dst):
os.makedirs(self.dst)
os.chdir(self.dst)
-
+
# pipe to handle extraction, e.g. rpm2cpio kernel.rpm | cpio -imud */vmlinux
p1 = Popen([self.rpm2cpio, self.rpm], stdout=PIPE)
- (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout,
- stdout=open(os.devnull, "w"),stderr=open(os.devnull,"w")).communicate()
+ (out, err) = Popen([self.cpio,self.cpio_args,self.filter], stdin=p1.stdout,
+ stdout=open(os.devnull, "w"),stderr=PIPE).communicate()
+ if return_results:
+ for item in out.stderr.readlines():
+ self.filter_results.append(os.path.join(self.dst,item))
+ return self.filter_results
return
commit a605acbf91a5c0b2fb123d675605dd6b4a5cab3b
Author: Adam Stokes <adam(a)conans.battleaxe>
Date: Tue Oct 21 21:34:48 2008 -0400
reworking casprint to not store debugs locally and use the debuginfo
rpm packages as main storage.
diff --git a/casprint b/casprint
index e8656da..eca970b 100755
--- a/casprint
+++ b/casprint
@@ -22,31 +22,20 @@ RPMS = config.get("settings","rpms")
DEBUGS = config.get("settings","debugs")
DPRINT = config.get("settings","dprint")
-class CasTimeStampHandler(object):
- def __init__(self):
- self.cmd = ["find", "-L", DEBUGS, "-iname", "vmlinux"]
- self.tool = CoreTool()
- self.util = Utilities()
- self.debugList = []
-
- def run(self):
- pipe = Popen(self.cmd, stdout=PIPE, stderr=PIPE)
- for line in pipe.stdout:
- vmlinux = line.strip()
- tstamp = CoreTool()
- timestamp = tstamp.timestamp(vmlinux)
- self.debugList.append((vmlinux,timestamp))
- sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),))
- self.util.save(self.debugList, RPMS)
- return
-
class CasDatabaseHandler(object):
def __init__(self):
self.util = Utilities()
if os.path.isfile(RPMS):
self.debugList = self.util.load(RPMS)
else:
- self.debugList = []
+ self.debugList = {}
+
+ def rpmExist(self, rpm):
+ """ checks existence of rpm in db
+ """
+ if self.debugList.has_key(rpm):
+ return True
+ return False
def run(self):
rpms = []
@@ -58,11 +47,19 @@ class CasDatabaseHandler(object):
totalRpms = len(rpms)
count = 0
for x in rpms:
- count = count + 1
- dst = os.path.join(DEBUGS, str(count))
- rpmTool = Tools()
- rpmTool.extract(x, dst)
- sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x)))
+ if not self.rpmExist(x):
+ count = count + 1
+ dst = os.path.join(DEBUGS, str(count))
+ rpmTool = Tools()
+ results = rpmTool.extract(x, dst)
+ sprint("(extracted) [%d/%d] %-100s" % (count, totalRpms, os.path.basename(x)))
+ for item in results:
+ vmlinux = item.strip()
+ stamper = CoreTool()
+ timestamp = stamper.timestamp(vmlinux)
+ self.debugList[x] = (vmlinux, timestamp)
+ sprint("(timestamp) %-100s" % (os.path.realpath(vmlinux),))
+ self.util.save(self.debugList, RPMS)
return
class CasprintApplication(object):
@@ -70,11 +67,9 @@ class CasprintApplication(object):
self.parse_options(args)
def parse_options(self, args):
- parser = optparse.OptionParser(usage="casprint -n -t")
- parser.add_option("-n","--new", dest="newCasDB",
- help="Build new CAS database", action="store_true")
- parser.add_option("-t","--timestamp", dest="timestamp",
- help="Timestamp existing debug kernels", action="store_true")
+ parser = optparse.OptionParser(usage="casprint -b")
+ parser.add_option("-b","--build", dest="buildDB",
+ help="Build CAS DB", action="store_true")
self.opts, args = parser.parse_args()
def run(self):
@@ -84,20 +79,17 @@ class CasprintApplication(object):
util = Utilities()
if os.getuid() is not 0:
raise RuntimeError, "You must be root(0), instead you are id(%d)" % (os.getuid())
- if not os.path.isdir(os.path.dirname(RPMS)):
+ if not os.path.isdir(os.path.dirname(RPMS)):
util.mkdir(RPMS)
- if not os.path.isdir(DEBUGS):
+ if not os.path.isdir(DEBUGS):
util.mkdir(DEBUGS)
- if self.opts.newCasDB:
- dprint("Running a new cas database instance.", DPRINT)
+ if self.opts.buildDB:
+ dprint("Building CAS DB instance.", DPRINT)
dbHandler = CasDatabaseHandler().run()
- elif self.opts.timestamp:
- dprint("Building an updated cache of timestamps on existing debugs.", DPRINT)
- timestampHandler = CasTimeStampHandler().run()
else:
- raise RuntimeError("You must define either -n for a new CAS database or \n"
- "-t to update existing debugs")
+ dprint("Please define -b switch to update the CAS database\n", DPRINT)
+ raise sys.exit(1)
if __name__=="__main__":
app = CasprintApplication(sys.argv[1:])
15 years, 6 months