r4445 - branches/scale_testing/mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-23 20:14:11 +0000 (Thu, 23 Dec 2010)
New Revision: 4445
Modified:
branches/scale_testing/mint/python/mint/update.py
Log:
Fix up null object stats cases. Apparently Python 2.4 deals with None + int differently than 2.6 (or running on grid0 directly causes a different execution path)
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2010-12-22 13:32:50 UTC (rev 4444)
+++ branches/scale_testing/mint/python/mint/update.py 2010-12-23 20:14:11 UTC (rev 4445)
@@ -116,6 +116,13 @@
self.objects_deleted_by_class = defaultdict(int)
self.objects_dropped_by_class = defaultdict(int)
+ for pkg in app.model._packages:
+ for cls in pkg._classes:
+ self.objects_created_by_class[cls] = 0
+ self.objects_updated_by_class[cls] = 0
+ self.objects_deleted_by_class[cls] = 0
+ self.objects_dropped_by_class[cls] = 0
+
self.sql_ops = 0
self.errors = 0
@@ -351,6 +358,8 @@
except:
cls = None
if cls != None:
+ if thread.stats.objects_dropped_by_class == None:
+ thread.stats.objects_dropped_by_class = 0
thread.stats.objects_dropped_by_class[cls] += 1
except:
@@ -510,6 +519,8 @@
self.model.print_event(3, "Created %s", obj)
stats.objects_created += 1
+ if stats.objects_created_by_class[cls] == None:
+ stats.objects_created_by_class[cls] = 0
stats.objects_created_by_class[cls] += 1
return obj
@@ -571,6 +582,8 @@
self.model.print_event(4, "Updated %s", obj)
stats.objects_updated += 1
+ if stats.objects_updated_by_class[cls] == None:
+ stats.objects_updated_by_class[cls] = 0
stats.objects_updated_by_class[cls] += 1
def delete_object(self, cursor, stats, obj):
@@ -579,6 +592,8 @@
self.model.print_event(3, "Deleted %s", obj)
stats.objects_deleted += 1
+ if stats.objects_deleted_by_class[cls] == None:
+ stats.objects_deleted_by_class[cls] = 0
stats.objects_deleted_by_class[obj._class] += 1
def process_properties(self, obj, columns, cursor):
13 years, 4 months
r4444 - branches/scale_testing/mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-22 13:32:50 +0000 (Wed, 22 Dec 2010)
New Revision: 4444
Modified:
branches/scale_testing/mint/python/mint/update.py
Log:
Tweak stats collecting, get rid of exceptions and bad values
on zero message intervals or dropped updates
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2010-12-21 20:15:51 UTC (rev 4443)
+++ branches/scale_testing/mint/python/mint/update.py 2010-12-22 13:32:50 UTC (rev 4444)
@@ -36,6 +36,10 @@
def enqueue(self, update):
update.enqueue_time = time.time()
+ # make sure these exist to avoid exceptions if processing
+ # is truncated for some reason
+ update.start_process = update.finish_process = \
+ update.after_process = update.after_commit = update.enqueue_time
self.updates.put(update)
@@ -149,6 +153,7 @@
update.after_commit - update.after_process
except:
print "record update exception"
+ print_exc()
def capture(self):
now = copy.copy(self)
@@ -261,7 +266,7 @@
def get_update_durations(self):
total = self.now.record_samples
- if total == 0:
+ if total == 0 or self.now.dequeued == self.then.dequeued:
create_dur = proc_dur = que_dur = init_proc_time = commit_time = 0
else:
create_dur = self.now.update_queue_duration / total
@@ -322,6 +327,7 @@
def process(self, thread):
self.start_process = time.time()
+
log.debug("Processing %s", self)
try:
@@ -330,10 +336,15 @@
thread.conn.commit()
self.after_commit = time.time()
except UpdateDropped:
+ # Don't want to throw off stats with negatives if
+ # these are never set. Assume the time to reach
+ # this point.
+ self.after_process = time.time()
+ self.after_commit = self.after_process
+
log.debug("Update dropped")
thread.conn.rollback()
-
thread.stats.dropped += 1
try:
cls = self.get_class()
@@ -341,7 +352,6 @@
cls = None
if cls != None:
thread.stats.objects_dropped_by_class[cls] += 1
- self.after_commit = time.time()
except:
log.exception("Update failed")
13 years, 4 months
r4443 - in branches/scale_testing: cumin/bin cumin/python/cumin mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-21 20:15:51 +0000 (Tue, 21 Dec 2010)
New Revision: 4443
Modified:
branches/scale_testing/cumin/bin/cumin-data
branches/scale_testing/cumin/python/cumin/config.py
branches/scale_testing/mint/python/mint/main.py
branches/scale_testing/mint/python/mint/session.py
branches/scale_testing/mint/python/mint/update.py
Log:
Changes to collect stats in CSV files for scale testing.
Modified: branches/scale_testing/cumin/bin/cumin-data
===================================================================
--- branches/scale_testing/cumin/bin/cumin-data 2010-12-20 14:12:38 UTC (rev 4442)
+++ branches/scale_testing/cumin/bin/cumin-data 2010-12-21 20:15:51 UTC (rev 4443)
@@ -16,6 +16,8 @@
parser = CuminOptionParser(values)
parser.add_option("--print-stats", action="store_true")
+ parser.add_option("--scale-stats", action="store_true")
+ parser.add_option("--object-stats", action="store_true")
parser.add_option("--print-events", type="int", default=0, metavar="LEVEL")
opts, args = parser.parse_args()
@@ -26,7 +28,7 @@
broker_uris = [x.strip() for x in opts.brokers.split(",")]
- mint = Mint(model_dir, broker_uris, opts.database)
+ mint = Mint(model_dir, broker_uris, opts.database, values.queue_size)
mint.print_event_level = opts.print_events
@@ -65,19 +67,40 @@
if opts.print_stats:
print "[Starred columns are the number of events per second]"
+ # set up for scale stats here
+ if opts.scale_stats:
+ stats.init_stats_path(mint, config.home)
+
+ if opts.print_stats or \
+ opts.scale_stats or \
+ opts.object_stats:
while True:
- if count % 20 == 0:
- stats.print_headings()
+ stats.capture()
+ if opts.print_stats:
+ if count % 20 == 0:
+ stats.print_headings()
+ count += 1
+ stats.print_values()
+
+ if opts.object_stats:
+ stats.print_values_by_class(mint)
- count += 1
+ if opts.scale_stats:
+ stats.write_values_by_class(mint, time.localtime())
+ stats.write_values(time.localtime())
- stats.print_values()
-
sleep(5)
else:
while True:
sleep(86400)
+ except Exception as inst:
+ print inst
+
finally:
+ # close the scale stats CSV file here:
+ if opts.scale_stats:
+ pass
+
mint.stop()
if __name__ == "__main__":
Modified: branches/scale_testing/cumin/python/cumin/config.py
===================================================================
--- branches/scale_testing/cumin/python/cumin/config.py 2010-12-20 14:12:38 UTC (rev 4442)
+++ branches/scale_testing/cumin/python/cumin/config.py 2010-12-21 20:15:51 UTC (rev 4443)
@@ -56,6 +56,9 @@
param = ConfigParameter(data, "vacuum-interval", int)
param.default = 60 * 60 # 1 hour
+ param = ConfigParameter(data, "queue-size", int)
+ param.default = 1000
+
def parse(self):
paths = list()
Modified: branches/scale_testing/mint/python/mint/main.py
===================================================================
--- branches/scale_testing/mint/python/mint/main.py 2010-12-20 14:12:38 UTC (rev 4442)
+++ branches/scale_testing/mint/python/mint/main.py 2010-12-21 20:15:51 UTC (rev 4443)
@@ -10,14 +10,14 @@
log = logging.getLogger("mint.main")
class Mint(object):
- def __init__(self, model_dir, broker_uris, database_dsn):
+ def __init__(self, model_dir, broker_uris, database_dsn, queue_max=1000):
self.model = MintModel(self, model_dir)
self.model.sql_logging_enabled = False
self.session = MintSession(self, broker_uris)
self.database = MintDatabase(self, database_dsn)
- self.update_thread = UpdateThread(self)
+ self.update_thread = UpdateThread(self,queue_max)
self.expire_enabled = True
self.expire_thread = ExpireThread(self)
Modified: branches/scale_testing/mint/python/mint/session.py
===================================================================
--- branches/scale_testing/mint/python/mint/session.py 2010-12-20 14:12:38 UTC (rev 4442)
+++ branches/scale_testing/mint/python/mint/session.py 2010-12-21 20:15:51 UTC (rev 4443)
@@ -64,14 +64,17 @@
def brokerConnected(self, qmf_broker):
message = "Broker %s:%i is connected"
self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+ self.model.app.update_thread.ignore()
def brokerInfo(self, qmf_broker):
message = "Broker info from %s:%i"
self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+ self.model.app.update_thread.ignore()
def brokerDisconnected(self, qmf_broker):
message = "Broker %s:%i is disconnected"
self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+ self.model.app.update_thread.ignore()
def newAgent(self, qmf_agent):
self.model.print_event(3, "Creating %s", qmf_agent)
@@ -94,9 +97,11 @@
def newPackage(self, name):
self.model.print_event(2, "New package %s", name)
+ self.model.app.update_thread.ignore()
def newClass(self, kind, classKey):
self.model.print_event(2, "New class %s", classKey)
+ self.model.app.update_thread.ignore()
def objectProps(self, broker, qmf_object):
up = ObjectUpdate(self.model, qmf_object)
@@ -108,6 +113,7 @@
def event(self, broker, event):
self.model.print_event(4, "New event %s from %s", broker, event)
+ self.model.app.update_thread.ignore()
def methodResponse(self, broker, seq, response):
message = "Method response for request %i received from %s"
@@ -121,3 +127,4 @@
callback(response.text, response.outArgs)
finally:
self.model.lock.release()
+ self.model.app.update_thread.method_response()
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2010-12-20 14:12:38 UTC (rev 4442)
+++ branches/scale_testing/mint/python/mint/update.py 2010-12-21 20:15:51 UTC (rev 4443)
@@ -1,6 +1,8 @@
import copy
import resource
import pickle
+import os.path
+import csv
from psycopg2 import IntegrityError, TimestampFromTicks
from psycopg2.extensions import cursor as Cursor
@@ -15,10 +17,10 @@
sample_window_max = 60 * 5
class UpdateThread(MintDaemonThread):
- def __init__(self, app):
+ def __init__(self, app, queue_max=1000):
super(UpdateThread, self).__init__(app)
- self.updates = ConcurrentQueue(maxsize=1000)
+ self.updates = ConcurrentQueue(maxsize=queue_max)
self.stats = UpdateStats(self.app)
self.conn = None
@@ -33,24 +35,37 @@
self.cursor.stats = self.stats
def enqueue(self, update):
+ update.enqueue_time = time.time()
+
self.updates.put(update)
-
+
self.stats.enqueued += 1
+ def ignore(self):
+ self.stats.ignore += 1
+
+ def method_response(self):
+ self.stats.method_response +=1
+
def run(self):
- while True:
- if self.stop_requested:
- break
+ try:
+ while True:
+ if self.stop_requested:
+ break
- try:
- update = self.updates.get(True, 1)
- except Empty:
- continue
+ try:
+ update = self.updates.get(True) #, 1)
+ except Empty:
+ continue
+ update.dequeue_time = time.time()
+ self.stats.dequeued += 1
- self.stats.dequeued += 1
+ update.process(self)
+ self.conn.commit()
+
+ except:
+ print "uh oh"
- update.process(self)
-
class UpdateStats(object):
group_names = ("Updates", "Agents", "Objects")
groups = "%43s | %32s | %32s |" % group_names
@@ -60,6 +75,7 @@
"*Created", "*Updated", "*Deleted",
"*Created", "*Updated", "*Deleted",
"*Sql Ops", "Errors", "Cpu (%)", "Mem (M)")
+
headings_fmt = \
"%10s %10s %10s %10s | " + \
"%10s %10s %10s | " + \
@@ -67,7 +83,6 @@
"%10s %10s %10s %10s"
headings = headings_fmt % heading_names
-
values_fmt = \
"%10i %10.1f %10.1f %10.1f | " + \
"%10.1f %10.1f %10.1f | " + \
@@ -81,6 +96,8 @@
self.enqueued = 0
self.dequeued = 0
self.dropped = 0
+ self.ignore = 0
+ self.method_response = 0
self.agents_created = 0
self.agents_updated = 0
@@ -93,6 +110,7 @@
self.objects_created_by_class = defaultdict(int)
self.objects_updated_by_class = defaultdict(int)
self.objects_deleted_by_class = defaultdict(int)
+ self.objects_dropped_by_class = defaultdict(int)
self.sql_ops = 0
self.errors = 0
@@ -101,11 +119,41 @@
self.cpu = 0
self.memory = 0
+ self.init_update_times()
+
+ def init_update_times(self):
+ # Track average time of update from arrival to
+ # processing complete. Refreshed per interval.
+ self.update_queue_duration = 0
+ self.update_process_duration = 0
+ self.queue_time = 0
+ self.initial_process_time = 0
+ self.commit_time = 0
+ self.record_samples = 0
+ self.init_update_flag = False
+
+ def record_update_times(self,update):
+ try:
+ if self.init_update_flag:
+ self.init_update_times()
+ self.record_samples += 1
+ self.update_queue_duration += \
+ update.start_process - update.create_time
+ self.update_process_duration += \
+ update.finish_process - update.start_process
+ self.queue_time += \
+ update.dequeue_time - update.enqueue_time
+ self.initial_process_time += \
+ update.after_process - update.start_process
+ self.commit_time += \
+ update.after_commit - update.after_process
+ except:
+ print "record update exception"
+
def capture(self):
now = copy.copy(self)
now.time = time.time()
-
rusage = resource.getrusage(resource.RUSAGE_SELF)
now.cpu = rusage[0] + rusage[1]
@@ -113,6 +161,7 @@
UpdateStats.then = UpdateStats.now
UpdateStats.now = now
+ self.init_update_flag = True
def get_resident_pages(self):
try:
@@ -126,12 +175,7 @@
print self.groups
print self.headings
- def print_values(self):
- self.capture()
-
- if not self.then:
- return
-
+ def _gather_values(self):
values = [self.now.enqueued - self.then.enqueued,
self.now.dequeued - self.then.dequeued,
self.now.dropped - self.then.dropped,
@@ -153,23 +197,118 @@
values.append(self.errors)
values.append(int((self.now.cpu - self.then.cpu) / secs * 100))
values.append(self.now.memory / 1000000.0)
+ return values
- print self.values_fmt % tuple(values)
+ def print_values(self):
+# moved capture call to cumin-data so that write_values can
+# be run as well
+# self.capture()
+ if not self.then:
+ return
- def print_values_by_class(self):
- names = ("Class", "Created", "Updated", "Deleted")
- print "%20s %10s %10s %10s" % names
+ print self.values_fmt % tuple(self._gather_values())
+ def print_values_by_class(self,mint):
+ names = ("Class", "Created", "Updated", "Deleted", "Dropped")
+ print "%20s %10s %10s %10s %10s" % names
+
for pkg in mint.model._packages:
for cls in pkg._classes:
- created = stats.created_by_class[cls]
- updated = stats.updated_by_class[cls]
- deleted = stats.deleted_by_class[cls]
+ created = self.objects_created_by_class[cls]
+ updated = self.objects_updated_by_class[cls]
+ deleted = self.objects_deleted_by_class[cls]
+ dropped = self.objects_dropped_by_class[cls]
+ if created or updated or deleted or dropped:
+ args = (cls._name, created, updated, deleted, dropped)
+ print "%-20s %10i %10i %10i %10i" % args
+
+ def init_stats_path(self, mint, path):
+ p = os.path.join(path, \
+ "stats_"+ time.strftime("%m%d%y_%H%M%S",time.localtime()))
+
+ if not os.path.exists(p):
+ os.makedirs(p)
+
+ # init the path for general stats
+ self.stats_path = os.path.join(p, "stats.csv")
+
+ # init the paths for stats by class
+ self.class_stats_paths = {}
+ for pkg in mint.model._packages:
+ for cls in pkg._classes:
+ self.class_stats_paths[cls._name] = \
+ os.path.join(p, cls._name+".csv")
+
+ def write_values_by_class(self, mint, localtime_):
+ names = ("Time", "Created", "Updated", "Deleted", "Dropped")
+
+ for pkg in mint.model._packages:
+ for cls in pkg._classes:
+ created = self.objects_created_by_class[cls]
+ updated = self.objects_updated_by_class[cls]
+ deleted = self.objects_deleted_by_class[cls]
+ dropped = self.objects_dropped_by_class[cls]
+
if created or updated or deleted:
- args = (cls._name, created, updated, deleted)
- print "%-20s %10i %10i %10i" % args
-
+ fname = self.class_stats_paths[cls._name]
+ write_header = not os.path.isfile(fname)
+ db = csv.writer(open(fname, "a+"))
+ if write_header:
+ db.writerow(names)
+ db.writerow((time.strftime("%H:%M:%S",localtime_),
+ created, updated, deleted, dropped))
+
+ def get_update_durations(self):
+ total = self.now.record_samples
+ if total == 0:
+ create_dur = proc_dur = que_dur = init_proc_time = commit_time = 0
+ else:
+ create_dur = self.now.update_queue_duration / total
+ proc_dur = self.now.update_process_duration / total
+ que_dur = self.now.queue_time / total
+ init_proc_time = self.now.initial_process_time / total
+ commit_time = self.now.commit_time / total
+ return (create_dur, proc_dur, que_dur, init_proc_time, commit_time)
+
+
+ def write_values(self, localtime_):
+ stats_headings = \
+ ("Time", "Total Msgs",
+ "Depth", "*Avg Que", "*Avg Create", "*Avg Proc",
+ "*Init Proc", "*Commit",
+ "*Enqueued", "*Dequeued", "*Dropped",
+ "*Create Agent", "*Update Agent", "*Delete Agent",
+ "*Create", "*Update", "*Delete",
+ "*Sql Ops", "Errors", "Cpu (%)", "Mem (M)")
+
+ if not self.then:
+ return
+ try:
+ values = self._gather_values()
+ values.insert(0, time.strftime("%H:%M:%S",localtime_))
+ values.insert(1, self.now.enqueued)
+
+ # include object update times
+ avg_create_dur,avg_process_dur,avg_que_dur, \
+ avg_init_proc,avg_commit = self.get_update_durations()
+
+ values.insert(3, avg_que_dur)
+ values.insert(4, avg_create_dur)
+ values.insert(5, avg_process_dur)
+ values.insert(6, avg_init_proc)
+ values.insert(7, avg_commit)
+
+ write_header = not os.path.isfile(self.stats_path)
+ db = csv.writer(open(self.stats_path, "a+"))
+ if write_header:
+ db.writerow(stats_headings)
+ db.writerow(values)
+
+ except Exception as inst:
+ print "exception in write values"
+ print inst
+
class UpdateCursor(Cursor):
def execute(self, sql, args=None):
super(UpdateCursor, self).execute(sql, args)
@@ -179,35 +318,50 @@
class Update(object):
def __init__(self, model):
self.model = model
+ self.create_time = time.time()
def process(self, thread):
+ self.start_process = time.time()
log.debug("Processing %s", self)
try:
self.do_process(thread.cursor, thread.stats)
-
+ self.after_process = time.time()
thread.conn.commit()
+ self.after_commit = time.time()
except UpdateDropped:
log.debug("Update dropped")
thread.conn.rollback()
thread.stats.dropped += 1
+ try:
+ cls = self.get_class()
+ except:
+ cls = None
+ if cls != None:
+ thread.stats.objects_dropped_by_class[cls] += 1
+ self.after_commit = time.time()
+
except:
log.exception("Update failed")
thread.conn.rollback()
-
thread.stats.errors += 1
- #print_exc()
+ print_exc()
if thread.halt_on_error:
raise
+ self.finish_process = time.time()
+ thread.stats.record_update_times(self)
def do_process(self, cursor, stats):
raise Exception("Not implemented")
+ def get_class(self):
+ return None
+
def __repr__(self):
return self.__class__.__name__
@@ -346,7 +500,7 @@
self.model.print_event(3, "Created %s", obj)
stats.objects_created += 1
- #stats.objects_created_by_class[cls] += 1
+ stats.objects_created_by_class[cls] += 1
return obj
@@ -380,7 +534,9 @@
# force a write if it's been too long, even if the values match
if object_columns \
- or obj._save_time < obj._qmf_update_time - timedelta(hours=1):
+ or (obj._save_time != None and \
+ obj._qmf_update_time != None and \
+ obj._save_time < obj._qmf_update_time - timedelta(hours=1)):
object_columns.append(cls.sql_table._qmf_update_time)
sql = cls.sql_update_object.emit(object_columns)
@@ -405,7 +561,7 @@
self.model.print_event(4, "Updated %s", obj)
stats.objects_updated += 1
- #stats.objects_updated_by_class[cls] += 1
+ stats.objects_updated_by_class[cls] += 1
def delete_object(self, cursor, stats, obj):
obj.delete(cursor)
@@ -413,7 +569,7 @@
self.model.print_event(3, "Deleted %s", obj)
stats.objects_deleted += 1
- #stats.objects_deleted_by_class[obj._class] += 1
+ stats.objects_deleted_by_class[obj._class] += 1
def process_properties(self, obj, columns, cursor):
cls = obj._class
@@ -605,7 +761,7 @@
count = cls.delete_selection(cursor, _qmf_agent_id=agent.id)
stats.objects_deleted += count
- #stats.objects_deleted_by_class[cls] += count
+ stats.objects_deleted_by_class[cls] += count
cursor.connection.commit()
13 years, 4 months
r4442 - trunk/cumin/python/cumin
by eallen@fedoraproject.org
Author: eallen
Date: 2010-12-20 14:12:38 +0000 (Mon, 20 Dec 2010)
New Revision: 4442
Modified:
trunk/cumin/python/cumin/model.py
Log:
Removing obsolete CuminClass and RemoteClass code
Modified: trunk/cumin/python/cumin/model.py
===================================================================
--- trunk/cumin/python/cumin/model.py 2010-12-20 13:53:33 UTC (rev 4441)
+++ trunk/cumin/python/cumin/model.py 2010-12-20 14:12:38 UTC (rev 4442)
@@ -667,189 +667,6 @@
self.ad_properties.append(prop)
self.ad_properties_by_name[prop.name] = prop
-class CuminClass(object):
- def __init__(self, model, name, mint_class):
- self.model = model
- self.cumin_name = name
- self.mint_class = mint_class
-
- self.properties = list()
- self.ad_properties = list()
- self.ad_properties_by_name = dict()
- self.stats = list()
- self.actions = list()
-
- self.tasks = list()
- self.tasks_by_class = dict()
-
- self.frame = None
-
- self.model.add_class(self)
-
- def add_property(self, prop):
- self.properties.append(prop)
- setattr(self, prop.name, prop)
-
- def add_ad_property(self, prop):
- self.ad_properties.append(prop)
- self.ad_properties_by_name[prop.name] = prop
-
- def add_stat(self, stat):
- self.stats.append(stat)
- setattr(self, stat.name, stat)
-
- def add_action(self, action):
- self.actions.append(action)
- setattr(self, action.name, action)
-
- def add_task(self, task):
- self.tasks.append(task)
- self.tasks_by_class[task.__class__] = task
-
- def init(self):
- log.debug("Initializing class %s", self)
-
- for prop in self.properties:
- prop.init()
-
- for ad_prop in self.ad_properties:
- ad_prop.init()
-
- for stat in self.stats:
- stat.init()
-
- for action in self.actions:
- action.init()
-
- for task in self.tasks:
- log.debug("Initializing task %s", task)
- task.init()
-
- def get_title(self, session):
- return "Object"
-
- def get_plural_title(self, session):
- return self.get_title(session) + "s"
-
- def get_icon_href(self, session):
- return "resource?name=action-36.png"
-
- def get_object_href(self, session, object):
- branch = session.branch()
- self.show_object(branch, object)
- return branch.marshal()
-
- def get_object_href_by_id(self, session, id):
- return self.get_object_href(session, Identifiable(id))
-
- def show_object(self, session, object):
- return self.frame.show_object(session, object)
-
- def get_object_title(self, session, object):
- title = self.get_title(session)
- name = self.get_object_name(object)
- return "%s '%s'" % (title, name)
-
- def get_object_name(self, object):
- return object.name
-
- def write_event_xml(self, writer, object):
- writer.write("<events errors=\"%i\" warnings=\"%i\"/>" % (0, 0))
-
- def write_stat_xml(self, session, writer, object):
- for stat in self.stats:
- stat.write_xml(writer, object)
-
- def write_xml(self, session, writer, object):
- object.sync()
-
- writer.write("<%s id=\"%s\" name=\"%s\">" % \
- (self.cumin_name, str(object.id),
- self.get_object_name(object)))
-
- self.write_event_xml(writer, object)
- self.write_stat_xml(session, writer, object)
-
- writer.write("</%s>" % self.cumin_name)
-
- def __str__(self):
- return "%s.%s" % (self.__module__, self.__class__.__name__)
-
-class RemoteClass(CuminClass):
- def __init__(self, model, name, mint_class, mint_stats_class):
- super(RemoteClass, self).__init__(model, name, mint_class)
-
- self.mint_stats_class = mint_stats_class
-
- prop = CuminProperty(self, "qmfObjectId")
- prop.title = "Management ID"
-
- prop = CuminProperty(self, "qmfCreateTime")
- prop.title = "Creation Time"
-
- prop = CuminProperty(self, "qmfDeleteTime")
- prop.title = "Deletion Time"
-
- prop = CuminProperty(self, "qmfUpdateTime")
- prop.title = "Last Updated"
- prop.summary = True
-
-class CuminGrid(RemoteClass):
- def __init__(self, model):
- super(CuminGrid, self).__init__(model, "grid", Grid, GridStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "Pool")
- prop.title = "Pool"
-
- prop = CuminProperty(self, "JobLimit")
- prop.title = "Job Limit"
-
- prop = CuminProperty(self, "SubmitLimit")
- prop.title = "Submit Limit"
-
- prop = CuminProperty(self, "GridResourceUnavailableTime")
- prop.title = "Grid Resource Unavailable Time"
-
- stat = CuminStat(self, "NumJobs")
- stat.title = "Num Jobs"
- stat.category = "general"
-
- stat = CuminStat(self, "SubmitsInProgress")
- stat.title = "Submits In Progress"
- stat.category = "general"
-
- stat = CuminStat(self, "SubmitsQueued")
- stat.title = "Submits Queued"
- stat.category = "general"
-
- stat = CuminStat(self, "SubmitsAllowed")
- stat.title = "Submits Allowed"
- stat.category = "general"
-
- stat = CuminStat(self, "SubmitsWanted")
- stat.title = "Submits Wanted"
- stat.category = "general"
-
- stat = CuminStat(self, "RunningJobs")
- stat.title = "Running Jobs"
- stat.category = "general"
-
- stat = CuminStat(self, "IdleJobs")
- stat.title = "Idle Jobs"
- stat.category = "general"
-
- def init(self):
- super(CuminGrid, self).init()
-
- def get_title(self, session):
- return "Grid"
-
- def get_icon_href(self, session):
- return "resource?name=pool-36.png"
-
# XXX "do_" on this doesn't make sense
def do_bind(session, queue_name, binding_info):
for exchange in binding_info:
@@ -863,573 +680,6 @@
binding_key=binding_key,
arguments=binding_info[exchange]["arguments"])
-class CuminQueue(RemoteClass):
- def __init__(self, model):
- super(CuminQueue, self).__init__(model, "queue", Queue, QueueStats)
-
- prop = CuminProperty(self, "name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "durable")
- prop.title = "Durable?"
- prop.summary = True
-
- prop = CuminProperty(self, "autoDelete")
- prop.title = "Auto Delete?"
- prop.summary = True
-
- prop = CuminProperty(self, "exclusive")
- prop.title = "Exclusive?"
- prop.summary = True
-
- prop = CuminProperty(self, "arguments")
- prop.title = "Arguments"
- prop.prefix = "qpid."
- prop.escape = False
-
- stat = CuminStat(self, "consumerCount")
- stat.title = "Consumers"
- stat.unit = "consumer"
- stat.highlow = True
-
- stat = CuminStat(self, "bindingCount")
- stat.title = "Bindings"
- stat.unit = "binding"
- stat.highlow = True
-
- stat = CuminStat(self, "msgDepth")
- stat.title = "Message Depth"
- stat.highlow = True
-
- stat = CuminStat(self, "msgTotalEnqueues")
- stat.title = "Msgs. Enqueued"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "msgTotalDequeues")
- stat.title = "Msgs. Dequeued"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "byteDepth")
- stat.title = "Byte Depth"
- stat.unit = "byte"
- stat.highlow = True
-
- stat = CuminStat(self, "byteTotalEnqueues")
- stat.title = "Bytes Enqueued"
- stat.unit = "byte"
- stat.category = "io"
-
- stat = CuminStat(self, "byteTotalDequeues")
- stat.title = "Bytes Dequeued"
- stat.unit = "byte"
- stat.category = "io"
-
- stat = CuminStat(self, "unackedMessages")
- stat.title = "Msgs. Unacked"
- stat.unit = "message"
- stat.category = "io"
- stat.highlow = True
-
- stat = CuminStat(self, "messageLatencyMin")
- stat.title = "Min. Msg. Latency"
- stat.unit = "nanosecond"
- stat.category = "io"
-
- stat = CuminStat(self, "messageLatencyMax")
- stat.title = "Max. Msg. Latency"
- stat.unit = "nanosecond"
- stat.category = "io"
-
- stat = CuminStat(self, "messageLatencyAverage")
- stat.title = "Avg. Msg. Latency"
- stat.unit = "nanosecond"
- stat.category = "io"
-
- # Transactional
-
- stat = CuminStat(self, "msgTxnEnqueues")
- stat.title = "Msgs. Enqueued"
- stat.unit = "message"
- stat.category = "io.txn"
-
- stat = CuminStat(self, "msgTxnDequeues")
- stat.title = "Msgs. Dequeued"
- stat.unit = "message"
- stat.category = "io.txn"
-
- stat = CuminStat(self, "byteTxnEnqueues")
- stat.title = "Bytes Enqueued"
- stat.unit = "byte"
- stat.category = "io.txn"
-
- stat = CuminStat(self, "byteTxnDequeues")
- stat.title = "Bytes Dequeued"
- stat.unit = "byte"
- stat.category = "io.txn"
-
- stat = CuminStat(self, "enqueueTxnStarts")
- stat.title = "Enq. Trans. Started"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "enqueueTxnCommits")
- stat.title = "Enq. Trans. Committed"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "enqueueTxnRejects")
- stat.title = "Enq. Trans. Rejected"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "enqueueTxnCount")
- stat.title = "Enq. Trans. Pending"
- stat.unit = "transaction"
- stat.category = "txn"
- stat.highlow = True
-
- stat = CuminStat(self, "dequeueTxnStarts")
- stat.title = "Deq. Trans. Started"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "dequeueTxnCommits")
- stat.title = "Deq. Trans. Committed"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "dequeueTxnRejects")
- stat.title = "Deq. Trans. Rejected"
- stat.unit = "transaction"
- stat.category = "txn"
-
- stat = CuminStat(self, "dequeueTxnCount")
- stat.title = "Deq. Trans. Pending"
- stat.unit = "transaction"
- stat.category = "txn"
- stat.highlow = True
-
- # Durable
-
- stat = CuminStat(self, "msgPersistEnqueues")
- stat.title = "Msgs. Enqueued"
- stat.unit = "message"
- stat.category = "io.durable"
-
- stat = CuminStat(self, "msgPersistDequeues")
- stat.title = "Msgs. Dequeued"
- stat.unit = "message"
- stat.category = "io.durable"
-
- stat = CuminStat(self, "bytePersistEnqueues")
- stat.title = "Bytes Enqueued"
- stat.unit = "byte"
- stat.category = "io.durable"
-
- stat = CuminStat(self, "bytePersistDequeues")
- stat.title = "Bytes Dequeued"
- stat.unit = "byte"
- stat.category = "io.durable"
-
- def init(self):
- super(CuminQueue, self).init()
-
- self.frame = self.model.frame.messaging.broker.queue
-
- def get_title(self, session):
- return "Queue"
-
- def get_icon_href(self, session):
- return "resource?name=queue-36.png"
-
-class CuminExchange(RemoteClass):
- def __init__(self, model):
- super(CuminExchange, self).__init__(model, "exchange",
- Exchange, ExchangeStats)
-
- prop = self.NameProperty(self, "name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "type")
- prop.title = "Type"
- prop.summary = True
-
- prop = CuminProperty(self, "durable")
- prop.title = "Durable?"
- prop.summary = True
-
- prop = CuminProperty(self, "arguments")
- prop.title = "Arguments"
- prop.prefix = "qpid."
- prop.escape = False
-
- stat = CuminStat(self, "producerCount")
- stat.title = "Producers"
- stat.unit = "producer"
- stat.highlow = True
-
- stat = CuminStat(self, "bindingCount")
- stat.title = "Bindings"
- stat.unit = "binding"
- stat.highlow = True
-
- stat = CuminStat(self, "msgReceives")
- stat.title = "Msgs. Received"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "msgRoutes")
- stat.title = "Msgs. Routed"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "msgDrops")
- stat.title = "Msgs. Dropped"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "byteReceives")
- stat.title = "Bytes Received"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "byteRoutes")
- stat.title = "Bytes Routed"
- stat.unit = "message"
- stat.category = "io"
-
- stat = CuminStat(self, "byteDrops")
- stat.title = "Bytes Dropped"
- stat.unit = "message"
- stat.category = "io"
-
- def init(self):
- super(CuminExchange, self).init()
-
- self.frame = self.model.frame.messaging.broker.exchange
-
- def get_title(self, session):
- return "Exchange"
-
- def get_icon_href(self, session):
- return "resource?name=exchange-36.png"
-
- def get_object_title(self, session, object):
- name = super(CuminExchange, self).get_object_name(object)
-
- if name:
- return super(CuminExchange, self).get_object_title(session, object)
- else:
- return self.get_object_name(object)
-
- def get_object_name(self, object):
- return object.name and object.name or "Default Exchange"
-
- class NameProperty(CuminProperty):
- def value(self, session, object):
- value = getattr(object, self.name, None)
- return value and value or "Default Exchange"
-
-class CuminBinding(RemoteClass):
- def __init__(self, model):
- super(CuminBinding, self).__init__(model, "binding",
- Binding, BindingStats)
-
- prop = CuminProperty(self, "bindingKey")
- prop.title = "Binding Key"
-
- stat = CuminStat(self, "msgMatched")
- stat.title = "Msgs. Matched"
- stat.unit = "message"
-
- def get_title(self, session):
- return "Binding"
-
- def get_object_name(self, binding):
- return "between %s and %s" % (binding.exchange.name, binding.queue.name)
-
-class CuminRoute(RemoteClass):
- def __init__(self, model):
- super(CuminRoute, self).__init__ \
- (model, "route", Bridge, BridgeStats)
-
- prop = CuminProperty(self, "key")
- prop.title = "Route Key"
-
- prop = CuminProperty(self, "tag")
- prop.title = "Tag"
-
- prop = CuminProperty(self, "excludes")
- prop.title = "Excludes"
-
- prop = CuminProperty(self, "durable")
- prop.title = "Durable"
-
- prop = CuminProperty(self, "dest")
- prop.title = "Exchange"
-
- def get_title(self, session):
- return "Route"
-
- def get_object_name(self, route):
- return route.src
-
-class CuminConnection(RemoteClass):
- def __init__(self, model):
- super(CuminConnection, self).__init__(model, "connection",
- ClientConnection,
- ClientConnectionStats)
-
- prop = CuminProperty(self, "address")
- prop.title = "Address"
-
- prop = CuminProperty(self, "SystemConnection")
- prop.title = "System Connection?"
-
- prop = CuminProperty(self, "authIdentity")
- prop.title = "Auth Identity?"
-
- prop = CuminProperty(self, "federationLink")
- prop.title = "Federation Link"
-
- stat = CuminStat(self, "closing")
- stat.title = "Closing Down"
- stat.category = "general"
-
- stat = CuminStat(self, "bytesFromClient")
- stat.title = "Bytes Sent"
- stat.unit = "byte"
- stat.category = "io"
-
- stat = CuminStat(self, "bytesToClient")
- stat.title = "Bytes Received"
- stat.unit = "byte"
- stat.category = "io"
-
- stat = CuminStat(self, "framesFromClient")
- stat.title = "Frames Sent"
- stat.unit = "frame"
- stat.category = "io"
-
- stat = CuminStat(self, "framesToClient")
- stat.title = "Frames Received"
- stat.unit = "frame"
- stat.category = "io"
-
- def init(self):
- super(CuminConnection, self).init()
-
- self.frame = self.model.frame.messaging.broker.connection
-
- def get_title(self, session):
- return "Connection"
-
- def get_icon_href(self, session):
- return "resource?name=client-36.png"
-
- def get_object_name(self, conn):
- return conn.address
-
-class CuminLink(RemoteClass):
- def __init__(self, model):
- super(CuminLink, self).__init__(model, "link", Link, LinkStats)
-
- prop = CuminProperty(self, "useSsl")
- prop.title = "Use SSL"
- prop.summary = True
-
- prop = CuminProperty(self, "durable")
- prop.title = "Durable"
- prop.summary = True
-
- stat = CuminStat(self, "closing")
- stat.title = "Closing Down"
-
- stat = CuminStat(self, "framesFromPeer")
- stat.title = "Frames from Peer"
- stat.unit = "frame"
- stat.category = "io"
-
- stat = CuminStat(self, "framesToPeer")
- stat.title = "Frames to Peer"
- stat.unit = "frame"
- stat.category = "io"
-
- stat = CuminStat(self, "bytesFromPeer")
- stat.title = "Bytes from Peer"
- stat.unit = "byte"
- stat.category = "io"
-
- stat = CuminStat(self, "bytesToPeer")
- stat.title = "Bytes to Peer"
- stat.unit = "byte"
- stat.category = "io"
-
- def get_title(self, session):
- return "Broker Link"
-
- def get_object_name(self, link):
- return "%s:%d" % (link.host, link.port)
-
-class CuminBrokerStoreModule(RemoteClass):
- def __init__(self, model):
- super(CuminBrokerStoreModule, self).__init__ \
- (model, "store", Store, StoreStats)
-
- prop = CuminProperty(self, "location")
- prop.title = "Location"
-
-class CuminJournal(RemoteClass):
- def __init__(self, model):
- super(CuminJournal, self).__init__(model, "journal",
- Journal, JournalStats)
-
- prop = CuminProperty(self, "name")
- prop.title = "Name"
- prop.summary = True
-
- prop = CuminProperty(self, "directory")
- prop.title = "Directory"
-
- stat = CuminStat(self, "initialFileCount")
- stat.title = "Initial File Count"
- stat.unit = "file"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "dataFileSize")
- stat.title = "Data File Size"
- stat.unit = "byte"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "recordDepth")
- stat.title = "Record Depth"
- stat.unit = "record"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "recordEnqueues")
- stat.title = "Record Enqueues"
- stat.unit = "record"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "outstandingAIOs")
- stat.title = "Outstanding AIOs"
- stat.unit = "aio"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "freeFileCount")
- stat.title = "Free Files"
- stat.unit = "file"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "availableFileCount")
- stat.title = "Avail. Files"
- stat.unit = "file"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "writeWaitFailures")
- stat.title = "Write Wait Failures"
- stat.unit = "failure"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "writeBusyFailures")
- stat.title = "Write Busy Failures"
- stat.unit = "failure"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "readRecordCount")
- stat.title = "Read Records"
- stat.unit = "record"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "readBusyFailures")
- stat.title = "Read Busy Failures"
- stat.unit = "failure"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "writePageCacheDepth")
- stat.title = "Write Page Cache Depth"
- stat.unit = "page"
- stat.category = "io.journal"
-
- stat = CuminStat(self, "readPageCacheDepth")
- stat.title = "Read Page Cache Depth"
- stat.unit = "page"
- stat.category = "io.journal"
-
-class CuminBrokerAclModule(RemoteClass):
- def __init__(self, model):
- super(CuminBrokerAclModule, self).__init__ \
- (model, "store", Acl, AclStats)
-
- prop = CuminProperty(self, "policyFile")
- prop.title = "Policy File"
-
- prop = CuminProperty(self, "enforcingAcl")
- prop.title = "Enforcing?"
-
- prop = CuminProperty(self, "transferAcl")
- prop.title = "Transfer?"
-
- prop = CuminProperty(self, "lastAclLoad")
- prop.title = "Last Loaded"
-
- stat = CuminStat(self, "aclDenyCount")
- stat.title = "Denials"
- stat.unit = "denial"
-
-class CuminBrokerClusterModule(RemoteClass):
- def __init__(self, model):
- super(CuminBrokerClusterModule, self).__init__ \
- (model, "broker_cluster_module", Cluster, ClusterStats)
-
- prop = CuminProperty(self, "clusterName")
- prop.title = "Name"
-
- prop = CuminProperty(self, "clusterID")
- prop.title = "ID"
-
- prop = CuminProperty(self, "publishedURL")
- prop.title = "URL"
-
- prop = CuminProperty(self, "clusterSize")
- prop.title = "Size"
-
- prop = CuminProperty(self, "status")
- prop.title = "Status"
-
- prop = CuminProperty(self, "members")
- prop.title = "Members"
-
-class CuminLimit(CuminClass):
- def __init__(self, model):
- super(CuminLimit, self).__init__ \
- (model, "limit", None)
-
- prop = CuminProperty(self, "name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "curr")
- prop.summary = True
- prop.title = "Current"
-
- prop = CuminProperty(self, "max")
- prop.summary = True
- prop.title = "Maximum"
-
- def init(self):
- super(CuminLimit, self).init()
-
- #self.frame = self.model.frame.grid.pool.limit
-
- def get_title(self, session):
- return "Concurrency Limit"
-
- def get_object_title(self, session, limit):
- title = self.get_title(session)
- return "%s '%s'" % (title, limit)
-
class JobMetaData(MetaData):
def __init__(self, name):
super(JobMetaData, self).__init__(name)
@@ -1759,181 +1009,6 @@
log.exception(e)
return self.do_wait()
-class CuminScheduler(RemoteClass):
- def __init__(self, model):
- super(CuminScheduler, self).__init__(model, "scheduler",
- Scheduler, SchedulerStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "JobQueueBirthdate")
- prop.title = "Job Queue Birthdate"
-
- prop = CuminProperty(self, "MaxJobsRunning")
- prop.title = "Max Jobs Running"
-
- prop = CuminProperty(self, "MyAddress")
- prop.title = "Contact Address"
-
- prop = CuminProperty(self, "PublicNetworkIpAddr")
- prop.title = "Public IP Address"
-
- prop = CuminProperty(self, "Machine")
- prop.title = "Machine"
- prop.summary = True
-
- prop = CuminProperty(self, "System")
- prop.title = "System"
-
- stat = CuminStat(self, "NumUsers")
- stat.title = "Users"
-
- stat = CuminStat(self, "TotalRunningJobs")
- stat.title = "Running Jobs"
-
- stat = CuminStat(self, "TotalIdleJobs")
- stat.title = "Idle Jobs"
-
- stat = CuminStat(self, "TotalHeldJobs")
- stat.title = "Held Jobs"
-
- stat = CuminStat(self, "TotalJobAds")
- stat.title = "Job Ads"
-
- stat = CuminStat(self, "TotalRemovedJobs")
- stat.title = "Removed Jobs"
-
- action = GetStartedAction(self, "GetStarted")
- action.navigable = False
-
- def init(self):
- super(CuminScheduler, self).init()
-
- # self.frame = self.model.frame.grid.pool.scheduler
-
- def get_title(self, session):
- return "Scheduler"
-
- def get_object_name(self, sched):
- return sched.Name
-
-class CuminSubmission(RemoteClass):
- def __init__(self, model):
- super(CuminSubmission, self).__init__(model, "submission",
- Submission, SubmissionStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- stat = CuminStat(self, "Idle")
- stat.title = "Idle Jobs"
-
- stat = CuminStat(self, "Running")
- stat.title = "Running Jobs"
-
- stat = CuminStat(self, "Removed")
- stat.title = "Removed Jobs"
-
- stat = CuminStat(self, "Completed")
- stat.title = "Completed Jobs"
-
- stat = CuminStat(self, "Held")
- stat.title = "Held Jobs"
-
- def get_title(self, session):
- return "Submission"
-
- def get_object_name(self, sub):
- return sub.Name
-
-class CuminSubmitter(RemoteClass):
- def __init__(self, model):
- super(CuminSubmitter, self).__init__(model, "submitter",
- Submitter, SubmitterStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- stat = CuminStat(self, "RunningJobs")
- stat.title = "Running Jobs"
-
- stat = CuminStat(self, "IdleJobs")
- stat.title = "Idle Jobs"
-
- stat = CuminStat(self, "HeldJobs")
- stat.title = "Held Jobs"
-
- def init(self):
- super(CuminSubmitter, self).init()
-
- #self.frame = self.model.frame.grid.pool.scheduler.submitter
-
- def get_title(self, session):
- return "Submitter"
-
- def get_object_name(self, sub):
- return sub.Name
-
- def show_object(self, session, sub):
- pool = self.model.get_main_pool()
-
- assert pool
-
- self.model.frame.grid.pool.set_object(session, pool)
-
- return self.frame.show_object(session, sub)
-
-class CuminCollector(RemoteClass):
- def __init__(self, model):
- super(CuminCollector, self).__init__(model, "collector",
- Collector, CollectorStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "PublicNetworkIpAddr")
- prop.title = "IP Address"
-
- action = GetStartedAction(self, "GetStarted")
- action.navigable = False
-
- stat = CuminStat(self, "RunningJobs")
- stat.title = "Running Jobs"
-
- stat = CuminStat(self, "IdleJobs")
- stat.title = "Idle Jobs"
-
- stat = CuminStat(self, "HostsClaimed")
- stat.title = "Claimed Slots"
-
- stat = CuminStat(self, "HostsUnclaimed")
- stat.title = "Unclaimed Slots"
-
- stat = CuminStat(self, "HostsOwner")
- stat.title = "Owner Slots"
-
- stat = CuminStat(self, "HostsTotal")
- stat.title = "Total Slots"
-
- def init(self):
- super(CuminCollector, self).init()
-
- #self.frame = self.model.frame.grid.pool.collector
-
- def get_title(self, session):
- return "Collector"
-
- def get_object_name(self, coll):
- return coll.Name
-
-class CuminSubject(CuminClass):
- def __init__(self, model):
- super(CuminSubject, self).__init__(model, "subject", Subject)
-
- def get_title(self, session):
- return "Subject"
-
class Pool(object):
def __init__(self, id):
self.id = id
13 years, 4 months
r4441 - trunk/wooly/python/wooly
by eallen@fedoraproject.org
Author: eallen
Date: 2010-12-20 13:53:33 +0000 (Mon, 20 Dec 2010)
New Revision: 4441
Modified:
trunk/wooly/python/wooly/datatable.py
Log:
Avoid potential exception if formatted values are None
Modified: trunk/wooly/python/wooly/datatable.py
===================================================================
--- trunk/wooly/python/wooly/datatable.py 2010-12-17 18:51:54 UTC (rev 4440)
+++ trunk/wooly/python/wooly/datatable.py 2010-12-20 13:53:33 UTC (rev 4441)
@@ -43,7 +43,7 @@
value = record[self.index]
if self.format is not None:
- value = self.format % value
+ value = value and self.format % value or None
return value
13 years, 4 months
r4440 - in trunk/cumin/python/cumin: . grid inventory messaging
by eallen@fedoraproject.org
Author: eallen
Date: 2010-12-17 18:51:54 +0000 (Fri, 17 Dec 2010)
New Revision: 4440
Modified:
trunk/cumin/python/cumin/grid/job.py
trunk/cumin/python/cumin/grid/negotiator.py
trunk/cumin/python/cumin/grid/pool.py
trunk/cumin/python/cumin/grid/scheduler.py
trunk/cumin/python/cumin/grid/slot.py
trunk/cumin/python/cumin/grid/submission.py
trunk/cumin/python/cumin/grid/submitter.py
trunk/cumin/python/cumin/inventory/system.py
trunk/cumin/python/cumin/messaging/binding.py
trunk/cumin/python/cumin/messaging/broker.py
trunk/cumin/python/cumin/messaging/brokerlink.py
trunk/cumin/python/cumin/messaging/connection.py
trunk/cumin/python/cumin/messaging/exchange.py
trunk/cumin/python/cumin/messaging/queue.py
trunk/cumin/python/cumin/messaging/subscription.py
trunk/cumin/python/cumin/model.py
trunk/cumin/python/cumin/objectselector.py
trunk/cumin/python/cumin/objectselector.strings
trunk/cumin/python/cumin/qmfadapter.py
Log:
-Added MonitorSelfStats to Negotiator and Submitter lists and overview pages.
-Added search boxes to ObjectTable lists to perform SQL LIKE searches
Modified: trunk/cumin/python/cumin/grid/job.py
===================================================================
--- trunk/cumin/python/cumin/grid/job.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/job.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -199,7 +199,7 @@
super(JobSelector, self).__init__(app, name, cls)
- self.adapter = JobSummariesAdapter(app, cls)
+ self.table.adapter = JobSummariesAdapter(app, cls)
self.submission = submission
frame = "main.grid.pool.submission.job"
@@ -220,6 +220,8 @@
self.qmf_response = Attribute(app, "qmf_response")
self.add_attribute(self.qmf_response)
+ self.add_search_filter(cls.GlobalJobId)
+
JobSelectionHold(app, self, "held")
JobSelectionRelease(app, self, "released")
JobSelectionRemove(app, self, "removed")
Modified: trunk/cumin/python/cumin/grid/negotiator.py
===================================================================
--- trunk/cumin/python/cumin/grid/negotiator.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/negotiator.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -45,6 +45,21 @@
self.add_attribute_column(cls.Machine)
self.add_attribute_column(cls.System)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfAge.name, cls.MonitorSelfAge)
+ self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfCPUUsage.name, cls.MonitorSelfCPUUsage)
+ self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfImageSize.name, cls.MonitorSelfImageSize)
+ self.add_column(stat)
+ #stat = MonitorSelfStatColumn(app, cls.MonitorSelfRegisteredSocketCount.name, cls.MonitorSelfRegisteredSocketCount)
+ #self.add_column(stat)
+ #stat = MonitorSelfStatColumn(app, cls.MonitorSelfResidentSetSize.name, cls.MonitorSelfResidentSetSize)
+ #self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfTime.name, cls.MonitorSelfTime)
+ self.add_column(stat)
+
+ self.add_search_filter(cls.Name)
+
#self.start = DaemonSelectionStart(app, self, "NEGOTIATOR")
#self.stop = DaemonSelectionStop(app, self, "NEGOTIATOR")
Modified: trunk/cumin/python/cumin/grid/pool.py
===================================================================
--- trunk/cumin/python/cumin/grid/pool.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/pool.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -53,7 +53,7 @@
when = "now() - interval '1 day'"
filter = SqlComparisonFilter(table._qmf_update_time, when, ">")
- self.adapter.query.add_filter(filter)
+ self.table.adapter.query.add_filter(filter)
class PoolNameColumn(ObjectLinkColumn):
def render_header_content(self, session):
Modified: trunk/cumin/python/cumin/grid/scheduler.py
===================================================================
--- trunk/cumin/python/cumin/grid/scheduler.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/scheduler.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -86,6 +86,17 @@
self.add_attribute_column(cls.TotalRunningJobs)
self.add_attribute_column(cls.TotalHeldJobs)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfAge.name, cls.MonitorSelfAge)
+ self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfCPUUsage.name, cls.MonitorSelfCPUUsage)
+ self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfImageSize.name, cls.MonitorSelfImageSize)
+ self.add_column(stat)
+ stat = MonitorSelfStatColumn(app, cls.MonitorSelfTime.name, cls.MonitorSelfTime)
+ self.add_column(stat)
+
+ self.add_search_filter(cls.Name)
+
#self.start = DaemonSelectionStart(app, self, "SCHEDD")
#self.stop = DaemonSelectionStop(app, self, "SCHEDD")
@@ -95,7 +106,10 @@
self.attrs = ("NumUsers", "TotalRunningJobs",
"TotalIdleJobs", "TotalHeldJobs",
- "TotalJobAds", "TotalRemovedJobs")
+ "TotalJobAds", "TotalRemovedJobs",
+ "MonitorSelfAge", "MonitorSelfCPUUsage",
+ "MonitorSelfImageSize","MonitorSelfRegisteredSocketCount",
+ "MonitorSelfResidentSetSize","MonitorSelfTime")
class SchedulerOverview(Widget):
def __init__(self, app, name, scheduler):
Modified: trunk/cumin/python/cumin/grid/slot.py
===================================================================
--- trunk/cumin/python/cumin/grid/slot.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/slot.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -24,6 +24,7 @@
self.pool = pool
self.add_filter(self.pool, cls.Pool, cls.Pool)
+ self.add_search_filter(cls.Name)
frame = "main.grid.pool.slot"
col = ObjectLinkColumn(app, "name", cls.Name, cls._id, frame)
@@ -451,7 +452,7 @@
return item[0][2] and "class='%s'" % item[0][2] or ""
class SlotStates(ItemSet):
- states = ("Unclaimed", "Claimed", "Owner", "Matched", "Preempting")
+ states = ("Unclaimed", "Claimed", "Unavailable", "Matched", "Preempting")
def do_get_items(self, session):
return self.states
Modified: trunk/cumin/python/cumin/grid/submission.py
===================================================================
--- trunk/cumin/python/cumin/grid/submission.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/submission.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -63,13 +63,15 @@
super(SubmissionSelector, self).__init__(app, name, cls)
- self.adapter = SubmissionData(app)
+ self.table.adapter = SubmissionData(app)
self.add_attribute_column(cls.Idle)
self.add_attribute_column(cls.Running)
self.add_attribute_column(cls.Completed)
self.add_attribute_column(cls.Held)
+ self.add_search_filter(cls.Name)
+
class PoolSubmissionSelector(SubmissionSelector):
def __init__(self, app, name, pool):
super(PoolSubmissionSelector, self).__init__(app, name)
@@ -108,7 +110,7 @@
super(PoolSubmissionSchedulerSelector, self).__init__(app, name, pool)
scheduler = app.model.com_redhat_grid.Scheduler
- self.SchedulerJoin(app, self.adapter.query, self.cls.sql_table,
+ self.SchedulerJoin(app, self.table.adapter.query, self.cls.sql_table,
self.cls.jobserverRef.sql_column, "jid")
frame = "main.grid.pool.scheduler"
Modified: trunk/cumin/python/cumin/grid/submitter.py
===================================================================
--- trunk/cumin/python/cumin/grid/submitter.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/grid/submitter.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -50,6 +50,8 @@
self.add_attribute_column(cls.RunningJobs)
self.add_attribute_column(cls.HeldJobs)
+ self.add_search_filter(cls.Name)
+
class SubmitterGeneralStatSet(StatSet):
def __init__(self, app, name, object):
super(SubmitterGeneralStatSet, self).__init__(app, name, object)
Modified: trunk/cumin/python/cumin/inventory/system.py
===================================================================
--- trunk/cumin/python/cumin/inventory/system.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/inventory/system.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -30,6 +30,8 @@
self.add_attribute_column(cls.memFree)
self.add_attribute_column(cls.loadAverage1Min)
+ self.add_search_filter(cls.nodeName)
+
class SystemFrame(ObjectFrame):
def __init__(self, app, name):
cls = app.model.com_redhat_sesame.Sysimage
Modified: trunk/cumin/python/cumin/messaging/binding.py
===================================================================
--- trunk/cumin/python/cumin/messaging/binding.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/binding.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -134,7 +134,7 @@
super(BindingSelector, self).__init__(app, name, binding)
- self.adapter = BindingData(app)
+ self.table.adapter = BindingData(app)
frame = "main.messaging.broker.binding"
col = ObjectLinkColumn \
@@ -156,6 +156,8 @@
self.remove = BindingSelectionRemove(app, self)
+ self.add_search_filter(binding.bindingKey)
+
class Exchange(ObjectLinkColumn):
def render_header_content(self, session):
return "Exchange"
Modified: trunk/cumin/python/cumin/messaging/broker.py
===================================================================
--- trunk/cumin/python/cumin/messaging/broker.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/broker.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -65,7 +65,7 @@
super(BrokerSelector, self).__init__(app, name, broker)
- self.adapter = BrokerData(app)
+ self.table.adapter = BrokerData(app)
self.group = SessionAttribute(self, "group")
Modified: trunk/cumin/python/cumin/messaging/brokerlink.py
===================================================================
--- trunk/cumin/python/cumin/messaging/brokerlink.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/brokerlink.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -73,6 +73,8 @@
self.add_attribute_column(cls.transport)
self.add_attribute_column(cls.durable)
+ self.add_search_filter(cls.host)
+
self.remove = BrokerLinkSelectionRemove(app, self)
# Address column XXX
@@ -100,6 +102,8 @@
self.add_attribute_column(cls.tag)
self.add_attribute_column(cls.excludes)
+ self.add_search_filter(cls.src)
+
self.remove = RouteSelectionRemove(app, self)
class RouteSelectionRemove(ObjectSelectorTask):
Modified: trunk/cumin/python/cumin/messaging/connection.py
===================================================================
--- trunk/cumin/python/cumin/messaging/connection.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/connection.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -74,6 +74,8 @@
self.add_attribute_column(cls.bytesFromClient)
self.add_attribute_column(cls.bytesToClient)
+ self.add_search_filter(cls.address)
+
self.close = ConnectionSelectionClose(app, self)
class ConnectionSelectionClose(ObjectSelectorTask):
@@ -194,6 +196,8 @@
self.add_attribute_column(cls.framesOutstanding)
self.add_attribute_column(cls.clientCredit)
+ self.add_search_filter(cls.name)
+
self.close = SessionSelectionClose(app, self)
self.detach = SessionSelectionDetach(app, self)
Modified: trunk/cumin/python/cumin/messaging/exchange.py
===================================================================
--- trunk/cumin/python/cumin/messaging/exchange.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/exchange.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -79,6 +79,7 @@
self.add_attribute_column(cls.byteRoutes)
self.add_reference_filter(vhost, cls.vhostRef)
+ self.add_search_filter(cls.name)
self.remove = ExchangeSelectionRemove(app, self)
Modified: trunk/cumin/python/cumin/messaging/queue.py
===================================================================
--- trunk/cumin/python/cumin/messaging/queue.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/queue.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -79,6 +79,7 @@
self.add_attribute_column(cls.byteDepth)
self.add_reference_filter(vhost, cls.vhostRef)
+ self.add_search_filter(cls.name)
self.remove = QueueSelectionRemove(app, self)
self.purge = QueueSelectionPurge(app, self)
Modified: trunk/cumin/python/cumin/messaging/subscription.py
===================================================================
--- trunk/cumin/python/cumin/messaging/subscription.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/messaging/subscription.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -21,6 +21,8 @@
self.add_attribute_column(cls.arguments)
self.add_attribute_column(cls.delivered)
+ self.add_search_filter(cls.name)
+
class SubscriptionFrame(ObjectFrame):
def __init__(self, app, name):
cls = app.model.org_apache_qpid_broker.Subscription
Modified: trunk/cumin/python/cumin/model.py
===================================================================
--- trunk/cumin/python/cumin/model.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/model.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -1927,57 +1927,6 @@
def get_object_name(self, coll):
return coll.Name
-class CuminNegotiator(RemoteClass):
- def __init__(self, model):
- super(CuminNegotiator, self).__init__(model, "negotiator",
- Negotiator, NegotiatorStats)
-
- prop = CuminProperty(self, "Name")
- prop.title = "Name"
-
- prop = CuminProperty(self, "Machine")
- prop.title = "Machine"
-
- prop = CuminProperty(self, "MyAddress")
- prop.title = "Address"
-
- prop = CuminProperty(self, "DaemonStartTime")
- prop.title = "Start Time"
-
- stat = CuminStat(self, "MonitorSelfAge")
- stat.title = "Age"
- stat.category = "general"
-
- stat = CuminStat(self, "MonitorSelfCPUUsage")
- stat.title = "CPU Usage"
- stat.category = "general"
-
- stat = CuminStat(self, "MonitorSelfImageSize")
- stat.title = "Image Size"
- stat.category = "general"
-
- stat = CuminStat(self, "MonitorSelfRegisteredSocketCount")
- stat.title = "Registered Socket Count"
- stat.category = "general"
-
- stat = CuminStat(self, "MonitorSelfResidentSetSize")
- stat.title = "Resident Set Size"
- stat.category = "general"
-
- action = GetStartedAction(self, "GetStarted")
- action.navigable = False
-
- def init(self):
- super(CuminNegotiator, self).init()
-
- #self.frame = self.model.frame.grid.pool.negotiator
-
- def get_title(self, session):
- return "Negotiator"
-
- def get_object_name(self, neg):
- return neg.Name
-
class CuminSubject(CuminClass):
def __init__(self, model):
super(CuminSubject, self).__init__(model, "subject", Subject)
Modified: trunk/cumin/python/cumin/objectselector.py
===================================================================
--- trunk/cumin/python/cumin/objectselector.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/objectselector.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -73,7 +73,7 @@
self.add_filter(attribute, this, that)
- def add_like_filter(self, attribute, this, type=SqlLikeFilter.BEGINS):
+ def add_like_filter(self, attribute, this, type=SqlLikeFilter.CONTAINS):
assert isinstance(attribute, Attribute), attribute
assert isinstance(this, RosemaryAttribute), this
@@ -88,8 +88,14 @@
for this, vattr, type in self.like_specs:
value = vattr.get(session)
- pre = type == SqlLikeFilter.CONTAINS and "%%" or ""
- values[this.name] = "%s%s%%" % (pre, value)
+ # if the user typed a %, don't add another
+ pre = ""
+ post = "%"
+ if not "%" in value:
+ pre = type == SqlLikeFilter.CONTAINS and "%%" or ""
+ else:
+ post = ""
+ values[this.name] = "%s%s%s" % (pre, value, post)
return values
@@ -119,11 +125,12 @@
else:
self.field = ObjectSqlField(self.table.adapter, self.attr)
-class ObjectSelector(ObjectTable, Form):
+class ObjectSelector(Form):
def __init__(self, app, name, cls):
- super(ObjectSelector, self).__init__(app, name, cls)
+ super(ObjectSelector, self).__init__(app, name)
- self.init_ids(app, cls)
+ self.update_enabled = False
+ self.cls = cls
self.switches = ObjectSelectorSwitches(app, "switches")
self.add_child(self.switches)
@@ -137,6 +144,9 @@
self.links = ObjectSelectorLinks(app, "links")
self.add_child(self.links)
+ self.table = ObjectSelectorTable(app, "table", cls)
+ self.add_child(self.table)
+
self.tasks = list()
def init(self):
@@ -150,6 +160,38 @@
self.buttons.add_child(button)
button.init()
+ def add_search_filter(self, this):
+ search = StringInput(self.app, "search")
+ search.param.default = ""
+
+ self.table.add_like_filter(search.param, this)
+ self.filters.add_child(search)
+
+ def add_filter(self, attribute, this, that=None):
+ self.table.add_filter(attribute, this, that)
+
+ def add_column(self, col):
+ self.table.add_column(col)
+
+ def add_attribute_column(self, attr):
+ self.table.add_attribute_column(attr)
+
+ def add_reference_filter(self, attribute, ref):
+ self.table.add_reference_filter(attribute, ref)
+
+ def insert_column(self, index, column):
+ self.table.insert_column(index, column)
+
+ def render_title(self, session):
+ return self.table.render_title(session)
+
+class ObjectSelectorTable(ObjectTable):
+ def __init__(self, app, name, cls):
+ super(ObjectSelectorTable, self).__init__(app, name, cls)
+
+ self.update_enabled = True
+ self.init_ids(app, cls)
+
def init_ids(self, app, cls):
item = IntegerParameter(app, "item")
@@ -345,3 +387,12 @@
def render_item_class(self, session, item):
return "item"
+
+class MonitorSelfStatColumn(ObjectTableColumn):
+ def render_header_content(self, session):
+ title = self.field.get_title(session)
+ parts = title.split()
+ if len(parts) > 2:
+ if parts[0] == "Monitor" and parts[1] == "Self":
+ return " ".join(parts[2:])
+ return title
Modified: trunk/cumin/python/cumin/objectselector.strings
===================================================================
--- trunk/cumin/python/cumin/objectselector.strings 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/objectselector.strings 2010-12-17 18:51:54 UTC (rev 4440)
@@ -7,8 +7,18 @@
div.ObjectSelectorFilters {
float: right;
+ padding-bottom: 0.25em;
}
+div.ObjectSelectorFilters ul {
+ float: left;
+ margin: 0;
+}
+
+div.ObjectSelectorFilters input {
+ font-size: 0.75em;
+}
+
div.ObjectSelectorButtons {
background-color: #e7e7f7;
padding: 0.35em 0.75em;
@@ -31,14 +41,22 @@
[ObjectSelector.html]
<div id="{id}" class="{class}">
{links}
-
- {filters}
-
{switches}
- <form method="post" action="?">
+ <form id="{id}.form" method="post" action="?">
+ {filters}
{buttons}
+ {table}
+
+ <div>{hidden_inputs}</div>
+ </form>
+ <script type="text/javascript">
+ cumin.restoreTableCheckboxes();
+ </script>
+</div>
+
+[ObjectSelectorTable.html]
<table id="{id}" class="{class}">
<style type="text/css">
{css}
@@ -48,14 +66,6 @@
<tbody>{rows}</tbody>
</table>
- <div>{hidden_inputs}</div>
- </form>
-
- <script type="text/javascript">
- cumin.restoreTableCheckboxes();
- </script>
-</div>
-
[ObjectSelectorControl.html]
<div class="{class}">
<ul>{widgets}</ul>
@@ -67,3 +77,9 @@
<ul>{widgets}</ul>
</div>
+
+[ObjectSelectorFilters.html]
+<div class="{class}">
+ <ul>{widgets}</ul>
+ <input type="submit" value="Search"/>
+</div><div style="clear:both;"></div>
Modified: trunk/cumin/python/cumin/qmfadapter.py
===================================================================
--- trunk/cumin/python/cumin/qmfadapter.py 2010-12-17 16:39:40 UTC (rev 4439)
+++ trunk/cumin/python/cumin/qmfadapter.py 2010-12-17 18:51:54 UTC (rev 4440)
@@ -94,6 +94,10 @@
self.fields_by_attr = dict()
+ def add_like_filter(self, attr):
+ #TODO: add sql LIKE filtering to QMF results
+ pass
+
class QmfField(DataAdapterField):
def __init__(self, adapter, column):
python_type = column.type.python_type
@@ -104,7 +108,6 @@
self.adapter.columns.append(column)
-
class ObjectQmfField(QmfField):
def __init__(self, adapter, attr):
assert isinstance(adapter, ObjectQmfAdapter), adapter
13 years, 4 months
r4439 - trunk/mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-17 16:39:40 +0000 (Fri, 17 Dec 2010)
New Revision: 4439
Modified:
trunk/mint/python/mint/update.py
Log:
Add a test for None value in check for forced write. Scale testing revealed errors generated when objects arrive with save_time or update_time equal to None.
Modified: trunk/mint/python/mint/update.py
===================================================================
--- trunk/mint/python/mint/update.py 2010-12-17 16:01:37 UTC (rev 4438)
+++ trunk/mint/python/mint/update.py 2010-12-17 16:39:40 UTC (rev 4439)
@@ -380,7 +380,9 @@
# force a write if it's been too long, even if the values match
if object_columns \
- or obj._save_time < obj._qmf_update_time - timedelta(hours=1):
+ or (obj._save_time != None and \
+ obj._qmf_update_time != None and \
+ obj._save_time < obj._qmf_update_time - timedelta(hours=1)):
object_columns.append(cls.sql_table._qmf_update_time)
sql = cls.sql_update_object.emit(object_columns)
13 years, 4 months
r4438 - branches
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-17 16:01:37 +0000 (Fri, 17 Dec 2010)
New Revision: 4438
Added:
branches/scale_testing/
Log:
Create a separate branch for instrumentation code for scale verification. May eventually be merged back into trunk.
Copied: branches/scale_testing (from rev 4437, trunk)
13 years, 4 months
r4437 - /
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-17 15:58:39 +0000 (Fri, 17 Dec 2010)
New Revision: 4437
Added:
branches/
Log:
Add a branch directory to the repository.
13 years, 4 months
r4436 - in trunk/cumin: bin python/cumin python/cumin/grid
by tmckay@fedoraproject.org
Author: tmckay
Date: 2010-12-15 14:11:58 +0000 (Wed, 15 Dec 2010)
New Revision: 4436
Modified:
trunk/cumin/bin/cumin-web
trunk/cumin/python/cumin/config.py
trunk/cumin/python/cumin/grid/submission.py
trunk/cumin/python/cumin/main.py
trunk/cumin/python/cumin/model.py
Log:
Add RequestMemory and RequestDisk fields to job submission form, with defaults.
Add RequestDisk field to VM job submission form, with default.
RequestMemory is already set to an expression for VM jobs.
These fields are necessary for scheduling to partitionable slots.
BZ 654305
Modified: trunk/cumin/bin/cumin-web
===================================================================
--- trunk/cumin/bin/cumin-web 2010-12-14 19:45:52 UTC (rev 4435)
+++ trunk/cumin/bin/cumin-web 2010-12-15 14:11:58 UTC (rev 4436)
@@ -32,6 +32,12 @@
cumin.user = values.web.user
cumin.update_interval = values.web.update_interval
+ # set default values for form inputs
+ cumin.set_form_defaults(values.web.request_memory,
+ values.web.request_memory_vm,
+ values.web.request_disk,
+ values.web.request_disk_vm)
+
cumin.check()
cumin.init()
Modified: trunk/cumin/python/cumin/config.py
===================================================================
--- trunk/cumin/python/cumin/config.py 2010-12-14 19:45:52 UTC (rev 4435)
+++ trunk/cumin/python/cumin/config.py 2010-12-15 14:11:58 UTC (rev 4436)
@@ -33,6 +33,15 @@
param = ConfigParameter(web, "user", str)
+ param = ConfigParameter(web, "request-memory", int)
+ param.default = 512 # MB
+ param = ConfigParameter(web, "request-memory-vm", int)
+ param.default = 512 # MB
+ param = ConfigParameter(web, "request-disk", int)
+ param.default = 1024 #MB
+ param = ConfigParameter(web, "request-disk-vm", int)
+ param.default = 5 * 1024 #MB
+
data = CuminConfigSection(self, "data")
data.log_file.default = os.path.join(self.home, "log", "data.log")
Modified: trunk/cumin/python/cumin/grid/submission.py
===================================================================
--- trunk/cumin/python/cumin/grid/submission.py 2010-12-14 19:45:52 UTC (rev 4435)
+++ trunk/cumin/python/cumin/grid/submission.py 2010-12-15 14:11:58 UTC (rev 4436)
@@ -326,6 +326,8 @@
stdout=None,
stderr=None,
usrlog=None,
+ request_mem_MB=None,
+ request_disk_KB=None,
attrs={}):
ad = dict()
@@ -345,6 +347,8 @@
put("Out", stdout)
put("Err", stderr)
put("UserLog", usrlog)
+ put("RequestMemory", request_mem_MB)
+ put("RequestDisk", request_disk_KB)
ad.update(standard_job_attributes)
ad.update(attrs)
@@ -416,6 +420,12 @@
self.usrlog.help = "User log file"
self.add_extra_field(self.usrlog)
+ self.request_mem_MB = self.RequestMemField(app, "requestmem")
+ self.add_extra_field(self.request_mem_MB)
+
+ self.request_disk_MB = self.RequestDiskField(app, "requestdisk")
+ self.add_extra_field(self.request_disk_MB)
+
#self.options = self.OptionsField(app, "options")
#self.add_extra_field(self.options)
@@ -423,6 +433,8 @@
self.add_extra_field(self.attrs)
def process_display(self, session):
+ self.request_mem_MB.set(session, self.app.form_defaults.request_memory)
+ self.request_disk_MB.set(session, self.app.form_defaults.request_disk)
self.scheduler.validate(session)
def process_submit(self, session):
@@ -441,6 +453,8 @@
stdout = self.stdout.get(session)
stderr = self.stderr.get(session)
usrlog = self.usrlog.get(session)
+ request_mem_MB = self.request_mem_MB.get(session)
+ request_disk_KB = self.request_disk_MB.get(session) * 1024
tokens = shlex.split(command)
@@ -459,6 +473,8 @@
stdout=stdout,
stderr=stderr,
usrlog=usrlog,
+ request_mem_MB=request_mem_MB,
+ request_disk_KB=request_disk_KB,
attrs=attrs)
self.task.exit_with_redirect(session, scheduler)
@@ -545,6 +561,22 @@
def render_title(self, session):
return "User Log"
+ class RequestMemField(IntegerField):
+ def render_title(self, session):
+ return "Request Memory"
+
+ def render_help(self, session):
+ return "Requested memory size (MB), required by " \
+ "partitionable slots."
+
+ class RequestDiskField(IntegerField):
+ def render_title(self, session):
+ return "Request Disk"
+
+ def render_help(self, session):
+ return "Requested disk space (MB), required by " \
+ "partitionable slots."
+
# class OptionsField(CheckboxField):
# def __init__(self, app, name):
# super(SubmissionAddForm.OptionsField, self).__init__(app, name)
@@ -573,7 +605,7 @@
return "Submit VM job"
def do_invoke(self, session, scheduler, invoc,
- description, image, memory, attrs={}):
+ description, image, memory, attrs={}, request_disk_KB=0):
ad = dict()
# General
@@ -588,6 +620,7 @@
"ceiling(ifThenElse(JobVMMemory =!= undefined," + \
"JobVMMemory, " + \
"ImageSize / 1024.000000))"
+ ad["RequestDisk"] = request_disk_KB
# VM
@@ -652,10 +685,11 @@
self.add_field(self.image)
self.memory = self.MemoryField(app, "memory")
- self.memory.help = "In megabytes"
self.memory.required = True
- self.memory.input.param.default = 512
self.add_extra_field(self.memory)
+
+ self.request_disk_MB = self.RequestDiskField(app, "requestdisk")
+ self.add_extra_field(self.request_disk_MB)
self.scheduler = JobSchedulerField(app, "scheduler", self.pool_name)
self.add_extra_field(self.scheduler)
@@ -665,6 +699,8 @@
def process_display(self, session):
self.scheduler.validate(session)
+ self.memory.set(session, self.app.form_defaults.request_memory_vm)
+ self.request_disk_MB.set(session, self.app.form_defaults.request_disk_vm)
def process_submit(self, session):
self.validate(session)
@@ -676,13 +712,15 @@
description = self.description.get(session)
image = self.image.get(session)
memory = self.memory.get(session)
+ request_disk_KB = self.request_disk_MB.get(session) * 1024
self.task.invoke(session,
scheduler,
description,
image,
memory,
- attrs=attrs)
+ attrs=attrs,
+ request_disk_KB=request_disk_KB)
self.task.exit_with_redirect(session, scheduler)
@@ -698,8 +736,15 @@
return "Memory"
def render_help(self, session):
- return "The VM's memory size in megabytes"
+ return "The VM's memory size (MB)"
+ class RequestDiskField(IntegerField):
+ def render_title(self, session):
+ return "Request Disk"
+
+ def render_help(self, session):
+ return "The VM's image size (MB), required by partitionable slots."
+
class DagJobSubmit(Task):
def __init__(self, app):
super(DagJobSubmit, self).__init__(app)
Modified: trunk/cumin/python/cumin/main.py
===================================================================
--- trunk/cumin/python/cumin/main.py 2010-12-14 19:45:52 UTC (rev 4435)
+++ trunk/cumin/python/cumin/main.py 2010-12-15 14:11:58 UTC (rev 4436)
@@ -54,6 +54,8 @@
self.operator_email = None
self.update_interval = 10
+ self.form_defaults = self.CuminFormDefaults()
+
# self.model.sql_logging_enabled = True
def check(self):
@@ -124,6 +126,23 @@
self.server.stop()
self.session.stop()
+ def set_form_defaults(self,
+ request_memory,
+ request_memory_vm,
+ request_disk,
+ request_disk_vm):
+ self.form_defaults.request_memory = request_memory
+ self.form_defaults.request_memory_vm = request_memory_vm
+ self.form_defaults.request_disk = request_disk
+ self.form_defaults.request_disk_vm = request_disk_vm
+
+ class CuminFormDefaults(object):
+ def __init__(self):
+ self.request_memory = 0
+ self.request_memory_vm = 0
+ self.request_disk = 0
+ self.request_disk_vm = 0
+
class CuminModule(object):
def __init__(self, app, name):
self.app = app
Modified: trunk/cumin/python/cumin/model.py
===================================================================
--- trunk/cumin/python/cumin/model.py 2010-12-14 19:45:52 UTC (rev 4435)
+++ trunk/cumin/python/cumin/model.py 2010-12-15 14:11:58 UTC (rev 4436)
@@ -1614,6 +1614,14 @@
prop.group = "Other"
prop.writable = False
+ prop = AdProperty(self, "RequestMemory")
+ prop.description = "Required for scheduling to partitionable slots."
+ prop.group = "Other"
+
+ prop = AdProperty(self, "RequestDisk")
+ prop.description = "Required for scheduling to partitionable slots."
+ prop.group = "Other"
+
class JobStatusProperty(AdProperty):
def render_status(self, session, status):
return JobStatusInfo.get_status_string(status)
13 years, 4 months