r4635 - branches/scale_testing/mint/python/mint
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-03-17 20:17:28 +0000 (Thu, 17 Mar 2011)
New Revision: 4635
Modified:
branches/scale_testing/mint/python/mint/update.py
Log:
Made latency for delete work.
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2011-03-17 19:30:33 UTC (rev 4634)
+++ branches/scale_testing/mint/python/mint/update.py 2011-03-17 20:17:28 UTC (rev 4635)
@@ -468,7 +468,7 @@
return
if delete_time != 0:
- self.delete_object(cursor, stats, obj)
+ self.delete_object(cursor, stats, obj, delete_time)
del agent.objects_by_id[obj._qmf_object_id]
@@ -669,7 +669,11 @@
stats.objects_updated += 1
stats.objects_updated_by_class[cls] += 1
- def delete_object(self, cursor, stats, obj):
+ def delete_object(self, cursor, stats, obj, delete_time):
+
+ self.latency_cls = obj._class._name
+ self.object_latency = time.time() - delete_time / 1000000000
+
obj.delete(cursor)
self.model.print_event(3, "Deleted %s", obj)
13 years, 2 months
r4634 - in branches/scale_testing: cumin/python/cumin mint/python/mint
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-03-17 19:30:33 +0000 (Thu, 17 Mar 2011)
New Revision: 4634
Modified:
branches/scale_testing/cumin/python/cumin/config.py
branches/scale_testing/mint/python/mint/update.py
Log:
Have to exclude deletes from cumulative latency totals for update/create
Modified: branches/scale_testing/cumin/python/cumin/config.py
===================================================================
--- branches/scale_testing/cumin/python/cumin/config.py 2011-03-17 18:29:41 UTC (rev 4633)
+++ branches/scale_testing/cumin/python/cumin/config.py 2011-03-17 19:30:33 UTC (rev 4634)
@@ -234,7 +234,7 @@
def setup_operational_logging(values):
- if values.bin_class != "":
+ if hasattr(values,"bin_class") and values.bin_class != "":
home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
enable_logging("mint_scale.update", "info", os.path.join(home, "log", "mint.update_" + str(values.my_bin) + ".log"))
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2011-03-17 18:29:41 UTC (rev 4633)
+++ branches/scale_testing/mint/python/mint/update.py 2011-03-17 19:30:33 UTC (rev 4634)
@@ -152,6 +152,7 @@
self.initial_process_time = 0
self.commit_time = 0
self.record_samples = 0
+ self.latency_samples = 0
self.depth = 0
for k, v in self.objects_latency_by_class.iteritems():
@@ -178,6 +179,7 @@
self.enqueued - self.dequeued
if update.latency_cls != None:
+ self.latency_samples += 1
self.objects_latency_by_class[update.latency_cls] += \
update.object_latency
@@ -302,11 +304,11 @@
def write_latency_by_class(self, mint, localtime_):
names = ("Time", "Latency")
- if self.now.record_samples == 0:
+ if self.now.latency_samples == 0:
return
for name, latency in self.objects_latency_by_class.iteritems():
- latency /= self.now.record_samples
+ latency /= self.now.latency_samples
fname = self.latency_stats[name]
write_header = not os.path.isfile(fname)
db = csv.writer(open(fname, "a+"))
13 years, 2 months
r4633 - in trunk/cumin/python/cumin: . grid
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-17 18:29:41 +0000 (Thu, 17 Mar 2011)
New Revision: 4633
Modified:
trunk/cumin/python/cumin/grid/dashboard.py
trunk/cumin/python/cumin/grid/dashboard.strings
trunk/cumin/python/cumin/objectselector.strings
Log:
Made the slot capacity page a vertical stats table instead of a objectselector table.
Moved the chart and csv icons to the bottom of the stats tables for consistency with the objectselector tables.
Modified: trunk/cumin/python/cumin/grid/dashboard.py
===================================================================
--- trunk/cumin/python/cumin/grid/dashboard.py 2011-03-17 18:20:16 UTC (rev 4632)
+++ trunk/cumin/python/cumin/grid/dashboard.py 2011-03-17 18:29:41 UTC (rev 4633)
@@ -15,6 +15,7 @@
from negotiator import NegotiatorAttribute
from cumin.grid.slotvis import PoolSlotMap, PoolSlotFlashVis
from wooly.template import WidgetTemplate
+from cumin.formats import fmt_bytes
strings = StringCatalog(__file__)
log = logging.getLogger("cumin.dashboard")
@@ -131,6 +132,8 @@
# if this gets defined, this widget will have a link to display
# the chart in a popup
self.chart_link = None
+ self.popup_chart = self.DefinitionSetPopup(app, "popup_chart")
+ self.add_child(self.popup_chart)
self.pie_tmpl = WidgetTemplate(self, "pie_html")
@@ -171,13 +174,9 @@
dt_class = self.render_legend_class(session, item)
return "total" is dt_class and " class=\"total\"" or ""
- # don't show a chart icon on hover if there is no chart
- def render_span_class(self, session):
- return self.chart_link and "chart" or "blank"
-
- def render_chart_href(self, session):
+ def render_popup_chart(self, session):
if self.chart_link:
- return self.page.get_popup_url(session, self.chart_link)
+ return self.popup_chart.render(session)
def render_chart(self, session):
if self.has_pie_chart:
@@ -199,8 +198,18 @@
def get_pie_slices(self, session):
# don't include total lines, convert None to 0
items = self.do_get_items(session)
- return [self.render_item_value(session, x) or 0 for x in items if not self.is_total(x)]
+ return [self.get_item_value(session, x) or 0 for x in items if not self.is_total(x)]
+ class DefinitionSetPopup(Widget):
+ def __init__(self, app, name):
+ super(PieStatSet.DefinitionSetPopup, self).__init__(app, name)
+
+ def render_chart_href(self, session):
+ return self.page.get_popup_url(session, self.parent.chart_link)
+
+ def render_title(self, session):
+ return self.parent.render_title(session)
+
class AliasSqlColumn(object):
def __init__(self, identifier, alias):
self.identifier = identifier
@@ -386,22 +395,25 @@
def do_get_items(self, session):
return self.columns
+ def get_item_index(self, item):
+ return self.sum_columns.index(item.name)
+
def is_total(self, item):
- index = self.sum_columns.index(item.name)
+ index = self.get_item_index(item)
return index == len(self.sum_columns) - 1
def render_item_title(self, session, item):
return item.title
- def render_item_value(self, session, item):
+ def get_item_value(self, session, item):
record = self.get_record(session)
- return record[self.sum_columns.index(item.name)] or 0
+ return record[self.get_item_index(item)] or 0
def render_html_title(self, session, item):
return item.description
def get_legend_class(self, session, item):
- index = self.sum_columns.index(item.name)
+ index = self.get_item_index(item)
return "legend%s" % rgb_to_string(*(PieChartPage.color_schemes[self.color_scheme][index]))
def get_record(self, session):
@@ -425,14 +437,73 @@
def render_title(self, session):
return "Job summary info"
-class DashboardCapacitySlotSummary(ObjectSelector):
+class DashboardCapacitySlotSummary(DashboardSummarySet):
+ PERCENT_COLUMN = "% Memory used"
def __init__(self, app, name, collector):
+ self.cls = app.model.com_redhat_grid.Slot
+ columns = [self.cls.Disk, self.cls.ImageSize, self.cls.Memory]
+
+ super(DashboardCapacitySlotSummary, self).__init__(app, name, collector, self.cls, columns, "slot.summary")
+
+ self.has_pie_chart = False
+ self.has_total = False
+
+ def render_title(self, session):
+ return "Slot capacity"
+
+ def do_get_items(self, session):
+ cols = list(self.columns)
+ cols.append(self.PERCENT_COLUMN)
+ return cols
+
+ def get_item_index(self, item):
+ if item is self.PERCENT_COLUMN:
+ return 0
+ return super(DashboardCapacitySlotSummary, self).get_item_index(item)
+
+ def is_total(self, item):
+ return False
+
+ def render_item_title(self, session, item):
+ if item is self.PERCENT_COLUMN:
+ return self.PERCENT_COLUMN
+ return item.title
+
+ def get_item_value(self, session, item):
+ record = self.get_record(session)
+ if item is self.PERCENT_COLUMN:
+ try:
+ used_index = self.get_item_index(self.cls.ImageSize)
+ total_index = self.get_item_index(self.cls.Memory)
+ used = record[used_index]
+ total = record[total_index]
+ return float(used/total) * 100.0
+ except:
+ return 0
+
+ return super(DashboardCapacitySlotSummary, self).get_item_value(session, item)
+
+ def render_formatted_value(self, session, item, value):
+ if item is self.PERCENT_COLUMN:
+ return "%0.02f%%" % value
+ return super(DashboardCapacitySlotSummary, self).render_formatted_value(session, item, value)
+
+ def render_html_title(self, session, item):
+ if item is self.PERCENT_COLUMN:
+ return item
+ return item.description
+
+ def get_legend_class(self, session, item):
+ return "blank"
+
+class DashboardCapacitySlotSummary_(ObjectSelector):
+ def __init__(self, app, name, collector):
cls = app.model.com_redhat_grid.Slot
super(DashboardCapacitySlotSummary, self).__init__(app, name, cls)
self.add_filter(collector, cls.Pool)
- stat = self.DiskTotalColumn(app, "disk", cls.Disk, "Disk total")
+ stat = self.DiskTotalColumn(app, "disk", cls.Disk, "Available disk")
stat.static_header = True
self.add_column(stat)
@@ -589,10 +660,13 @@
_, record = item
return record[0]
- def render_item_value(self, session, item):
+ def get_item_value(self, session, item):
_, record = item
return record[1]
+ def render_item_value(self, session, item):
+ return self.get_item_value(session, item)
+
def render_html_title(self, session, item):
return None
@@ -625,13 +699,6 @@
index = self.attrs.index(item[0].name)
return "legend%s" % rgb_to_string(*(PieChartPage.color_schemes[self.color_scheme][index]))
- def render_item_value(self, session, item):
- _, value = item
- if value is None:
- return 0
- return super(NegotiatorDefinitionStatSet, self).render_item_value(session, item)
-
-
class NegotiatorJobThroughput(Widget):
def __init__(self, app, name, collector):
super(NegotiatorJobThroughput, self).__init__(app, name)
Modified: trunk/cumin/python/cumin/grid/dashboard.strings
===================================================================
--- trunk/cumin/python/cumin/grid/dashboard.strings 2011-03-17 18:20:16 UTC (rev 4632)
+++ trunk/cumin/python/cumin/grid/dashboard.strings 2011-03-17 18:29:41 UTC (rev 4633)
@@ -55,7 +55,6 @@
}
[DashboardCapacity.html]
<div id="{id}" class="{class}">
-<h3>{title}</h3>
{slot_capacity}
</div>
@@ -73,7 +72,7 @@
border-bottom: 1px solid #999999;
clear: left;
float: right;
- margin: 0 0 1em;
+ margin: 0 0 0.5em;
padding: 0;
}
.DefinitionSet.hastotal dl {
@@ -126,31 +125,31 @@
display: block;
}
-.DefinitionSet span.chart {
- display: block;
- float: right;
- width: 20px;
- height: 15px;
- background: url(resource?name=chart-20.png) repeat;
- margin-right: 4px;
- cursor: pointer;
-}
-
.DefinitionSet span.chart.blank {
cursor: default;
visibility: hidden;
}
-.DefinitionSet div.ExportButton img {
+.DefinitionSet div.icons img {
margin: 0 4px;
+ float: none;
+ cursor: pointer;
}
-.DefinitionSet div.ExportButton span {
- display: none;
+.DefinitionSet div.icons span {
+ font-style: italic;
+ font-size: 80%;
+ padding-right: 4px;
+ position: relative;
+ top: -2px;
}
.DefinitionSet div.icons {
- margin-top: 0.5em;
+ margin-bottom: 1em;
+ height: 1em;
+ padding: 0.5em;
+ background-color: #F0F0F0;
+ clear: right;
}
/* hide the chart title since we have one in the popup title bar */
@@ -163,17 +162,25 @@
<style type="text/css">
{legend_styles}
</style>
- <div class="icons">
- {export}
- <span class="{span_class} chart" title="Click to show a timeseries chart" onclick="showHistoryChart('{title}', '{chart_href}')"></span>
- </div>
<h3>{title}</h3>
<dl>
{items}
</dl>
{chart}
+ <div class="icons">
+ {export}
+ {popup_chart}
+ </div>
</div>
+[DefinitionSetPopup.html]
+<div class="{class}">
+ <span>Show a timeseries chart</span>
+ <img src="resource?name=chart-20.png"
+ onclick="return showHistoryChart('{title}', '{chart_href}')"
+ title="Click to show a timeseries chart" alt="" border="0"/>
+</div>
+
[DefinitionSet.pie_html]
<img src="{pie_src}" alt="pie_chart" title="{title}" width="{width}" height="{height}"/>
@@ -191,11 +198,11 @@
div.NegotiatorJobThroughput div.timeseries,
div.NegotiatorJobThroughput div.stats {
- float: left;
+ float: right;
}
div.NegotiatorJobThroughput div.stats {
- margin-left: 14em;
+ margin-right: 5.3em;
}
[NegotiatorJobThroughput.html]
@@ -225,13 +232,6 @@
<div style="clear:both;"></div>
</div>
-[NegotiatorStatSet.item_html]
-<tr title="{html_title}" class="item">
- <th><span class="{chart_class}" onclick="showHistoryChart('{item_title}', '{chart_href}')" title="{span_title}"></span>{item_title}</th>
- <td class="ralign"> {item_value}</td>
-</tr>
-
-
[Visualization.css]
div#flashSlotMap, div#pngSlotMap {
display: none;
Modified: trunk/cumin/python/cumin/objectselector.strings
===================================================================
--- trunk/cumin/python/cumin/objectselector.strings 2011-03-17 18:20:16 UTC (rev 4632)
+++ trunk/cumin/python/cumin/objectselector.strings 2011-03-17 18:29:41 UTC (rev 4633)
@@ -119,7 +119,7 @@
}
[ExportButton.html]
-<div class="{class}"><span>Save to CSV File</span> <a href="{href}"><img src="resource?name=save.png" alt="save as csv" title="Save table as CSV file" valign="middle" border="0"/></a></div>
+<div class="{class}"><span>Save to CSV file</span> <a href="{href}"><img src="resource?name=save.png" alt="save as csv" title="Save table as CSV file" valign="middle" border="0"/></a></div>
[ObjectQmfSelector.css]
div.TaskInvocationSet ul {
13 years, 2 months
r4632 - trunk/cumin/python/cumin
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-17 18:20:16 +0000 (Thu, 17 Mar 2011)
New Revision: 4632
Modified:
trunk/cumin/python/cumin/stat.py
Log:
Format stats using the unit when available
Modified: trunk/cumin/python/cumin/stat.py
===================================================================
--- trunk/cumin/python/cumin/stat.py 2011-03-17 18:19:33 UTC (rev 4631)
+++ trunk/cumin/python/cumin/stat.py 2011-03-17 18:20:16 UTC (rev 4632)
@@ -39,7 +39,7 @@
return stats
def render_item_title(self, session, item):
- stat, value = item
+ stat, _ = item
return stat.title
def render_html_title(self, session, item):
@@ -47,7 +47,33 @@
return stat.description
def render_item_value(self, session, item):
- stat, value = item
+ value = self.get_item_value(session, item)
+ return self.render_formatted_value(session, item, value)
+
+ def get_item_value(self, session, item):
+ _, value = item
+ return value
+
+ def render_formatted_value(self, session, item, value):
+ def fmt_b(value):
+ return value is not None and fmt_bytes(value) or None
+
+ def fmt_kb(value):
+ return value is not None and fmt_bytes(value * 1024) or None
+
+ def fmt_mb(value):
+ return value is not None and fmt_bytes(value * 1024 * 1024) or None
+
+ try:
+ format = item.unit
+ if format:
+ if format == "MiB":
+ return fmt_mb(value)
+ elif format == "KiB":
+ return fmt_kb(value)
+ except:
+ pass
+
return CuminStatistic.fmt_value(value)
class NewStatSet(ItemSet):
13 years, 2 months
r4631 - trunk/cumin/model
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-17 18:19:33 +0000 (Thu, 17 Mar 2011)
New Revision: 4631
Modified:
trunk/cumin/model/rosemary.xml
Log:
Override title for MonitorSelfRegisteredSocketCount for Scheduler
Modified: trunk/cumin/model/rosemary.xml
===================================================================
--- trunk/cumin/model/rosemary.xml 2011-03-17 18:18:45 UTC (rev 4630)
+++ trunk/cumin/model/rosemary.xml 2011-03-17 18:19:33 UTC (rev 4631)
@@ -310,6 +310,10 @@
<statistic name="MonitorSelfImageSize">
<title>Image size</title>
</statistic>
+
+ <statistic name="MonitorSelfRegisteredSocketCount">
+ <title>Registered socket count</title>
+ </statistic>
</class>
<class name="Collector">
13 years, 2 months
r4630 - trunk/rosemary/python/rosemary
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-17 18:18:45 +0000 (Thu, 17 Mar 2011)
New Revision: 4630
Modified:
trunk/rosemary/python/rosemary/model.py
Log:
Remove multiple consecutive spaces from the schema descriptions
Modified: trunk/rosemary/python/rosemary/model.py
===================================================================
--- trunk/rosemary/python/rosemary/model.py 2011-03-17 17:13:55 UTC (rev 4629)
+++ trunk/rosemary/python/rosemary/model.py 2011-03-17 18:18:45 UTC (rev 4630)
@@ -539,7 +539,7 @@
self.unit = elem.get("unit")
self.index = elem.get("index", "n") == "y" and True
self.optional = elem.get("optional", "n") == "y" and True
- self.description = elem.get("desc")
+ self.description = clean_description(elem.get("desc"))
def extend(self, elem):
self.title = elem.findtext("title")
@@ -675,7 +675,7 @@
self.arguments_by_name = dict()
def load(self, elem):
- self.description = elem.get("desc")
+ self.description = clean_description(elem.get("desc"))
for child in elem.findall("arg"):
arg = RosemaryArgument(self, child.get("name"))
@@ -703,7 +703,7 @@
def load(self, elem):
self.type = elem.get("type")
self.direction = elem.get("dir")
- self.description = elem.get("desc")
+ self.description = clean_description(elem.get("desc"))
def extend(self, elem):
pass
@@ -839,3 +839,6 @@
prev = curr
return "".join(chars)
+
+def clean_description(description):
+ return description and " ".join(description.split()) or description
13 years, 2 months
r4629 - in branches/scale_testing: cumin/bin mint/python/mint
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-03-17 17:13:55 +0000 (Thu, 17 Mar 2011)
New Revision: 4629
Modified:
branches/scale_testing/cumin/bin/cumin-data
branches/scale_testing/mint/python/mint/update.py
Log:
Add facility for tracking average latency by class in csv files
Modified: branches/scale_testing/cumin/bin/cumin-data
===================================================================
--- branches/scale_testing/cumin/bin/cumin-data 2011-03-16 20:29:58 UTC (rev 4628)
+++ branches/scale_testing/cumin/bin/cumin-data 2011-03-17 17:13:55 UTC (rev 4629)
@@ -144,6 +144,7 @@
if opts.scale_stats:
stats.write_values_by_class(mint, time.localtime())
+ stats.write_latency_by_class(mint, time.localtime())
stats.write_values(time.localtime())
sleep(5)
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2011-03-16 20:29:58 UTC (rev 4628)
+++ branches/scale_testing/mint/python/mint/update.py 2011-03-17 17:13:55 UTC (rev 4629)
@@ -62,6 +62,8 @@
update = self.updates.get(True) #, 1)
except Empty:
continue
+ update.object_latency = 0
+ update.latency_cls = None
update.dequeue_time = time.time()
self.stats.dequeued += 1
@@ -116,6 +118,8 @@
self.objects_deleted_by_class = defaultdict(int)
self.objects_dropped_by_class = defaultdict(int)
+ self.objects_latency_by_class = defaultdict(int)
+
for pkg in app.model._packages:
for cls in pkg._classes:
self.objects_created_by_class[cls] = 0
@@ -123,6 +127,8 @@
self.objects_deleted_by_class[cls] = 0
self.objects_dropped_by_class[cls] = 0
+ self.objects_latency_by_class[cls._name] = 0
+
self.sql_ops = 0
self.errors = 0
@@ -147,6 +153,10 @@
self.commit_time = 0
self.record_samples = 0
self.depth = 0
+
+ for k, v in self.objects_latency_by_class.iteritems():
+ self.objects_latency_by_class[k] = 0
+
self.init_update_flag = False
def record_update_times(self,update):
@@ -166,6 +176,11 @@
update.after_commit - update.after_process
self.depth += \
self.enqueued - self.dequeued
+
+ if update.latency_cls != None:
+ self.objects_latency_by_class[update.latency_cls] += \
+ update.object_latency
+
except:
print "record update exception"
print_exc()
@@ -258,10 +273,13 @@
# init the paths for stats by class
self.class_stats_paths = {}
+ self.latency_stats = {}
for pkg in mint.model._packages:
for cls in pkg._classes:
self.class_stats_paths[cls._name] = \
os.path.join(p, cls._name+".csv")
+ self.latency_stats[cls._name] = \
+ os.path.join(p, cls._name+"latency.csv")
def write_values_by_class(self, mint, localtime_):
names = ("Time", "Created", "Updated", "Deleted", "Dropped")
@@ -282,6 +300,20 @@
db.writerow((time.strftime("%H:%M:%S",localtime_),
created, updated, deleted, dropped))
+ def write_latency_by_class(self, mint, localtime_):
+ names = ("Time", "Latency")
+ if self.now.record_samples == 0:
+ return
+
+ for name, latency in self.objects_latency_by_class.iteritems():
+ latency /= self.now.record_samples
+ fname = self.latency_stats[name]
+ write_header = not os.path.isfile(fname)
+ db = csv.writer(open(fname, "a+"))
+ if write_header:
+ db.writerow(names)
+ db.writerow((time.strftime("%H:%M:%S",localtime_),latency))
+
def get_update_durations(self):
total = self.now.record_samples
if total == 0 or self.now.dequeued == self.then.dequeued:
@@ -493,6 +525,10 @@
update_time, create_time, delete_time = self.qmf_object.getTimestamps()
bin_time = create_time / 1000000000
+ self.latency_cls = cls._name
+ self.object_latency = time.time() - update_time / 1000000000
+
+
# experimental binning code
if stats.binMod > 0 and cls._name == stats.binClass:
bin_time = int(floor(bin_time))
@@ -572,6 +608,9 @@
update_time, create_time, delete_time = self.qmf_object.getTimestamps()
bin_time = create_time / 1000000000
+ self.latency_cls = obj._class._name
+ self.object_latency = time.time() - update_time / 1000000000
+
# experimental binning code
if stats.binMod > 0 and obj._class._name == stats.binClass:
bin_time = int(floor(bin_time))
13 years, 2 months
r4628 - in branches/scale_testing: cumin/bin cumin/python/cumin mint/python/mint
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-03-16 20:29:58 +0000 (Wed, 16 Mar 2011)
New Revision: 4628
Modified:
branches/scale_testing/cumin/bin/cumin-data
branches/scale_testing/cumin/python/cumin/config.py
branches/scale_testing/mint/python/mint/main.py
branches/scale_testing/mint/python/mint/update.py
Log:
Allowing binning of QMF objects within class based on modulo
of object creation time (proof of concept for intraclass partitioning)
Modified: branches/scale_testing/cumin/bin/cumin-data
===================================================================
--- branches/scale_testing/cumin/bin/cumin-data 2011-03-16 20:11:51 UTC (rev 4627)
+++ branches/scale_testing/cumin/bin/cumin-data 2011-03-16 20:29:58 UTC (rev 4628)
@@ -97,7 +97,8 @@
broker_uris = [x.strip() for x in opts.brokers.split(",")]
- mint = Mint(model_dir, broker_uris, opts.database)
+ # test binning code for slots
+ mint = Mint(model_dir, broker_uris, opts.database, binMod = opts.bin_mod, myBin = opts.my_bin, binClass = opts.bin_class)
mint.print_event_level = opts.print_events
@@ -118,10 +119,11 @@
if opts.init_only:
return
+
+ stats = mint.update_thread.stats
mint.start()
- stats = mint.update_thread.stats
count = 0
if opts.print_stats:
Modified: branches/scale_testing/cumin/python/cumin/config.py
===================================================================
--- branches/scale_testing/cumin/python/cumin/config.py 2011-03-16 20:11:51 UTC (rev 4627)
+++ branches/scale_testing/cumin/python/cumin/config.py 2011-03-16 20:29:58 UTC (rev 4628)
@@ -92,6 +92,10 @@
param = ConfigParameter(data, "vacuum-interval", int)
param.default = 60 * 60 # 1 hour
+ param = ConfigParameter(data, "my-bin", int)
+ param = ConfigParameter(data, "bin-mod", int)
+ param = ConfigParameter(data, "bin-class", str)
+
class CuminConfigSection(ConfigSection):
def __init__(self, config, name, strict_section=False):
super(CuminConfigSection, self).__init__(config, name, strict_section)
@@ -134,7 +138,8 @@
def _apply_default(self, section, name):
try:
val = getattr(section, name)
- self.set_default(name, val)
+ if val != None:
+ self.set_default(name, val)
except:
pass
@@ -172,7 +177,18 @@
self.add_option("--print-stats", action="store_true")
self.add_option("--print-events", type="int", default=0, metavar="LEVEL")
self.add_option("--scale-stats", action="store_true", default=False)
+ self.add_option("--bin-mod", type="int", default=0)
+ self.add_option("--my-bin", type="int", default=0)
+ self.add_option("--bin-class", type=str, default="")
+ def set_defaults_from_section(self, section):
+ super(CuminDataOptionParser, self).set_defaults_from_section(section)
+
+ # For web, these options also have defaults in config files
+ self._apply_default(section, "bin_mod")
+ self._apply_default(section, "my_bin")
+ self._apply_default(section, "bin_class")
+
def get_configuration(parser, ConfigType):
# First parse should return a "section" value
opts, args = parser.parse_args()
@@ -217,6 +233,11 @@
enable_logging(name, "warn", sys.stderr)
def setup_operational_logging(values):
+
+ if values.bin_class != "":
+ home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
+ enable_logging("mint_scale.update", "info", os.path.join(home, "log", "mint.update_" + str(values.my_bin) + ".log"))
+
for name in _logging_modules:
disable_logging(name)
enable_logging(name, values.log_level, values.log_file,
Modified: branches/scale_testing/mint/python/mint/main.py
===================================================================
--- branches/scale_testing/mint/python/mint/main.py 2011-03-16 20:11:51 UTC (rev 4627)
+++ branches/scale_testing/mint/python/mint/main.py 2011-03-16 20:29:58 UTC (rev 4628)
@@ -10,14 +10,14 @@
log = logging.getLogger("mint.main")
class Mint(object):
- def __init__(self, model_dir, broker_uris, database_dsn):
+ def __init__(self, model_dir, broker_uris, database_dsn, binMod = 0, myBin = 0, binClass = ""):
self.model = MintModel(self, model_dir)
self.model.sql_logging_enabled = False
self.session = MintSession(self, broker_uris)
self.database = MintDatabase(self, database_dsn)
- self.update_thread = UpdateThread(self)
+ self.update_thread = UpdateThread(self, binMod = binMod, myBin = myBin, binClass = binClass)
self.expire_enabled = True
self.expire_thread = ExpireThread(self)
Modified: branches/scale_testing/mint/python/mint/update.py
===================================================================
--- branches/scale_testing/mint/python/mint/update.py 2011-03-16 20:11:51 UTC (rev 4627)
+++ branches/scale_testing/mint/python/mint/update.py 2011-03-16 20:29:58 UTC (rev 4628)
@@ -3,6 +3,7 @@
import pickle
import os.path
import csv
+from math import floor
from psycopg2 import IntegrityError, TimestampFromTicks
from psycopg2.extensions import cursor as Cursor
@@ -11,17 +12,17 @@
from model import *
from util import *
-log = logging.getLogger("mint.update")
+log = logging.getLogger("mint_scale.update")
sample_window_min = 60
sample_window_max = 60 * 5
class UpdateThread(MintDaemonThread):
- def __init__(self, app, queue_max=1000):
+ def __init__(self, app, queue_max=1000, binMod = 0, myBin = 0, binClass = ""):
super(UpdateThread, self).__init__(app)
self.updates = ConcurrentQueue(maxsize=queue_max)
- self.stats = UpdateStats(self.app)
+ self.stats = UpdateStats(self.app, binMod, myBin, binClass)
self.conn = None
self.cursor = None
@@ -95,7 +96,7 @@
then = None
now = None
- def __init__(self, app):
+ def __init__(self, app, binMod = 0, myBin = 0, binClass = ""):
self.enqueued = 0
self.dequeued = 0
self.dropped = 0
@@ -130,6 +131,12 @@
self.memory = 0
self.init_update_times()
+ self.slot_create_bins = defaultdict(int)
+ self.slot_update_bins = defaultdict(int)
+ self.myBin = myBin
+ self.binMod = binMod
+ self.binClass = binClass
+
def init_update_times(self):
# Track average time of update from arrival to
# processing complete. Refreshed per interval.
@@ -484,6 +491,20 @@
def create_object(self, cursor, stats, cls):
update_time, create_time, delete_time = self.qmf_object.getTimestamps()
+ bin_time = create_time / 1000000000
+
+ # experimental binning code
+ if stats.binMod > 0 and cls._name == stats.binClass:
+ bin_time = int(floor(bin_time))
+ slotBin = bin_time % stats.binMod
+ if slotBin != stats.myBin:
+ #log.info("Create, skip creation " + stats.binClass)
+ return
+ else:
+ stats.slot_create_bins[slotBin] += 1
+ if stats.slot_create_bins[slotBin] % 50 == 0:
+ log.info("Create, bin counts, " + stats.binClass + " " + str(stats.slot_create_bins))
+
create_time = datetime.fromtimestamp(create_time / 1000000000)
update_time = datetime.fromtimestamp(update_time / 1000000000)
@@ -549,6 +570,20 @@
def update_object(self, cursor, stats, obj):
update_time, create_time, delete_time = self.qmf_object.getTimestamps()
+ bin_time = create_time / 1000000000
+
+ # experimental binning code
+ if stats.binMod > 0 and obj._class._name == stats.binClass:
+ bin_time = int(floor(bin_time))
+ slotBin = bin_time % stats.binMod
+ if slotBin != stats.myBin:
+ #log.info("Update, skip " + stats.binClass)
+ raise UpdateDropped()
+ else:
+ stats.slot_update_bins[slotBin] += 1
+ if stats.slot_update_bins[slotBin] % 50 == 0:
+ log.info("Update, bin counts, " + stats.binClass + " " + str(stats.slot_update_bins))
+
update_time = datetime.fromtimestamp(update_time / 1000000000)
obj._qmf_update_time = update_time
13 years, 2 months
r4627 - trunk/cumin/python/cumin/grid
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-16 20:11:51 +0000 (Wed, 16 Mar 2011)
New Revision: 4627
Modified:
trunk/cumin/python/cumin/grid/dashboard.strings
Log:
Implement csv export on dashboard stats tables.
Modified: trunk/cumin/python/cumin/grid/dashboard.strings
===================================================================
--- trunk/cumin/python/cumin/grid/dashboard.strings 2011-03-16 20:11:08 UTC (rev 4626)
+++ trunk/cumin/python/cumin/grid/dashboard.strings 2011-03-16 20:11:51 UTC (rev 4627)
@@ -126,21 +126,33 @@
display: block;
}
-.DefinitionSet h3 span {
+.DefinitionSet span.chart {
display: block;
float: right;
width: 20px;
height: 15px;
background: url(resource?name=chart-20.png) repeat;
- margin-right:1em;
+ margin-right: 4px;
cursor: pointer;
}
-.DefinitionSet h3 span.blank {
+.DefinitionSet span.chart.blank {
cursor: default;
visibility: hidden;
}
+.DefinitionSet div.ExportButton img {
+ margin: 0 4px;
+}
+
+.DefinitionSet div.ExportButton span {
+ display: none;
+}
+
+.DefinitionSet div.icons {
+ margin-top: 0.5em;
+}
+
/* hide the chart title since we have one in the popup title bar */
div.DefaultStickyWin div.StatValueChart h2 {
display: none;
@@ -151,7 +163,11 @@
<style type="text/css">
{legend_styles}
</style>
- <h3>{title}<span class="{span_class}" title="Click to show a timeseries chart" onclick="showHistoryChart('{title}', '{chart_href}')"></span></h3>
+ <div class="icons">
+ {export}
+ <span class="{span_class} chart" title="Click to show a timeseries chart" onclick="showHistoryChart('{title}', '{chart_href}')"></span>
+ </div>
+ <h3>{title}</h3>
<dl>
{items}
</dl>
13 years, 2 months
r4626 - in trunk/cumin/python/cumin: . grid
by eallen＠fedoraproject.org
Author: eallen
Date: 2011-03-16 20:11:08 +0000 (Wed, 16 Mar 2011)
New Revision: 4626
Modified:
trunk/cumin/python/cumin/grid/dashboard.py
trunk/cumin/python/cumin/objectselector.py
Log:
Allow Stat tables to be exported as csv files
Modified: trunk/cumin/python/cumin/grid/dashboard.py
===================================================================
--- trunk/cumin/python/cumin/grid/dashboard.py 2011-03-16 17:03:54 UTC (rev 4625)
+++ trunk/cumin/python/cumin/grid/dashboard.py 2011-03-16 20:11:08 UTC (rev 4626)
@@ -10,7 +10,7 @@
from cumin.stat import StatSet, PieChartPage, StatFlashChart
from rosemary.sqlquery import SqlQueryOptions
from cumin.objectselector import ObjectSelector, MonitorSelfStatColumn, ObjectTableColumn, ObjectTable,\
- ObjectLinkColumn
+ ObjectLinkColumn, CsvStatsExporter, ExportButton
from cumin.util import rgb_to_string
from negotiator import NegotiatorAttribute
from cumin.grid.slotvis import PoolSlotMap, PoolSlotFlashVis
@@ -104,8 +104,14 @@
return "Slot capacity"
class DefinitionSet(StatSet):
- pass
+ def __init__(self, app, name, object, title):
+ super(DefinitionSet, self).__init__(app, name, object)
+ exporter = CsvStatsExporter(self.app, self.name + "_csv", (object,), self)
+
+ self.export = ExportButton(self.app, "export", (object,), exporter, title)
+ self.add_child(self.export)
+
# Displays a pie chart and a set of stats as the legend
# The legend is a definition list with the dt being the stat name
# and the dd being the value
@@ -113,8 +119,8 @@
# the stat isn't put in the pie chart
class PieStatSet(DefinitionSet):
radius = 60
- def __init__(self, app, name, object):
- super(PieStatSet, self).__init__(app, name, object)
+ def __init__(self, app, name, object, title="stats"):
+ super(PieStatSet, self).__init__(app, name, object, title)
self.color_scheme = PieChartPage.RAINBOW
@@ -362,8 +368,8 @@
return records[0]
class DashboardSummarySet(PieStatSet):
- def __init__(self, app, name, collector, cls, columns):
- super(DashboardSummarySet, self).__init__(app, name, collector)
+ def __init__(self, app, name, collector, cls, columns, title):
+ super(DashboardSummarySet, self).__init__(app, name, collector, title)
self.record = Attribute(app, "totals")
self.add_attribute(self.record)
@@ -377,12 +383,6 @@
self.has_total = True
self.color_scheme = PieChartPage.BLUES
- def do_process(self, session):
- record = self.data.get_record(session)
- self.record.set(session, record)
-
- super(DashboardSummarySet, self).do_process(session)
-
def do_get_items(self, session):
return self.columns
@@ -394,7 +394,7 @@
return item.title
def render_item_value(self, session, item):
- record = self.record.get(session)
+ record = self.get_record(session)
return record[self.sum_columns.index(item.name)] or 0
def render_html_title(self, session, item):
@@ -404,13 +404,21 @@
index = self.sum_columns.index(item.name)
return "legend%s" % rgb_to_string(*(PieChartPage.color_schemes[self.color_scheme][index]))
+ def get_record(self, session):
+ record = self.record.get(session)
+ if not record:
+ record = self.data.get_record(session)
+ self.record.set(session, record)
+
+ return record
+
class DashboardOverviewJobSummary(DashboardSummarySet):
def __init__(self, app, name, collector):
cls = app.model.com_redhat_grid.Scheduler
columns = [cls.TotalRunningJobs, cls.TotalHeldJobs, cls.TotalIdleJobs,
cls.TotalRemovedJobs, cls.TotalJobAds]
- super(DashboardOverviewJobSummary, self).__init__(app, name, collector, cls, columns)
+ super(DashboardOverviewJobSummary, self).__init__(app, name, collector, cls, columns, "scheduler.jobs")
self.color_scheme = PieChartPage.GREENS
@@ -453,7 +461,7 @@
class GridSlotsSummary(PieStatSet):
def __init__(self, app, name, collector):
- super(GridSlotsSummary, self).__init__(app, name, collector)
+ super(GridSlotsSummary, self).__init__(app, name, collector, "collector.hosts")
self.attrs = ["HostsClaimed", "HostsUnclaimed",
"HostsOwner", "HostsTotal"]
@@ -545,7 +553,7 @@
class GridOSBreakdown(PieStatSet):
def __init__(self, app, name, collector):
- super(GridOSBreakdown, self).__init__(app, name, collector)
+ super(GridOSBreakdown, self).__init__(app, name, collector, "slot.os")
self.os_records = Attribute(app, "os_breakdown")
self.add_attribute(self.os_records)
@@ -555,14 +563,15 @@
self.color_scheme = PieChartPage.GROUP2
self.has_total = True
- def do_process(self, session):
- os_records = self.os_data.get_records(session)
- self.os_records.set(session, os_records)
+ def get_records(self, session):
+ os_records = self.os_records.get(session)
+ if not os_records:
+ os_records = self.os_data.get_records(session)
+ self.os_records.set(session, os_records)
+ return os_records
- super(GridOSBreakdown, self).do_process(session)
-
def do_render(self, session):
- records = self.os_records.get(session)
+ records = self.get_records(session)
if len(records):
return super(GridOSBreakdown, self).do_render(session)
@@ -570,7 +579,7 @@
return "Slot breakdown by OS"
def do_get_items(self, session):
- return [(i, x) for i, x in enumerate(self.os_records.get(session))]
+ return [(i, x) for i, x in enumerate(self.get_records(session))]
def is_total(self, item):
_, record = item
@@ -598,9 +607,9 @@
return "legend%s" % rgb_to_string(*(PieChartPage.color_schemes[self.color_scheme][real_index]))
class NegotiatorDefinitionStatSet(PieStatSet):
- def __init__(self, app, name, collector):
+ def __init__(self, app, name, collector, title):
self.negotiator = NegotiatorAttribute(app, "neg", collector)
- super(NegotiatorDefinitionStatSet, self).__init__(app, name, self.negotiator)
+ super(NegotiatorDefinitionStatSet, self).__init__(app, name, self.negotiator, title)
self.add_attribute(self.negotiator)
self.chart_link = StatFlashChart(app, "popup", self.negotiator)
@@ -643,7 +652,7 @@
class JobThroughput(NegotiatorDefinitionStatSet):
def __init__(self, app, name, attrs, collector):
- super(JobThroughput, self).__init__(app, name, collector)
+ super(JobThroughput, self).__init__(app, name, collector, "job.throughput")
self.attrs = attrs
@@ -661,7 +670,7 @@
class NegotiatorCycles(NegotiatorDefinitionStatSet):
def __init__(self, app, name, collector):
- super(NegotiatorCycles, self).__init__(app, name, collector)
+ super(NegotiatorCycles, self).__init__(app, name, collector, "negotiator.cycles")
self.attrs = ["Phase1Duration", "Phase2Duration", "Phase3Duration", "Phase4Duration", "Duration"]
self.chart_link.stats = self.attrs
@@ -674,7 +683,7 @@
class NegotiatorSlots(NegotiatorDefinitionStatSet):
def __init__(self, app, name, collector):
- super(NegotiatorSlots, self).__init__(app, name, collector)
+ super(NegotiatorSlots, self).__init__(app, name, collector, "negotiator.slots")
self.attrs = ["TrimmedSlots", "CandidateSlots", "TotalSlots"]
self.chart_link.stats = self.attrs
Modified: trunk/cumin/python/cumin/objectselector.py
===================================================================
--- trunk/cumin/python/cumin/objectselector.py 2011-03-16 17:03:54 UTC (rev 4625)
+++ trunk/cumin/python/cumin/objectselector.py 2011-03-16 20:11:08 UTC (rev 4626)
@@ -120,14 +120,14 @@
return self.export.render(session)
class CsvExporter(Widget):
- def __init__(self, app, name, args, selector):
+ def __init__(self, app, name, args, data_source):
super(CsvExporter, self).__init__(app, name)
# 0 or more RosemaryAttributes that need to be set
# for the selector's table to get it's data
self.objects = list(args)
- self.selector = selector
+ self.data_source = data_source
# avoid name collisions on csv page
while self.name in self.app.export_page.modes.children_by_name:
@@ -136,16 +136,6 @@
self.app.export_page.modes.add_mode(self)
def render(self, session):
- table = self.selector.table
-
- values = table.get_data_values(session)
- options = table.get_data_options(session)
- # get all the records, starting at the beginning
- options.limit = None
- options.offset = 0
-
- data = table.adapter.get_data(values, options)
-
writer = Writer()
# put out the column headers
@@ -153,6 +143,7 @@
writer.write("\n")
# put out the data
+ data = self.get_data(session)
for record in data:
writer.write(",".join(self.render_csv_cells(session, record)))
writer.write("\n")
@@ -160,22 +151,62 @@
return writer.to_string()
def render_csv_header(self, session):
+ return []
+
+ def render_csv_cells(self, session, record):
+ return []
+
+ def _safe_value(self, value):
+ # Cconvert None to ""
+ # Ensure there are no commas. There is no standard way to handle this so
+ # we are just replacing each comma with a space
+ return value is not None and str(value).replace(",", " ") or ""
+
+class CsvStatsExporter(CsvExporter):
+ # stats only have 1 row of data
+ # each col in that row is a stat
+ def get_data(self, session):
+ return (1,)
+
+ # the col header is the stat title
+ def render_csv_header(self, session):
+ items = self.data_source.do_get_items(session)
+ return [self._safe_value(self.data_source.render_item_title(session, x)) for x in items]
+
+ # the col value is the stat value
+ def render_csv_cells(self, session, record):
+ items = self.data_source.do_get_items(session)
+ return [self._safe_value(self.data_source.render_item_value(session, x)) for x in items]
+
+class CsvTableExporter(CsvExporter):
+ def get_data(self, session):
+ table = self.data_source.table
+
+ values = table.get_data_values(session)
+ options = table.get_data_options(session)
+ # get all the records, starting at the beginning
+ options.limit = None
+ options.offset = 0
+
+ return table.adapter.get_data(values, options)
+
+ def render_csv_header(self, session):
cells = list()
- for column in self.selector.table.get_visible_columns(session):
+ for column in self.data_source.table.get_visible_columns(session):
if column.name != "id":
value = column.render_header_content(session)
- value = value is not None and str(value).replace(",", " ") or ""
+ value = self._safe_value(value)
cells.append(value)
return cells
def render_csv_cells(self, session, record):
cells = list()
- for column in self.selector.table.get_visible_columns(session):
+ for column in self.data_source.table.get_visible_columns(session):
# don't output the checkbox column since it's just a database id
if column.name != "id":
value = column.cell.render_content(session, record)
- value = value is not None and str(value).replace(",", " ") or ""
+ value = self._safe_value(value)
cells.append(value)
return cells
@@ -257,7 +288,7 @@
def enable_csv_export(self, *args):
assert self.table
- exporter = CsvExporter(self.app, self.name + "_csv", args, self)
+ exporter = CsvTableExporter(self.app, self.name + "_csv", args, self)
self.table.export = ExportButton(self.app, "export", args, exporter, self.cls._title)
self.table.add_child(self.table.export)
13 years, 2 months