Author: tmckay
Date: 2012-03-30 20:27:05 +0000 (Fri, 30 Mar 2012)
New Revision: 5280
Added:
branches/play_plumage/cumin/bin/cumin-report
branches/play_plumage/cumin/model/admin/
branches/play_plumage/cumin/model/admin/cumin.xml
branches/play_plumage/cumin/model/admin/rosemary.xml
branches/play_plumage/cumin/model/plumage/
branches/play_plumage/cumin/model/plumage/plumage.xml
branches/play_plumage/cumin/model/plumage/rosemary.xml
branches/play_plumage/mint/python/mint/plumage/
branches/play_plumage/mint/python/mint/plumage/__init__.py
branches/play_plumage/mint/python/mint/plumage/main.py
branches/play_plumage/mint/python/mint/plumage/session.py
branches/play_plumage/mint/python/mint/plumage/update.py
Removed:
branches/play_plumage/cumin/model/plumage.xml
Modified:
branches/play_plumage/cumin/bin/cumin
branches/play_plumage/cumin/bin/cumin-admin
branches/play_plumage/cumin/bin/cumin-data
branches/play_plumage/cumin/bin/cumin-web
branches/play_plumage/cumin/etc/module.profile
branches/play_plumage/cumin/model/cumin.xml
branches/play_plumage/cumin/model/rosemary.xml
branches/play_plumage/cumin/python/cumin/config.py
branches/play_plumage/cumin/python/cumin/main.py
branches/play_plumage/cumin/python/cumin/messaging/brokergroup.py
branches/play_plumage/cumin/python/cumin/model.py
branches/play_plumage/cumin/python/cumin/parameters.py
branches/play_plumage/mint/python/mint/model.py
branches/play_plumage/rosemary/python/rosemary/model.py
Log:
Initial framework for cumin-report.
Fair amout of refactoring around config.py, rosemary/model
Modified: branches/play_plumage/cumin/bin/cumin
===================================================================
--- branches/play_plumage/cumin/bin/cumin 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/bin/cumin 2012-03-30 20:27:05 UTC (rev 5280)
@@ -80,18 +80,28 @@
help="Configuration section names for cumin-data instances."\
"\nEach value implies a separate cumin-data instance.")
+ parser.add_option("--reports", dest="reports", default=values.reports,
+ help="Configuration section names for cumin-report instances."\
+ "\nEach value implies a separate cumin-report instance.")
+
parser.add_option("--console", dest="console", action="store_true", default=False,
help="Log to stderr rather than master.log, no IO redirection for children.")
+ parser.add_option("--web-options", dest="web_options", default="", type=str,
+ help="Additional options string to pass to web instances."\
+ "\nEnclose in quotes, options must be --option form, splits on spaces."\
+ '\nExample: web_options="--debug --port=12345"')
+
parser.add_option("--data-options", dest="data_options", default="", type=str,
help="Additional options string to pass to data instances."\
"\nEnclose in quotes, options must be --option form, splits on spaces."\
'\nExample: data_options="--print-events=5 --print-stats"')
- parser.add_option("--web-options", dest="web_options", default="", type=str,
- help="Additional options string to pass to web instances."\
- "\nEnclose in quotes, options must be --option form, splits on spaces."\
- '\nExample: web_options="--debug --port=12345"')
+ parser.add_option("--report-options", dest="report_options", default="", type=str,
+ help="Additional options string to pass to report instances."\
+ "\nEnclose in quotes, options must be --option form, splits on spaces."\
+ '\nExample: report_options="--print-events=5 --print-stats"')
+
parser.add_option("--syslog", dest="syslog", action="store_true", default=False,
help="Log general error notfications to syslog. Intended for systemd")
@@ -139,6 +149,11 @@
for instance in options.datas.split(','):
args, prog_string = get_args("cumin-data", instance, options.init_only, console, options.data_options)
apps.append([None, args, prog_string])
+
+ if len(options.reports) > 0:
+ for instance in options.reports.split(','):
+ args, prog_string = get_args("cumin-report", instance, options.init_only, console, options.report_options)
+ apps.append([None, args, prog_string])
# Launch and babysit, do not restart if options.init_only is set
complete = 0
Modified: branches/play_plumage/cumin/bin/cumin-admin
===================================================================
--- branches/play_plumage/cumin/bin/cumin-admin 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/bin/cumin-admin 2012-03-30 20:27:05 UTC (rev 5280)
@@ -65,10 +65,9 @@
print "Command '%s' is unknown" % name
sys.exit(1)
- broker_uris = [x.strip() for x in opts.brokers.split(",")]
authmech = [x.strip() for x in values.common.auth.split(";")]
- app = Cumin(config.get_home(), broker_uris, opts.database, authmech=authmech)
+ app = Cumin(config.get_home(), [], opts.database, authmech=authmech)
app.check()
Modified: branches/play_plumage/cumin/bin/cumin-data
===================================================================
--- branches/play_plumage/cumin/bin/cumin-data 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/bin/cumin-data 2012-03-30 20:27:05 UTC (rev 5280)
@@ -137,7 +137,7 @@
mint = None
setup_initial_logging()
- parser = CuminOptionParser()
+ parser = BrokeredOptionParser()
# Add additional parameters for data
parser.add_option("--print-stats", action="store_true", default=False)
@@ -169,7 +169,8 @@
log.error("Extra arguments:" + "".join([" "+arg for arg in args]))
raise ArgError
- model_dir = os.path.join(config.home, "model")
+
+ model_dir = [os.path.join(config.home, x) for x in ("model", "model/admin")]
broker_uris = [x.strip() for x in opts.brokers.split(",")]
Added: branches/play_plumage/cumin/bin/cumin-report
===================================================================
--- branches/play_plumage/cumin/bin/cumin-report (rev 0)
+++ branches/play_plumage/cumin/bin/cumin-report 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,243 @@
+#!/usr/bin/python
+
+import os
+import sys
+from traceback import print_exc
+
+home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
+sys.path.append(os.path.join(home, "python"))
+
+from cumin.config import *
+from cumin.util import *
+from mint.plumage.main import Plumage
+from parsley.loggingex import PipeLogThread
+from psycopg2 import OperationalError
+from cumin.admin import SchemaVersion, SchemaMissing
+from cumin.errors import CuminErrors
+
+def restore_IO():
+ sys.stderr = sys.__stderr__
+ sys.stdout = sys.__stdout__
+
+def process_classes(mint, values, section_name, on_empty=None):
+
+ return_code = CuminErrors.NO_ERROR
+ pkgs = set()
+ if values and len(values) > 0:
+
+ for cls_str in values.split(","):
+ pair = cls_str.strip().split(":")
+ if len(pair) == 2:
+ pname = pair[0]
+ cname = pair[1]
+ else:
+ log.error("Configuration section '%s',"\
+ " class name '%s' is badly formed"\
+ % (section_name, cls_str.strip()))
+ return_code = CuminErrors.PARSE_ERROR
+ break
+
+ try:
+ pkg = mint.model._packages_by_name[pname]
+ if cname == "*":
+ pkgs = pkgs.union(set(pkg._classes))
+ else:
+ try:
+ cls = pkg._classes_by_name[cname]
+ pkgs.add(cls)
+ except KeyError:
+ log.warning("Configuration section '%s',"\
+ " class '%s' is not contained in package"\
+ " '%s'" % (section_name, cname, pname))
+ except KeyError:
+ log.warning("Configuration section '%s',"\
+ " package '%s' not found" % (section_name, pname))
+ elif on_empty == "all":
+ for pkg in mint.model._packages:
+ pkgs = pkgs.union(set(pkg._classes))
+
+ return return_code, pkgs
+
+def adjust_return(passed_init, ret):
+ # Shift non-zer0 return codes left 1 bit
+ # and OR in whether or not init passed
+ if ret != 0:
+ ret = ret << 1 | passed_init
+ return ret
+
+def main():
+ passed_init = 1
+ return_code = CuminErrors.NO_ERROR
+
+ # Do our own simple option check so we can redirect IO early
+ # without worrying about other options or the behavior of optParse
+ opts = check_for_options(["--section", "--daemon"], sys.argv[1:])
+ if type(opts["--section"]) is str:
+ section_name = opts["--section"]
+ else:
+ section_name = "report"
+
+ # If the --daemon option has been set, redirect stderr and stdio through
+ # pipes. Launch a thread to read those pipes and direct the content to
+ # files with rollover control.
+ pipeThread = None
+ if opts["--daemon"]:
+ try:
+ cumin_home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
+ err_file = os.path.join(cumin_home, "log", section_name+".stderr")
+ out_file = os.path.join(cumin_home, "log", section_name+".stdout")
+
+ # Open pipes and redirect
+ err_r, err_w = os.pipe()
+ out_r, out_w = os.pipe()
+ pipeThread = PipeLogThread(descriptors=[err_r, out_r],
+ paths=[err_file, out_file],
+ call_on_fail=restore_IO)
+ pipeThread.start()
+ sys.stderr = os.fdopen(err_w, "w", 0)
+ sys.stdout = os.fdopen(out_w, "w", 0)
+ except:
+ print_exc()
+ pipeThread = None
+
+ class ArgError(Exception):
+ pass
+
+ try:
+ mint = None
+ setup_initial_logging()
+
+ parser = PlumageOptionParser()
+
+ # Add additional parameters for report
+ parser.add_option("--print-stats", action="store_true", default=False)
+ parser.add_option("--print-events", type="int", default=0, metavar="LEVEL")
+ parser.add_option("--section", default="report")
+ parser.add_option("--daemon", action="store_true", default=False)
+ parser.add_option("--no-vacuum", dest="vacuum_enabled", action="store_false")
+ parser.add_option("--no-expire", dest="expire_enabled", action="store_false")
+
+ # Get options
+ opts, args = parser.parse_args()
+
+ # --section controls which section is read from the config file
+ # If a section other than "report" is specified, require it to exist
+ config = CuminReportConfig(opts.section, strict_section = opts.section != "report")
+
+ # There might be other sections returned from config.parse() but
+ # currently we don't care about them...
+ values = getattr(config.parse(), opts.section)
+
+ # Use the config values as defaults for unspecified options
+ apply_defaults(values, opts)
+
+ setup_operational_logging(opts,
+ values.log_max_mb,
+ values.log_max_archives)
+
+ if len(args) != 0:
+ log.error("Extra arguments:" + "".join([" "+arg for arg in args]))
+ raise ArgError
+
+ model_dir = [os.path.join(config.home, x) for x in ("model/admin", "model/plumage")]
+
+ broker_uris = []
+# broker_uris = [x.strip() for x in opts.brokers.split(",")]
+
+ mint = Plumage(model_dir, broker_uris, opts.database)
+
+ mint.print_event_level = opts.print_events
+
+ mint.expire_enabled = opts.expire_enabled
+ mint.expire_thread.interval = values.expire_interval
+ mint.expire_thread.threshold = values.expire_threshold
+
+ mint.vacuum_enabled = opts.vacuum_enabled
+ mint.vacuum_thread.interval = values.vacuum_interval
+
+ mint.check()
+ mint.init()
+
+ # Handle QMF class binding options
+ if values.include_classes or values.exclude_classes:
+ return_code, includes = process_classes(mint,
+ values.include_classes,
+ opts.section, on_empty="all")
+
+ return_code, excludes = process_classes(mint,
+ values.exclude_classes,
+ opts.section)
+
+ mint.qmf_classes = includes.difference(excludes)
+
+
+ # If init_only was set or we failed init, don't proceed...
+ if not opts.init_only and not return_code:
+ passed_init = 0
+
+ mint.start()
+ stats = mint.update_thread.stats
+ count = 0
+
+ if opts.print_stats:
+ print "[Starred columns are the number of events per second]"
+
+ while True:
+ if count % 20 == 0:
+ stats.print_headings()
+
+ count += 1
+
+ stats.print_values()
+
+ sleep(5)
+ else:
+ while True:
+ sleep(86400)
+
+ except KeyboardInterrupt:
+ log.info("Received shutdown signal")
+
+ except SystemExit:
+ if "--help" not in sys.argv:
+ log.error("Error in options")
+ return_code = CuminErrors.PARSE_ERROR
+
+ except ArgError:
+ return_code = CuminErrors.PARSE_ERROR
+
+ except OperationalError:
+ # Failed to talk to the database on check()
+ log.info("Run 'cumin-database check' as root for more information.")
+ return_code = CuminErrors.DATABASE_ERROR
+
+ except SchemaMissing:
+ log.info("Run 'cumin-admin create-schema' as root")
+ return_code = CuminErrors.SCHEMA_ERROR
+
+ except SchemaVersion:
+ log.info("Run 'cumin-admin upgrade-schema' as root")
+ return_code = CuminErrors.SCHEMA_VER_ERROR
+
+ except:
+ print_exc()
+ return_code = CuminErrors.UNHANDLED_ERROR
+
+ if mint:
+ mint.stop()
+ if pipeThread:
+ pipeThread.stop()
+ log.info("about to call logging shutdown")
+ logging.shutdown()
+ return adjust_return(passed_init, return_code)
+
+def make_ctrl_c(sig, frame):
+ raise KeyboardInterrupt
+
+if __name__ == "__main__":
+ try:
+ import signal
+ signal.signal(signal.SIGTERM, make_ctrl_c)
+ sys.exit(main())
+ except KeyboardInterrupt:
+ sys.exit(0)
Property changes on: branches/play_plumage/cumin/bin/cumin-report
___________________________________________________________________
Added: svn:executable
+ *
Modified: branches/play_plumage/cumin/bin/cumin-web
===================================================================
--- branches/play_plumage/cumin/bin/cumin-web 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/bin/cumin-web 2012-03-30 20:27:05 UTC (rev 5280)
@@ -95,7 +95,7 @@
setup_initial_logging()
- parser = CuminOptionParser()
+ parser = BrokeredOptionParser()
# Add additional parameters for web
parser.add_option("--host")
Modified: branches/play_plumage/cumin/etc/module.profile
===================================================================
--- branches/play_plumage/cumin/etc/module.profile 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/etc/module.profile 2012-03-30 20:27:05 UTC (rev 5280)
@@ -1,3 +1,3 @@
export PYTHONPATH="${PWD}/python:${PYTHONPATH}"
-export PATH="${PWD}/bin:${PATH}"
+export PATH="${PWD}/bin:${PWD}/bin/test:${PATH}"
export CUMIN_HOME="${PWD}/instance"
Added: branches/play_plumage/cumin/model/admin/cumin.xml
===================================================================
--- branches/play_plumage/cumin/model/admin/cumin.xml (rev 0)
+++ branches/play_plumage/cumin/model/admin/cumin.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,21 @@
+<model>
+ <package name="com.redhat.cumin">
+ <class name="Info">
+ <property name="schema_version" type="sstr"/>
+ </class>
+
+ <class name="User">
+ <property name="name" type="sstr" index="y"/>
+ <property name="password" type="sstr"/>
+ </class>
+
+ <class name="Role">
+ <property name="name" type="sstr" index="y"/>
+ </class>
+
+ <class name="UserRoleMapping">
+ <property name="user" references="User" index="y"/>
+ <property name="role" references="Role" index="y"/>
+ </class>
+ </package>
+</model>
Added: branches/play_plumage/cumin/model/admin/rosemary.xml
===================================================================
--- branches/play_plumage/cumin/model/admin/rosemary.xml (rev 0)
+++ branches/play_plumage/cumin/model/admin/rosemary.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,7 @@
+<model>
+ <package name="com.redhat.cumin" persistent="y">
+ <class name="User">
+ <property name="name" unique="y"/>
+ </class>
+ </package>
+</model>
Modified: branches/play_plumage/cumin/model/cumin.xml
===================================================================
--- branches/play_plumage/cumin/model/cumin.xml 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/model/cumin.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -1,5 +1,5 @@
<model>
- <package name="com.redhat.cumin">
+ <package name="com.redhat.cumin.messaging">
<class name="BrokerGroup">
<property name="name" type="sstr"/>
<property name="description" type="lstr" optional="y"/>
@@ -9,24 +9,6 @@
<property name="broker" type="objId" references="org.apache.qpid.broker:Broker"/>
<property name="group" type="objId" references="BrokerGroup"/>
</class>
-
- <class name="Info">
- <property name="schema_version" type="sstr"/>
- </class>
-
- <class name="User">
- <property name="name" type="sstr" index="y"/>
- <property name="password" type="sstr"/>
- </class>
-
- <class name="Role">
- <property name="name" type="sstr" index="y"/>
- </class>
-
- <class name="UserRoleMapping">
- <property name="user" references="User" index="y"/>
- <property name="role" references="Role" index="y"/>
- </class>
</package>
<package name="com.redhat.cumin.grid">
Added: branches/play_plumage/cumin/model/plumage/plumage.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage/plumage.xml (rev 0)
+++ branches/play_plumage/cumin/model/plumage/plumage.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<schema package="com.redhat.grid.plumage">
+ <class name="OSUtil" qmf_headers="False">
+ <property name="uuid" index="y" type="uuid" access="RC" desc="UUID of System Image"/>
+
+ <property name="osName" type="sstr" access="RO" desc="Operating System Name" />
+
+ <statistic name="avail" type="uint32" unit="uint32" />
+ <statistic name="used" type="uint32" unit="uint32" />
+ </class>
+</schema>
Added: branches/play_plumage/cumin/model/plumage/rosemary.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage/rosemary.xml (rev 0)
+++ branches/play_plumage/cumin/model/plumage/rosemary.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,18 @@
+<model>
+ <package name="com.redhat.grid.plumage">
+ <class name="OSUtil">
+ <property name="osName">
+ <title>OS</title>
+ </property>
+
+ <statistic name="used">
+ <title>Used</title>
+ </statistic>
+
+ <statistic name="avail">
+ <title>Available</title>
+ </statistic>
+ </class>
+ </package>
+
+</model>
Deleted: branches/play_plumage/cumin/model/plumage.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage.xml 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/model/plumage.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<schema package="com.redhat.grid.plumage">
- <class name="OSUtil" qmf_headers="False">
- <property name="uuid" index="y" type="uuid" access="RC" desc="UUID of System Image"/>
-
- <property name="osName" type="sstr" access="RO" desc="Operating System Name" />
-
- <statistic name="avail" type="uint32" unit="uint32" />
- <statistic name="used" type="uint32" unit="uint32" />
- </class>
-</schema>
Modified: branches/play_plumage/cumin/model/rosemary.xml
===================================================================
--- branches/play_plumage/cumin/model/rosemary.xml 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/model/rosemary.xml 2012-03-30 20:27:05 UTC (rev 5280)
@@ -1,10 +1,4 @@
<model>
- <package name="com.redhat.cumin" persistent="y">
- <class name="User">
- <property name="name" unique="y"/>
- </class>
- </package>
-
<package name="com.redhat.cumin.grid">
<class name="JobSummary">
<property name="GlobalJobId">
@@ -204,7 +198,7 @@
</class>
</package>
- <package name="com.redhat.cumin">
+ <package name="com.redhat.cumin.messaging">
<class name="BrokerGroup">
<title>Broker group</title>
@@ -547,20 +541,4 @@
</class>
</package>
- <package name="com.redhat.grid.plumage">
- <class name="OSUtil">
- <property name="osName">
- <title>OS</title>
- </property>
-
- <statistic name="used">
- <title>Used</title>
- </statistic>
-
- <statistic name="avail">
- <title>Available</title>
- </statistic>
- </class>
- </package>
-
</model>
Modified: branches/play_plumage/cumin/python/cumin/config.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/config.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/python/cumin/config.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -16,90 +16,7 @@
if not os.path.isdir(self.home):
raise Exception("Home path '%s' is not a directory")
-
- test = CuminConfigSection(self, "test")
- test.log_file.default = os.path.join(self.home, "log", "test.log")
-
- def create_web_section(self, name, strict_section):
- web = CuminConfigSection(self, name, strict_section)
- web.log_file.default = os.path.join(self.home, "log", name + ".log")
-
- param = ConfigParameter(web, "update-interval", int)
- param.default = 10
-
- param = ConfigParameter(web, "max-qmf-table-sort", int)
- param.default = 1000
-
- param = ConfigParameter(web, "host", str)
- param.default = "localhost"
-
- param = ConfigParameter(web, "port", int)
- param.default = 45672
-
- param = ConfigParameter(web, "operator-email", str)
-
- param = ConfigParameter(web, "user", str)
-
- param = ConfigParameter(web, "request-memory", int)
- param.default = 512 # MB
- param = ConfigParameter(web, "request-memory-vm", int)
- param.default = 512 # MB
- param = ConfigParameter(web, "request-disk", int)
- param.default = 1024 #MB
- param = ConfigParameter(web, "request-disk-vm", int)
- param.default = 5 * 1024 #MB
-
- param = ConfigParameter(web, "persona", str)
- param.default = "grid"
- param = ConfigParameter(web, "fast-view-attributes", str)
- param.default = "JobStatus,Cmd,Args,ExitStatus,JobStartDate,LastRemoteHost,LastJobStatus,Owner"
-
- param = ConfigParameter(web, "notification-timeout", int)
- param.default = 180
-
- # Hidden parameter used to force html doctype rather than xhtml
- # This is hopefully a temporary workaround so that selenium can
- # be used to do some automated testing against cumin
- param = ConfigParameter(web, "force-html-doctype", bool)
- param.default = False
-
- def create_data_section(self, name, strict_section):
- data = CuminConfigSection(self, name, strict_section)
- data.log_file.default = os.path.join(self.home, "log", name + ".log")
-
- param = ConfigParameter(data, "include-classes", str)
- param = ConfigParameter(data, "exclude-classes", str)
- param = ConfigParameter(data, "agents", str)
-
- param = ConfigParameter(data, "expire-enabled", bool)
- param.default = True
-
- param = ConfigParameter(data, "expire-interval", int)
- param.default = 60 * 60 # 1 hour
-
- param = ConfigParameter(data, "expire-threshold", int)
- param.default = 24 * 60 * 60 # 1 day
-
- param = ConfigParameter(data, "vacuum-enabled", bool)
- param.default = True
-
- param = ConfigParameter(data, "vacuum-interval", int)
- param.default = 60 * 60 # 1 hour
-
- def create_master_section(self, name, strict_section):
- '''
- Creates a blank master section.
- CuminConfigSection values are not interesting to the master
- '''
-
- master = ConfigSection(self, name, strict_section)
- param = ConfigParameter(master, "datas", str)
- param.default = "data"
-
- param = ConfigParameter(master, "webs", str)
- param.default = "web"
-
def get_home(self):
return self.home
@@ -130,20 +47,59 @@
def __init__(self, section_name="web", strict_section="False"):
super(CuminWebConfig, self).__init__()
- self.create_web_section(section_name, strict_section)
+ web = CuminWebConfigSection(self, section_name, strict_section)
+ web.log_file.default = os.path.join(self.home, "log",
+ section_name + ".log")
+
+def _common_data_section(self, data, name):
+ data.log_file.default = os.path.join(self.home, "log", name + ".log")
+
+ param = ConfigParameter(data, "include-classes", str)
+ param = ConfigParameter(data, "exclude-classes", str)
+
+ param = ConfigParameter(data, "expire-enabled", bool)
+ param.default = True
+
+ param = ConfigParameter(data, "expire-interval", int)
+ param.default = 60 * 60 # 1 hour
+
+ param = ConfigParameter(data, "expire-threshold", int)
+ param.default = 24 * 60 * 60 # 1 day
+
+ param = ConfigParameter(data, "vacuum-enabled", bool)
+ param.default = True
+
+ param = ConfigParameter(data, "vacuum-interval", int)
+ param.default = 60 * 60 # 1 hour
class CuminDataConfig(CuminConfig):
def __init__(self, section_name="data", strict_section=False):
super(CuminDataConfig, self).__init__()
- self.create_data_section(section_name, strict_section)
+ data = CuminDataConfigSection(self, section_name, strict_section)
+ _common_data_section(self, data, section_name)
+class CuminReportConfig(CuminConfig):
+ def __init__(self, section_name="report", strict_section=False):
+ super(CuminReportConfig, self).__init__()
+
+ data = CuminDataConfigSection(self, section_name, strict_section)
+ _common_data_section(self, data, section_name)
+
class CuminMasterConfig(CuminConfig):
def __init__(self, section_name="master", strict_section=False):
super(CuminMasterConfig, self).__init__()
- self.create_master_section(section_name, strict_section)
+ master = ConfigSection(self, section_name, strict_section)
+ param = ConfigParameter(master, "datas", str)
+ param.default = "data"
+ param = ConfigParameter(master, "webs", str)
+ param.default = "web"
+
+ param = ConfigParameter(master, "reports", str)
+ param.default = ""
+
class CuminConfigSection(ConfigSection):
def __init__(self, config, name, strict_section=False):
super(CuminConfigSection, self).__init__(config, name, strict_section)
@@ -151,16 +107,42 @@
param = ConfigParameter(self, "database", str)
param.default = "dbname=cumin user=cumin host=localhost"
+ # Put this here, because authentication is something that
+ # might need to be done commonly
+ param = ConfigParameter(self, "auth", str)
+ param.default = 'internal'
+
+ self.log_file = ConfigParameter(self, "log-file", str)
+
+ param = ConfigParameter(self, "log-level", str)
+ param.default = "info"
+
+ param = ConfigParameter(self, "log-max-mb", float)
+ param.default = 10
+
+ param = ConfigParameter(self, "log-max-archives", int)
+ param.default = 1
+
+ param = ConfigParameter(self, "debug", bool)
+ param.default = False
+
+class BrokeredConfigSection(CuminConfigSection):
+ def __init__(self, config, name, strict_section=False):
+ super(BrokeredConfigSection, self).__init__(config, name,
+ strict_section)
+
param = ConfigParameter(self, "brokers", str)
param.default = "amqp://localhost"
- param = ConfigParameter(self, "auth", str)
- param.default = 'internal'
-
# Leave default set to None, which is equivalent to
# previous behavior
param = ConfigParameter(self, "sasl-mech-list", str)
+class CuminWebConfigSection(BrokeredConfigSection):
+ def __init__(self, config, name, strict_section=False):
+ super(CuminWebConfigSection, self).__init__(config, name,
+ strict_section)
+
param = ConfigParameter(self, "wallaby-broker", str)
param.default = ""
@@ -190,20 +172,60 @@
param = ConfigParameter(self, "aviary-suds-logs", bool)
param.default = False
- self.log_file = ConfigParameter(self, "log-file", str)
+ param = ConfigParameter(self, "update-interval", int)
+ param.default = 10
- param = ConfigParameter(self, "log-level", str)
- param.default = "info"
+ param = ConfigParameter(self, "max-qmf-table-sort", int)
+ param.default = 1000
- param = ConfigParameter(self, "log-max-mb", float)
- param.default = 10
+ param = ConfigParameter(self, "host", str)
+ param.default = "localhost"
- param = ConfigParameter(self, "log-max-archives", int)
- param.default = 1
+ param = ConfigParameter(self, "port", int)
+ param.default = 45672
- param = ConfigParameter(self, "debug", bool)
+ param = ConfigParameter(self, "operator-email", str)
+
+ param = ConfigParameter(self, "user", str)
+
+ param = ConfigParameter(self, "request-memory", int)
+ param.default = 512 # MB
+ param = ConfigParameter(self, "request-memory-vm", int)
+ param.default = 512 # MB
+ param = ConfigParameter(self, "request-disk", int)
+ param.default = 1024 #MB
+ param = ConfigParameter(self, "request-disk-vm", int)
+ param.default = 5 * 1024 #MB
+
+ param = ConfigParameter(self, "persona", str)
+ param.default = "grid"
+
+ param = ConfigParameter(self, "fast-view-attributes", str)
+ param.default = "JobStatus,Cmd,Args,ExitStatus,JobStartDate,LastRemoteHost,LastJobStatus,Owner"
+
+ param = ConfigParameter(self, "notification-timeout", int)
+ param.default = 180
+
+ # Hidden parameter used to force html doctype rather than xhtml
+ # This is hopefully a temporary workaround so that selenium can
+ # be used to do some automated testing against cumin
+ param = ConfigParameter(self, "force-html-doctype", bool)
param.default = False
+class CuminDataConfigSection(BrokeredConfigSection):
+ def __init__(self, config, name, strict_section=False):
+ super(CuminDataConfigSection, self).__init__(config, name,
+ strict_section)
+ param = ConfigParameter(self, "agents", str)
+
+class CuminReportConfigSection(CuminConfigSection):
+ def __init__(self, config, name, strict_section=False):
+ super(CuminReportConfigSection, self).__init__(config, name,
+ strict_section)
+
+ param = ConfigParameter(self, "plumage_server", str)
+ param.default = "localhost"
+
class CuminOptionParser(OptionParser,object):
def __init__(self):
OptionParser.__init__(self)
@@ -214,10 +236,25 @@
# section named in --section, via apply_defaults()
self.add_option("--debug", action="store_true")
self.add_option("--database")
- self.add_option("--brokers")
self.add_option("--log-file")
self.add_option("--log-level")
+class BrokeredOptionParser(CuminOptionParser):
+ def __init__(self):
+ CuminOptionParser.__init__(self)
+
+ # Defaults for these come later from the config
+ # section named in --section, via apply_defaults()
+ self.add_option("--brokers")
+
+class PlumageOptionParser(CuminOptionParser):
+ def __init__(self):
+ CuminOptionParser.__init__(self)
+
+ # Defaults for these come later from the config
+ # section named in --section, via apply_defaults()
+ self.add_option("--server")
+
def apply_defaults(values, opts):
for k, v in opts.__dict__.iteritems():
if v == None and k in values:
Modified: branches/play_plumage/cumin/python/cumin/main.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/main.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/python/cumin/main.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -39,7 +39,7 @@
self.home = home
self.authmech = authmech
- model_dir = os.path.join(self.home, "model")
+ model_dir = [os.path.join(self.home, x) for x in ("model/admin", "model")]
self.model = CuminModel(self, model_dir)
self.session = CuminSession(self, broker_uris)
Modified: branches/play_plumage/cumin/python/cumin/messaging/brokergroup.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/messaging/brokergroup.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/python/cumin/messaging/brokergroup.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -18,7 +18,7 @@
class BrokerGroupSelector(ObjectSelector):
def __init__(self, app, name):
- cls = app.model.com_redhat_cumin.BrokerGroup
+ cls = app.model.com_redhat_cumin_messaging.BrokerGroup
super(BrokerGroupSelector, self).__init__(app, name, cls)
@@ -54,7 +54,7 @@
class BrokerGroupFrame(ObjectFrame):
def __init__(self, app, name):
- cls = app.model.com_redhat_cumin.BrokerGroup
+ cls = app.model.com_redhat_cumin_messaging.BrokerGroup
super(BrokerGroupFrame, self).__init__(app, name, cls)
@@ -106,7 +106,7 @@
try:
cursor = conn.cursor()
- cls = self.app.model.com_redhat_cumin.BrokerGroup
+ cls = self.app.model.com_redhat_cumin_messaging.BrokerGroup
group = cls.create_object(cursor)
group.name = name
Modified: branches/play_plumage/cumin/python/cumin/model.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/model.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/python/cumin/model.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -40,9 +40,12 @@
def check(self):
log.info("Checking %s", self)
- assert os.path.isdir(self.model_dir)
+ if not type(self.model_dir) in (list, tuple):
+ self.model_dir = [self.model_dir]
- log.debug("Model dir exists at '%s'", self.model_dir)
+ for dirs in self.model_dir:
+ assert os.path.isdir(dirs)
+ log.debug("Model dir exists at '%s'", dirs)
def init(self):
log.info("Initializing %s", self)
Modified: branches/play_plumage/cumin/python/cumin/parameters.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/parameters.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/cumin/python/cumin/parameters.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -82,7 +82,7 @@
class NewBrokerGroupParameter(Parameter):
def do_unmarshal(self, string):
id = int(string)
- cls = self.app.model.com_redhat_cumin.BrokerGroup
+ cls = self.app.model.com_redhat_cumin_messaging.BrokerGroup
cursor = self.app.database.get_read_cursor()
Modified: branches/play_plumage/mint/python/mint/model.py
===================================================================
--- branches/play_plumage/mint/python/mint/model.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/mint/python/mint/model.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -20,11 +20,14 @@
def check(self):
log.info("Checking %s", self)
+
+ if not type(self.model_dir) in (tuple, list):
+ self.model_dir = [self.model_dir]
- assert os.path.isdir(self.model_dir)
+ for dirs in self.model_dir:
+ assert os.path.isdir(dirs)
+ log.debug("Model dir exists at '%s'", dirs)
- log.debug("Model dir exists at '%s'", self.model_dir)
-
def init(self):
log.info("Initializing %s", self)
Added: branches/play_plumage/mint/python/mint/plumage/__init__.py
===================================================================
Added: branches/play_plumage/mint/python/mint/plumage/main.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/main.py (rev 0)
+++ branches/play_plumage/mint/python/mint/plumage/main.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,85 @@
+from mint.database import MintDatabase
+from mint.expire import ExpireThread
+from mint.model import MintModel
+from mint.vacuum import VacuumThread
+from session import PlumageSession
+from update import UpdateThread
+from cumin.admin import CuminAdmin
+#from util import *
+
+import logging
+
+log = logging.getLogger("mint.plumage")
+
+class Plumage(object):
+ def __init__(self, model_dir, server_url, database_dsn):
+
+ # Can we just use a different model dir here?
+ # That would control visible packages, vacuuming, etc.
+ self.model = MintModel(self, model_dir)
+ self.model.sql_logging_enabled = False
+
+ self.database = MintDatabase(self, database_dsn)
+ self.admin = CuminAdmin(self)
+ self.update_thread = UpdateThread(self)
+ self.session = PlumageSession(self, server_url)
+
+ self.expire_enabled = False
+ self.expire_thread = ExpireThread(self)
+
+ self.vacuum_enabled = False
+ self.vacuum_thread = VacuumThread(self)
+
+ self.print_event_level = 0
+
+ self.packages = []
+
+ def check(self):
+ log.info("Checking %s", self)
+
+ self.model.check()
+ self.database.check()
+
+ def init(self):
+ log.info("Initializing %s", self)
+
+ def state(cond):
+ return cond and "enabled" or "disabled"
+
+ log.info("Expiration is %s", state(self.expire_enabled))
+ log.info("Vacuum is %s", state(self.vacuum_enabled))
+
+ self.model.init()
+ self.database.init()
+
+ self.update_thread.init()
+ self.session.init()
+ #self.expire_thread.init()
+ #self.vacuum_thread.init()
+
+ def start(self):
+ log.info("Starting %s", self)
+
+ self.update_thread.start()
+ self.session.start()
+
+ #if self.expire_enabled:
+ # self.expire_thread.start()
+
+ #if self.vacuum_enabled:
+ # self.vacuum_thread.start()
+
+ def stop(self):
+ log.info("Stopping %s", self)
+
+ self.update_thread.stop()
+ self.session.stop()
+
+ #if self.expire_enabled:
+ # self.expire_thread.stop()
+
+ #if self.vacuum_enabled:
+ # self.vacuum_thread.stop()
+
+ def __repr__(self):
+ return self.__class__.__name__
Added: branches/play_plumage/mint/python/mint/plumage/session.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/session.py (rev 0)
+++ branches/play_plumage/mint/python/mint/plumage/session.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,89 @@
+import logging
+
+from mint.util import MintDaemonThread
+from time import sleep
+
+log = logging.getLogger("mint.plumage.session")
+
+class PlumageSession(object):
+ def __init__(self, app, server_url):
+ self.app = app
+ self.server_url = server_url
+
+ def check(self):
+ log.info("Checking %s", self)
+
+ def init(self):
+ log.info("Initializing %s", self)
+
+ def start(self):
+ log.info("Starting %s", self)
+
+ def stop(self):
+ log.info("Stopping %s", self)
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, self.server_url)
+
+class PlumageSessionThread(MintDaemonThread):
+ def __init__(self, app):
+ super(PumageSessionThread, self).__init__(app)
+
+ def run(self):
+ while True:
+ if self.stop_requested:
+ break
+
+ # We create objects here
+ sleep(5)
+
+'''
+class MintConsole(Console):
+ def __init__(self, model):
+ self.model = model
+
+ def brokerConnected(self, qmf_broker):
+ message = "Broker %s:%i is connected"
+ self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+
+ def brokerInfo(self, qmf_broker):
+ message = "Broker info from %s:%i"
+ self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+
+ def brokerDisconnected(self, qmf_broker):
+ message = "Broker %s:%i is disconnected"
+ self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
+
+ def newAgent(self, qmf_agent):
+ self.model.print_event(3, "Creating %s", qmf_agent)
+
+ up = AgentUpdate(self.model, qmf_agent)
+ self.model.app.update_thread.enqueue(up)
+
+ def delAgent(self, qmf_agent):
+ self.model.print_event(3, "Deleting %s", qmf_agent)
+
+ up = AgentDelete(self.model, qmf_agent)
+ self.model.app.update_thread.enqueue(up)
+
+ def heartbeat(self, qmf_agent, timestamp):
+ message = "Heartbeat from %s at %s"
+ self.model.print_event(5, message, qmf_agent, timestamp)
+
+ up = AgentUpdate(self.model, qmf_agent)
+ self.model.app.update_thread.enqueue(up)
+
+ def newPackage(self, name):
+ self.model.print_event(2, "New package %s", name)
+
+ def newClass(self, kind, classKey):
+ self.model.print_event(2, "New class %s", classKey)
+
+ def objectProps(self, broker, qmf_object):
+ up = ObjectUpdate(self.model, qmf_object)
+ self.model.app.update_thread.enqueue(up)
+
+ def objectStats(self, broker, qmf_object):
+ up = ObjectUpdate(self.model, qmf_object)
+ self.model.app.update_thread.enqueue(up)
+'''
Added: branches/play_plumage/mint/python/mint/plumage/update.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/update.py (rev 0)
+++ branches/play_plumage/mint/python/mint/plumage/update.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -0,0 +1,363 @@
+import copy
+import resource
+import pickle
+
+from psycopg2 import IntegrityError, TimestampFromTicks
+from psycopg2.extensions import cursor as Cursor
+from rosemary.model import *
+
+from mint.model import *
+from mint.util import *
+
+log = logging.getLogger("mint.plumage.update")
+
+sample_window_min = 60
+sample_window_max = 60 * 5
+
+class UpdateThread(MintDaemonThread):
+ def __init__(self, app):
+ super(UpdateThread, self).__init__(app)
+
+ self.updates = ConcurrentQueue(maxsize=1000)
+ self.stats = UpdateStats(self.app)
+
+ self.conn = None
+ self.cursor = None
+
+ self.halt_on_error = False
+
+ def _delete_all_objects(self):
+ def loop_body(cls):
+ if cls._storage != "none":
+ if cls.check_persistent():
+ log.debug("Skipping persistent class " + str(cls))
+ else:
+ cls.delete_selection(self.cursor)
+ self.cursor.connection.commit()
+
+ log.debug("Delete all objects by bound packages " + str(self.app.packages))
+ for pkg in self.app.packages:
+ for cls in pkg._classes:
+ loop_body(cls)
+
+ def init(self):
+ self.conn = self.app.database.get_connection()
+
+ self.cursor = self.conn.cursor(cursor_factory=UpdateCursor)
+ self.cursor.stats = self.stats
+
+ def enqueue(self, update):
+ self.updates.put(update)
+
+ self.stats.enqueued += 1
+
+ def run(self):
+ while True:
+ if self.stop_requested:
+ break
+
+ try:
+ update = self.updates.get(True, 1)
+ except Empty:
+ continue
+
+ self.stats.dequeued += 1
+
+ update.process(self)
+
+class UpdateStats(object):
+
+ def __init__(self, app):
+ self.enqueued = 0
+ self.dequeued = 0
+ self.dropped = 0
+
+ self.objects_created = 0
+ self.objects_updated = 0
+ self.objects_deleted = 0
+
+ self.objects_created_by_class = defaultdict(int)
+ self.objects_updated_by_class = defaultdict(int)
+ self.objects_deleted_by_class = defaultdict(int)
+
+ self.sql_ops = 0
+ self.errors = 0
+
+ self.time = None
+ self.cpu = 0
+ self.memory = 0
+
+class UpdateCursor(Cursor):
+ def execute(self, sql, args=None):
+ super(UpdateCursor, self).execute(sql, args)
+
+ self.stats.sql_ops += 1
+
+class Update(object):
+ def __init__(self, model):
+ self.model = model
+
+ def process(self, thread):
+ log.debug("Processing %s", self)
+ try:
+ self.do_process(thread.cursor, thread.stats, thread.app.packages)
+ thread.conn.commit()
+ except UpdateDropped:
+ log.debug("Update dropped")
+
+ thread.conn.rollback()
+
+ thread.stats.dropped += 1
+ except:
+ log.debug("Update failed", exc_info=True)
+
+ thread.conn.rollback()
+
+ thread.stats.errors += 1
+
+ if thread.halt_on_error:
+ raise
+
+ def do_process(self, cursor, stats, packages):
+ raise Exception("Not implemented")
+
+ def __repr__(self):
+ return self.__class__.__name__
+
+class ObjectUpdate(Update):
+ def __init__(self, model, plumage_object):
+ super(ObjectUpdate, self).__init__(model)
+
+ self.plumage_object = plumage_object
+
+ def do_process(self, cursor, stats, bound_classes, bound_packages):
+ cls = self.get_class()
+ object_id = self.get_object_id()
+
+ # Okay, we want to do a lookup by object id here.
+ # If we can't find it, we create it. Agent is
+ # a mint Agent class.
+ # We need a lookup by field on machine.
+
+ try:
+ obj = agent.get_object( cursor, cls, object_id)
+ except RosemaryNotFound:
+ obj = self.create_object(cursor, stats, cls)
+ return
+
+ self.update_object(cursor, stats, obj)
+
+ def maybe_drop_sample(self, obj):
+ properties = self.qmf_object.getProperties()
+ statistics = self.qmf_object.getStatistics()
+
+ if not properties and statistics:
+ # Just stats; do we want it?
+ # if stats.enqueued - stats.dequeued > 500:
+
+ now = time.time()
+ update = self.qmf_object.getTimestamps()[0] / 1000000000
+ sample = obj._sample_time
+
+ if update < now - sample_window_max:
+ # The sample is too old
+ raise UpdateDropped()
+
+ if sample and sample > now - sample_window_min:
+ # The samples are too fidelitous
+ raise UpdateDropped()
+
+ def create_object(self, cursor, stats, cls):
+ obj = cls.create_object(cursor)
+
+ object_columns = list()
+ sample_columns = list()
+
+ table = cls.sql_table
+
+ object_columns.append(table._id)
+
+ self.process_properties(obj, object_columns, cursor)
+ self.process_statistics(obj, object_columns, sample_columns)
+
+ statements = list()
+
+ sql = cls.sql_insert_object.emit(object_columns)
+ statements.append(sql)
+
+ if sample_columns:
+ sql = cls.sql_samples_insert.emit(sample_columns)
+ statements.append(sql)
+ obj._sample_time = time.time()
+
+ sql = "; ".join(statements)
+ self.execute_sql(cursor, sql, obj.__dict__)
+ obj._save_time = datetime.now()
+ self.model.print_event(3, "Created %s", obj)
+
+ stats.objects_created += 1
+ #stats.objects_created_by_class[cls] += 1
+
+ return obj
+
+ def update_object(self, cursor, stats, obj):
+ object_columns = list()
+ sample_columns = list()
+
+ self.process_properties(obj, object_columns, cursor)
+ self.process_statistics(obj, object_columns, sample_columns)
+
+ statements = list()
+ cls = obj._class
+
+ # force a write if it's been too long, even if the values match
+ if object_columns:
+ sql = cls.sql_update_object.emit(object_columns)
+ statements.append(sql)
+
+ if sample_columns:
+ sql = cls.sql_samples_insert.emit(sample_columns)
+ statements.append(sql)
+
+ obj._sample_time = time.time()
+
+ if not statements:
+ raise UpdateDropped()
+
+ sql = "; ".join(statements)
+ self.execute_sql(cursor, sql, obj.__dict__)
+
+ self.model.print_event(4, "Updated %s", obj)
+
+ stats.objects_updated += 1
+ #stats.objects_updated_by_class[cls] += 1
+
+ def process_properties(self, obj, columns, cursor):
+ cls = obj._class
+
+ # getProperties comes from console.py
+ # list of tuples, first is an instance
+ # which can be spoofed, second is a value for the property
+
+ for prop, value in self.qmf_object.getProperties():
+ try:
+ col, nvalue = self.process_value(cls, prop, value)
+ except MappingException, e:
+ log.debug(e)
+ continue
+
+ # XXX This optimization will be obsolete when QMF does it
+ # instead
+ if nvalue == getattr(obj, col.name):
+ continue
+
+ setattr(obj, col.name, nvalue)
+ columns.append(col)
+
+ def process_value(self, cls, prop, value):
+ try:
+ col = cls._properties_by_name[prop.name].sql_column
+ except KeyError:
+ raise MappingException("Property %s is unknown" % prop)
+
+ if value is not None:
+ value = transform_value(prop, value)
+
+ return col, value
+
+ def process_statistics(self, obj, update_columns, insert_columns):
+ build_columns = list()
+ saw_change = False
+
+
+ # getStatistics comes from console.py
+ # list of tuples, first is an instance
+ # which can be spoofed, second is a value for the property
+
+ for stat, value in self.qmf_object.getStatistics():
+ try:
+ col = obj._class._statistics_by_name[stat.name].sql_column
+ except KeyError:
+ log.debug("Statistic %s is unknown", stat)
+
+ continue
+
+ if value is not None:
+ value = transform_value(stat, value)
+
+ # Don't write unchanged values
+ #
+ # XXX This optimization will be obsolete when QMF does it
+ # instead
+ if value != getattr(obj, col.name):
+ setattr(obj, col.name, value)
+ update_columns.append(col)
+ saw_change = True
+
+ # If we do end up seeing a value change, we will
+ # need to insert an entire row so build it up
+ build_columns.append(col)
+
+ if saw_change:
+ insert_columns.extend(build_columns)
+
+ def execute_sql(self, cursor, text, args):
+ try:
+ cursor.execute(text, args)
+ except:
+ log.debug("%s failed sql execute", self, exc_info=True)
+ log.error("Sql execute failed")
+ log.error("Sql text: %s", text)
+ log.error("Sql values:")
+
+ for item in sorted(args.items()):
+ log.error(" %-34s %r", *item)
+
+ log.error("Sql row count: %i", cursor.rowcount)
+
+ log.error("object properties:")
+
+ # see process_properties
+ for item in sorted(self.qmf_object.getProperties()):
+ log.error(" %-34s %r", *item)
+
+ log.error("object statistics:")
+
+ # see process statistics
+ for item in sorted(self.qmf_object.getStatistics()):
+ log.error(" %-34s %r", *item)
+
+ raise
+
+ def __repr__(self):
+ name = self.__class__.__name__
+ return name
+
+ #agent_id = self.get_agent_id()
+ #cls = self.qmf_object.getClassKey().getClassName()
+ #obj_id = self.get_object_id()
+
+ #return "%s(%s,%s,%s)" % (name, agent_id, cls, obj_id)
+
+class MappingException(Exception):
+ pass
+
+def transform_default(value):
+ return value
+
+def transform_timestamp(value):
+ if value != 0:
+ return datetime.fromtimestamp(value / 1000000000)
+
+def transform_pickle(value):
+ return pickle.dumps(x)
+
+transformers = list([transform_default for x in range(32)])
+
+transformers[8] = transform_timestamp
+transformers[10] = str
+transformers[14] = str
+transformers[15] = transform_pickle
+
+def transform_value(attr, value):
+ return transformers[attr.type](value)
Modified: branches/play_plumage/rosemary/python/rosemary/model.py
===================================================================
--- branches/play_plumage/rosemary/python/rosemary/model.py 2012-03-30 17:17:00 UTC (rev 5279)
+++ branches/play_plumage/rosemary/python/rosemary/model.py 2012-03-30 20:27:05 UTC (rev 5280)
@@ -16,25 +16,29 @@
self.sql_logging_enabled = False
- def load_model_dir(self, path):
- assert os.path.isdir(path)
+ def load_model_dir(self, paths):
+ # Let this work across multiple directories
+ if not type(paths) in (list, tuple):
+ paths = [paths]
- extensions = os.path.join(path, "rosemary.xml")
+ for path in paths:
+ assert os.path.isdir(path)
+ extensions = os.path.join(path, "rosemary.xml")
- for name in os.listdir(path):
- file_path = os.path.join(path, name)
+ for name in os.listdir(path):
+ file_path = os.path.join(path, name)
- if not os.path.isfile(file_path):
- continue
+ if not os.path.isfile(file_path):
+ continue
- if file_path == extensions:
- continue
+ if file_path == extensions:
+ continue
- if file_path.endswith(".xml"):
- self.load_model_file(file_path)
+ if file_path.endswith(".xml"):
+ self.load_model_file(file_path)
- if os.path.isfile(extensions):
- self.load_extensions(extensions)
+ if os.path.isfile(extensions):
+ self.load_extensions(extensions)
def load_model_file(self, path):
tree = ElementTree()
