Branch 'v1.4' - 2 commits - server/sqlobject-patches server/UPGRADING.txt
by Matt Domsch
server/UPGRADING.txt | 49 ----------
server/sqlobject-patches/sqlobject-sqlite-executescript.patch | 21 ----
2 files changed, 70 deletions(-)
New commits:
commit d4408b78a3b88ef78b81caf44d491027645b3f47
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Apr 27 11:28:05 2011 -0500
we don't need this sqlobject patch as we aren't using tg-admin upgrade anymore
diff --git a/server/sqlobject-patches/sqlobject-sqlite-executescript.patch b/server/sqlobject-patches/sqlobject-sqlite-executescript.patch
deleted file mode 100644
index 05e2be6..0000000
--- a/server/sqlobject-patches/sqlobject-sqlite-executescript.patch
+++ /dev/null
@@ -1,21 +0,0 @@
-This patch allows multi-line SQL scripts, executed with tg-admin sql
-upgrade, to complete if your backend database is sqlite.
-
-filed upstream as
-http://sourceforge.net/tracker/?func=detail&aid=2883211&group_id=74338&atid=540674
-
-
---- sqlobject/sqlite/sqliteconnection.py.~1~ 2008-03-07 09:54:30.000000000 -0600
-+++ sqlobject/sqlite/sqliteconnection.py 2009-10-21 08:03:40.000000000 -0500
-@@ -172,7 +172,10 @@
- if self.debug:
- self.printDebug(conn, query, 'QueryR')
- try:
-- return cursor.execute(query)
-+ if query.count('\n') <= 1:
-+ return cursor.execute(query)
-+ else:
-+ return cursor.executescript(query)
- except self.module.OperationalError, e:
- raise OperationalError(ErrorMessage(e))
- except self.module.IntegrityError, e:
commit fe3754a0a487d6719c7588e731ccd79acf9bf4b2
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Apr 27 11:26:07 2011 -0500
remove UPGRADING.txt - we have a new method now
diff --git a/server/UPGRADING.txt b/server/UPGRADING.txt
deleted file mode 100644
index 3bf2825..0000000
--- a/server/UPGRADING.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-On occasion, the schema of the MirrorManager database is updated to
-add new functionality. As a mirrormanager administrator, you have
-several choices to deal with this:
-
-1) drop all your existing tables and data, and start fresh with the
-new schema. This erases all information in your database. You've
-been warned. You can:
-
-$ tg-admin sql drop
-$ tg-admin sql create
-
-2) upgrade your database's schema without destroying your data.
-However, this requires that your database already has a table
-sqlobject_db_version, which is created when running:
-
-$ tg-admin sql record --force-db-version=someversion
-
-This gets tricky, as the 'someversion' value isn't automatically
-figured out - you have to specify it manually. If you look in
-mirrormanager/sqlobject-history/, you see a bunch of directories
-corresponding to the schema as it has changed over time. For example:
-
-mirrormanager/sqlobject-history/2009-02-23
-mirrormanager/sqlobject-history/2009-04-06
-mirrormanager/sqlobject-history/2009-10-21
-
-In each directory you will find a file, mirrormanager_version.txt,
-which shows the version of MirrorManager which first used this
-schema. You need to force the database version once, like so:
-
-$ tg-admin sql record --force-db-version 2009-10-21
-
-thereafter you can upgrade to the next version by doing:
-
-$ tg-admin sql upgrade
-
-
-If you must upgrade across several versions, you have to use the
---upgrade-to argument, like so:
-
-# starting from 2009-02-23, 2009-10-21 is latest
-$ tg-admin sql upgrade --upgrade-to=2009-04-06
-$ tg-admin sql upgrade --upgrade-to=2009-10-21
-
-No, SQLObject doesn't make this easier. SQLAlchemy may.
-
-N.B. python-sqlobject-0.10.2 has a bug that is triggered on upgrade if
-you are using a sqlite database. See the sqlobject-patches/ directory
-for a patch to fix this.
13 years
Branch 'v1.4' - server/mirrormanager server/update-schema
by Matt Domsch
server/mirrormanager/schema_updates/__init__.py | 75 ++++++++++++++++++--
server/mirrormanager/schema_updates/oldtables.py | 35 +++++++++
server/mirrormanager/schema_updates/update_1_4_0.py | 65 -----------------
server/update-schema | 40 ++++++++++
4 files changed, 144 insertions(+), 71 deletions(-)
New commits:
commit ab0e395a5a4c8a124acfc0165905ed8772f1513c
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Apr 26 18:05:21 2011 -0500
fix schema updates, add update-schema tool
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index a072a7b..a8141f9 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -1,11 +1,74 @@
-import update_1_2_10
-import update_1_3_2
-import update_1_4_0
+from sqlobject import *
+from sqlobject.sqlbuilder import *
+from turbogears import identity, config
+from mirrormanager.model import *
+from sqlobject import SQLObject, BoolCol, IntCol
+from oldtables import *
+
+from turbogears.database import PackageHub
+hub = PackageHub("mirrormanager")
+__connection__ = hub
+
+changes = {}
+
+def change_tables():
+ Location.createTable(ifNotExists=True)
+ FileGroup.createTable(ifNotExists=True)
+
+ if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
+ 'emailOnAdd' not in OldSite.sqlmeta.columns:
+ OldSite.sqlmeta.addColumn(BoolCol("emailOnDrop", default=False), changeSchema=True)
+ OldSite.sqlmeta.addColumn(BoolCol("emailOnAdd", default=False), changeSchema=True)
+ changes['site.email_on_drop_add'] = True
+
+
+ if 'dnsCountryHost' not in OldHost.sqlmeta.columns:
+ OldHost.sqlmeta.addColumn(BoolCol("dnsCountryHost", default=False), changeSchema=True)
+ changes['host.dns_country_host'] = True
+
+ if 'sortorder' not in OldVersion.sqlmeta.columns and \
+ 'codename' not in OldVersion.sqlmeta.columns:
+ OldVersion.sqlmeta.addColumn(IntCol("sortorder", default=0), changeSchema=True)
+ OldVersion.sqlmeta.addColumn(UnicodeCol("codename", default=None), changeSchema=True)
+ changes['version.sortorder_codename'] = True
+
+ if 'publiclist' not in OldProduct.sqlmeta.columns:
+ OldProduct.sqlmeta.addColumn(BoolCol("publiclist", default=True), changeSchema=True)
+ changes['product.publiclist'] = True
+
+ if 'geoDnsDomain' not in OldCategory.sqlmeta.columns:
+ OldCategory.sqlmeta.addColumn(UnicodeCol("GeoDNSDomain", default=None), changeSchema=True)
+ changes['category.geo_dns_domain'] = True
+
+def fill_columns():
+ if changes.get('site.email_on_drop_add'):
+ for s in Site.select():
+ s.emailOnDrop=False
+ s.emailOnAdd=False
+
+ if changes.get('host.dns_country_host'):
+ for h in Host.select():
+ h.dnsCountryHost = False
+
+ if changes.get('version.sortorder_codename'):
+ for v in Version.select():
+ v.sortorder = 0
+ v.codename = None
+
+ if changes.get('product.publiclist'):
+ for p in Product.select():
+ p.publiclist = True
+
+ if changes.get('category.geo_dns_domain'):
+ for c in Category.select():
+ c.GeoDNSDomain = None
+
+
+
def update():
"""Fills newly created database columns with information.
Run this after using tg-admin sql upgrade.
"""
- update_1_2_10.update()
- update_1_3_2.update()
- update_1_4_0.update()
+ change_tables()
+ fill_columns()
diff --git a/server/mirrormanager/schema_updates/oldtables.py b/server/mirrormanager/schema_updates/oldtables.py
new file mode 100644
index 0000000..b7779f3
--- /dev/null
+++ b/server/mirrormanager/schema_updates/oldtables.py
@@ -0,0 +1,35 @@
+from sqlobject import *
+
+from turbogears.database import PackageHub
+hub = PackageHub("mirrormanager")
+__connection__ = hub
+
+# upgrade methodology borrowed from
+# http://www.mail-archive.com/sqlobject-discuss@lists.sourceforge.net/msg04...
+
+class OldSite(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'site'
+
+class OldHost(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'host'
+
+class OldVersion(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'version'
+
+class OldProduct(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'product'
+
+class OldCategory(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'category'
+
+
diff --git a/server/mirrormanager/schema_updates/update_1_4_0.py b/server/mirrormanager/schema_updates/update_1_4_0.py
deleted file mode 100644
index a5629b5..0000000
--- a/server/mirrormanager/schema_updates/update_1_4_0.py
+++ /dev/null
@@ -1,65 +0,0 @@
-from mirrormanager.model import Location, FileGroup
-from sqlobject import SQLObject, BoolCol, IntCol
-
-# upgrade methodology borrowed from
-# http://www.mail-archive.com/sqlobject-discuss@lists.sourceforge.net/msg04...
-
-class OldSite(SQLObject):
- class sqlmeta:
- fromDatabase = True
- table = 'site'
-
-class OldHost(SQLObject):
- class sqlmeta:
- fromDatabase = True
- table = 'host'
-
-class OldVersion(SQLObject):
- class sqlmeta:
- fromDatabase = True
- table = 'version'
-
-class OldProduct(SQLObject):
- class sqlmeta:
- fromDatabase = True
- table = 'product'
-
-class OldCategory(SQLObject):
- class sqlmeta:
- fromDatabase = True
- table = 'category'
-
-def update():
- Location.createTable(ifNotExists=True)
- FileGroup.createTable(ifNotExists=True)
-
- if 'email_on_drop' not in OldSite.sqlmeta.columns and \
- 'email_on_add' not in OldSite.sqlmeta.columns:
- OldSite.addColumn(BoolCol("emailOnDrop", default=False), changeSchema=True)
- OldSite.addColumn(BoolCol("emailOnAdd", default=False), changeSchema=True)
- for s in OldSite.select():
- s.emailOnDrop=False
- s.emailOnAdd=False
-
- if 'dns_country_host' not in OldHost.sqlmeta.columns:
- OldHost.addColumn(BoolCol("dnsCountryHost", default=False), changeSchema=True)
- for h in OldHost.select():
- h.dnsCountryHost = False
-
- if 'sortorder' not in OldVersion.sqlmeta.columns and \
- 'codename' not in OldVersion.sqlmeta.columns:
- OldVersion.addColumn(IntCol("sortorder", default=0, changeSchema=True))
- OldVersion.addColumn(UnicodeCol("codename", default=None), changeSchema=True)
- for v in OldVersion.select():
- v.sortorder = 0
- v.codename = None
-
- if 'publiclist' not in OldProduct.sqlmeta.columns:
- OldProduct.addColumn(BoolCol("publiclist", default=True), changeSchema=True)
- for p in OldProduct.select():
- p.publiclist = True
-
- if 'geo_dns_domain' not in OldCategory.sqlmeta.columns:
- OldCategory.addColumn(UnicodeCol("GeoDNSDomain", default=None), changeSchema=True)
- for c in OldCategory.select():
- c.GeoDNSDomain = None
diff --git a/server/update-schema b/server/update-schema
new file mode 100755
index 0000000..ca421cc
--- /dev/null
+++ b/server/update-schema
@@ -0,0 +1,40 @@
+#!/usr/bin/python
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+from optparse import OptionParser
+import sys
+from sqlobject import *
+import turbogears
+from mirrormanager.model import *
+
+from turbogears.database import PackageHub
+hub = __connection__ = None
+
+options = None
+
+
+def main():
+ global options
+ parser = OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='/etc/mirrormanager/prod.cfg',
+ help="TurboGears config file to use (default=/etc/mirrormanager/prod.cfg)")
+
+ (options, args) = parser.parse_args()
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+ global hub
+ global __connection__
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ import mirrormanager.schema_updates
+ mirrormanager.schema_updates.update()
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())
+
13 years
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 478043767b735a158ff0c3b0ffd98e7f88e9ef80
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Apr 26 15:34:38 2011 -0500
add missing comma
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 8f05356..c01ba4e 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -1268,7 +1268,7 @@ class Root(controllers.RootController):
"locations":Location.select(orderBy='name'),
"num_sites": num_sites,
"num_private_sites":num_private_sites,
- "num_public_sites":(num_sites-num_private_sites)
+ "num_public_sites":(num_sites-num_private_sites),
"num_hosts":num_hosts,
"num_private_hosts":num_private_hosts,
"num_public_hosts":(num_hosts - num_private_hosts)
13 years
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
New commits:
commit ed972730963a1c618faa5fb6b3110c860e1c4859
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Apr 23 22:26:43 2011 -0500
mirrorlist_server: fix host netblocks lookup
1) host_netblocks_tree (with an s!)
2) don't walk the list, use the tree as it is intended, deleting found
nodes and searching again. This should eliminate any slowdown due to
the host netblocks list getting too long.
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 440fb5b..e2d19e6 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -190,13 +190,15 @@ def client_netblocks(ip):
except:
return result
- # fast lookup in the tree; if present, find all the netblocks in the list
- # rather than just the "best" one
- node = host_netblock_tree.search_best(ip.strNormal()))
- if node is not None:
- for k,v in host_netblock_cache.iteritems():
- if clientIP in k:
- result.extend(v)
+ # fast lookup in the tree; if present, find all the netblocks by deleting the found one and searching again
+ # this is safe w/o copying the tree again only because this is the only place the tree is used, and
+ # we'll get a new copy of the tree from our parent the next time it fork()s.
+ node = host_netblocks_tree.search_best(ip.strNormal())
+ while node is not None:
+ result.extend(node.data['hosts'])
+ prefix = node.prefix
+ host_netblocks_tree.delete(prefix)
+ node = host_netblocks_tree.search_best(ip.strNormal()))
return result
def trim_by_client_country(s, clientCountry):
@@ -612,7 +614,8 @@ def do_mirrorlist(kwargs):
def setup_host_netblocks_tree():
tree = radix.Radix()
for k, v in host_netblock_cache.iteritems():
- tree.add(k.strNormal())
+ node = tree.add(k.strNormal())
+ node.data['hosts'] = v
return tree
def setup_netblocks(netblocks_file):
13 years, 1 month