Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 502f065773418a66b0d3204e8658dbfb018a9102
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Apr 30 17:18:50 2011 -0500
mirrorlist_server: on invalid requested repo, return repo=%s&arch=%s
fixes https://bugzilla.redhat.com/show_bug.cgi?id=701049
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 751dc0a..d5d078c 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -464,7 +464,7 @@ def do_mirrorlist(kwargs):
repo_information += "# following repositories are available:\n"
for i in repos:
if i[0] is not None and i[1] is not None:
- repo_information += "# repo=%s, arch=%s\n" % i
+ repo_information += "# repo=%s&arch=%s\n" % i
return return_error(kwargs, message=repo_information)
13 years
Branch 'v1.4' - server/errorcheck
by Matt Domsch
server/errorcheck | 47 ++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 44 insertions(+), 3 deletions(-)
New commits:
commit fb055060dbe308f9dee1fc265c8810217872a83b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Apr 30 00:17:47 2011 -0500
errorcheck: add tests; fix fails-to-run bugs
diff --git a/server/errorcheck b/server/errorcheck
index 82355da..5b740ba 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -22,7 +22,12 @@ import dns.resolver
from IPy import IP
from fedora.client.fas2 import AccountSystem
import turbomail
-from turbomail.startup import start_extension, shutdown_extension
+try:
+ from turbomail.adapters.tg1 import start_extension, shutdown_extension
+except ImportError:
+ # fall back to older method
+ from turbomail.startup import start_extension, shutdown_extension
+
from turbogears.database import PackageHub
from turbogears import config
@@ -164,7 +169,7 @@ def check_host(sr, host):
return
ips = name_to_ips(host.name)
- if len(ips) == 0 && not host.is_private():
+ if len(ips) == 0 and not host.is_private():
msg = u"Host name %s not found in DNS. Please use a FQDN for your host name. You will also need to make the corresponding change in your report_mirror.conf file." % (host.name)
sr.messages.append(msg)
else:
@@ -355,6 +360,40 @@ Site: %(site)s %(siteurl)s
send_mail(sender, to, subject, body, cc=cc)
+def check_objects():
+ for p in Product.select():
+ if not p.name:
+ print "Error: Product %s has no name" % p
+
+ for a in Arch.select():
+ if not a.name:
+ print "Error: Arch %s has no name" % a
+
+ for c in Category.select():
+ if not c.name:
+ print "Error: Category %s has no name" % c
+ if not c.product:
+ print "Error: Category %s has no product" % c
+ if not c.topdir:
+ print "Error: Category %s has no topdir" % c
+
+ for v in Version.select():
+ if not v.name:
+ print "Error: Version %s has no name" % v
+ if not v.product:
+ print "Error: Version %s has no product" % v
+
+ for r in Repository.select():
+ if not r.category:
+ print "Error: Repository %s has no category" % r
+ if not r.version:
+ print "Error: Repository %s has no version" % r
+ if not r.arch:
+ print "Error: Repository %s has no arch" % r
+ if not r.directory:
+ print "Error: Repository %s has no directory" % r
+
+
def doit():
global options
global internet2_tree
@@ -362,6 +401,8 @@ def doit():
internet2_tree = setup_netblocks(internet2_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file)
+ check_objects()
+
for site in Site.select():
sr = check_site(site)
if sr is not None and options.email: send_site_emails(sr, options.cc)
@@ -390,7 +431,7 @@ def main():
parser.add_option("-e", "--email",
dest="email", action="store_true", default=False,
help="Send emails to Site admins")
- parser.add_option("-cc",
+ parser.add_option("--cc",
dest="cc", default=[],
help="Email address to include on cc:")
13 years
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/model.py | 9 +++++++++
server/mirrormanager/schema_updates/__init__.py | 11 +----------
2 files changed, 10 insertions(+), 10 deletions(-)
New commits:
commit 249c59c29ece8f783f6759657f2d61ffa2a9f8e5
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Apr 29 23:55:02 2011 -0500
create HostLocation intermediate table in model
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 8838302..29d2949 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -899,6 +899,15 @@ class Location(SQLObject):
self.removeHost(h)
SQLObject.destroySelf(self)
+# manual creation of the RelatedJoin table so we can guarantee uniqueness
+class HostLocation(SQLObject):
+ class sqlmeta:
+ table = 'host_location'
+ host = ForeignKey('Host')
+ location = ForeignKey('Location')
+ hlidx = DatabaseIndex('host', 'location', unique=True)
+
+
class FileGroup(SQLObject):
class sqlmeta:
cacheValues = False
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index 234f188..22dd19d 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -11,21 +11,12 @@ __connection__ = hub
changes = {}
-# this class won't appear in the model, it's only used to create the table in the database
-class NewHostLocation(SQLObject):
- class sqlmeta:
- table = 'host_location'
- host = ForeignKey('Host')
- location = ForeignKey('Location')
- index = DatabaseIndex('host', 'location', unique=True)
-
-
def change_tables():
global changes
Location.createTable(ifNotExists=True)
FileGroup.createTable(ifNotExists=True)
- NewHostLocation.createTable(ifNotExists=True)
+ HostLocation.createTable(ifNotExists=True)
if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
'emailOnAdd' not in OldSite.sqlmeta.columns:
13 years
Branch 'v1.4' - 2 commits - server/update-master-directory-list
by Matt Domsch
server/update-master-directory-list | 33 +++++++++++++++++++++------------
1 file changed, 21 insertions(+), 12 deletions(-)
New commits:
commit 6733901175ba722737c985366bc8979e22d0a5cd
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Apr 29 10:19:19 2011 -0500
umdl: lookup category early, pass it down
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index 8bada71..a25a99d 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -368,8 +368,7 @@ def sync_category_directories(category, category_directories):
make_repomd_file_details(dir)
ageFileDetails()
-def parse_rsync_listing(cname, f):
- category = Category.byName(cname)
+def parse_rsync_listing(category, f):
category_directories = {}
for line in f:
line.strip()
@@ -387,7 +386,7 @@ def parse_rsync_listing(cname, f):
-def sync_directories_using_rsync(rsyncpath, cname, extra_rsync_args=None):
+def sync_directories_using_rsync(rsyncpath, category, extra_rsync_args=None):
cmd = "rsync --temp-dir=/tmp -r --exclude=.snapshot --exclude='*.~tmp~'"
if extra_rsync_args is not None:
cmd += ' ' + extra_rsync_args
@@ -396,23 +395,22 @@ def sync_directories_using_rsync(rsyncpath, cname, extra_rsync_args=None):
devnull = open('/dev/null', 'r')
p = Popen(cmd, shell=True, stdin=devnull, stdout=PIPE, bufsize=1, close_fds=True)
except:
- print "Category %s: Unable to run %s" % (cname, args)
+ print "Category %s: Unable to run %s" % (category.name, args)
devnull.close()
return
- parse_rsync_listing(cname, p.stdout)
+ parse_rsync_listing(category, p.stdout)
devnull.close()
-def sync_directories_from_file(filename, cname):
+def sync_directories_from_file(filename, category):
f = open(filename, 'r')
- parse_rsync_listing(cname, f)
+ parse_rsync_listing(category, f)
f.close()
-def sync_directories_from_directory(directory, cname, excludes=[]):
+def sync_directories_from_directory(directory, category, excludes=[]):
global unreadable_dirs
# drop trailing slashes from path
directory = directory.rstrip('/')
- category = Category.byName(cname)
category_directories = {}
topdir = category.topdir
topdirName = topdir.name
@@ -510,14 +508,14 @@ def main():
continue
if i['type'] == 'rsync':
- sync_directories_using_rsync(i['url'], cname, options)
+ sync_directories_using_rsync(i['url'], category, options)
if i['type'] == 'file':
- sync_directories_from_file(i['url'], cname)
+ sync_directories_from_file(i['url'], category)
if i['type'] == 'directory':
excludes = i.get('excludes', [])
- sync_directories_from_directory(i['path'], cname, excludes)
+ sync_directories_from_directory(i['path'], category, excludes)
remove_pidfile(pidfile)
print "Ending umdl %s" % (datetime.datetime.utcnow().isoformat())
commit 24b5c76e5b3c92ca2a62d61fae34749e0cb0c1f9
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Apr 29 10:15:26 2011 -0500
catch errors in umdl.master_directories earlier, avoids later failures
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index a7e7f9e..8bada71 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -498,15 +498,26 @@ def main():
except KeyError:
options = None
+ cname = i['category']
+ try:
+ category = Category.byName(cname)
+ except SQLObjectNotFound:
+ print 'Error: umdl.master_directories Category %s does not exist in the database, skipping' % (cname)
+ continue
+
+ if category.product is None:
+ print 'Error: umdl.master_directories Category %s has null Product, skipping' % (cname)
+ continue
+
if i['type'] == 'rsync':
- sync_directories_using_rsync(i['url'], i['category'], options)
+ sync_directories_using_rsync(i['url'], cname, options)
if i['type'] == 'file':
- sync_directories_from_file(i['url'], i['category'])
+ sync_directories_from_file(i['url'], cname)
if i['type'] == 'directory':
excludes = i.get('excludes', [])
- sync_directories_from_directory(i['path'], i['category'], excludes)
+ sync_directories_from_directory(i['path'], cname, excludes)
remove_pidfile(pidfile)
print "Ending umdl %s" % (datetime.datetime.utcnow().isoformat())
13 years
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/schema_updates/__init__.py | 1 +
1 file changed, 1 insertion(+)
New commits:
commit 79c72e2a9ff89b1244a56c033ee0911cbc5f56d6
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Apr 29 01:21:08 2011 -0500
add uniqueness index to host_location
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index 4490464..234f188 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -17,6 +17,7 @@ class NewHostLocation(SQLObject):
table = 'host_location'
host = ForeignKey('Host')
location = ForeignKey('Location')
+ index = DatabaseIndex('host', 'location', unique=True)
def change_tables():
13 years
Branch 'v1.4' - 2 commits - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 3 --
server/mirrormanager/schema_updates/__init__.py | 11 ++++++++-
server/mirrormanager/templates/adminview.kid | 28 ------------------------
3 files changed, 10 insertions(+), 32 deletions(-)
New commits:
commit 6a65f14581980e2ffa2ebec6b8d8f77c3d98c9cb
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Apr 28 17:47:30 2011 -0500
create host_location table in the database on upgrade
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index f2d49d7..4490464 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -11,11 +11,20 @@ __connection__ = hub
changes = {}
+# this class won't appear in the model, it's only used to create the table in the database
+class NewHostLocation(SQLObject):
+ class sqlmeta:
+ table = 'host_location'
+ host = ForeignKey('Host')
+ location = ForeignKey('Location')
+
+
def change_tables():
global changes
Location.createTable(ifNotExists=True)
FileGroup.createTable(ifNotExists=True)
+ NewHostLocation.createTable(ifNotExists=True)
if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
'emailOnAdd' not in OldSite.sqlmeta.columns:
@@ -23,7 +32,7 @@ def change_tables():
OldSite.sqlmeta.addColumn(BoolCol("emailOnAdd", default=False), changeSchema=True)
changes['site.email_on_drop_add'] = True
-
+ # Host
if 'dnsCountryHost' not in OldHost.sqlmeta.columns:
OldHost.sqlmeta.addColumn(BoolCol("dnsCountryHost", default=False), changeSchema=True)
changes['host.dns_country_host'] = True
commit b73415723a171edc898dcfe07e4a4f2eb2b2c9fd
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Apr 28 16:19:57 2011 -0500
shorten adminview by dropping directories, repositories, and netblocks long lists
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 95f3321..b06d455 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -1258,11 +1258,8 @@ class Root(controllers.RootController):
"arches":Arch.select(),
"products":Product.select(),
"versions":Version.select(),
- "directories":Directory.select(orderBy='name'),
"categories":Category.select(),
- "repositories":Repository.select(orderBy='name'),
"embargoed_countries":EmbargoedCountry.select(),
- "netblocks":HostNetblock.select(orderBy='host_id'),
"repository_redirects":RepositoryRedirect.select(orderBy='fromRepo'),
"country_continent_redirects":CountryContinentRedirect.select(orderBy='country'),
"locations":Location.select(orderBy='name'),
diff --git a/server/mirrormanager/templates/adminview.kid b/server/mirrormanager/templates/adminview.kid
index 089295a..50d1740 100644
--- a/server/mirrormanager/templates/adminview.kid
+++ b/server/mirrormanager/templates/adminview.kid
@@ -65,25 +65,6 @@ Nothing to see here, move along.
</li>
</ul>
</div>
-<div id="Repositories">
- <h3>Repositories</h3>
- <ul>
- <li py:for="r in repositories">
- <a href="${tg.url('/repository/'+str(r.id)+'/')}">
- <span py:replace="r.directory.name">Repository Name
- </span></a>
- <a href="${tg.url('/repository/'+str(r.id)+'/delete')}">[Delete]</a>
- </li>
- </ul>
-</div>
-<div id="Directories">
- <h3>Directories</h3>
- <ul>
- <li py:for="d in directories">
- <span py:replace="d.name">Directory Name</span>
- </li>
- </ul>
-</div>
<div id="Embargoed Countries">
<h3>Embargoed Counries <a href="${tg.url('/embargoed_country/0/new')}">[Add]</a></h3>
<ul>
@@ -93,15 +74,6 @@ Nothing to see here, move along.
</li>
</ul>
</div>
-<div id="Netblocks">
-<h3>Netblocks</h3>
-<ul>
- <li py:for="n in netblocks">
- <a href="${tg.url('/host/'+str(n.host.id)+'/')}"><span py:replace="n.host.name">Host Name</span></a>
- <span py:replace="n.netblock">Netblock</span>
- </li>
-</ul>
-</div>
<div id="Repository Redirects">
<h3>Repository Redirects</h3>
<ul>
13 years
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 56bdcb1706a9b5fe781ed623417698fce3f125e0
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Apr 28 16:13:11 2011 -0500
fix adminview statistics
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index c01ba4e..95f3321 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -1248,7 +1248,7 @@ class Root(controllers.RootController):
num_sites = Site.select().count()
num_private_sites = 0
for s in Site.select():
- if site.private: num_private_sites = num_private_sites + 1
+ if s.private: num_private_sites = num_private_sites + 1
num_hosts = Host.select().count()
num_private_hosts = 0
for h in Host.select():
13 years
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
New commits:
commit 79df6c0e95148c96e08a17198bc49278ea63456e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Apr 27 18:13:14 2011 -0500
ip -> clientIP
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 9a35bde..751dc0a 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -193,12 +193,12 @@ def client_netblocks(ip):
# fast lookup in the tree; if present, find all the netblocks by deleting the found one and searching again
# this is safe w/o copying the tree again only because this is the only place the tree is used, and
# we'll get a new copy of the tree from our parent the next time it fork()s.
- node = host_netblocks_tree.search_best(ip.strNormal())
+ node = host_netblocks_tree.search_best(clientIP.strNormal())
while node is not None:
result.extend(node.data['hosts'])
prefix = node.prefix
host_netblocks_tree.delete(prefix)
- node = host_netblocks_tree.search_best(ip.strNormal())
+ node = host_netblocks_tree.search_best(clientIP.strNormal())
return result
def trim_by_client_country(s, clientCountry):
13 years
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 2c4ac04da63559f82ddbe962fa90dfbaa631159f
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Apr 28 05:04:27 2011 -0500
fix typo
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index e2d19e6..9a35bde 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -198,7 +198,7 @@ def client_netblocks(ip):
result.extend(node.data['hosts'])
prefix = node.prefix
host_netblocks_tree.delete(prefix)
- node = host_netblocks_tree.search_best(ip.strNormal()))
+ node = host_netblocks_tree.search_best(ip.strNormal())
return result
def trim_by_client_country(s, clientCountry):
13 years
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/schema_updates/__init__.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
New commits:
commit fcd3d551f658fa729d42e5aacf764ebafbb009da
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Apr 27 14:18:56 2011 -0500
use global
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index a8141f9..f2d49d7 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -12,6 +12,8 @@ __connection__ = hub
changes = {}
def change_tables():
+ global changes
+
Location.createTable(ifNotExists=True)
FileGroup.createTable(ifNotExists=True)
@@ -40,7 +42,9 @@ def change_tables():
OldCategory.sqlmeta.addColumn(UnicodeCol("GeoDNSDomain", default=None), changeSchema=True)
changes['category.geo_dns_domain'] = True
-def fill_columns():
+def fill_new_columns():
+ global changes
+
if changes.get('site.email_on_drop_add'):
for s in Site.select():
s.emailOnDrop=False
@@ -71,4 +75,4 @@ def update():
Run this after using tg-admin sql upgrade.
"""
change_tables()
- fill_columns()
+ fill_new_columns()
13 years