mirrormanager-commits
December 2014
mirrormanager-commits@lists.fedorahosted.org
- 1 participants
- 4 discussions
[mirrormanager] master: update-EC2-netblocks: set name=None creating new HostNetblocks (e645277)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit e645277f40786b3eaad765d320cc45d0a9d9252b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Dec 5 12:39:41 2014 -0600
update-EC2-netblocks: set name=None creating new HostNetblocks
For some reason, we have to provide the name. I guess there's no
default value being set.
>---------------------------------------------------------------
server/update-EC2-netblocks | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/update-EC2-netblocks b/server/update-EC2-netblocks
index e57b0a8..c23b7a1 100755
--- a/server/update-EC2-netblocks
+++ b/server/update-EC2-netblocks
@@ -66,7 +66,7 @@ def doit():
if not host_has_netblock(hosts_by_region, region, ip_prefix):
print "Added host %s netblock %s" % (host.name, ip_prefix)
if not options.dry_run:
- nb = HostNetblock(host=host, netblock=ip_prefix) # this adds the entry to the database, mark as not stale
+ nb = HostNetblock(host=host, netblock=ip_prefix, name=None) # this adds the entry to the database, mark as not stale
hosts_by_region[region]['netblocks'][ip_prefix] = {'host_netblock_id' : nb.id,
'stale':False,
}
9 years, 5 months
- 1
- 0
[mirrormanager] master: bugfixes in update-EC2-netblocks (8f36390)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 8f363903e330257b10aece7cb8005b9b7c1c42bb
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Dec 5 12:09:28 2014 -0600
bugfixes in update-EC2-netblocks
>---------------------------------------------------------------
server/update-EC2-netblocks | 30 +++++++++++++++++-------------
1 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/server/update-EC2-netblocks b/server/update-EC2-netblocks
index 8f3ec1d..e57b0a8 100755
--- a/server/update-EC2-netblocks
+++ b/server/update-EC2-netblocks
@@ -43,42 +43,46 @@ def get_ip_ranges():
def parse_ip_ranges(ipranges_str):
return json.loads(ipranges_str)
-def host_has_netblock(hosts_by_region, host, netblock):
- return netblock in hosts_by_region[host]['netblocks']
+def host_has_netblock(hosts_by_region, region, netblock):
+ return netblock in hosts_by_region[region]['netblocks']
def doit():
hosts_by_region = s3_mirrors()
ipranges_str = get_ip_ranges()
ipranges = parse_ip_ranges(ipranges_str)
- for p in ipranges.prefixes:
+ for p in ipranges['prefixes']:
service = p['service']
region = p['region']
ip_prefix = p['ip_prefix']
if service != "EC2":
continue
+ if region == "GLOBAL":
+ continue
if region in hosts_by_region: # ignore regions we don't have a mirror in
- host = hosts_by_region[region]
+ h = hosts_by_region[region]
+ host = h['host']
if not host_has_netblock(hosts_by_region, region, ip_prefix):
- print "Added region %s netblock %s" % (region, ip_prefix)
+ print "Added host %s netblock %s" % (host.name, ip_prefix)
if not options.dry_run:
nb = HostNetblock(host=host, netblock=ip_prefix) # this adds the entry to the database, mark as not stale
- hosts_by_region[region]['netblocks'][ip_prefix] = {'host_netblock_id' : nb.id,
- 'stale':False,
- }
+ hosts_by_region[region]['netblocks'][ip_prefix] = {'host_netblock_id' : nb.id,
+ 'stale':False,
+ }
else:
# found the netblock in our database, mark it as not stale
hosts_by_region[region]['netblocks'][ip_prefix]['stale'] = False
# delete stale netblock entries from the database
- for h in hosts_by_region:
- for netblock in hosts_by_region[h]['netblocks']:
- if hosts_by_region[h]['netblocks'][netblock]['stale']: # delete this, it's no longer on Amazon's list
- print "Deleted region %s netblock %s" % (region, netblock)
- nb = HostNetblock.get(hosts_by_region[h]['netblocks'][netblock]['host_netblock_id'])
+ for region, h in hosts_by_region.items():
+ for netblock in hosts_by_region[region]['netblocks']:
+ if hosts_by_region[region]['netblocks'][netblock]['stale']: # delete this, it's no longer on Amazon's list
+ host = h['host']
+ print "Deleted host %s netblock %s" % (host.name, netblock)
if not options.dry_run:
+ nb = HostNetblock.get(hosts_by_region[region]['netblocks'][netblock]['host_netblock_id'])
nb.destroySelf()
9 years, 5 months
- 1
- 0
[mirrormanager] master: Merge branch 'master' of ssh+git://fedorahosted.org/git/mirrormanager (1a5922b)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 1a5922b087dd03a84a8404a80cbbd37528e9361b
Merge: 82ebdfe 9b8611a
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Dec 5 11:19:07 2014 -0600
Merge branch 'master' of ssh+git://fedorahosted.org/git/mirrormanager
>---------------------------------------------------------------
Makefile | 4 +-
client/report_mirror | 2 +-
mirrorlist-server/apache/mirrorlist-server.conf | 12 ++++++++
mirrormanager.spec.in | 4 ++
server/dev.cfg | 4 ++
server/mirrormanager/controllers.py | 6 ++--
server/mirrormanager/lib.py | 2 +-
server/mirrormanager/model.py | 17 ++++++++---
server/mirrormanager/repomap.py | 35 +++++++++++++++++++++++
server/prod.cfg.example | 3 ++
10 files changed, 77 insertions(+), 12 deletions(-)
diff --git a/Makefile b/Makefile
index 2fd533b..2a5e4fa 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
-RELEASE_DATE := "18-Dec-2013"
+RELEASE_DATE := "21-Jan-2014"
RELEASE_MAJOR := 1
RELEASE_MINOR := 4
-RELEASE_EXTRALEVEL := .3
+RELEASE_EXTRALEVEL := .4
RELEASE_NAME := mirrormanager
RELEASE_VERSION := $(RELEASE_MAJOR).$(RELEASE_MINOR)$(RELEASE_EXTRALEVEL)
RELEASE_STRING := $(RELEASE_NAME)-$(RELEASE_VERSION)
diff --git a/client/report_mirror b/client/report_mirror
index 00813be..fe64a93 100755
--- a/client/report_mirror
+++ b/client/report_mirror
@@ -290,7 +290,7 @@ def main():
print server.checkin(data)
except socket.error, m:
print "Error checking in: %s. Please try again later." % (m[1])
- except ProtocolError:
+ except xmlrpclib.ProtocolError:
print "Error checking in: Service Temporarily Unavailable. Please try again later."
sys.exit(1)
except xmlrpclib.Fault:
diff --git a/mirrorlist-server/apache/mirrorlist-server.conf b/mirrorlist-server/apache/mirrorlist-server.conf
index 2858c63..9295787 100644
--- a/mirrorlist-server/apache/mirrorlist-server.conf
+++ b/mirrorlist-server/apache/mirrorlist-server.conf
@@ -40,3 +40,15 @@ WSGIScriptAlias /mirrorlist /usr/share/mirrormanager/mirrorlist-server/mirrorlis
Allow from all
</IfModule>
</Location>
+
+<Location /publiclist>
+ <IfModule mod_authz_core.c>
+ # Apache 2.4
+ Require all granted
+ </IfModule>
+ <IfModule !mod_authz_core.c>
+ # Apache 2.2
+ Order deny,allow
+ Allow from all
+ </IfModule>
+</Location>
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in
index 6c70c11..b6cc80f 100644
--- a/mirrormanager.spec.in
+++ b/mirrormanager.spec.in
@@ -62,6 +62,7 @@ getent passwd mirrormanager >/dev/null || \
-c "MirrorManager" mirrormanager
exit 0
+
%files
%defattr(-,root,root,-)
%{_datadir}/%{name}
@@ -97,6 +98,9 @@ exit 0
%doc LICENSES LICENSE-GPLv2
%changelog
+* Tue Jan 21 2014 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.4-1
+- upstream bugfix release
+
* Wed Dec 18 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.3-1
- upstream bugfix release
- Reduce Fedora-isms to make it more usable for other distributions
diff --git a/server/dev.cfg b/server/dev.cfg
index 6391da1..262a478 100644
--- a/server/dev.cfg
+++ b/server/dev.cfg
@@ -108,6 +108,10 @@ sqlobject.dburi="notrans_postgres://postgres@127.0.0.1/mirrormanager"
# for Windows users, sqlite URIs look like:
# sqlobject.dburi="sqlite:///drive_letter:/path/to/file"
+# Note: If you are using a database on another server, over a network socket,
+# and selinux is enabled, you will need to run:
+# sudo setsebool -P httpd_can_network_connect_db 1
+
# SERVER
# Some server parameters that you may want to tweak
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 756ce69..aae01b8 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -914,13 +914,13 @@ class Root(controllers.RootController):
num_prefix = num_prefix_components(stripprefix)
except KeyError:
message=u'Missing categories, since, or stripprefix arguments'
- return project_dict('rsync_acl', includes=[], excludes=excludes, message=message)
+ return project_dict('rsyncFilter', includes=[], excludes=excludes, message=message)
try:
since = int(since)
except:
message=u'value of argument since is not an integer'
- return project_dict('rsync_acl', includes=[], excludes=excludes, message=message)
+ return project_dict('rsyncFilter', includes=[], excludes=excludes, message=message)
includes = set()
categories_requested = c.split(',')
@@ -935,7 +935,7 @@ class Root(controllers.RootController):
# add trailing slash as rsync wants it
for i in xrange(len(includes)):
includes[i] += u'/'
- return project_dict('rsync_acl', includes=includes, excludes=excludes, message=message)
+ return project_dict('rsyncFilter', includes=includes, excludes=excludes, message=message)
def fedora_login(self, forward_url=None, previous_url=None, *args, **kw):
login_dict = fc_login(forward_url, previous_url, args, kw)
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index b78884a..a88fbd8 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -15,7 +15,7 @@ def createErrorString(tg_errors):
if type(tg_errors) == types.DictType:
for param, inv in tg_errors.items():
- if type(inv) == types.StringType:
+ if type(inv) == types.StringType or type(inv) == types.UnicodeType:
errors.append("%s: %s"%(param, inv))
else:
errors.append("%s(%s): %s"%(param, inv.value, inv.msg))
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 8ea815e..2484a8a 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -291,15 +291,22 @@ class Host(SQLObject):
dname = hc.category.topdir.name
# Don't create an entry for a directory the database doesn't know about
+ # and if a crawler created it so we hit a unique violation, then we don't have to
try:
- dir = Directory.byName(dname)
- hcdir = HostCategoryDir(host_category=hc, path=d, directory=dir)
+ directory = Directory.byName(dname)
+ hcdir = HostCategoryDir(host_category=hc, path=d, directory=directory)
added += 1
except:
pass
- for d in HostCategoryDir.selectBy(host_category=hc):
- if d.path not in config[c]['dirtree'].keys():
- d.destroySelf()
+ for hcdir in hc.dirs:
+ # handle disappearing hcdirs, deleted by other processes
+ try:
+ hcdirpath = hcdir.path
+ except: continue
+ if hcdirpath not in config[c]['dirtree'].keys():
+ try:
+ hcdir.destroySelf()
+ except: pass
deleted += 1
message += "Category %s directories updated: %s added: %s deleted %s\n" % (category.name, marked_up2date, added, deleted)
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index 15d7d22..dde7ac6 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -37,6 +37,9 @@ def repo_prefix(path, category, ver):
isUpdatesReleased = u'updates' in path
isEverything = u'Everything' in path
isFedora = u'Fedora' in path
+ isCloud = u'Cloud' in path
+ isServer = u'Server' in path
+ isWorkstation = u'Workstation' in path
isEpel = (category.name == u'Fedora EPEL')
@@ -92,6 +95,38 @@ def repo_prefix(path, category, ver):
else:
# fedora-install-
prefix = u'fedora-install-%s' % version
+ elif isCloud:
+ if isDebug or isSource:
+ # ignore releases/$version/Cloud/$arch/debug/
+ # ignore releases/$version/Cloud/source/SRPMS/
+ prefix = None
+ else:
+ # fedora-cloud-
+ prefix = u'fedora-cloud-%s' % version
+ elif isServer:
+ if isDebug or isSource:
+ # ignore releases/$version/Server/$arch/debug/
+ # ignore releases/$version/Server/source/SRPMS/
+ prefix = None
+ else:
+ # fedora-server-
+ prefix = u'fedora-server-%s' % version
+ elif isWorkstation:
+ if isDebug or isSource:
+ # ignore releases/$version/Workstation/$arch/debug/
+ # ignore releases/$version/Workstation/source/SRPMS/
+ prefix = None
+ else:
+ # fedora-workstation-
+ prefix = u'fedora-workstation-%s' % version
+ else: # development/NN branch day
+ # fedora-
+ if isDebug:
+ prefix = u'fedora-debug-%s' % version
+ elif isSource:
+ prefix = u'fedora-source-%s' % version
+ else:
+ prefix=u'fedora-%s' % version
elif isUpdatesReleased:
# updates-released-
if isDebug:
diff --git a/server/prod.cfg.example b/server/prod.cfg.example
index e4a2fea..c8ff204 100644
--- a/server/prod.cfg.example
+++ b/server/prod.cfg.example
@@ -57,6 +57,9 @@ mrr.repos = ''' {
# sqlobject.dburi="notrans_postgres://mirrormanager@127.0.0.1/mirrormanager"
# sqlobject.dburi="notrans_mysql://username:password@hostname:port/databasename"
# Note: MirrorManager doesn't use Transactions, so preceed with 'notrans_' as shown.
+# Note: If you are using a database on another server, over a network socket,
+# and selinux is enabled, you will need to run:
+# sudo setsebool -P httpd_can_network_connect_db 1
sqlobject.dburi="notrans_sqlite:///var/lib/mirrormanager/sqlite.db"
9 years, 5 months
- 1
- 0
[mirrormanager] master: add update-EC2-netblocks (82ebdfe)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 82ebdfe82fe8777f6064dba407890afaa25f98f0
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Dec 5 11:18:05 2014 -0600
add update-EC2-netblocks
>---------------------------------------------------------------
server/update-EC2-netblocks | 109 +++++++++++++++++++++++++++++++++++++++++++
1 files changed, 109 insertions(+), 0 deletions(-)
diff --git a/server/update-EC2-netblocks b/server/update-EC2-netblocks
new file mode 100755
index 0000000..8f3ec1d
--- /dev/null
+++ b/server/update-EC2-netblocks
@@ -0,0 +1,109 @@
+#!/usr/bin/python
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+# std imports
+import optparse
+import sys
+import json
+import urlgrabber
+# TurboGears imports
+import turbogears
+from mirrormanager.model import Site, HostNetblock
+from turbogears.database import PackageHub
+hub = __connection__ = None
+options = None
+
+def parse_out_region(hostname):
+ hostname = hostname.lstrip('s3-mirror-')
+ hostname = hostname.rstrip('.fedoraproject.org')
+ return hostname
+
+def s3_mirrors():
+ hosts_by_region = {}
+ site = Site.byName("Red Hat")
+ for host in site.hosts:
+ if host.name.startswith("s3"):
+ region = parse_out_region(host.name)
+ hosts_by_region[region] = {'host': host,
+ 'netblocks': {},
+ }
+ for nb in host.netblocks:
+ hosts_by_region[region]['netblocks'][nb.netblock] = {'host_netblock_id' : nb.id,
+ 'stale':True,
+ }
+
+ return hosts_by_region
+
+
+def get_ip_ranges():
+ return urlgrabber.urlread('https://ip-ranges.amazonaws.com/ip-ranges.json')
+
+def parse_ip_ranges(ipranges_str):
+ return json.loads(ipranges_str)
+
+def host_has_netblock(hosts_by_region, host, netblock):
+ return netblock in hosts_by_region[host]['netblocks']
+
+def doit():
+ hosts_by_region = s3_mirrors()
+ ipranges_str = get_ip_ranges()
+ ipranges = parse_ip_ranges(ipranges_str)
+
+ for p in ipranges.prefixes:
+ service = p['service']
+ region = p['region']
+ ip_prefix = p['ip_prefix']
+ if service != "EC2":
+ continue
+
+ if region in hosts_by_region: # ignore regions we don't have a mirror in
+ host = hosts_by_region[region]
+ if not host_has_netblock(hosts_by_region, region, ip_prefix):
+ print "Added region %s netblock %s" % (region, ip_prefix)
+ if not options.dry_run:
+ nb = HostNetblock(host=host, netblock=ip_prefix) # this adds the entry to the database, mark as not stale
+ hosts_by_region[region]['netblocks'][ip_prefix] = {'host_netblock_id' : nb.id,
+ 'stale':False,
+ }
+ else:
+ # found the netblock in our database, mark it as not stale
+ hosts_by_region[region]['netblocks'][ip_prefix]['stale'] = False
+
+
+ # delete stale netblock entries from the database
+ for h in hosts_by_region:
+ for netblock in hosts_by_region[h]['netblocks']:
+ if hosts_by_region[h]['netblocks'][netblock]['stale']: # delete this, it's no longer on Amazon's list
+ print "Deleted region %s netblock %s" % (region, netblock)
+ nb = HostNetblock.get(hosts_by_region[h]['netblocks'][netblock]['host_netblock_id'])
+ if not options.dry_run:
+ nb.destroySelf()
+
+
+def main():
+ global options
+ parser = optparse.OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='/etc/mirrormanager/prod.cfg',
+ help="TurboGears config file to use (default=/etc/mirrormanager/prod.cfg)")
+ parser.add_option("-n", "--dry-run",
+ dest="dry_run", action="store_true", default=False)
+
+
+ (options, args) = parser.parse_args()
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+ global hub
+ global __connection__
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ doit()
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())
+
9 years, 5 months
- 1
- 0
- ← Newer
- 1
- Older →