mirrorlist-server/mirrorlist_client.wsgi
by Matt Domsch
mirrorlist-server/mirrorlist_client.wsgi | 6 ++++++
1 file changed, 6 insertions(+)
New commits:
commit 621af2882e984459a23d1e7af3ef1854ea6f0ba1
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 11 22:57:19 2010 -0500
mirrorlist_client: sanitize input into UTF-8
Users may put all sorts of odd URL-escaped characters onto the URLs.
When this happens, mirrorlist_server.py child thread would crash
trying to write the byte string with escaped '\x80' characters inside
it into the error report header. When the child crashes,
mirrorlist_client.wsgi would spin forever eating 100% CPU waiting for
a response from the server that would never come.
This patch sanitizes the input query args into a byte string that only
contains UTF-8 characters. This fixes this particular source of crash
in the server and subsequent hang of the client. It would still be
good if the client could time out or otherwise recognize when the
server is never coming back.
diff --git a/mirrorlist-server/mirrorlist_client.wsgi b/mirrorlist-server/mirrorlist_client.wsgi
index 9dbdf7a..dd6d1d9 100755
--- a/mirrorlist-server/mirrorlist_client.wsgi
+++ b/mirrorlist-server/mirrorlist_client.wsgi
@@ -83,6 +83,12 @@ def request_setup(request):
pathinfo = request.environ['PATH_INFO']
if scriptname == '/metalink' or pathinfo == '/metalink':
d['metalink'] = True
+
+ for k, v in d.iteritems():
+ try:
+ d[k] = unicode(v, 'utf8', 'ignore').encode('utf8')
+ except:
+ pass
return d
def accept_encoding_gzip(request):
14 years
mirrorlist-server/test
by Matt Domsch
mirrorlist-server/test/server_tester.py | 29 ++++++++++++++++++++---------
1 file changed, 20 insertions(+), 9 deletions(-)
New commits:
commit 4ad121b2d49316c892207bb1c4a4ca8616435956
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 11 21:38:11 2010 -0500
mirrorlist tester: more printing, run forever
diff --git a/mirrorlist-server/test/server_tester.py b/mirrorlist-server/test/server_tester.py
index d7d95cc..1e46310 100755
--- a/mirrorlist-server/test/server_tester.py
+++ b/mirrorlist-server/test/server_tester.py
@@ -4,31 +4,39 @@
# by Matt Domsch <Matt_Domsch(a)dell.com>
# Licensed under the MIT/X11 license
-import socket
+import socket, os
import cPickle as pickle
from string import zfill, atoi
+from datetime import datetime
-socketfile = '/tmp/mirrormanager_mirrorlist_server.sock'
+socketfile = '/var/run/mirrormanager/mirrorlist_server.sock'
+
+pid = os.getpid()
+connectTime = None
def do_mirrorlist(d):
try:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ start = datetime.utcnow()
s.connect(socketfile)
+ end = datetime.utcnow()
except:
raise
+ global connectTime
+ connectTime = (end - start)
p = pickle.dumps(d)
size = len(p)
- print "writing size %s to server" % size
+ #print "writing size %s to server" % size
s.sendall(zfill('%s' % size, 10))
# write the pickle
- print "writing the pickle"
+ #print "writing the pickle"
s.sendall(p)
s.shutdown(socket.SHUT_WR)
del p
- print "reading result pickle size back"
+ #print "reading result pickle size back"
readlen = 0
resultsize = ''
while readlen < 10:
@@ -36,7 +44,7 @@ def do_mirrorlist(d):
readlen = len(resultsize)
resultsize = atoi(resultsize)
- print "reading %s bytes of the results list" % resultsize
+ #print "reading %s bytes of the results list" % resultsize
readlen = 0
p = ''
while readlen < resultsize:
@@ -55,9 +63,12 @@ import random
client_ip = "%s.%s.%s.%s" % (random.randint(0,255), random.randint(0,255), random.randint(0,255), random.randint(0,255))
-d = {'repo':'fedora-7',
+d = {'repo':'fedora-12',
'arch':'i386',
'client_ip':client_ip}
-print do_mirrorlist(d)
-
+while 1:
+ start = datetime.utcnow()
+ result = do_mirrorlist(d)
+ end = datetime.utcnow()
+ print "[%s] connect: %s total: %s" % (pid, connectTime, (end-start))
14 years
mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 1 -
1 file changed, 1 deletion(-)
New commits:
commit d82f20b10c755e5ce40d67ca7ea4a6dba9e37d34
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon May 10 23:56:09 2010 -0500
mirrorlist_server: don't ignore SIGCHLD
Amazing that this ever worked in the first place. Ignoring SIGCHLD
causes the parent's active_children list to grow without bound. This
is also probably the cause of our long-term memory size growth. The
parent really needs to catch SIGCHLD in order to do its reaping.
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 2ade357..0de7132 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -814,7 +814,6 @@ def main():
open_geoip_databases()
read_caches()
signal.signal(signal.SIGHUP, sighup_handler)
- signal.signal(signal.SIGCHLD, signal.SIG_IGN)
ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler)
ss.serve_forever()
14 years
mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 45d401446bfecba768fdf4f26409bf291172f7bc
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon May 10 15:23:57 2010 -0500
mirrorlist_server: set request_queue_size earlier
While the docs say that request_queue_size can be a per-instance
value, in reality it's used during ForkingUnixStreamServer __init__,
meaning it needs to override the default class attribute instead.
Moving this up means that connections aren't blocking after about 5
are already running (default), and mirrorlist_client can now connect
in ~200us like one would expect, rather than seconds or tens of
seconds like we were seeing when lots (say, 40+) clients were
connecting simultaneously.
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 8825a1a..2ade357 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -725,6 +725,7 @@ def sighup_handler(signum, frame):
signal.signal(signal.SIGHUP, sighup_handler)
class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer):
+ request_queue_size = 300
def finish_request(self, request, client_address):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
BaseServer.finish_request(self, request, client_address)
@@ -815,7 +816,6 @@ def main():
signal.signal(signal.SIGHUP, sighup_handler)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler)
- ss.request_queue_size = 300
ss.serve_forever()
try:
14 years
2 commits - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
New commits:
commit a171c433e2be047020131698a277530a13ebbd31
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun May 9 08:54:27 2010 -0500
mirrorlist_server: increase socket request queue length to 300 (still limited by kernel options to 128)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 9ce4e43..8825a1a 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -815,7 +815,7 @@ def main():
signal.signal(signal.SIGHUP, sighup_handler)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler)
- ss.request_queue_size = 100
+ ss.request_queue_size = 300
ss.serve_forever()
try:
commit b1d814d3dbbfe2c15a6d661abc4e77740bbb0d35
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun May 9 08:53:36 2010 -0500
mirrorlist_server: ignore SIGHUP in child - it'll exit soon enough
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 67dbf65..9ce4e43 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -661,6 +661,7 @@ def errordoc(metalink, message):
class MirrorlistHandler(StreamRequestHandler):
def handle(self):
+ signal.signal(signal.SIGHUP, signal.SIG_IGN)
random.seed()
try:
# read size of incoming pickle
14 years
3 commits - server/crawler server/crawler_perhost server/dev.cfg TODO
by Matt Domsch
TODO | 2 -
server/crawler | 7 +++--
server/crawler_perhost | 68 ++++++++++++++++++++++++++++++++++++++++++++-----
server/dev.cfg | 5 +++
4 files changed, 72 insertions(+), 10 deletions(-)
New commits:
commit f5da33ec1bf29b65f6a2f80e583e181cf4a295ef
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 09:34:57 2010 -0400
crawler: send email when marking a whole host not up2date
This does not send mail to each individual admin yet. It also
requires prod.cfg to be updated. See dev.cfg for the crawler.*
options.
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 881fc4b..eda28aa 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -6,6 +6,7 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys, os, signal, socket
import turbogears
+from turbogears import config
from mirrormanager.model import *
from datetime import datetime
import ftplib
@@ -16,6 +17,7 @@ from optparse import OptionParser
import threading
import urlgrabber
import hashlib
+from smtplib import SMTP
from turbogears.database import PackageHub
hostid = None
@@ -454,17 +456,62 @@ def dienow():
os.kill(mypid, signal.SIGUSR1)
os._exit(0)
-def mark_not_up2date(host, reason="Unknown"):
+def send_email(host, report_str, exc):
+ SMTP_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+ msg = """From: %s
+To: %s
+Subject: %s MirrorManager crawler report
+Date: %s
+
+""" % (config.get('crawler.mail_from'),
+ config.get('crawler.admin_mail_to'),
+ host.name,
+ time.strftime(SMTP_DATE_FORMAT))
+
+ msg += report_str + '\n'
+ msg += 'Log can be found at %s/%s.log\n' % (config.get('crawler.logdir'), str(host.id))
+ if exc is not None:
+ msg += "Exception info: type %s; value %s\n" % (exc[0], exc[1])
+ msg += str(exc[2])
+ try:
+ smtp = SMTP(config.get('crawler.smtp_host'),
+ config.get('crawler.smtp_port'))
+
+ username = config.get('crawler.smtp_username')
+ password = config.get('crawler.smtp_password')
+
+ if username and password:
+ smtp.login(username, password)
+
+ smtp.sendmail(config.get('crawler.smtp_from'),
+ config.get('crawler.admin_mail_to'),
+ msg)
+ except Exception, e:
+ print "Error sending email"
+ print e
+ print "Email message follows:"
+ print msg
+
+ try:
+ smtp.quit()
+ except:
+ pass
+
+def mark_not_up2date(exc, host, reason="Unknown", mail=True):
host.set_not_up2date()
host.lastCrawled = datetime.utcnow()
- print >> logfile, "Host marked not up2date: %s" % reason
-
+ msg = "Host marked not up2date: %s" % reason
+ print >> logfile, msg
+ if exc is not None:
+ print >> logfile, exc[0], exc[1], exc[2]
+ if mail:
+ send_email(host, msg, exc)
def sigusr1_handler(signum, frame):
if signum == signal.SIGUSR1:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
try:
- mark_not_up2date(Host.get(hostid), "Crawler timed out before completing. Host is likely overloaded.")
+ mark_not_up2date(None, Host.get(hostid), "Crawler timed out before completing. Host is likely overloaded.")
except:
pass
os._exit(1)
@@ -561,7 +608,7 @@ def per_host(host, options=None):
pass
except:
timer.cancel()
- mark_not_up2date(host, "Unknown exception raised. This is a bug in the MM crawler.")
+ mark_not_up2date(sys.exc_info(), host, "Unknown exception raised. This is a bug in the MM crawler.")
sys.exit(1)
hoststate.close()
@@ -571,7 +618,7 @@ def per_host(host, options=None):
if len(host_category_dirs) > 0:
sync_hcds(host, host_category_dirs)
else:
- mark_not_up2date(host, "No host category directories found. Check that your Host Category URLs are correct.")
+ mark_not_up2date(None, host, "No host category directories found. Check that your Host Category URLs are correct.")
def main():
global mypid
diff --git a/server/dev.cfg b/server/dev.cfg
index a02cd89..962453d 100644
--- a/server/dev.cfg
+++ b/server/dev.cfg
@@ -62,6 +62,11 @@ mrr.repos = ''' {
'updates-testing-source-f%s':'rawhide-source'
} '''
+crawler.mail_from = 'mirror-admin(a)fedoraproject.org'
+crawler.admin_mail_to = 'mirror-admin(a)fedoraproject.org'
+crawler.smtp_host = 'localhost'
+crawler.smtp_port = 25
+crawler.logdir = 'https://admin.fedoraproject.org/mirrormanager/crawler'
# This is where all of your settings go for your development environment
# Settings that are the same for both development and production
commit 80c22e55482fbc83cdedcaf0437e4c0004782e73
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 08:30:53 2010 -0400
crawler: crawl private i2 hosts that also say i2_clients are ok
diff --git a/TODO b/TODO
index 6c93a5b..19041ed 100644
--- a/TODO
+++ b/TODO
@@ -17,8 +17,6 @@
* cronjob to mark private mirrors as not-up2date, some amount of time
after the content has changed but they haven't run report_mirror.
-* can the crawler check private, but serving I2 clients, mirrors?
-
* metalinks for multiple CDs/files
- Just like torrents, we should have a single metalink that downloads
all the CDs in a set via one URL.
diff --git a/server/crawler b/server/crawler
index eaabaa0..3cb3753 100755
--- a/server/crawler
+++ b/server/crawler
@@ -63,9 +63,12 @@ def doit(options, include_private=False, threads=10, config=None, logdir=None):
commonargs.append('--canary')
for h in Host.select():
- if not h.is_active() or (not include_private and h.is_private()):
+ if not h.is_active():
continue
-
+ if not include_private and h.is_private():
+ if not (h.internet2 and h.internet2_clients):
+ continue
+
args = commonargs + ['--hostid=%s' % h.id]
if logdir is not None:
logfilename = os.path.join(logdir, str(h.id) + '.log')
commit 7c92f194683059a3b70903d81a933dcac34168a1
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 08:21:44 2010 -0400
crawler: implement canary mode
diff --git a/server/crawler_perhost b/server/crawler_perhost
index ebc3fa6..881fc4b 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -487,6 +487,11 @@ def sighup_handler(signum, frame):
def per_host(host, options=None):
+ """Canary mode looks for 2 things:
+ directory.path ends in 'iso' or directory.path ends in 'repodata'. In
+ this case it checks for availability of each of the files in those
+ directories.
+ """
global must_dienow
host = Host.get(host)
done_host=False
@@ -530,6 +535,10 @@ def per_host(host, options=None):
if not d.readable:
continue
+ if options.canary:
+ if not (d.name.endswith('/repodata') or d.name.endswith('/iso')):
+ continue
+
dirname = d.name[categoryPrefixLen:]
url = '%s/%s' % (categoryUrl, dirname)
14 years