Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master commit ba5f493afaff4b6c14bfb65773499e24788cfaec Author: Valentin Gologuzov <vgologuz(a)redhat.com&gt; Date: Tue Nov 4 11:29:50 2014 +0100 [backend] pass results_baseurl to createrepo backend/backend/actions.py | 44 +++++++++++++++----------- backend/backend/createrepo.py | 10 +++--- backend/backend/dispatcher.py | 1 + backend/backend/mockremote.py | 52 ++++++++++++++++++++----------- backend/copr-be.py | 3 +- backend/tests/test_action.py | 68 +++++++++++++++++++++++++++++----------- 6 files changed, 116 insertions(+), 62 deletions(-) diff --git a/backend/backend/actions.py b/backend/backend/actions.py index a6d80c1..8e8fd04 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -24,7 +24,9 @@ class Action(object): """ - def __init__(self, events, action, lock, frontend_callback, destdir, front_url): + def __init__(self, events, action, lock, + frontend_callback, destdir, + front_url, results_root_url): super(Action, self).__init__() self.frontend_callback = frontend_callback self.destdir = destdir @@ -32,6 +34,7 @@ class Action(object): self.events = events self.lock = lock self.front_url = front_url + self.results_root_url = results_root_url def add_event(self, what): self.events.put({"when": time.time(), "who": "action", "what": what}) @@ -91,11 +94,13 @@ class Action(object): def handle_delete_build(self): self.add_event("Action delete build") project = self.data["old_value"] + ext_data = json.loads(self.data["data"]) + username = ext_data["username"] + projectname = ext_data["projectname"] - packages = [os.path.basename(x).replace(".src.rpm", "") for x in \ - ext_data["pkgs"].split()] - # self.data["data"].split()] + packages = [os.path.basename(x).replace(".src.rpm", "") + for x in ext_data["pkgs"].split()] path = os.path.join(self.destdir, project) @@ -118,10 +123,10 @@ class Action(object): # than I delete the failed, it would delete the succeeded # files as well - that would be wrong. for pkg in packages: - if self.data["object_type"] == "build-succeeded" or \ - (self.data["object_type"] == "build-failed" and - os.path.exists(os.path.join(path, chroot, pkg, "fail")) - ): + if self.data["object_type"] == "build-succeeded" or ( + self.data["object_type"] == "build-failed" and + os.path.exists(os.path.join(path, chroot, pkg, "fail"))): + pkg_path = os.path.join(path, chroot, pkg) if os.path.isdir(pkg_path): self.add_event("Removing build {0}".format(pkg_path)) @@ -132,16 +137,19 @@ class Action(object): "Package {0} dir not found in chroot {1}" .format(pkg, chroot)) - if altered: - self.add_event("Running createrepo") - _, _, err = createrepo( - path=os.path.join(path, chroot), lock=self.lock, - front_url=self.front_url, - username=ext_data["username"], projectname=ext_data["projectname"] - ) - if err.strip(): - self.add_event( - "Error making local repo: {0}".format(err)) + if altered: + self.add_event("Running createrepo") + + result_base_url = "/".join([self.results_root_url, username, + projectname, chroot]) + _, _, err = createrepo( + path=os.path.join(path, chroot), lock=self.lock, + front_url=self.front_url, base_url=result_base_url, + username=username, projectname=projectname + ) + if err.strip(): + self.add_event( + "Error making local repo: {0}".format(err)) log_path = os.path.join( path, chroot, diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 3dedb3d..d28e0f9 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -69,11 +69,11 @@ def createrepo(path, front_url, username, projectname, base_url=None, lock=None) Creates repo depending on the project setting "auto_createrepo". When enabled creates `repodata` at the provided path, otherwise - :param str path: directory with rpms - :param str front_url: url to the copr frontend - :param str username: copr project owner username - :param str projectname: copr project name - :param str base_url: base_url to access rpms independently of repomd location + :param path: directory with rpms + :param front_url: url to the copr frontend + :param username: copr project owner username + :param projectname: copr project name + :param base_url: base_url to access rpms independently of repomd location :param Multiprocessing.Lock lock: [optional] global copr-backend lock :return: tuple(returncode, stdout, stderr) produced by `createrepo_c` diff --git a/backend/backend/dispatcher.py b/backend/backend/dispatcher.py index f305d0c..3ddc781 100644 --- a/backend/backend/dispatcher.py +++ b/backend/backend/dispatcher.py @@ -574,6 +574,7 @@ class Worker(multiprocessing.Process): callback=CliLogCallBack( quiet=True, logfn=chrootlogfile), front_url=self.opts.frontend_base_url, + results_base_url=self.opts.results_baseurl ) build_details = mr.build_pkgs(job.pkgs) diff --git a/backend/backend/mockremote.py b/backend/backend/mockremote.py index 919fbd9..ff97db2 100755 --- a/backend/backend/mockremote.py +++ b/backend/backend/mockremote.py @@ -539,7 +539,8 @@ class MockRemote(object): def __init__(self, builder=None, user=DEF_USER, job=None, cont=False, recurse=False, repos=None, callback=None, remote_basedir=DEF_REMOTE_BASEDIR, remote_tempdir=None, - macros=None, lock=None, do_sign=False, front_url=None): + macros=None, lock=None, do_sign=False, + front_url=None, results_base_url=None): """ @@ -592,6 +593,7 @@ class MockRemote(object): self.lock = lock self.do_sign = do_sign self.front_url = front_url + self.results_base_url = results_base_url or u'' if not self.callback: self.callback = DefaultCallBack() @@ -685,6 +687,30 @@ class MockRemote(object): fcntl.flock(r_log, fcntl.LOCK_UN) r_log.close() + def do_createrepo(self, chroot_dir): + base_url = "/".join([self.results_base_url, self.job.project_owner, + self.job.project_name, self.job.chroot]) + self.callback.log("Createrepo:: owner: {}; project: {}; front url: {}; path: {}; base_url: {}".format( + self.job.project_owner, self.job.project_name, self.front_url, chroot_dir, base_url + )) + + + _, _, err = createrepo( + path=chroot_dir, + front_url=self.front_url, + base_url=base_url, + username=self.job.project_owner, + projectname=self.job.project_name, + lock=self.lock, + ) + if err.strip(): + self.callback.error( + "Error making local repo: {0}".format(chroot_dir)) + + self.callback.error(str(err)) + # FIXME - maybe clean up .repodata and .olddata + # here? + def build_pkgs(self, pkgs=None): if not pkgs: @@ -780,23 +806,7 @@ class MockRemote(object): built_pkgs.append(pkg) # createrepo with the new pkgs - self.callback.log("Createrepo:: owner: {}; project: {}; front url: {}; path: {}".format( - self.job.project_owner, self.job.project_name, self.front_url, chroot_dir - )) - _, _, err = createrepo( - path=chroot_dir, - front_url=self.front_url, - username=self.job.project_owner, - projectname=self.job.project_name, - lock=self.lock, - ) - if err.strip(): - self.callback.error( - "Error making local repo: {0}".format(chroot_dir)) - - self.callback.error(str(err)) - # FIXME - maybe clean up .repodata and .olddata - # here? + self.do_createrepo(chroot_dir, ) if self.failed: if len(self.failed) != len(to_be_built): @@ -858,7 +868,10 @@ def parse_args(args): action="store_true", help="output very little to the terminal") parser.add_option("-f", "--front_url", dest="front_url", - help="copr frontend url") + help="copr frontend url") + parser.add_option("--results_url", dest="results_base_url", + help="backend base url for built packages") + opts, args = parser.parse_args(args) @@ -930,6 +943,7 @@ def main(args): do_sign=opts.do_sign, callback=callback, front_url=opts.front_url, + results_base_url=opts.results_base_url, ) # FIXMES diff --git a/backend/copr-be.py b/backend/copr-be.py index 0869059..380a081 100755 --- a/backend/copr-be.py +++ b/backend/copr-be.py @@ -116,7 +116,8 @@ class CoprJobGrab(multiprocessing.Process): for action in r_json["actions"]: ao = Action(self.events, action, self.lock, destdir=self.opts.destdir, frontend_callback=FrontendCallback(self.opts, self.events), - front_url=self.opts.frontend_base_url) + front_url=self.opts.frontend_base_url, + results_root_url=self.opts.results_baseurl) ao.run() def run(self): diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py index c5edb7a..7040530 100644 --- a/backend/tests/test_action.py +++ b/backend/tests/test_action.py @@ -20,14 +20,14 @@ else: import logging -logging.basicConfig( - level=logging.INFO, - format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', - datefmt='%H:%M:%S' -) - -log = logging.getLogger() -log.info("Logger initiated") +# logging.basicConfig( +# level=logging.INFO, +# format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', +# datefmt='%H:%M:%S' +# ) +# +# log = logging.getLogger() +# log.info("Logger initiated") from backend.actions import Action, ActionType, ActionResult @@ -39,6 +39,10 @@ else: import Queue as queue from Queue import Empty as EmptyQueue + +RESULTS_ROOT_URL = "http://example.com/results" + + @mock.patch("backend.actions.time") class TestAction(object): @@ -80,8 +84,15 @@ class TestAction(object): return self.tmp_dir_name def test_action_event(self, mc_time): - test_action = Action(events=self.test_q, action={}, lock=None, - frontend_callback=None, destdir=None, front_url=None) + test_action = Action( + events=self.test_q, + action={}, lock=None, + frontend_callback=None, + destdir=None, + front_url=None, + results_root_url=RESULTS_ROOT_URL + ) + with pytest.raises(EmptyQueue): test_action.events.get_nowait() @@ -105,7 +116,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=None, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() assert not mc_front_cb.called @@ -136,7 +148,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=tmp_dir, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] @@ -167,7 +180,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=os.path.join(tmp_dir, "dir-not-exists"), - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] @@ -197,7 +211,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=tmp_dir, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() result_dict = mc_front_cb.update.call_args[0][0]["actions"][0] @@ -227,7 +242,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=os.path.join(tmp_dir, "dir-not-exists"), - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() @@ -262,7 +278,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=tmp_dir, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run() @@ -303,7 +320,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=tmp_dir, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) with mock.patch("backend.actions.shutil") as mc_shutil: test_action.run() @@ -362,6 +380,7 @@ class TestAction(object): frontend_callback=mc_front_cb, destdir=tmp_dir, front_url=None, + results_root_url=RESULTS_ROOT_URL ) assert os.path.exists(foo_pkg_dir) @@ -384,7 +403,7 @@ class TestAction(object): def assert_fedora20(): assert_what_from_queue(self.test_q, msg_list=[ "Removing build ", - "Running createrepo", + #"Running createrepo", "Package bar dir not found in chroot fedora20", "Running createrepo", ]) @@ -407,6 +426,16 @@ class TestAction(object): with pytest.raises(EmptyQueue): self.test_q.get_nowait() + create_repo_expected_call = mock.call( + username=u'foo', + projectname=u'bar', + base_url=u'http://example.com/results/foo/bar/fedora20';, + lock=None, + path='{}/old_dir/fedora20'.format(self.tmp_dir_name), + front_url=None + ) + assert mc_createrepo.call_args == create_repo_expected_call + @mock.patch("backend.actions.createrepo") def test_delete_build_succeeded_createrepo_error(self, mc_createrepo, mc_time): mc_time.time.return_value = self.test_time @@ -439,7 +468,8 @@ class TestAction(object): events=self.test_q, lock=None, frontend_callback=mc_front_cb, destdir=tmp_dir, - front_url=None + front_url=None, + results_root_url=RESULTS_ROOT_URL ) test_action.run()