January 2015 - copr-commits - Fedora Mailing-Lists

[copr] master: [backend] [RHBZ:#1182106] JobGrabber dies when action raises an exception. (dfd78b4)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit dfd78b450886e61a6c9237949a6487b5edaf7718 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Jan 14 14:14:51 2015 +0100 [backend] [RHBZ:#1182106] JobGrabber dies when action raises an exception. >--------------------------------------------------------------- backend/backend/daemons/job_grab.py | 6 +++++- backend/tests/deamons/test_job_grab.py | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletions(-) diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py index 6353682..71c4341 100644 --- a/backend/backend/daemons/job_grab.py +++ b/backend/backend/daemons/job_grab.py @@ -142,7 +142,11 @@ class CoprJobGrab(Process): self.event("{0} actions returned".format(len(r_json["actions"]))) for action in r_json["actions"]: - self.process_action(action) + try: + self.process_action(action) + except Exception as error: + self.event("Error during processing action `{}`: {}" + .format(action, error)) def run(self): """ diff --git a/backend/tests/deamons/test_job_grab.py b/backend/tests/deamons/test_job_grab.py index a0dbe3c..09d1884 100644 --- a/backend/tests/deamons/test_job_grab.py +++ b/backend/tests/deamons/test_job_grab.py @@ -265,6 +265,33 @@ class TestJobGrab(object): expected_calls = [call(action_1), call(action_2)] assert self.jg.process_action.call_args_list == expected_calls + @mock.patch("backend.daemons.job_grab.get") + def test_regression_load_tasks_actions_(self, mc_get, init_jg): + """ + https://bugzilla.redhat.com/show_bug.cgi?id=1182106 + """ + action_1 = MagicMock() + action_2 = MagicMock() + mc_get.return_value.json.return_value = { + "actions": [ + action_1, + action_2, + ], + "builds": [], + } + + self.jg.process_build_task = MagicMock() + self.jg.event = MagicMock() + self.jg.process_action = MagicMock() + + # load_tasks should suppress this error + self.jg.process_action.side_effect = IOError() + + self.jg.load_tasks() + + expected_calls = [call(action_1), call(action_2)] + assert self.jg.process_action.call_args_list == expected_calls + def test_run(self, mc_time, mc_setproctitle, init_jg): self.jg.connect_queues = MagicMock() self.jg.load_tasks = MagicMock()

9 years, 3 months

1
0
0 / 0

[copr] master: [frontend] new command AddDebugUser for manage script (f6e55ec)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit f6e55ecf0fbcc3675b14ae3f4f678cdf3a137b27 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Jan 13 15:52:06 2015 +0100 [frontend] new command AddDebugUser for manage script >--------------------------------------------------------------- frontend/coprs_frontend/manage.py | 57 ++++++++++++++++++++++++++++++++++++- 1 files changed, 56 insertions(+), 1 deletions(-) diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py index e6bf99b..489c7f0 100755 --- a/frontend/coprs_frontend/manage.py +++ b/frontend/coprs_frontend/manage.py @@ -3,15 +3,17 @@ import argparse import os import subprocess +import datetime import flask -from flask.ext.script import Manager, Command, Option, Group +from flask_script import Manager, Command, Option, Group from coprs import app from coprs import db from coprs import exceptions from coprs import models from coprs.logic import coprs_logic +from coprs.views.misc import create_user_wrapper from coprs.whoosheers import CoprUserWhoosheer @@ -180,6 +182,57 @@ class DisplayChrootsCommand(Command): ) +class AddDebugUserCommand(Command): + + """ + Adds user for debug/testing purpose. + You shouldn't use this on production instance + """ + + def run(self, name, mail, **kwargs): + user = models.User(username=name, mail=mail) + + if kwargs["admin"]: + user.admin = True + if kwargs["no_admin"]: + user.admin = False + if kwargs["proven"]: + user.proven = True + if kwargs["no_proven"]: + user.proven = False + # + # if kwargs["api_token"]: + # user.api_token = kwargs["api_token"] + # user.api_token_expiration = datetime.date(2030, 1, 1) + # if kwargs["api_login"]: + # user.api_token = kwargs["api_login"] + # + + db.session.add(create_user_wrapper(user, mail)) + db.session.commit() + + option_list = ( + Option("name"), + Option("mail"), + Option("--api_token", default=None, required=False), + Option("--api_login", default=None, required=False), + Group( + Option("--admin", + action="store_true"), + Option("--no-admin", + action="store_true"), + exclusive=True + ), + Group( + Option("--proven", + action="store_true"), + Option("--no-proven", + action="store_true"), + exclusive=True + ), + ) + + class AlterUserCommand(Command): def run(self, name, **kwargs): @@ -198,6 +251,7 @@ class AlterUserCommand(Command): if kwargs["no_proven"]: user.proven = False + db.session.add(user) db.session.commit() option_list = ( @@ -250,6 +304,7 @@ manager.add_command("alter_chroot", AlterChrootCommand()) manager.add_command("display_chroots", DisplayChrootsCommand()) manager.add_command("drop_chroot", DropChrootCommand()) manager.add_command("alter_user", AlterUserCommand()) +manager.add_command("add_debug_user", AddDebugUserCommand()) manager.add_command("update_indexes", UpdateIndexesCommand()) if __name__ == "__main__":

9 years, 3 months

1
0
0 / 0

[copr] master: update url of playbooks (f7cca7d)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit f7cca7dad7992d993a9b925ddd59f38a76451d87 Author: Miroslav Suchý <msuchy(a)redhat.com> Date: Fri Jan 9 11:24:21 2015 +0100 update url of playbooks >--------------------------------------------------------------- copr-setup.txt | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/copr-setup.txt b/copr-setup.txt index a8b5dfe..fca0cc9 100644 --- a/copr-setup.txt +++ b/copr-setup.txt @@ -34,18 +34,18 @@ ansible playbook: FEDORA PLAYBOOK: 1. spawn instance playbook - - http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio... + http://infrastructure.fedoraproject.org/infra/ansible/roles/copr/backend/... - just an ansible playbook which sets up and provisions an instance in our case it is using eucalyptus but it doesn't have to be any cloud provider - it just needs to be able to get an ip an ssh into it - cloud providers or private clouds just make more sense 2. provisioning dir [TODO - this playbook is old and need to be updated] - http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio... + http://infrastructure.fedoraproject.org/infra/ansible/roles/copr/backend/... - what gets pushed to the other side - ssh keys, users, mock configs, etc 3. terminate instance playbook - - http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio... + http://infrastructure.fedoraproject.org/infra/ansible/roles/copr/backend/... 4. create ~copr/.ssh/id_rsa with ssh key to VM guests

9 years, 3 months

1
0
0 / 0

[copr] master: [backend] Moved .sh scripts into /usr/bin/ Renamed copr{-, _}be.py. (c380764)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c38076409006974963e4788d839b0b880cad522c Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 9 11:19:57 2015 +0100 [backend] Moved .sh scripts into /usr/bin/ Renamed copr{-,_}be.py. >--------------------------------------------------------------- backend/conf/crontab/copr-backend | 2 +- backend/copr-backend.service | 2 +- backend/copr-backend.spec | 4 +++- backend/run/{copr-be.py => copr_be.py} | 0 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend index a8cd38b..68ba1a4 100644 --- a/backend/conf/crontab/copr-backend +++ b/backend/conf/crontab/copr-backend @@ -1,3 +1,3 @@ #!/usr/bin/sh -runuser -c "PYTHONUNBUFFERED=1 python /usr/share/copr/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1 +runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1 diff --git a/backend/copr-backend.service b/backend/copr-backend.service index 83abcfe..1c2b58b 100644 --- a/backend/copr-backend.service +++ b/backend/copr-backend.service @@ -8,7 +8,7 @@ PIDFile=/var/run/copr-backend/copr-be.pid EnvironmentFile=/home/copr/cloud/ec2rc.variable User=copr Group=copr -ExecStart=/usr/share/copr/copr-be.py +ExecStart=/usr/bin/copr_be.py ExecReload=/bin/kill -HUP $MAINPID [Install] diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 0569167..b7bf343 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -118,9 +118,10 @@ install -d %{buildroot}/%{_tmpfilesdir} install -d %{buildroot}/%{_sbindir} install -d %{buildroot}%{_sysconfdir}/cron.daily install -d %{buildroot}%{_sysconfdir}/sudoers.d +install -d %{buildroot}%{_bindir}/ cp -a backend/* %{buildroot}%{_datadir}/copr/backend -cp -a run/* %{buildroot}%{_datadir}/copr/ +cp -a run/* %{buildroot}%{_bindir}/ cp -a conf/copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf install -p -m 755 conf/crontab/copr-backend %{buildroot}%{_sysconfdir}/cron.daily/copr-backend @@ -190,6 +191,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf %{_unitdir}/copr-backend.service %{_tmpfilesdir}/copr-backend.conf +%{_bindir}/ %config(noreplace) %{_sysconfdir}/cron.daily/copr-backend diff --git a/backend/run/copr-be.py b/backend/run/copr_be.py similarity index 100% rename from backend/run/copr-be.py rename to backend/run/copr_be.py

9 years, 3 months

1
0
0 / 0

[copr] tag 'copr-backend-1.54-1' created

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-backend-1.54-1 Referencing: c8ca413b7936085ab6d70f79b1977e0e4e294e8d

9 years, 3 months

1
0
0 / 0

[copr] master: Automatic commit of package [copr-backend] release [1.54-1]. (f657dcc)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit f657dcc0c22e2c967a8d3f0a281a548af7b9d17b Author: Miroslav Suchý <msuchy(a)redhat.com> Date: Wed Jan 7 17:07:49 2015 +0100 Automatic commit of package [copr-backend] release [1.54-1]. >--------------------------------------------------------------- backend/copr-backend.spec | 8 +++++++- rel-eng/packages/copr-backend | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index d8a1fa4..0569167 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -3,7 +3,7 @@ %endif Name: copr-backend -Version: 1.53 +Version: 1.54 Release: 1%{?dist} Summary: Backend for Copr @@ -202,6 +202,12 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %exclude %{_pkgdocdir}/playbooks %changelog +* Wed Jan 07 2015 Miroslav Suchý <msuchy(a)redhat.com> 1.54-1 +- 1179713 - workaround for 1179806 +- run script unbufferred otherwise log is written after full block +- express that it is n-th projects +- fix permissions on prune script + * Mon Jan 05 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.53-1 - [backend, frontend] [RHBZ:#1176364] Wrong value for the build timeout. diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend index e040ff5..d7c4d2c 100644 --- a/rel-eng/packages/copr-backend +++ b/rel-eng/packages/copr-backend @@ -1 +1 @@ -1.53-1 backend/ +1.54-1 backend/

9 years, 3 months

1
0
0 / 0

[copr] master: 1179713 - workaround for 1179806 (e4caa6e)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit e4caa6ecaf5cdc1204abc3ccfb3caf06c22a3756 Author: Miroslav Suchý <msuchy(a)redhat.com> Date: Wed Jan 7 16:36:37 2015 +0100 1179713 - workaround for 1179806 job.result always end with /, so this result with // in url >--------------------------------------------------------------- backend/backend/daemons/dispatcher.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index 6c980ff..ca777a8 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -558,9 +558,8 @@ class Worker(multiprocessing.Process): self.callback.log("Building pkgs: {0}".format(job.pkg)) chroot_repos = list(job.repos) - chroot_repos.append(job.results + '/' + job.chroot) - # for RHBZ: #1150954 - chroot_repos.append(job.results + '/' + job.chroot + '/devel') + chroot_repos.append(job.results + job.chroot + '/') + chroot_repos.append(job.results + job.chroot + '/devel/') chroot_logfile = "{0}/build-{1}.log".format( chroot_destdir, job.build_id)

9 years, 3 months

1
0
0 / 0

[copr] master: run script unbufferred otherwise log is written after full block (e167082)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit e167082ba4ab39a02c822d4c0e1ce1fe2ee4df9e Author: Miroslav Suchý <miroslav(a)suchy.cz> Date: Wed Jan 7 01:12:14 2015 +0100 run script unbufferred otherwise log is written after full block >--------------------------------------------------------------- backend/conf/crontab/copr-backend | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend index a376be4..a8cd38b 100644 --- a/backend/conf/crontab/copr-backend +++ b/backend/conf/crontab/copr-backend @@ -1,3 +1,3 @@ #!/usr/bin/sh -runuser -c "python /usr/share/copr/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1 +runuser -c "PYTHONUNBUFFERED=1 python /usr/share/copr/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1

9 years, 4 months

1
0
0 / 0

[copr] master: express that it is n-th projects (22c42e4)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 22c42e4c68e3b5abd5e5f47db92bf62c33b5d910 Author: Miroslav Suchý <miroslav(a)suchy.cz> Date: Wed Jan 7 00:56:04 2015 +0100 express that it is n-th projects >--------------------------------------------------------------- backend/run/copr_prune_results.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index 1baf229..bc79816 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -84,7 +84,7 @@ def main(): prune_project(opts, project_path, username, projectname) counter += 1 - print("Pruned {} projects".format(counter)) + print("Pruned {}. projects".format(counter)) print("Pruning finished")

9 years, 4 months

1
0
0 / 0

[copr] master: fix permissions on prune script (9b0a918)

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 9b0a918954db338037a3574e671ccdfce7c0b2da Author: Miroslav Suchý <miroslav(a)suchy.cz> Date: Wed Jan 7 00:55:09 2015 +0100 fix permissions on prune script >--------------------------------------------------------------- backend/conf/crontab/copr-backend | 2 +- backend/run/copr_prune_results.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend index 2260f7a..a376be4 100644 --- a/backend/conf/crontab/copr-backend +++ b/backend/conf/crontab/copr-backend @@ -1,3 +1,3 @@ #!/usr/bin/sh -python /usr/share/copr/copr_prune_results.py >> /var/log/copr/prune_old.log 2>&1 +runuser -c "python /usr/share/copr/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1 diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index 2ab428e..1baf229 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -16,8 +16,7 @@ log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) DEF_DAYS = 14 -DEF_PRUNE_SCRIPT = "copr_prune_old_builds.sh" -DEF_PRUNE_SCRIPT = "/usr/sbin/copr_prune_old_builds.sh" +DEF_PRUNE_SCRIPT = "/usr/share/copr/copr_prune_old_builds.sh" def list_subdir(path):

9 years, 4 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits January 2015