[copr] master: [backend] cleanup (5605335)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 5605335fcd57f28c47af0ae9ba0c6e3102e5b906
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Nov 6 17:05:21 2014 +0100
[backend] cleanup
>---------------------------------------------------------------
backend/run/copr_prune_results.py | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index eec6df4..8fdfdc4 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -40,13 +40,11 @@ def prune_project(opts, path, username, projectname):
# run prune project sh
- #cmd = ["/usr/bin/copr_prune.sh", path, DAYS]
days = getattr(opts, "prune_days", DEF_DAYS)
prune_script = getattr(opts, "prune_script", DEF_PRUNE_SCRIPT)
cmd = map(str, [prune_script, path, days])
- print("CMD {}".format(cmd))
handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = handle.communicate()
@@ -69,7 +67,6 @@ def prune_project(opts, path, username, projectname):
.format(username, projectname, exception))
-
def main():
config_file= os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
opts = BackendConfigReader(config_file).read()
9 years, 6 months
[copr] master: [backend] bugfixes for prune-results (6882237)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 688223708ef1c657ff8fb55e18ab2ff86e18a996
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Nov 6 16:40:52 2014 +0100
[backend] bugfixes for prune-results
>---------------------------------------------------------------
backend/copr-be.conf.example | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/copr-be.conf.example b/backend/copr-be.conf.example
index 0f1f461..ff79e6d 100644
--- a/backend/copr-be.conf.example
+++ b/backend/copr-be.conf.example
@@ -99,7 +99,7 @@ worker_logdir=/var/log/copr/workers/
# minimum age for builds to be pruned
prune_days=14
# path to executable script to clean old build
-prune_script=/usr/sbin/copr_prune_old_builds.sh
+prune_script=/usr/share/copr/copr_prune_old_builds.sh
[builder]
# default is 1800
9 years, 6 months
[copr] master: [backend, python-copr] fixed tests (2e0cce8)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 2e0cce8ab0bab1c158065aef60fa502faddad7de
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Nov 6 15:55:29 2014 +0100
[backend, python-copr] fixed tests
>---------------------------------------------------------------
backend/tests/test_createrepo.py | 4 ++--
cli/copr_cli/main.py | 13 +++++++------
test_suite.sh | 6 +++---
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/backend/tests/test_createrepo.py b/backend/tests/test_createrepo.py
index 40ec041..86e6fe1 100644
--- a/backend/tests/test_createrepo.py
+++ b/backend/tests/test_createrepo.py
@@ -25,7 +25,7 @@ else:
from backend.createrepo import createrepo, createrepo_unsafe
@mock.patch('backend.createrepo.createrepo_unsafe')
-(a)mock.patch('backend.createrepo.CoprClient')
+(a)mock.patch('backend.helpers.CoprClient')
def test_createrepo_conditional_true(mc_client, mc_create_unsafe):
mc_client.return_value.get_project_details.return_value = MagicMock(data={"detail": {}})
@@ -42,7 +42,7 @@ def test_createrepo_conditional_true(mc_client, mc_create_unsafe):
mc_create_unsafe.reset_mock()
@mock.patch('backend.createrepo.createrepo_unsafe')
-(a)mock.patch('backend.createrepo.CoprClient')
+(a)mock.patch('backend.helpers.CoprClient')
def test_createrepo_conditional_false(mc_client, mc_create_unsafe):
mc_client.return_value.get_project_details.return_value = MagicMock(data={"detail": {"auto_createrepo": False}})
diff --git a/cli/copr_cli/main.py b/cli/copr_cli/main.py
index 3998fa3..7c2ac72 100644
--- a/cli/copr_cli/main.py
+++ b/cli/copr_cli/main.py
@@ -379,6 +379,12 @@ def main(argv=sys.argv[1:]):
except KeyboardInterrupt:
sys.stderr.write("\nInterrupted by user.")
sys.exit(1)
+ except copr_exceptions.CoprBuildException as e:
+ sys.stderr.write("\nBuild error: {0}\n".format(e))
+ sys.exit(4)
+ except copr_exceptions.CoprUnknownResponseException as e:
+ sys.stderr.write("\nError: {0}\n".format(e))
+ sys.exit(5)
except copr_exceptions.CoprRequestException as e:
sys.stderr.write("\nSomething went wrong:")
sys.stderr.write("\nError: {0}\n".format(e))
@@ -389,12 +395,7 @@ def main(argv=sys.argv[1:]):
except copr_exceptions.CoprException as e:
sys.stderr.write("\nError: {0}\n".format(e))
sys.exit(3)
- except copr_exceptions.CoprBuildException as e:
- sys.stderr.write("\nBuild error: {0}\n".format(e))
- sys.exit(4)
- except copr_exceptions.CoprUnknownResponseException as e:
- sys.stderr.write("\nError: {0}\n".format(e))
- sys.exit(5)
+
# except Exception as e:
# print "Error: {0}".format(e)
# sys.exit(100)
diff --git a/test_suite.sh b/test_suite.sh
index ab7468b..7089b21 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -23,6 +23,9 @@ pip install -r keygen/requirements.txt
mkdir -p _report
+python -m pytest python/copr/test --junitxml=_report/python-copr.junit.xml --cov-report xml --cov python/copr/client $@
+mv {,_report/python-copr.}coverage.xml
+
COPR_CONFIG="$(pwd)/frontend/coprs_frontend/config/copr_unit_test.conf" \
python -m pytest frontend/coprs_frontend/tests --junitxml=_report/frontend.junit.xml --cov-report xml --cov frontend/coprs_frontend/coprs $@
mv {,_report/frontend.}coverage.xml
@@ -30,9 +33,6 @@ mv {,_report/frontend.}coverage.xml
PYTHONPATH=backend/run:backend:python:$PYTHONPATH python -m pytest backend/tests --junitxml=_report/backend.junit.xml --cov-report xml --cov backend/backend $@
mv {,_report/backend.}coverage.xml
-python -m pytest python/copr/test --junitxml=_report/python-copr.junit.xml --cov-report xml --cov python/copr/client $@
-mv {,_report/python-copr.}coverage.xml
-
PYTHONPATH=python/:cli/:$PYTHONPATH python -m pytest cli/tests --junitxml=_report/cli.junit.xml --cov-report xml --cov cli/copr_cli $@
mv {,_report/cli.}coverage.xml
9 years, 6 months
[copr] master: [pyhton-copr] fixed Exception enheritance (8e5bc30)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 8e5bc300d5aa4e6fa46e9695aba70f5e7b93322a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Nov 6 15:35:15 2014 +0100
[pyhton-copr] fixed Exception enheritance
>---------------------------------------------------------------
python/copr/client/exceptions.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/python/copr/client/exceptions.py b/python/copr/client/exceptions.py
index 6936548..73f832c 100644
--- a/python/copr/client/exceptions.py
+++ b/python/copr/client/exceptions.py
@@ -25,21 +25,21 @@ class CoprConfigException(CoprException):
pass
-class CoprRequestException(Exception):
+class CoprRequestException(CoprException):
""" Exception thrown when the request is bad. For example,
the user provided wrong project name or build ID.
"""
pass
-class CoprBuildException(Exception):
+class CoprBuildException(CoprException):
""" Exception thrown when one or more builds fail and client is waiting
for the result.
"""
pass
-class CoprUnknownResponseException(Exception):
+class CoprUnknownResponseException(CoprException):
""" Exception thrown when the response is unknown to client.
It usually means that something is broken.
"""
9 years, 6 months
[copr] master: [backend] copr-prune-repo should respect auto_createrepo option (2f547f9)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 2f547f9cf20f2804336f091dd6f32ebc800a3c33
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Nov 6 14:21:23 2014 +0100
[backend] copr-prune-repo should respect auto_createrepo option
>---------------------------------------------------------------
backend/backend/createrepo.py | 16 +----
backend/backend/helpers.py | 155 +++++++++++++++++++++++++++++++++-
backend/copr-backend.spec | 18 ++--
backend/copr-be.conf.example | 5 +
backend/copr-be.local.conf | 4 +
backend/copr-prune-repo | 55 ------------
backend/crontab/copr-backend | 2 +-
backend/{ => run}/copr-be.py | 102 ++++-------------------
backend/run/copr_prune_old_builds.sh | 52 +++++++++++
backend/run/copr_prune_results.py | 103 ++++++++++++++++++++++
10 files changed, 343 insertions(+), 169 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index 3c3998d..0e8e4ac 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -1,12 +1,8 @@
-import sys
-
import os
import subprocess
from subprocess import Popen
-from copr.client import CoprClient
-
-from backend.helpers import SortedOptParser
+from .helpers import get_auto_createrepo_status
def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
@@ -54,16 +50,6 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
return cmd.returncode, out, err
-def get_auto_createrepo_status(front_url, username, projectname):
- client = CoprClient(copr_url=front_url)
- result = client.get_project_details(projectname, username)
-
- if "auto_createrepo" in result.data["detail"]:
- return bool(result.data["detail"]["auto_createrepo"])
- else:
- return True
-
-
def createrepo(path, front_url, username, projectname, base_url=None, lock=None):
"""
Creates repo depending on the project setting "auto_createrepo".
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index db30689..0992eb6 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -1,7 +1,17 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
from operator import methodcaller
import optparse
+import ConfigParser
+import os
+
+from bunch import Bunch
+from copr.client import CoprClient
-__author__ = 'vgologuz'
+from .exceptions import CoprBackendError
class SortedOptParser(optparse.OptionParser):
@@ -10,4 +20,145 @@ class SortedOptParser(optparse.OptionParser):
def format_help(self, formatter=None):
self.option_list.sort(key=methodcaller("get_opt_string"))
- return optparse.OptionParser.format_help(self, formatter=None)
+ return optparse.OptionParser.format_help(self)
+
+
+def _get_conf(cp, section, option, default, mode=None):
+ """
+ To make returning items from config parser less irritating
+
+ :param mode: convert obtained value, possible modes:
+ - None (default): do nothing
+ - "bool" or "boolean"
+ - "int"
+ - "float"
+ """
+
+ if cp.has_section(section) and cp.has_option(section, option):
+ if mode is None:
+ return cp.get(section, option)
+ elif mode in ["bool", "boolean"]:
+ return cp.getboolean(section, option)
+ elif mode == "int":
+ return cp.getint(section, option)
+ elif mode == "float":
+ return cp.getfloat(section, option)
+ elif mode == "path":
+ path = cp.get(section, option)
+ if path.startswith("~"):
+ path = os.path.expanduser(path)
+ path = os.path.abspath(path)
+ path = os.path.normpath(path)
+
+ return path
+ return default
+
+
+class BackendConfigReader(object):
+ def __init__(self, config_file=None, ext_opts=None):
+ self.config_file = config_file or "/etc/copr/copr-be.conf"
+ self.ext_opts = ext_opts
+
+ def read(self):
+ try:
+ opts = self._read_unsafe()
+ if self.ext_opts:
+ for key, value in self.ext_opts.items():
+ setattr(opts, key, value)
+
+ if not opts.destdir:
+ raise CoprBackendError(
+ "Incomplete Config - must specify"
+ " destdir in configuration")
+
+ return opts
+
+ except ConfigParser.Error as e:
+ raise CoprBackendError(
+ "Error parsing config file: {0}: {1}".format(
+ self.config_file, e))
+
+ def _read_unsafe(self):
+ cp = ConfigParser.ConfigParser()
+ cp.read(self.config_file)
+
+ opts = Bunch()
+ opts.results_baseurl = _get_conf(
+ cp, "backend", "results_baseurl", "http://copr")
+
+ #TODO: this should be built from frontend_base_url + '/backend'
+ opts.frontend_url = _get_conf(
+ cp, "backend", "frontend_url", "http://coprs/rest/api")
+
+ # We need this to access public api
+ opts.frontend_base_url = _get_conf(
+ cp, "backend", "frontend_base_url", "http://coprs/")
+
+ opts.frontend_auth = _get_conf(
+ cp, "backend", "frontend_auth", "PASSWORDHERE")
+
+ opts.do_sign = _get_conf(
+ cp, "backend", "do_sign", False, mode="bool")
+
+ opts.build_groups_count = _get_conf(
+ cp, "backend", "build_groups", 1, mode="int")
+
+ opts.build_groups = []
+ for group_id in range(int(opts.build_groups_count)):
+ group = {
+ "id": int(group_id),
+ "name": _get_conf(
+ cp, "backend", "group{0}_name".format(group_id), "PC"),
+ "archs": _get_conf(
+ cp, "backend", "group{0}_archs".format(group_id),
+ "i386,x86_64").split(","),
+ "spawn_playbook": _get_conf(
+ cp, "backend", "group{0}_spawn_playbook".format(group_id),
+ "/srv/copr-work/provision/builderpb-PC.yml"),
+ "terminate_playbook": _get_conf(
+ cp, "backend",
+ "group{0}_terminate_playbook".format(group_id),
+ "/srv/copr-work/provision/terminatepb-PC.yml"),
+ "max_workers": int(_get_conf(
+ cp, "backend", "group{0}_max_workers".format(group_id), 8))
+ }
+ opts.build_groups.append(group)
+
+ opts.destdir = _get_conf(cp, "backend", "destdir", None, mode="path")
+
+
+ opts.exit_on_worker = _get_conf(
+ cp, "backend", "exit_on_worker", False, mode="bool")
+ opts.fedmsg_enabled = _get_conf(
+ cp, "backend", "fedmsg_enabled", False, mode="bool")
+ opts.sleeptime = _get_conf(
+ cp, "backend", "sleeptime", 10, mode="int")
+ opts.timeout = _get_conf(
+ cp, "builder", "timeout", 1800, mode="int")
+ opts.logfile = _get_conf(
+ cp, "backend", "logfile", "/var/log/copr/backend.log")
+ opts.verbose = _get_conf(
+ cp, "backend", "verbose", False, mode="bool")
+ opts.worker_logdir = _get_conf(
+ cp, "backend", "worker_logdir", "/var/log/copr/workers/")
+ opts.spawn_vars = _get_conf(cp, "backend", "spawn_vars", None)
+ opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", None)
+
+ opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int")
+ opts.prune_script = _get_conf(cp, "backend", "prune_script", None, mode="path")
+
+ # thoughts for later
+ # ssh key for connecting to builders?
+ # cloud key stuff?
+ #
+ return opts
+
+
+def get_auto_createrepo_status(front_url, username, projectname):
+ client = CoprClient(copr_url=front_url)
+ result = client.get_project_details(projectname, username)
+
+ if "auto_createrepo" in result.data["detail"]:
+ return bool(result.data["detail"]["auto_createrepo"])
+ else:
+ return True
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 2ac7a41..8b9a8c6 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -104,9 +104,9 @@ install -d %{buildroot}%{_sysconfdir}/cron.daily
install -d %{buildroot}%{_sysconfdir}/sudoers.d
cp -a backend/* %{buildroot}%{_datadir}/copr/backend
-cp -a copr-be.py %{buildroot}%{_datadir}/copr/
+cp -a run/* %{buildroot}%{_datadir}/copr/
cp -a copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf
-install -p -m 755 copr-prune-repo %{buildroot}%{_sbindir}/copr-prune-repo
+
install -p -m 755 crontab/copr-backend %{buildroot}%{_sysconfdir}/cron.daily/copr-backend
cp -a dist/lighttpd/* %{buildroot}%{_pkgdocdir}/lighttpd/
@@ -115,6 +115,8 @@ cp -a tmpfiles.d/* %{buildroot}/%{_tmpfilesdir}
# for ghost files
touch %{buildroot}%{_var}/log/copr/copr.log
+touch %{buildroot}%{_var}/log/copr/prune_old.log
+
for i in `seq 7`; do
touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log
done
@@ -144,7 +146,8 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%files
%doc LICENSE
-%dir %{_datadir}/copr
+
+%{_datadir}/copr/*
%dir %{_sharedstatedir}/copr
%dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/jobs/
%dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/
@@ -153,7 +156,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%dir %attr(0755, copr, copr) %{_var}/log/copr/workers
%dir %attr(0755, copr, copr) %{_var}/run/copr-backend
-%ghost %{_var}/log/copr/copr.log
+%ghost %{_var}/log/copr/*.log
%ghost %{_var}/log/copr/workers/worker-*.log
%ghost %{_var}/run/copr-backend/copr-be.pid
@@ -165,13 +168,8 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf
%{_unitdir}/copr-backend.service
%{_tmpfilesdir}/copr-backend.conf
-%{_sbindir}/copr-prune-repo
-%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
-
-
-%{_datadir}/copr/backend
-%{_datadir}/copr/copr-be.py*
+%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
%config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr
diff --git a/backend/copr-be.conf.example b/backend/copr-be.conf.example
index 0ae59bc..0f1f461 100644
--- a/backend/copr-be.conf.example
+++ b/backend/copr-be.conf.example
@@ -96,6 +96,11 @@ worker_logdir=/var/log/copr/workers/
# signer host and correct /etc/sign.conf
#do_sign=false
+# minimum age for builds to be pruned
+prune_days=14
+# path to executable script to clean old build
+prune_script=/usr/sbin/copr_prune_old_builds.sh
+
[builder]
# default is 1800
timeout=3600
diff --git a/backend/copr-be.local.conf b/backend/copr-be.local.conf
index 3885b04..76df89c 100644
--- a/backend/copr-be.local.conf
+++ b/backend/copr-be.local.conf
@@ -98,6 +98,10 @@ worker_logdir=/tmp/log/copr/workers/
verbose=true
+# minimum age for builds to be pruned
+prune_days=14
+# path to executable script to clean old build
+prune_script=./copr_prune_old_builds.sh
[builder]
# default is 1800
diff --git a/backend/copr-prune-repo b/backend/copr-prune-repo
deleted file mode 100755
index 7b5ac79..0000000
--- a/backend/copr-prune-repo
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/bash
-
-DAYS=14
-RESULTDIR=$(awk -F "=" '/^destdir/ {print $2}' /etc/copr/copr-be.conf)
-
-if [ ! -d $RESULTDIR ]; then
- echo "ERROR: destdir ($RESULTDIR) in /etc/copr/copr-be.conf does not exist."
- exit 1
-fi
-
-pushd $RESULTDIR >/dev/null || { echo "Could not change to $RESULTDIR"; exit 1; }
-
-for REPO in $( ls -d */*/* | grep -v '/repodata$' ); do
- pushd $REPO >/dev/null || continue
-
- # Removing failed builds
- for FAILED in $(find -name 'fail' -mtime +$DAYS); do
- rm -rf $(dirname $FAILED)
- echo -n .
- done
-
- # Query latest (sucessfull) packages
- LATEST_PKGS=$(mktemp)
- ERR_LOG=$(mktemp)
- # "yum clean metadata" does not work on this custom repos
- rm -rf $(find /var/tmp -name query)
- rm -rf /var/tmp/createrepo*
- rm -rf /var/cache/yum/*
- repoquery --repofrompath=query,$RESULTDIR/$REPO --repoid=query -a --location 2>$ERR_LOG \
- | cut -c8- > $LATEST_PKGS
- # remove directory if it does not have repodata at all
- grep 'Cannot retrieve repository metadata' $ERR_LOG && rm -rf $RESULTDIR/$REPO
-
- # Remove builds older then $DAYS days and which have newer builds available
- for SUCCESS in $(find -name success -mtime +$DAYS); do
- DIR=$(basename $(dirname $SUCCESS))
- for PACKAGE in $(ls $DIR); do
- grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
- done
- # package was not found in $LATEST_PKGS
- rm -rf $DIR
- echo -n .
- done
-
- rm -f $LATEST_PKGS $ERR_LOG
- # unless we previosly deleted the repo, regen metadata
- if [ -d $(pwd) ]; then
- runuser -c "createrepo_c --update -q --ignore-lock $(pwd)" - copr &>/dev/null || runuser -c "createrepo_c --update -q --ignore-lock $(pwd)" - copr
- fi
- popd >/dev/null
-done
-popd >/dev/null
-rm -rf /var/tmp/createrepo*
-echo
-echo Done
diff --git a/backend/crontab/copr-backend b/backend/crontab/copr-backend
index 586d45b..2260f7a 100644
--- a/backend/crontab/copr-backend
+++ b/backend/crontab/copr-backend
@@ -1,3 +1,3 @@
#!/usr/bin/sh
-/usr/sbin/copr-prune-repo >/var/log/copr.log 2>&1
+python /usr/share/copr/copr_prune_results.py >> /var/log/copr/prune_old.log 2>&1
diff --git a/backend/copr-be.py b/backend/run/copr-be.py
similarity index 77%
rename from backend/copr-be.py
rename to backend/run/copr-be.py
index 74d74a1..e1fea3b 100755
--- a/backend/copr-be.py
+++ b/backend/run/copr-be.py
@@ -1,5 +1,10 @@
#!/usr/bin/python -ttu
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
import ConfigParser
import grp
import lockfile
@@ -25,6 +30,7 @@ from backend.exceptions import CoprBackendError
from backend.dispatcher import Worker
from backend.actions import Action
from backend.callback import FrontendCallback
+from backend.helpers import BackendConfigReader
def _get_conf(cp, section, option, default, mode=None):
@@ -203,7 +209,12 @@ class CoprBackend(object):
self.ext_opts = ext_opts # to stow our cli options for read_conf()
self.workers_by_group_id = defaultdict(list)
self.max_worker_num_by_group_id = defaultdict(int)
- self.opts = self.read_conf()
+
+ #self.opts = self.read_conf()
+ self.config_reader = BackendConfigReader(self.config_file, self.ext_opts)
+ self.opts = None
+ self.update_conf()
+
self.lock = multiprocessing.Lock()
self.task_queues = []
@@ -240,90 +251,8 @@ class CoprBackend(object):
def event(self, what):
self.events.put({"when": time.time(), "who": "main", "what": what})
- def read_conf(self):
- "read in config file - return Bunch of config data"
- opts = Bunch()
- cp = ConfigParser.ConfigParser()
- try:
- cp.read(self.config_file)
- opts.results_baseurl = _get_conf(
- cp, "backend", "results_baseurl", "http://copr")
-
- #TODO: this should be built from frontend_base_url + '/backend'
- opts.frontend_url = _get_conf(
- cp, "backend", "frontend_url", "http://coprs/rest/api")
-
- # We need this to access public api
- opts.frontend_base_url = _get_conf(
- cp, "backend", "frontend_base_url", "http://coprs/")
-
- opts.frontend_auth = _get_conf(
- cp, "backend", "frontend_auth", "PASSWORDHERE")
-
- opts.build_groups_count = _get_conf(
- cp, "backend", "build_groups", 1, mode="int")
-
- opts.do_sign = _get_conf(
- cp, "backend", "do_sign", False, mode="bool")
-
- opts.build_groups = []
- for group_id in range(int(opts.build_groups_count)):
- group = {
- "id": int(group_id),
- "name": _get_conf(
- cp, "backend", "group{0}_name".format(group_id), "PC"),
- "archs": _get_conf(
- cp, "backend", "group{0}_archs".format(group_id),
- "i386,x86_64").split(","),
- "spawn_playbook": _get_conf(
- cp, "backend", "group{0}_spawn_playbook".format(group_id),
- "/srv/copr-work/provision/builderpb-PC.yml"),
- "terminate_playbook": _get_conf(
- cp, "backend",
- "group{0}_terminate_playbook".format(group_id),
- "/srv/copr-work/provision/terminatepb-PC.yml"),
- "max_workers": int(_get_conf(
- cp, "backend", "group{0}_max_workers".format(group_id), 8))
- }
- opts.build_groups.append(group)
-
- opts.destdir = _get_conf(cp, "backend", "destdir", None)
- opts.exit_on_worker = _get_conf(
- cp, "backend", "exit_on_worker", False, mode="bool")
- opts.fedmsg_enabled = _get_conf(
- cp, "backend", "fedmsg_enabled", False, mode="bool")
- opts.sleeptime = _get_conf(
- cp, "backend", "sleeptime", 10, mode="int")
- opts.timeout = _get_conf(
- cp, "builder", "timeout", 1800, mode="int")
- opts.logfile = _get_conf(
- cp, "backend", "logfile", "/var/log/copr/backend.log")
- opts.verbose = _get_conf(
- cp, "backend", "verbose", False, mode="bool")
- opts.worker_logdir = _get_conf(
- cp, "backend", "worker_logdir", "/var/log/copr/workers/")
- opts.spawn_vars = _get_conf(cp, "backend", "spawn_vars", None)
- opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars",
- None)
-
- # thoughts for later
- # ssh key for connecting to builders?
- # cloud key stuff?
- #
- except ConfigParser.Error as e:
- raise CoprBackendError(
- "Error parsing config file: {0}: {1}".format(
- self.config_file, e))
-
- if not opts.destdir:
- raise CoprBackendError(
- "Incomplete Config - must specify"
- " destdir in configuration")
-
- if self.ext_opts:
- for v in self.ext_opts:
- setattr(opts, v, self.ext_opts.get(v))
- return opts
+ def update_conf(self):
+ self.opts = self.config_reader.read()
def clean_task_queues(self):
try:
@@ -338,7 +267,8 @@ class CoprBackend(object):
self.abort = False
while not self.abort:
# re-read config into opts
- self.opts = self.read_conf()
+ #self.opts = self.read_conf()
+ self.update_conf()
for group in self.opts.build_groups:
group_id = group["id"]
diff --git a/backend/run/copr_prune_old_builds.sh b/backend/run/copr_prune_old_builds.sh
new file mode 100755
index 0000000..30f280d
--- /dev/null
+++ b/backend/run/copr_prune_old_builds.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+REPOPATH=$1
+DAYS=$2
+
+
+pushd $REPOPATH >/dev/null
+
+
+for CHROOT in $(ls $REPOPATH); do
+ pushd $CHROOT >/dev/null
+
+ # remove old failed builds
+ for FAILED in $(find -name 'fail' -mtime +$DAYS); do
+ rm -rf $(dirname $FAILED)
+ echo -n .
+ done
+
+ # query latest (sucessfull) packages
+ LATEST_PKGS=$(mktemp)
+ ERR_LOG=$(mktemp)
+ # "yum clean metadata" does not work on this custom repos
+ rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null
+ rm -rf /var/tmp/createrepo* &> /dev/null
+ rm -rf /var/cache/yum/* &> /dev/null
+
+ repoquery --repofrompath=query,$REPOPATH/$CHROOT --repoid=query -a --location 2>$ERR_LOG \
+ | cut -c8- > $LATEST_PKGS
+
+ # remove directory if it does not have repodata at all
+ if $(grep --quiet 'Cannot retrieve repository metadata' $ERR_LOG) || $(grep --quiet "Can't download or revert repomd.xml" $ERR_LOG) ; then
+ # don't remove when /devel presents
+ if [ ! -d $REPOPATH/$CHROOT/devel ]; then
+ rm -rf $REPOPATH/$CHROOT/
+ fi
+ else
+ # Remove builds older then $DAYS days and which have newer builds available
+ for SUCCESS in $(find -name success -mtime +$DAYS); do
+ DIR=$(basename $(dirname $SUCCESS))
+ for PACKAGE in $(ls $DIR); do
+ grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2
+ done
+ # package was not found in $LATEST_PKGS
+ rm -rf $DIR
+ echo -n .
+ done
+ fi
+
+ rm -f $LATEST_PKGS $ERR_LOG
+ popd >/dev/null
+done
+popd >/dev/null
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
new file mode 100644
index 0000000..eec6df4
--- /dev/null
+++ b/backend/run/copr_prune_results.py
@@ -0,0 +1,103 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
+import os
+import logging
+from subprocess import Popen, PIPE
+
+from copr.client.exceptions import CoprException, CoprRequestException
+
+from backend.helpers import BackendConfigReader, get_auto_createrepo_status
+from backend.createrepo import createrepo_unsafe
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+DEF_DAYS = 14
+DEF_PRUNE_SCRIPT = "copr_prune_old_builds.sh"
+DEF_PRUNE_SCRIPT = "/usr/sbin/copr_prune_old_builds.sh"
+
+
+def list_subdir(path):
+ dir_names = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
+ return dir_names, map(lambda x: os.path.join(path, x), dir_names)
+
+
+def prune_project(opts, path, username, projectname):
+ log.debug("Going to prune {}/{}".format(username, projectname))
+ # get ACR
+ try:
+ if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname):
+ log.debug("Skipped {}/{} since auto createrepo option is disabled"
+ .format(username, projectname))
+ return
+ except (CoprException, CoprRequestException) as exception:
+ log.debug("Failed to get project details for {}/{} with error: {}".format(
+ username, projectname, exception))
+ return
+
+
+ # run prune project sh
+ #cmd = ["/usr/bin/copr_prune.sh", path, DAYS]
+ days = getattr(opts, "prune_days", DEF_DAYS)
+ prune_script = getattr(opts, "prune_script", DEF_PRUNE_SCRIPT)
+
+ cmd = map(str, [prune_script, path, days])
+
+ print("CMD {}".format(cmd))
+ handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = handle.communicate()
+
+ if handle.returncode != 0:
+ print("Failed to prune old builds for copr {}/{}".format(username, projectname))
+ print("STDOUT: \n{}".format(stdout.decode()))
+ print("STDERR: \n{}".format(stderr.decode()))
+ return
+
+ # run createrepo
+ log.debug("Prune done for {}/{}".format(username, projectname))
+ try:
+ retcode, stdout, stderr = createrepo_unsafe(path)
+ if retcode != 0:
+ print("Createrepo for {}/{} failed".format(username, projectname))
+ print("STDOUT: \n{}".format(stdout.decode()))
+ print("STDERR: \n{}".format(stderr.decode()))
+ except Exception as exception:
+ print("Createrepo for {}/{} failed with error: {}"
+ .format(username, projectname, exception))
+
+
+
+def main():
+ config_file= os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf")
+ opts = BackendConfigReader(config_file).read()
+
+ results_dir = opts.destdir
+ log.info("Pruning results dir: {} ".format(results_dir))
+ user_dir_names, user_dirs = list_subdir(results_dir)
+
+ print("Going to process total number: {} of user's directories".format(len(user_dir_names)))
+ log.info("Going to process user's directories: {}".format(user_dir_names))
+
+ counter = 0
+ for username, subpath in zip(user_dir_names, user_dirs):
+ log.debug("For user `{}` exploring path: {}".format(username, subpath))
+ for projectname, project_path in zip(*list_subdir(subpath)):
+ log.debug("Exploring project `{}` with path: {}".format(projectname, project_path))
+ prune_project(opts, project_path, username, projectname)
+
+ counter += 1
+ print("Pruned {} projects".format(counter))
+
+ print("Pruning finished")
+
+
+if __name__ == "__main__":
+ # logging.basicConfig(
+ # level=logging.DEBUG,
+ # format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
+ # datefmt='%H:%M:%S'
+ # )
+ main()
9 years, 6 months
[copr] master: [backend] bugfix: repeated config reads produced constantly growing lists (c8a7ee7)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c8a7ee764de925b608a4ccbfc2545863c2fd042b
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Nov 5 15:36:35 2014 +0100
[backend] bugfix: repeated config reads produced constantly growing lists
>---------------------------------------------------------------
backend/copr-be.py | 53 +++++++++++++++++++++++++++------------------------
1 files changed, 28 insertions(+), 25 deletions(-)
diff --git a/backend/copr-be.py b/backend/copr-be.py
index 380a081..74d74a1 100755
--- a/backend/copr-be.py
+++ b/backend/copr-be.py
@@ -1,13 +1,6 @@
#!/usr/bin/python -ttu
-
-
-from bunch import Bunch
-from retask.task import Task
-from retask.queue import Queue
-from retask import ConnectionError
import ConfigParser
-import daemon
import grp
import lockfile
import logging
@@ -15,11 +8,18 @@ import multiprocessing
import optparse
import os
import pwd
-import requests
-import setproctitle
import signal
import sys
import time
+from collections import defaultdict
+
+import daemon
+import requests
+import setproctitle
+from bunch import Bunch
+from retask.task import Task
+from retask.queue import Queue
+from retask import ConnectionError
from backend.exceptions import CoprBackendError
from backend.dispatcher import Worker
@@ -201,8 +201,8 @@ class CoprBackend(object):
self.config_file = config_file
self.ext_opts = ext_opts # to stow our cli options for read_conf()
- self.worker_num = []
- self.workers = []
+ self.workers_by_group_id = defaultdict(list)
+ self.max_worker_num_by_group_id = defaultdict(int)
self.opts = self.read_conf()
self.lock = multiprocessing.Lock()
@@ -286,8 +286,6 @@ class CoprBackend(object):
cp, "backend", "group{0}_max_workers".format(group_id), 8))
}
opts.build_groups.append(group)
- self.worker_num.append(0)
- self.workers.append([])
opts.destdir = _get_conf(cp, "backend", "destdir", None)
opts.exit_on_worker = _get_conf(
@@ -348,14 +346,17 @@ class CoprBackend(object):
group["name"],
self.task_queues[group_id].length))
# this handles starting/growing the number of workers
- if len(self.workers[group_id]) < group["max_workers"]:
+ if len(self.workers_by_group_id[group_id]) < group["max_workers"]:
self.event("Spinning up more workers")
- for _ in range(group["max_workers"] - len(self.workers[group_id])):
- self.worker_num[group_id] += 1
+ for _ in range(group["max_workers"] - len(self.workers_by_group_id[group_id])):
+ self.max_worker_num_by_group_id[group_id] += 1
w = Worker(
- self.opts, self.events, self.worker_num[group_id], group_id,
- lock=self.lock)
- self.workers[group_id].append(w)
+ self.opts, self.events,
+ self.max_worker_num_by_group_id[group_id],
+ group_id, lock=self.lock
+ )
+
+ self.workers_by_group_id[group_id].append(w)
w.start()
self.event("Finished starting worker processes")
# FIXME - prune out workers
@@ -366,16 +367,18 @@ class CoprBackend(object):
# FIXME - if a worker bombs out - we need to check them
# and startup a new one if it happens
# check for dead workers and abort
- for w in self.workers[group_id]:
+ preserved_workers = []
+ for w in self.workers_by_group_id[group_id]:
if not w.is_alive():
- self.event("Worker {0} died unexpectedly".format(
- w.worker_num))
+ self.event("Worker {0} died unexpectedly".format(w.worker_num))
if self.opts.exit_on_worker:
raise CoprBackendError(
"Worker died unexpectedly, exiting")
else:
- self.workers[group_id].remove(w) # it is not working anymore
w.terminate() # kill it with a fire
+ else:
+ preserved_workers.append(w)
+ self.workers_by_group_id[group_id] = preserved_workers
time.sleep(self.opts.sleeptime)
@@ -388,8 +391,8 @@ class CoprBackend(object):
self.abort = True
for group in self.opts.build_groups:
group_id = group["id"]
- for w in self.workers[group_id]:
- self.workers[group_id].remove(w)
+ for w in self.workers_by_group_id[group_id]:
+ self.workers_by_group_id[group_id].remove(w)
w.terminate()
self.clean_task_queues()
9 years, 6 months
[copr] master: [backend] fixed CoprClient constructor call (39929ab)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 39929abe69430de435369f7100f897dffe8c5382
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Nov 5 14:53:19 2014 +0100
[backend] fixed CoprClient constructor call
>---------------------------------------------------------------
backend/backend/createrepo.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index d28e0f9..3c3998d 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -55,7 +55,7 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
def get_auto_createrepo_status(front_url, username, projectname):
- client = CoprClient({"copr_url": front_url})
+ client = CoprClient(copr_url=front_url)
result = client.get_project_details(projectname, username)
if "auto_createrepo" in result.data["detail"]:
9 years, 6 months
[copr] master: [python] fixed poor decision abou CoprClient constructor, now it accepts kwargs arguments instead of config dict (db3e8e3)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit db3e8e3ee379542a0298cef25d949863df1ead9d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Nov 4 16:54:37 2014 +0100
[python] fixed poor decision abou CoprClient constructor, now it accepts kwargs arguments instead of config dict
>---------------------------------------------------------------
cli/copr_cli/main.py | 8 ++++----
cli/tests/test_cli.py | 4 ++--
python/copr/README.rst | 12 ++++++------
python/copr/client/client.py | 25 +++++++++++++++----------
python/copr/test/test_client.py | 4 ++--
python/docs/Examples.rst | 7 +++++++
6 files changed, 36 insertions(+), 24 deletions(-)
diff --git a/cli/copr_cli/main.py b/cli/copr_cli/main.py
index 4752905..3998fa3 100644
--- a/cli/copr_cli/main.py
+++ b/cli/copr_cli/main.py
@@ -41,10 +41,10 @@ class Commands(object):
except (copr_exceptions.CoprNoConfException,
copr_exceptions.CoprConfigException):
print(no_config_warning)
- self.client = CoprClient({
- "copr_url": "http://copr.fedoraproject.org",
- "no_config": True
- })
+ self.client = CoprClient(
+ copr_url=u"http://copr.fedoraproject.org",
+ no_config=True
+ )
def requires_api_auth(func):
""" Decorator that checks config presence
diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py
index e9b7e20..d9bed34 100644
--- a/cli/tests/test_cli.py
+++ b/cli/tests/test_cli.py
@@ -199,7 +199,7 @@ def test_list_project(mock_cc, capsys):
# no config
mock_cc.create_from_file_config.side_effect = CoprNoConfException()
- mocked_client = MagicMock(CoprClient(dict(no_config=True)))
+ mocked_client = MagicMock(CoprClient(no_config=True))
control_response = CoprResponse(client=None, method="", data=response_data,
parsers=[ProjectListParser, CommonMsgErrorOutParser])
@@ -229,7 +229,7 @@ def test_list_project_no_username(mock_cc, capsys):
@mock.patch('copr_cli.main.CoprClient')
def test_list_project_no_username2(mock_cc, capsys):
- mock_cc.create_from_file_config.return_value = CoprClient(defaultdict())
+ mock_cc.create_from_file_config.return_value = CoprClient()
with pytest.raises(SystemExit) as err:
main.main(argv=["list"])
diff --git a/python/copr/README.rst b/python/copr/README.rst
index c624ebb..8b3f708 100644
--- a/python/copr/README.rst
+++ b/python/copr/README.rst
@@ -26,12 +26,12 @@ Usage:
::
from python_copr.main import CoprClient
- client = CoprClient({
- "login": "<login from /api>",
- "token": "<token from /api>",
- "username": "<copr username>",
- "copr_url": "<url copr instance ; optional>"
- })
+ client = CoprClient(
+ login="<login from /api>",
+ token="<token from /api>",
+ username="<copr username>",
+ copr_url="<url copr instance ; optional>"
+ )
Alternatively you could use configuration file:
- Create the file ``~/.config/copr``
diff --git a/python/copr/client/client.py b/python/copr/client/client.py
index ad59f76..8f7824f 100644
--- a/python/copr/client/client.py
+++ b/python/copr/client/client.py
@@ -56,22 +56,27 @@ class CoprClient(object):
:ivar unicode copr_url: used as copr projects root
Could be created:
- - directly by providing a `dict` with credentionals
+ - directly
- using static method :py:meth:`CoprClient.create_from_file_config`
"""
- def __init__(self, config=None):
+ def __init__(self, username=None, login=None, token=None, copr_url=None,
+ no_config=False):
"""
- :param dict config: Dictionary with client configuration.
+ :param unicode username: username used by default for all requests
+ :param unicode login: user login, used for identification
+ :param unicode token: copr api token
+ :param unicode copr_url: used as copr projects root
+ :param bool no_config: helper flag to indicate that no config was provided
"""
- self.token = config.get("token")
- self.login = config.get("login")
- self.username = config.get("username")
- self.copr_url = config.get("copr_url", "http://copr.fedoraproject.org/")
+ self.token = token
+ self.login = login
+ self.username = username
+ self.copr_url = copr_url or "http://copr.fedoraproject.org/"
- self.no_config = config.get("no_config", False)
+ self.no_config = no_config
def __str__(self):
return "<Copr client. username: {0}, api url: {1}, " \
@@ -109,7 +114,7 @@ class CoprClient(object):
log.warning(
"No configuration file '~/.config/copr' found. "
"See man copr-cli for more information")
-
+ config["no_config"] = True
if not ignore_error:
raise CoprNoConfException()
else:
@@ -124,7 +129,7 @@ class CoprClient(object):
if not ignore_error:
raise CoprConfigException(
"Bad configuration file: {0}".format(err))
- return CoprClient(config=config)
+ return CoprClient(**config)
def _fetch(self, url, data=None, projectname=None, username=None,
method=None, skip_auth=False, on_error_response=None):
diff --git a/python/copr/test/test_client.py b/python/copr/test/test_client.py
index 6d73f97..642f2f9 100644
--- a/python/copr/test/test_client.py
+++ b/python/copr/test/test_client.py
@@ -13,12 +13,12 @@ config_location = os.path.join(resource_location, "copr_cli.conf")
def test_client_from_dict():
- cl = CoprClient(dict(
+ cl = CoprClient(
login="api-login",
username="user_name",
token="api-token",
copr_url="http://copr-fe-dev.cloud.fedoraproject.org"
- ))
+ )
assert isinstance(cl, CoprClient)
assert cl.login == "api-login"
diff --git a/python/docs/Examples.rst b/python/docs/Examples.rst
index 540ca2b..37171e2 100644
--- a/python/docs/Examples.rst
+++ b/python/docs/Examples.rst
@@ -3,8 +3,15 @@ Create client
::
from copr.client import CoprClient
+
+ # using default config location ~/.config/copr
cl = CoprClient.create_from_file_config()
+ # using config at ~/.config/alt_copr
+ cl = CoprClient.create_from_file_config("~/.config/alt_copr")
+
+ # directly
+ cl = CoprClient(username="foo", login="abcd", token="efgk", copr_url="http://example.com/")
Get projects list
-----------------
9 years, 6 months
[copr] master: [frontend] show "createrepo" action only when user disable auto_createrepo (07222f6)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 07222f6f4fbd2a24bdbc74863e0dd7fae9f6b734
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Nov 4 15:19:35 2014 +0100
[frontend] show "createrepo" action only when user disable auto_createrepo
>---------------------------------------------------------------
.../coprs/templates/coprs/_coprs_forms.html | 5 +----
.../coprs/templates/coprs/detail/overview.html | 5 +++--
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
index cae838b..81c9193 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
@@ -21,9 +21,6 @@
{{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.') }}
{{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.') }}
-
-
-
<dd>You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTML is forbidden..</dd>
<dt class="field-label required">Chroots</dt>
@@ -66,7 +63,7 @@
<label for="auto_createrepo">Build release options</label>
</dt>
<dd>
- Automatically re-create repository metadata:
+ Create repository metadata automatically:
{{ form.auto_createrepo }}
</dd>
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index f011242..a88dc9a 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -85,17 +85,18 @@
</ul>
{% endif %}
- {% if g.user and g.user.can_edit(copr) %}
+ {% if g.user and g.user.can_edit(copr) and not copr.auto_createrepo %}
<div>
<h2>Release options</h2>
+
<form action="{{
url_for(
'coprs_ns.copr_createrepo',
username=copr.owner.name,
coprname=copr.name
) }}" method="post" >
-
+ Create repository metodata forcible:
<button type="submit">Run createrepo</button>
</form>
</div>
9 years, 6 months
[copr] master: [copr-cli] print debug logs only when user provides "--debug" option (1c69b2c)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 1c69b2c91ef0bdc7cf4dd824eb27f3c7b8532c66
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Tue Nov 4 13:47:07 2014 +0100
[copr-cli] print debug logs only when user provides "--debug" option
>---------------------------------------------------------------
cli/copr_cli/main.py | 28 ++++++++++++++++++----------
cli/run_tests.sh | 2 +-
cli/tests/test_cli.py | 36 +++++++++++++++++++++++++-----------
3 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/cli/copr_cli/main.py b/cli/copr_cli/main.py
index d384e3a..4752905 100644
--- a/cli/copr_cli/main.py
+++ b/cli/copr_cli/main.py
@@ -13,6 +13,12 @@ import datetime
import time
from collections import defaultdict
+
+import logging
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
from copr import CoprClient
import copr.client.exceptions as copr_exceptions
@@ -251,6 +257,9 @@ def setup_parser():
parser.add_argument("--version", action="version",
version="copr {0}".format(__version__))
+ parser.add_argument("--debug", dest="debug", action="store_true",
+ help="Enable debug output")
+
subparsers = parser.add_subparsers(title="actions")
# create the parser for the "list" command
@@ -346,16 +355,13 @@ def setup_parser():
return parser
-import logging
-
-logging.basicConfig(
- level=logging.DEBUG,
- format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
- datefmt='%H:%M:%S'
-)
-
-log = logging.getLogger()
-log.info("Logger initiated")
+def enable_debug():
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
+ datefmt='%H:%M:%S'
+ )
+ log.debug("# Debug log enabled #")
def main(argv=sys.argv[1:]):
@@ -364,6 +370,8 @@ def main(argv=sys.argv[1:]):
parser = setup_parser()
# Parse the commandline
arg = parser.parse_args(argv)
+ if arg.debug:
+ enable_debug()
commands = Commands()
getattr(commands, arg.func)(arg)
diff --git a/cli/run_tests.sh b/cli/run_tests.sh
index 326c3c4..a66094f 100755
--- a/cli/run_tests.sh
+++ b/cli/run_tests.sh
@@ -1,4 +1,4 @@
#!/bin/sh
-PYTHONPATH=./copr_cli:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./copr_cli/ tests $@
+PYTHONPATH=./copr_cli:$PYTHONPATH python -B -m pytest --cov-report term-missing --cov ./copr_cli/ tests $@
#PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report term-missing --cov ./copr_cli/ $@
diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py
index ef952f6..e9b7e20 100644
--- a/cli/tests/test_cli.py
+++ b/cli/tests/test_cli.py
@@ -25,17 +25,17 @@ if six.PY3:
else:
import mock
from mock import MagicMock
-
-import logging
-
-logging.basicConfig(
- level=logging.INFO,
- format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
- datefmt='%H:%M:%S'
-)
-
-log = logging.getLogger()
-log.info("Logger initiated")
+#
+# import logging
+#
+# logging.basicConfig(
+# level=logging.INFO,
+# format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
+# datefmt='%H:%M:%S'
+# )
+#
+# log = logging.getLogger()
+# log.info("Logger initiated")
from copr_cli import main
@@ -285,6 +285,20 @@ def test_status_response(mock_cc, capsys):
out, err = capsys.readouterr()
assert "{}\n".format(response_status) in out
+(a)mock.patch('copr_cli.main.CoprClient')
+def test_debug_by_status_response(mock_cc, capsys):
+ response_status = "foobar"
+
+ mock_client = MagicMock(no_config=False)
+ mock_client.get_build_details.return_value = \
+ MagicMock(status=response_status)
+ mock_cc.create_from_file_config.return_value = mock_client
+
+ main.main(argv=["--debug", "status", "123"])
+ stdout, stderr = capsys.readouterr()
+ assert "{}\n".format(response_status) in stdout
+ assert "Debug log enabled " in stderr
+
@mock.patch('copr_cli.main.CoprClient')
def test_status_response_no_args(mock_cc, capsys):
9 years, 6 months