[copr] master: alter vm_name= regexp (a0c53cb)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit a0c53cbb9f704aa1c1f2bfddf07f840a3a18787e
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Thu Jun 11 09:48:48 2015 +0200
alter vm_name= regexp
so it does not correctly parse string:
Domain vm26 started\n\nVM_IP=147.229.8.206\nvm_name=rh-power-vm26.fit.vutbr.cz
>---------------------------------------------------------------
backend/backend/vm_manage/spawn.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/backend/vm_manage/spawn.py b/backend/backend/vm_manage/spawn.py
index ae8c1f6..3a797a4 100644
--- a/backend/backend/vm_manage/spawn.py
+++ b/backend/backend/vm_manage/spawn.py
@@ -48,7 +48,7 @@ def spawn_instance(spawn_playbook, log):
if not match:
raise CoprSpawnFailError("No ip in the result, trying again")
ipaddr = match.group(1)
- match = re.search(r'vm_name=([^\{\}"]+)', result, re.MULTILINE)
+ match = re.search(r'vm_name=([^\{\}"\n]+)', result, re.MULTILINE)
if match:
vm_name = match.group(1)
8 years, 10 months
[copr] master: [python] Bug 1188874 - better unicode handling (0380a2b)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 0380a2b5f240fd7471d4a85db42306254dfa0e50
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jun 12 14:37:03 2015 +0200
[python] Bug 1188874 - better unicode handling
>---------------------------------------------------------------
python/copr/client/client.py | 11 ++++++-----
python/copr/client/responses.py | 29 ++++++++++++-----------------
python/copr/client/util.py | 15 +++++++++++++++
3 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/python/copr/client/client.py b/python/copr/client/client.py
index e7f1cec..c71e190 100644
--- a/python/copr/client/client.py
+++ b/python/copr/client/client.py
@@ -37,6 +37,7 @@ from .parsers import fabric_simple_fields_parser, ProjectListParser, \
CommonMsgErrorOutParser, NewBuildListParser, ProjectChrootsParser, \
ProjectDetailsFieldsParser
+from .util import UnicodeMixin
# TODO: add deco to check that login/token are provided
# and raise correct error
@@ -50,7 +51,7 @@ from .parsers import fabric_simple_fields_parser, ProjectListParser, \
# """
-class CoprClient(object):
+class CoprClient(UnicodeMixin):
""" Main interface to the copr service
:ivar unicode username: username used by default for all requests
@@ -81,11 +82,11 @@ class CoprClient(object):
self.no_config = no_config
- def __str__(self):
+ def __unicode__(self):
return (
- "<Copr client. username: {0}, api url: {1}, "
- "login presents: {2}, token presents: {3}>"
- ).format(self.username, self.api_url, bool(self.login), bool(self.token))
+ u"<Copr client. username: {}, api url: {}, login presents: {}, token presents: {}>"
+ .format(self.username, self.api_url, bool(self.login), bool(self.token))
+ )
@property
def api_url(self):
diff --git a/python/copr/client/responses.py b/python/copr/client/responses.py
index 3719988..4866d0a 100644
--- a/python/copr/client/responses.py
+++ b/python/copr/client/responses.py
@@ -15,9 +15,10 @@ import weakref
# TODO: Add Response for collections?
# TODO: what about copr_project-> chroot_list, build_list, repos_list
#
+from .util import UnicodeMixin
-class CoprResponse(object):
+class CoprResponse(UnicodeMixin):
"""
Wrapper for Copr api responses
@@ -60,11 +61,8 @@ class CoprResponse(object):
return value
raise KeyError(str(item))
- def __str__(self):
- return str(self.data)
-
def __unicode__(self):
- return unicode(self.data)
+ return self.data
class BaseHandle(object):
@@ -225,7 +223,7 @@ class ProjectChrootHandle(BaseHandle):
self.projectname, self.chrootname, pkgs=pkgs)
-class ProjectWrapper(object):
+class ProjectWrapper(UnicodeMixin):
"""
Helper class to represent project objects
@@ -250,25 +248,22 @@ class ProjectWrapper(object):
self.handle = ProjectHandle(client=client, projectname=projectname,
username=username, response=None)
- def __str__(self):
- return unicode(self).encode('utf-8')
-
def __unicode__(self):
out = list()
- out.append("Name: {0}".format(self.projectname))
- out.append(" Description: {0}".format(self.description))
+ out.append(u"Name: {}".format(self.projectname))
+ out.append(u" Description: {}".format(self.description))
if self.yum_repos:
- out.append(" Yum repo(s):")
+ out.append(u" Yum repo(s):")
for k in sorted(self.yum_repos.keys()):
- out.append(" {0}: {1}".format(k, self.yum_repos[k]))
+ out.append(u" {}: {}".format(k, self.yum_repos[k]))
if self.additional_repos:
- out.append(" Additional repo: {0}".format(self.additional_repos))
+ out.append(u" Additional repo: {0}".format(self.additional_repos))
if self.instructions:
- out.append(" Instructions: {0}".format(self.instructions))
+ out.append(u" Instructions: {0}".format(self.instructions))
- out.append("")
- return "\n".join(out)
+ out.append(u"")
+ return u"\n".join(out)
class BuildWrapper(object):
diff --git a/python/copr/client/util.py b/python/copr/client/util.py
new file mode 100644
index 0000000..632ebe8
--- /dev/null
+++ b/python/copr/client/util.py
@@ -0,0 +1,15 @@
+# coding: utf-8
+import sys
+
+
+class UnicodeMixin(object):
+ # http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
+ """Mixin class to handle defining the proper __str__/__unicode__
+ methods in Python 2 or 3."""
+
+ if sys.version_info[0] >= 3: # Python 3
+ def __str__(self):
+ return self.__unicode__()
+ else: # Python 2
+ def __str__(self):
+ return self.__unicode__().encode('utf8')
8 years, 10 months
[copr] master: [cli] test unicode representation of ProjectWrapper (RhBug:1188874) (abc7cfb)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit abc7cfbee90c820269bb76c4d660dbf18460cb72
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Wed Jun 10 21:57:43 2015 +0200
[cli] test unicode representation of ProjectWrapper (RhBug:1188874)
>---------------------------------------------------------------
python/copr/test/test_resources.py | 11 +++++++++++
1 files changed, 11 insertions(+), 0 deletions(-)
diff --git a/python/copr/test/test_resources.py b/python/copr/test/test_resources.py
new file mode 100644
index 0000000..95c7ace
--- /dev/null
+++ b/python/copr/test/test_resources.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from copr.client.responses import ProjectWrapper
+
+def test_project_wrapper_unicode():
+ project = ProjectWrapper("<client>", "<username>", "<projectname>",
+ description="ěščřžýáíé",
+ instructions="ěščřžýáíé")
+ assert str(project)
8 years, 10 months
[copr] master: [backend] have just one backup directory per results directory (71adcbe)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 71adcbe7e6d957ed419626cf81fe6c6cbc9e7b8a
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Wed Jun 10 12:56:48 2015 +0200
[backend] have just one backup directory per results directory
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 8 +-------
1 files changed, 1 insertions(+), 7 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index be446f2..0b3b834 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -2,7 +2,6 @@ from datetime import datetime
import json
import logging
import os
-import re
import sys
import time
import fcntl
@@ -374,12 +373,7 @@ class Worker(multiprocessing.Process):
Create backup directory and move there results from previous build.
Backup directory will be called ``build-<id>`` and located in result directory
"""
- with open(os.path.join(resdir, "build.info"), 'r') as build_info:
- content = build_info.read()
- prev_id = re.search("{}(.*){}".format("build_id=", "\n"), content).group(1)
- prev_id = "0" * (8 - len(prev_id)) + prev_id
-
- backupdir = os.path.join(resdir, "build-{}".format(prev_id))
+ backupdir = os.path.join(resdir, "prev_build_backup")
self.log.info("Cleaning target directory, results from previous build storing in {}"
.format(backupdir))
8 years, 11 months
[copr] master: [backend] polishing Bug 1195867 - Move or delete logs when rebuilding failed build. (41a5992)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 41a599298253f80836935f9337a5a73a673be9fb
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jun 12 14:11:22 2015 +0200
[backend] polishing Bug 1195867 - Move or delete logs when rebuilding failed build.
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 53 ++++++++++++++++-----------------
backend/backend/job.py | 8 +++++
2 files changed, 34 insertions(+), 27 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 8ffcf73..5e3b0c3 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -281,27 +281,20 @@ class Worker(multiprocessing.Process):
"""
self._announce_start(job)
- self.update_process_title(suffix="Task: {} chroot: {} build started".format(job.build_id, job.chroot))
+ self.update_process_title(suffix="Task: {} chroot: {} build started"
+ .format(job.build_id, job.chroot))
status = BuildStatus.SUCCEEDED
- chroot_destdir = os.path.normpath("{}/{}".format(job.destdir, job.chroot))
# setup our target dir locally
- if not os.path.exists(chroot_destdir):
+ if not os.path.exists(job.chroot_dir):
try:
- os.makedirs(chroot_destdir)
- except (OSError, IOError) as e:
- msg = "Could not make results dir" \
- " for job: {0} - {1}".format(chroot_destdir, str(e))
-
- self.log.exception(msg)
+ os.makedirs(job.chroot_dir)
+ except (OSError, IOError):
+ self.log.exception("Could not make results dir for job: {}"
+ .format(job.chroot_dir))
status = BuildStatus.FAILURE
- pdn = os.path.basename(job.pkg).replace(".src.rpm", "")
- resdir = os.path.join(chroot_destdir, pdn)
- if os.path.exists(resdir) and os.listdir(resdir) != []:
- # Target have already been attempted to build
- # We should backup the results and clean the directory
- self._clean_result_directory(resdir)
+ self.clean_result_directory(job)
if status == BuildStatus.SUCCEEDED:
# FIXME
@@ -322,7 +315,7 @@ class Worker(multiprocessing.Process):
chroot_repos.append(job.results + job.chroot + '/')
chroot_repos.append(job.results + job.chroot + '/devel/')
- chroot_logfile = "{}/build-{:08d}.log".format(chroot_destdir, job.build_id)
+ chroot_logfile = "{}/build-{:08d}.log".format(job.chroot_dir, job.build_id)
build_logger = create_file_logger("{}.builder.mr".format(self.logger_name),
chroot_logfile, fmt=build_log_format)
@@ -369,25 +362,31 @@ class Worker(multiprocessing.Process):
self.update_process_title(suffix="Task: {} chroot: {} done"
.format(job.build_id, job.chroot))
- def _clean_result_directory(self, resdir):
+ def clean_result_directory(self, job):
"""
Create backup directory and move there results from previous build.
- Backup directory will be called ``build-<id>`` and located in result directory
"""
- backupdir = os.path.join(resdir, "prev_build_backup")
+ if not os.path.exists(job.results_dir) or os.listdir(job.results_dir) != []:
+ return
+
+ backup_dir_name = "prev_build_backup"
+ backup_dir = os.path.join(job.results_dir, backup_dir_name)
self.log.info("Cleaning target directory, results from previous build storing in {}"
- .format(backupdir))
+ .format(backup_dir))
- if not os.path.exists(backupdir):
- os.makedirs(backupdir)
+ if not os.path.exists(backup_dir):
+ os.makedirs(backup_dir)
- files = filter(lambda x: x != os.path.basename(backupdir), os.listdir(resdir))
+ files = (x for x in os.listdir(job.results_dir) if x != backup_dir_name)
for filename in files:
- file_path = os.path.join(resdir, filename)
- if os.path.isfile(file_path) and file_path.endswith((".info", ".log", ".log.gz")):
- os.rename(file_path, os.path.join(backupdir, filename))
+ file_path = os.path.join(job.results_dir, filename)
+ if os.path.isfile(file_path):
+ if file_path.endswith((".info", ".log", ".log.gz")):
+ os.rename(file_path, os.path.join(backup_dir, filename))
+ else:
+ os.remove(file_path)
else:
- os.remove(file_path) if os.path.isfile(file_path) else shutil.rmtree(file_path)
+ shutil.rmtree(file_path)
def update_process_title(self, suffix=None):
title = "worker-{} {} ".format(self.group_name, self.worker_num)
diff --git a/backend/backend/job.py b/backend/backend/job.py
index c395430..630cc7e 100644
--- a/backend/backend/job.py
+++ b/backend/backend/job.py
@@ -82,6 +82,14 @@ class BuildJob(object):
else:
return self.pkg
+ @property
+ def chroot_dir(self):
+ return os.path.normpath("{}/{}".format(self.destdir, self.chroot))
+
+ @property
+ def results_dir(self):
+ return os.path.join(self.chroot_dir, self.pkg_name)
+
def update(self, data_dict):
"""
8 years, 11 months
[copr] master: [backend] backup only info and log files (b4e4152)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit b4e415258d2c51b09dcd4f8d4b50d72a2a6bf6d0
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Wed Jun 10 15:19:54 2015 +0200
[backend] backup only info and log files
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 11 ++++++++---
1 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 0b3b834..8ffcf73 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -5,6 +5,7 @@ import os
import sys
import time
import fcntl
+import shutil
import multiprocessing
from setproctitle import setproctitle
@@ -380,9 +381,13 @@ class Worker(multiprocessing.Process):
if not os.path.exists(backupdir):
os.makedirs(backupdir)
- for filename in os.listdir(resdir):
- if os.path.isfile(os.path.join(resdir, filename)):
- os.rename(os.path.join(resdir, filename), os.path.join(backupdir, filename))
+ files = filter(lambda x: x != os.path.basename(backupdir), os.listdir(resdir))
+ for filename in files:
+ file_path = os.path.join(resdir, filename)
+ if os.path.isfile(file_path) and file_path.endswith((".info", ".log", ".log.gz")):
+ os.rename(file_path, os.path.join(backupdir, filename))
+ else:
+ os.remove(file_path) if os.path.isfile(file_path) else shutil.rmtree(file_path)
def update_process_title(self, suffix=None):
title = "worker-{} {} ".format(self.group_name, self.worker_num)
8 years, 11 months
[copr] master: [backend] clean results from previous build (d4064b5)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit d4064b529335a50a01a58aa1d0e50f668b36de42
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Tue Jun 9 20:28:46 2015 +0200
[backend] clean results from previous build
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 29 +++++++++++++++++++++++++++++
1 files changed, 29 insertions(+), 0 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 44d25d5..be446f2 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -2,6 +2,7 @@ from datetime import datetime
import json
import logging
import os
+import re
import sys
import time
import fcntl
@@ -295,6 +296,13 @@ class Worker(multiprocessing.Process):
self.log.exception(msg)
status = BuildStatus.FAILURE
+ pdn = os.path.basename(job.pkg).replace(".src.rpm", "")
+ resdir = os.path.join(chroot_destdir, pdn)
+ if os.path.exists(resdir) and os.listdir(resdir) != []:
+ # Target have already been attempted to build
+ # We should backup the results and clean the directory
+ self._clean_result_directory(resdir)
+
if status == BuildStatus.SUCCEEDED:
# FIXME
# need a plugin hook or some mechanism to check random
@@ -361,6 +369,27 @@ class Worker(multiprocessing.Process):
self.update_process_title(suffix="Task: {} chroot: {} done"
.format(job.build_id, job.chroot))
+ def _clean_result_directory(self, resdir):
+ """
+ Create backup directory and move there results from previous build.
+ Backup directory will be called ``build-<id>`` and located in result directory
+ """
+ with open(os.path.join(resdir, "build.info"), 'r') as build_info:
+ content = build_info.read()
+ prev_id = re.search("{}(.*){}".format("build_id=", "\n"), content).group(1)
+ prev_id = "0" * (8 - len(prev_id)) + prev_id
+
+ backupdir = os.path.join(resdir, "build-{}".format(prev_id))
+ self.log.info("Cleaning target directory, results from previous build storing in {}"
+ .format(backupdir))
+
+ if not os.path.exists(backupdir):
+ os.makedirs(backupdir)
+
+ for filename in os.listdir(resdir):
+ if os.path.isfile(os.path.join(resdir, filename)):
+ os.rename(os.path.join(resdir, filename), os.path.join(backupdir, filename))
+
def update_process_title(self, suffix=None):
title = "worker-{} {} ".format(self.group_name, self.worker_num)
if self.vm_ip:
8 years, 11 months
[copr] master: alter IP= regexp (597410e)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 597410e2330a923a824acc0de1cd3083a93d7296
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Thu Jun 11 09:45:00 2015 +0200
alter IP= regexp
so it does not correctly parse string:
Domain vm26 started\n\nVM_IP=147.229.8.206\nvm_name=rh-power-vm26.fit.vutbr.cz
>---------------------------------------------------------------
backend/backend/vm_manage/spawn.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/backend/vm_manage/spawn.py b/backend/backend/vm_manage/spawn.py
index be22dd6..ae8c1f6 100644
--- a/backend/backend/vm_manage/spawn.py
+++ b/backend/backend/vm_manage/spawn.py
@@ -43,7 +43,7 @@ def spawn_instance(spawn_playbook, log):
if not result:
raise CoprSpawnFailError("No result, trying again")
- match = re.search(r'IP=([^\{\}"]+)', result, re.MULTILINE)
+ match = re.search(r'IP=([^\{\}"\n]+)', result, re.MULTILINE)
if not match:
raise CoprSpawnFailError("No ip in the result, trying again")
8 years, 11 months
[copr] hotfix_1188874: [cli] test unicode representation of ProjectWrapper (RhBug:1188874) (5d1962e)
by frostyx＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : hotfix_1188874
>---------------------------------------------------------------
commit 5d1962e0990e8057877ae6232148781be55f08f4
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Wed Jun 10 21:57:43 2015 +0200
[cli] test unicode representation of ProjectWrapper (RhBug:1188874)
>---------------------------------------------------------------
python/copr/test/test_resources.py | 11 +++++++++++
1 files changed, 11 insertions(+), 0 deletions(-)
diff --git a/python/copr/test/test_resources.py b/python/copr/test/test_resources.py
new file mode 100644
index 0000000..95c7ace
--- /dev/null
+++ b/python/copr/test/test_resources.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from copr.client.responses import ProjectWrapper
+
+def test_project_wrapper_unicode():
+ project = ProjectWrapper("<client>", "<username>", "<projectname>",
+ description="ěščřžýáíé",
+ instructions="ěščřžýáíé")
+ assert str(project)
8 years, 11 months
[copr] hotfix_1195867: [backend] backup only info and log files (9770306)
by frostyx＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : hotfix_1195867
>---------------------------------------------------------------
commit 977030603e52a325a2e8ae5711bc2eb4a5bcaedb
Author: Jakub Kadlčík <jkadlcik(a)redhat.com>
Date: Wed Jun 10 15:19:54 2015 +0200
[backend] backup only info and log files
>---------------------------------------------------------------
backend/backend/daemons/dispatcher.py | 11 ++++++++---
1 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 0b3b834..8ffcf73 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -5,6 +5,7 @@ import os
import sys
import time
import fcntl
+import shutil
import multiprocessing
from setproctitle import setproctitle
@@ -380,9 +381,13 @@ class Worker(multiprocessing.Process):
if not os.path.exists(backupdir):
os.makedirs(backupdir)
- for filename in os.listdir(resdir):
- if os.path.isfile(os.path.join(resdir, filename)):
- os.rename(os.path.join(resdir, filename), os.path.join(backupdir, filename))
+ files = filter(lambda x: x != os.path.basename(backupdir), os.listdir(resdir))
+ for filename in files:
+ file_path = os.path.join(resdir, filename)
+ if os.path.isfile(file_path) and file_path.endswith((".info", ".log", ".log.gz")):
+ os.rename(file_path, os.path.join(backupdir, filename))
+ else:
+ os.remove(file_path) if os.path.isfile(file_path) else shutil.rmtree(file_path)
def update_process_title(self, suffix=None):
title = "worker-{} {} ".format(self.group_name, self.worker_num)
8 years, 11 months