October 2014 - copr-commits - Fedora Mailing-Lists

[copr] master: [backend] minor changes in createrepo (c3b8286)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c3b82862bf8a59617139d611e07ad78fd50a39e4 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Oct 23 15:12:34 2014 +0200 [backend] minor changes in createrepo >--------------------------------------------------------------- {cli => backend}/.coveragerc | 0 backend/backend/createrepo.py | 10 ++++++---- backend/run/copr_create_repo.py | 18 +++++++++++------- backend/run_test.sh | 2 +- backend/tests/test_createrepo.py | 5 ++--- backend/tests/test_run_copr_create_repo.py | 27 +++++++++++++++++---------- 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/cli/.coveragerc b/backend/.coveragerc similarity index 100% copy from cli/.coveragerc copy to backend/.coveragerc diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 0c1bbfe..0319c8e 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -43,8 +43,9 @@ def createrepo_unsafe(path, lock=None): def get_auto_createrepo_status(front_url, username, projectname): client = CoprClient({"copr_url": front_url}) result = client.get_project_details(projectname, username) - if "auto_createrepo" in result.data: - return bool(result.data["auto_createrepo"]) + + if "auto_createrepo" in result.data["detail"]: + return bool(result.data["detail"]["auto_createrepo"]) else: return True @@ -53,6 +54,7 @@ def createrepo(path, front_url, username, projectname, lock=None): # TODO: add means of logging if get_auto_createrepo_status(front_url, username, projectname): - createrepo_unsafe(path, lock) - + return createrepo_unsafe(path, lock) + else: + return None diff --git a/backend/run/copr_create_repo.py b/backend/run/copr_create_repo.py index 56022ed..ea11a1a 100644 --- a/backend/run/copr_create_repo.py +++ b/backend/run/copr_create_repo.py @@ -6,9 +6,7 @@ from backend.createrepo import createrepo from backend.helpers import SortedOptParser - def main(args): - print(args) parser = SortedOptParser( "mockremote -u user_name -p copr_project repo_dir") @@ -19,8 +17,8 @@ def main(args): parser.add_option("-p", "--project", dest="project", help="copr project name") - parser.add_option("-a", "--api_url", dest="api_url", - help="copr frontend api url") + parser.add_option("-f", "--front_url", dest="front_url", + help="copr frontend url") opts, args = parser.parse_args(args) @@ -32,7 +30,7 @@ def main(args): print("No project was specified, exiting", file=sys.stderr) sys.exit(1) - if not opts.api_url: + if not opts.front_url: print("No api url was specified, exiting", file=sys.stderr) sys.exit(1) @@ -40,8 +38,14 @@ def main(args): print("No directory with repo was specified, exiting", file=sys.stderr) sys.exit(1) - createrepo(path=args[0], front_url=opts.api_url, - username=opts.user, projectname=opts.project) + result = createrepo(path=args[0], front_url=opts.front_url, + username=opts.user, projectname=opts.project) + if not result: + print("Createrepo was skipped") + else: + retcode, stdout, stderr = result + print("STDOUT: {}".format(stdout)) + print("STDERR: {}".format(stderr)) if __name__ == "__main__": diff --git a/backend/run_test.sh b/backend/run_test.sh index fa1d6ef..dfa8b45 100755 --- a/backend/run_test.sh +++ b/backend/run_test.sh @@ -1,4 +1,4 @@ #!/bin/sh -PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend ./tests/ $@ +PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend --cov ./run ./tests/ $@ #PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report term-missing --cov ./copr_cli/ $@ diff --git a/backend/tests/test_createrepo.py b/backend/tests/test_createrepo.py index 0d48e51..23dfad6 100644 --- a/backend/tests/test_createrepo.py +++ b/backend/tests/test_createrepo.py @@ -27,7 +27,7 @@ from backend.createrepo import createrepo, createrepo_unsafe @mock.patch('backend.createrepo.createrepo_unsafe') @mock.patch('backend.createrepo.CoprClient') def test_createrepo_conditional(mc_client, mc_create_unsafe): - mc_client.return_value.get_project_details.return_value = MagicMock(data={}) + mc_client.return_value.get_project_details.return_value = MagicMock(data={"detail": {}}) createrepo(path="/tmp/", front_url="http://example.com/api", username="foo", projectname="bar", lock=None) @@ -35,7 +35,7 @@ def test_createrepo_conditional(mc_client, mc_create_unsafe): mc_create_unsafe.reset_mock() for val in [True, False]: - mc_client.return_value.get_project_details.return_value = MagicMock(data={"auto_createrepo": val}) + mc_client.return_value.get_project_details.return_value = MagicMock(data={"detail": {"auto_createrepo": val}}) createrepo(path="/tmp/", front_url="http://example.com/api", username="foo", projectname="bar", lock=None) @@ -129,5 +129,4 @@ class TestCreaterepoUnsafe(object): createrepo_unsafe(path, None) - print(mc_popen.call_args) assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1) diff --git a/backend/tests/test_run_copr_create_repo.py b/backend/tests/test_run_copr_create_repo.py index 4bc4781..c5e0bd9 100644 --- a/backend/tests/test_run_copr_create_repo.py +++ b/backend/tests/test_run_copr_create_repo.py @@ -26,16 +26,23 @@ from copr_create_repo import main as createrepo_main @mock.patch("copr_create_repo.createrepo") class TestArgParser(object): - def test_arg_parser_good_input(self, mc_main, capsys): - args = ['-u', 'foo', '-p', 'bar', '-a', 'http://example.com/api/', '/tmp'] + def test_arg_parser_good_input(self, mc_createrepo, capsys): + args = ['-u', 'foo', '-p', 'bar', '-f', 'http://example.com/api/', '/tmp'] + mc_createrepo.return_value = 0, "", "" createrepo_main(args) - stdout, stderr = capsys.readouterr() - assert mc_main.call_args == mock.call(username='foo', projectname='bar', - front_url='http://example.com/api/', path='/tmp') + + assert mc_createrepo.call_args == mock.call(username='foo', projectname='bar', + front_url='http://example.com/api/', path='/tmp') + + mc_createrepo.return_value = None + createrepo_main(args) + + assert mc_createrepo.call_args == mock.call(username='foo', projectname='bar', + front_url='http://example.com/api/', path='/tmp') def test_arg_parser_missing_path(self, mc_main, capsys): - args = ['-u', 'foo', '-p', 'bar', '-a', 'http://example.com/api/'] + args = ['-u', 'foo', '-p', 'bar', '-f', 'http://example.com/api/'] with pytest.raises(SystemExit) as err: createrepo_main(args) @@ -45,7 +52,7 @@ class TestArgParser(object): assert "No directory" in stderr def test_arg_parser_missing_user(self, mc_main, capsys): - args = ['-p', 'bar', '-a', 'http://example.com/api/', '/tmp'] + args = ['-p', 'bar', '-f', 'http://example.com/api/', '/tmp'] with pytest.raises(SystemExit) as err: createrepo_main(args) @@ -56,7 +63,7 @@ class TestArgParser(object): def test_arg_parser_missing_project(self, mc_main, capsys): - args = ['-u', 'foo', '-a', 'http://example.com/api/', '/tmp'] + args = ['-u', 'foo', '-f', 'http://example.com/api/', '/tmp'] with pytest.raises(SystemExit) as err: createrepo_main(args) @@ -65,7 +72,7 @@ class TestArgParser(object): stdout, stderr = capsys.readouterr() assert "No project" in stderr - def test_arg_parser_missing_project(self, mc_main, capsys): + def test_arg_parser_missing_front(self, mc_main, capsys): args = ['-u', 'foo', '-p', 'bar', '/tmp'] with pytest.raises(SystemExit) as err: @@ -73,5 +80,5 @@ class TestArgParser(object): assert err.value.code == 1 stdout, stderr = capsys.readouterr() - assert "No api url" in stderr + assert "No front url" in stderr

9 years, 6 months

1
0
0 / 0

[copr] master: [backend] [WIP] cli to run createrepo (9320666)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 93206669285b1c868190a2ec64f6c74d5fd4b101 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Oct 23 13:23:55 2014 +0200 [backend] [WIP] cli to run createrepo >--------------------------------------------------------------- backend/backend/createrepo.py | 8 ++- backend/backend/helpers.py | 13 +++++ backend/backend/mockremote.py | 17 +----- backend/requirements.txt | 5 ++- backend/run/copr_create_repo.py | 52 +++++++++++++++++++ backend/tests/test_run_copr_create_repo.py | 76 ++++++++++++++++++++++++++++ test_suite.sh | 5 +- 7 files changed, 157 insertions(+), 19 deletions(-) diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 85598b3..dca44a9 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -1,3 +1,6 @@ +import sys +from backend.helpers import SortedOptParser + __author__ = 'vgologuz' import os @@ -49,7 +52,8 @@ def get_auto_createrepo_status(front_url, username, projectname): def createrepo(path, front_url, username, projectname, lock=None): # TODO: add means of logging - if get_auto_createrepo_status(front_url, username, projectname): - createrepo_unsafe(path, lock) + print(" ".join(map(str, [path, front_url, username, projectname]))) + #if get_auto_createrepo_status(front_url, username, projectname): + # createrepo_unsafe(path, lock) diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py new file mode 100644 index 0000000..db30689 --- /dev/null +++ b/backend/backend/helpers.py @@ -0,0 +1,13 @@ +from operator import methodcaller +import optparse + +__author__ = 'vgologuz' + + +class SortedOptParser(optparse.OptionParser): + + """Optparser which sorts the options by opt before outputting --help""" + + def format_help(self, formatter=None): + self.option_list.sort(key=methodcaller("get_opt_string")) + return optparse.OptionParser.format_help(self, formatter=None) diff --git a/backend/backend/mockremote.py b/backend/backend/mockremote.py index 5f90727..bb5d8cb 100755 --- a/backend/backend/mockremote.py +++ b/backend/backend/mockremote.py @@ -29,24 +29,22 @@ from __future__ import unicode_literals from __future__ import division from __future__ import absolute_import - import sys import time import fcntl import pipes import socket import urllib -import optparse import subprocess -from operator import methodcaller - import os -import ansible.runner +import ansible.runner +from .helpers import SortedOptParser from .exceptions import MockRemoteError, BuilderError from .sign import sign_rpms_in_dir, get_pubkey + # where we should execute mockchain from on the remote mockchain = "/usr/bin/mockchain" # rsync path @@ -62,15 +60,6 @@ DEF_MACROS = {} DEF_BUILDROOT_PKGS = "" -class SortedOptParser(optparse.OptionParser): - - """Optparser which sorts the options by opt before outputting --help""" - - def format_help(self, formatter=None): - self.option_list.sort(key=methodcaller("get_opt_string")) - return optparse.OptionParser.format_help(self, formatter=None) - - def createrepo(path, lock=None): comm = ['/usr/bin/createrepo_c', '--database', '--ignore-lock'] if os.path.exists(path + '/repodata/repomd.xml'): diff --git a/backend/requirements.txt b/backend/requirements.txt index 9a1d6d4..539e50a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,7 @@ +PyYAML ansible setproctitle +redis retask -python-deamon +python-daemon +bunch diff --git a/backend/run/copr_create_repo.py b/backend/run/copr_create_repo.py new file mode 100644 index 0000000..e1d4942 --- /dev/null +++ b/backend/run/copr_create_repo.py @@ -0,0 +1,52 @@ +from __future__ import print_function + +import sys + +from backend.createrepo import createrepo +from backend.helpers import SortedOptParser + + + +def main(args): + print(args) + + parser = SortedOptParser( + "mockremote -u user_name -p copr_project repo_dir") + + parser.add_option("-u", "--user", dest="user", + help="copr project owner username") + + parser.add_option("-p", "--project", dest="project", + help="copr project name") + + parser.add_option("-a", "--api_url", dest="api_url", + help="copr frontend api url") + + opts, args = parser.parse_args(args) + + if not opts.user: + print("No user was specified, exiting", file=sys.stderr) + sys.exit(1) + + if not opts.project: + print("No project was specified, exiting", file=sys.stderr) + sys.exit(1) + + if not opts.api_url: + print("No api url was specified, exiting", file=sys.stderr) + sys.exit(1) + + if not args: + print("No directory with repo was specified, exiting", file=sys.stderr) + sys.exit(1) + + createrepo(path=args[0], front_url=opts.api_url, + username=opts.user, projectname=opts.project) + #print(args) + +if __name__ == "__main__": + """ + Provides cli interface for conditional execution of createrepo_c + depending on user setting `auto_createrepo` + """ + main(sys.argv[1:]) diff --git a/backend/tests/test_run_copr_create_repo.py b/backend/tests/test_run_copr_create_repo.py new file mode 100644 index 0000000..14417c5 --- /dev/null +++ b/backend/tests/test_run_copr_create_repo.py @@ -0,0 +1,76 @@ +import sys + +import os +import argparse +from collections import defaultdict +import json +from pprint import pprint +from _pytest.capture import capsys +import pytest + +import six + +if six.PY3: + from unittest import mock + from unittest.mock import MagicMock +else: + import mock + from mock import MagicMock + + +sys.path.insert(0, "../run") + +from copr_create_repo import main as createrepo_main + + +(a)mock.patch("copr_create_repo.createrepo") +class TestArgParser(object): + + def test_arg_parser_good_input(self, mc_main, capsys): + args = ['-u', 'foo', '-p', 'bar', '-a', 'http://example.com/api/', '/tmp'] + + createrepo_main(args) + assert mc_main.call_args == mock.call(username='foo', projectname='bar', + front_url='http://example.com/api/', path='/tmp') + + def test_arg_parser_missing_path(self, mc_main, capsys): + args = ['-u', 'foo', '-p', 'bar', '-a', 'http://example.com/api/'] + + with pytest.raises(SystemExit) as err: + createrepo_main(args) + + assert err.value.code == 1 + stdout, stderr = capsys.readouterr() + assert "No directory" in stderr + + def test_arg_parser_missing_user(self, mc_main, capsys): + args = ['-p', 'bar', '-a', 'http://example.com/api/', '/tmp'] + + with pytest.raises(SystemExit) as err: + createrepo_main(args) + + assert err.value.code == 1 + stdout, stderr = capsys.readouterr() + assert "No user" in stderr + + + def test_arg_parser_missing_project(self, mc_main, capsys): + args = ['-u', 'foo', '-a', 'http://example.com/api/', '/tmp'] + + with pytest.raises(SystemExit) as err: + createrepo_main(args) + + assert err.value.code == 1 + stdout, stderr = capsys.readouterr() + assert "No project" in stderr + + def test_arg_parser_missing_project(self, mc_main, capsys): + args = ['-u', 'foo', '-p', 'bar', '/tmp'] + + with pytest.raises(SystemExit) as err: + createrepo_main(args) + + assert err.value.code == 1 + stdout, stderr = capsys.readouterr() + assert "No api url" in stderr + diff --git a/test_suite.sh b/test_suite.sh index d98e3b4..7daaf59 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -8,7 +8,7 @@ source _venv/bin/activate # sphinx flask flask-script SQLAlchemy==0.8.7 flask-whooshee Flask-OpenID Flask-SQLAlchemy==1.0 Flask-WTF blinker pytz markdown pyLibravatar pydns flexmock whoosh decorator -pip install pytest mock pytest-cov ipdb +pip install pytest mock pytest-cov ipdb redis bunch PyYAML cp -rv /usr/lib/python2.7/site-packages/rpmUtils _venv/lib/python2.7/site-packages/ @@ -17,6 +17,7 @@ cp -rv /usr/lib64/python2.7/site-packages/rpm _venv/lib/python2.7/site-packages/ pip install -r python/requirements.txt pip install -r cli/requirements.txt pip install -r frontend/requirements.txt +pip install -r backend/requirements.txt pip install -r keygen/requirements.txt @@ -26,7 +27,7 @@ COPR_CONFIG="$(pwd)/frontend/coprs_frontend/config/copr_unit_test.conf" \ python -m pytest frontend/coprs_frontend/tests --junitxml=_report/frontend.junit.xml --cov-report xml --cov frontend/coprs_frontend/coprs $@ mv {,_report/frontend.}coverage.xml -python -m pytest backend/tests --junitxml=_report/backend.junit.xml --cov-report xml --cov backend/backend $@ +PYTHONPATH=backend/run:backend:python:$PYTHONPATH python -m pytest backend/tests --junitxml=_report/backend.junit.xml --cov-report xml --cov backend/backend $@ mv {,_report/backend.}coverage.xml python -m pytest python/copr/test --junitxml=_report/python-copr.junit.xml --cov-report xml --cov python/copr/client $@

9 years, 6 months

1
0
0 / 0

[copr] master: [fronted] adding option to disable auto invokation of createrepo (ffe7b42)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit ffe7b4273ebd323b06092f8ee1dd1d88dc0b908f Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Oct 21 19:09:24 2014 +0200 [fronted] adding option to disable auto invokation of createrepo >--------------------------------------------------------------- ...b1b27f9_copr_added_attribute_auto_createrepo.py | 24 ++++++++++++++++++++ frontend/coprs_frontend/coprs/forms.py | 6 +++- frontend/coprs_frontend/coprs/models.py | 4 +++ .../coprs/templates/coprs/_coprs_forms.html | 10 ++++++++ .../coprs/views/api_ns/api_general.py | 15 +++++++----- .../coprs/views/coprs_ns/coprs_general.py | 1 + 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/frontend/coprs_frontend/alembic/versions/1d5b5b1b27f9_copr_added_attribute_auto_createrepo.py b/frontend/coprs_frontend/alembic/versions/1d5b5b1b27f9_copr_added_attribute_auto_createrepo.py new file mode 100644 index 0000000..5e7bdea --- /dev/null +++ b/frontend/coprs_frontend/alembic/versions/1d5b5b1b27f9_copr_added_attribute_auto_createrepo.py @@ -0,0 +1,24 @@ +"""Copr added attribute 'auto_createrepo' + +Revision ID: 1d5b5b1b27f9 +Revises: 2a4242380f24 +Create Date: 2014-10-21 14:32:45.062257 + +""" + +# revision identifiers, used by Alembic. +revision = '1d5b5b1b27f9' +down_revision = '2a4242380f24' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('copr', sa.Column('auto_createrepo', sa.Boolean(), + default=True, server_default="true", nullable=False)) + + +def downgrade(): + op.drop_column('copr', 'auto_createrepo') + diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index e6e962e..591b38c 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -99,10 +99,10 @@ class CoprFormFactory(object): # that name already exists id = wtforms.HiddenField() - name = wtforms.TextField( + name = wtforms.StringField( "Name", validators=[ - wtforms.validators.Required(), + wtforms.validators.DataRequired(), wtforms.validators.Regexp( re.compile(r"^[\w.-]+$"), message="Name must contain only letters," @@ -125,6 +125,8 @@ class CoprFormFactory(object): validators=[UrlListValidator()], filters=[StringListFilter()]) + auto_createrepo = wtforms.BooleanField() + @property def selected_chroots(self): selected = [] diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index f2d8838..08356a3 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -118,6 +118,7 @@ class User(db.Model, helpers.Serializer): except IOError: return "" + class Copr(db.Model, helpers.Serializer): """ @@ -138,6 +139,9 @@ class Copr(db.Model, helpers.Serializer): deleted = db.Column(db.Boolean, default=False) playground = db.Column(db.Boolean, default=False) + # should copr run `createrepo` each time when build packages are changed + auto_createrepo = db.Column(db.Boolean, default=True) + # relations owner_id = db.Column(db.Integer, db.ForeignKey("user.id")) owner = db.relationship("User", backref=db.backref("coprs")) diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html index 910968c..f7ece5e 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html @@ -20,8 +20,14 @@ {% endif %} {{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.') }} {{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.') }} + + + + <dd>You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTML is forbidden..</dd> <dt class="field-label required">Chroots</dt> + + {% if form._mock_chroots_error %} <p class="form-error">{{ form._mock_chroots_error }}</p> {% endif %} @@ -48,6 +54,10 @@ {% if copr is none %}{# we're creating the copr, so display initial builds area #} {{ render_field(form.initial_pkgs, rows=5, cols=50, placeholder='Optional - list of src.rpm to build initially. Can be skipped and submitted later.') }} {% endif %} + + {{ render_field(form.auto_createrepo, label="Generate repo data after build ") }} + + <dt><input type="submit" value="{% if copr %}Update{% else %}Create{% endif %}"></dt> </dl> </form> diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index b1d13d6..3875883 100644 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -225,12 +225,15 @@ def api_coprs_by_owner_detail(username, coprname): yum_repos[release] = fix_protocol_for_backend( urlparse.urljoin(build.results, release + '/')) break - output["detail"] = {"name": copr.name, - "additional_repos": copr.repos, - "yum_repos": yum_repos, - "description": copr.description, - "instructions": copr.instructions, - "last_modified": builds_logic.BuildsLogic.last_modified(copr)} + output["detail"] = { + "name": copr.name, + "additional_repos": copr.repos, + "yum_repos": yum_repos, + "description": copr.description, + "instructions": copr.instructions, + "last_modified": builds_logic.BuildsLogic.last_modified(copr), + "auto_createrepo": copr.auto_createrepo, + } else: output = {"output": "notok", "error": "Copr with name {0} does not exist.".format(coprname)} httpcode = 500 diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index dc14a90..dab0160 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -259,6 +259,7 @@ def copr_update(username, coprname): copr.repos = form.repos.data.replace("\n", " ") copr.description = form.description.data copr.instructions = form.instructions.data + copr.auto_createrepo = form.auto_createrepo.data coprs_logic.CoprChrootsLogic.update_from_names( flask.g.user, copr, form.selected_chroots)

9 years, 6 months

1
0
0 / 0

[copr] master: [backend] added tests for createrepo (f5ec954)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit f5ec9542ecfe773940b73c527423ca0aba79a5f9 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Oct 23 14:34:26 2014 +0200 [backend] added tests for createrepo >--------------------------------------------------------------- backend/backend/createrepo.py | 15 ++-- backend/run/copr_create_repo.py | 2 +- backend/run_test.sh | 2 +- backend/tests/test_createrepo.py | 133 ++++++++++++++++++++++++++++ backend/tests/test_run_copr_create_repo.py | 1 + 5 files changed, 143 insertions(+), 10 deletions(-) diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index dca44a9..0c1bbfe 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -1,13 +1,13 @@ import sys -from backend.helpers import SortedOptParser - -__author__ = 'vgologuz' import os import subprocess +from subprocess import Popen from copr.client import CoprClient +from backend.helpers import SortedOptParser + def createrepo_unsafe(path, lock=None): """ @@ -31,10 +31,10 @@ def createrepo_unsafe(path, lock=None): if lock: with lock.acquire(): - cmd = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() else: - cmd = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() return cmd.returncode, out, err @@ -52,8 +52,7 @@ def get_auto_createrepo_status(front_url, username, projectname): def createrepo(path, front_url, username, projectname, lock=None): # TODO: add means of logging - print(" ".join(map(str, [path, front_url, username, projectname]))) - #if get_auto_createrepo_status(front_url, username, projectname): - # createrepo_unsafe(path, lock) + if get_auto_createrepo_status(front_url, username, projectname): + createrepo_unsafe(path, lock) diff --git a/backend/run/copr_create_repo.py b/backend/run/copr_create_repo.py index e1d4942..56022ed 100644 --- a/backend/run/copr_create_repo.py +++ b/backend/run/copr_create_repo.py @@ -42,7 +42,7 @@ def main(args): createrepo(path=args[0], front_url=opts.api_url, username=opts.user, projectname=opts.project) - #print(args) + if __name__ == "__main__": """ diff --git a/backend/run_test.sh b/backend/run_test.sh index 03aa25a..fa1d6ef 100755 --- a/backend/run_test.sh +++ b/backend/run_test.sh @@ -1,4 +1,4 @@ #!/bin/sh -PYTHONPATH=.:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend/actions.py ./tests/ $@ +PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest -s --cov-report term-missing --cov ./backend ./tests/ $@ #PYTHONPATH=../python/:./copr_cli:$PYTHONPATH python3 -B -m pytest --cov-report term-missing --cov ./copr_cli/ $@ diff --git a/backend/tests/test_createrepo.py b/backend/tests/test_createrepo.py new file mode 100644 index 0000000..0d48e51 --- /dev/null +++ b/backend/tests/test_createrepo.py @@ -0,0 +1,133 @@ +__author__ = 'vgologuz' + +import os + +from collections import defaultdict +import json +from pprint import pprint +from _pytest.capture import capsys +import pytest +import copy +import tempfile +import shutil + +import six +import time + +if six.PY3: + from unittest import mock + from unittest.mock import MagicMock +else: + import mock + from mock import MagicMock + + +from backend.createrepo import createrepo, createrepo_unsafe + +(a)mock.patch('backend.createrepo.createrepo_unsafe') +(a)mock.patch('backend.createrepo.CoprClient') +def test_createrepo_conditional(mc_client, mc_create_unsafe): + mc_client.return_value.get_project_details.return_value = MagicMock(data={}) + + createrepo(path="/tmp/", front_url="http://example.com/api", + username="foo", projectname="bar", lock=None) + assert mc_create_unsafe.called # old behaviour + mc_create_unsafe.reset_mock() + + for val in [True, False]: + mc_client.return_value.get_project_details.return_value = MagicMock(data={"auto_createrepo": val}) + + createrepo(path="/tmp/", front_url="http://example.com/api", + username="foo", projectname="bar", lock=None) + assert val == mc_create_unsafe.called + mc_create_unsafe.reset_mock() + + +(a)mock.patch('backend.createrepo.Popen') +(a)mock.patch('backend.createrepo.CoprClient') +class TestCreaterepoUnsafe(object): + def setup_method(self, method): + self.tmp_dir_name = self.make_temp_dir() + self.test_time = time.time() + + def teardown_method(self, method): + self.rm_tmp_dir() + + def rm_tmp_dir(self): + if self.tmp_dir_name: + shutil.rmtree(self.tmp_dir_name) + self.tmp_dir_name = None + + def make_temp_dir(self): + root_tmp_dir = tempfile.gettempdir() + subdir = "test_createrepo_{}".format(time.time()) + self.tmp_dir_name = os.path.join(root_tmp_dir, subdir) + + os.mkdir(self.tmp_dir_name) + + return self.tmp_dir_name + + def test_createrepo_unsafe_lock_usage(self, mc_client, mc_popen): + mocked_lock = MagicMock() + + self.shared_state = dict(in_lock=False, lock_status=None) + + def enter_lock(*args, **kwargs): + self.shared_state["in_lock"] = True + + def exit_lock(*args, **kwargs): + self.shared_state["in_lock"] = False + + #mock_cmd_result = MagicMock() + + def popen_side_effect(*args, **kwargs): + self.shared_state["lock_status"] = copy.copy(self.shared_state["in_lock"]) + return mock.DEFAULT + + mocked_lock.acquire.return_value.__enter__.side_effect = enter_lock + mocked_lock.acquire.return_value.__exit__.side_effect = exit_lock + + mc_popen.side_effect = popen_side_effect + mc_popen.return_value.communicate.return_value = ("", "") + + createrepo_unsafe(self.tmp_dir_name, lock=mocked_lock) + assert self.shared_state["lock_status"] == True + + self.shared_state["lock_status"] = None + createrepo_unsafe(self.tmp_dir_name, lock=None) + assert self.shared_state["lock_status"] == False + + def test_createrepo_generated_commands_existing_repodata(self, mc_client, mc_popen): + mc_popen.return_value.communicate.return_value = ("", "") + path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") + expected_epel_5 = ['/usr/bin/createrepo_c', '--database', + '--ignore-lock', '--update', '-s', 'sha', '--checksum', 'md5', path_epel_5] + path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") + expected_fedora = ['/usr/bin/createrepo_c', '--database', + '--ignore-lock', '--update', path_fedora] + for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: + os.makedirs(path) + + repo_path = os.path.join(path, "repodata") + os.makedirs(repo_path) + with open(os.path.join(repo_path, "repomd.xml"), "w") as handle: + handle.write("1") + + createrepo_unsafe(path, None) + assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1) + + def test_createrepo_generated_commands(self, mc_client, mc_popen): + mc_popen.return_value.communicate.return_value = ("", "") + path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5") + expected_epel_5 = ['/usr/bin/createrepo_c', '--database', + '--ignore-lock', '-s', 'sha', '--checksum', 'md5', path_epel_5] + path_fedora = os.path.join(self.tmp_dir_name, "fedora-21") + expected_fedora = ['/usr/bin/createrepo_c', '--database', + '--ignore-lock', path_fedora] + for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]: + os.makedirs(path) + + + createrepo_unsafe(path, None) + print(mc_popen.call_args) + assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1) diff --git a/backend/tests/test_run_copr_create_repo.py b/backend/tests/test_run_copr_create_repo.py index 14417c5..4bc4781 100644 --- a/backend/tests/test_run_copr_create_repo.py +++ b/backend/tests/test_run_copr_create_repo.py @@ -30,6 +30,7 @@ class TestArgParser(object): args = ['-u', 'foo', '-p', 'bar', '-a', 'http://example.com/api/', '/tmp'] createrepo_main(args) + stdout, stderr = capsys.readouterr() assert mc_main.call_args == mock.call(username='foo', projectname='bar', front_url='http://example.com/api/', path='/tmp')

9 years, 6 months

1
0
0 / 0

[copr] master: [backend] [WIP] conditional execution of createrepo_c (8768167)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 8768167733f45a3c23da3e8b5356c40e54d88c7b Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Oct 22 13:22:00 2014 +0200 [backend] [WIP] conditional execution of createrepo_c >--------------------------------------------------------------- backend/backend/createrepo.py | 55 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 55 insertions(+), 0 deletions(-) diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py new file mode 100644 index 0000000..85598b3 --- /dev/null +++ b/backend/backend/createrepo.py @@ -0,0 +1,55 @@ +__author__ = 'vgologuz' + +import os +import subprocess + +from copr.client import CoprClient + + +def createrepo_unsafe(path, lock=None): + """ + Run createrepo_c on the given path + + Warning! This function doesn't check user preferences. + In most cases use `createrepo(...)` + + :param string path: target location to create repo + :param lock: [optional] + :return tuple: (return_code, stdout, stderr) + """ + + comm = ['/usr/bin/createrepo_c', '--database', '--ignore-lock'] + if os.path.exists(path + '/repodata/repomd.xml'): + comm.append("--update") + if "epel-5" in path: + # this is because rhel-5 doesn't know sha256 + comm.extend(['-s', 'sha', '--checksum', 'md5']) + comm.append(path) + + if lock: + with lock.acquire(): + cmd = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + else: + cmd = subprocess.Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + + return cmd.returncode, out, err + + +def get_auto_createrepo_status(front_url, username, projectname): + client = CoprClient({"copr_url": front_url}) + result = client.get_project_details(projectname, username) + if "auto_createrepo" in result.data: + return bool(result.data["auto_createrepo"]) + else: + return True + + +def createrepo(path, front_url, username, projectname, lock=None): + # TODO: add means of logging + + if get_auto_createrepo_status(front_url, username, projectname): + createrepo_unsafe(path, lock) + +

9 years, 6 months

1
0
0 / 0

[copr] tag 'copr-keygen-1.60-1' created

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-keygen-1.60-1 Referencing: 7ccfa9ad4fc68630c71552a189d594dc39abe3ca

9 years, 6 months

1
0
0 / 0

[copr] master: Automatic commit of package [copr-keygen] release [1.60-1]. (a426038)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit a426038c0d1b5021a55c6587adc6183a57412c45 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Oct 24 16:22:06 2014 +0200 Automatic commit of package [copr-keygen] release [1.60-1]. >--------------------------------------------------------------- keygen/copr-keygen.spec | 5 ++++- rel-eng/packages/copr-keygen | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/keygen/copr-keygen.spec b/keygen/copr-keygen.spec index 356fe47..1a5a1fd 100644 --- a/keygen/copr-keygen.spec +++ b/keygen/copr-keygen.spec @@ -7,7 +7,7 @@ # this package doesn't support rhel7- since it doesn't have python3* packages Name: copr-keygen -Version: 1.59 +Version: 1.60 Release: 1%{?dist} Summary: Part of Copr build system. Aux service that generate keys for signd @@ -180,6 +180,9 @@ service httpd condrestart %changelog +* Fri Oct 24 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.60-1 +- [keygen] added option "--no-auto-check-trustdb" to gpg wrapper + * Tue Oct 21 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.59-1 - [keygen] replaced Python RotatedFileHandler with logrotate.d - [keygen] [.spec] fixes to follow Fedora packaging guidelines diff --git a/rel-eng/packages/copr-keygen b/rel-eng/packages/copr-keygen index 9367980..17a7956 100644 --- a/rel-eng/packages/copr-keygen +++ b/rel-eng/packages/copr-keygen @@ -1 +1 @@ -1.59-1 keygen/ +1.60-1 keygen/

9 years, 6 months

1
0
0 / 0

[copr] master: [keygen] added option "--no-auto-check-trustdb" to gpg wrapper. should fix long time to access large keyring after any change. (9d71f2d)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 9d71f2d1c453ee621a2952d42b1bbe6a66b8f5a0 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Oct 24 14:49:54 2014 +0200 [keygen] added option "--no-auto-check-trustdb" to gpg wrapper. should fix long time to access large keyring after any change. >--------------------------------------------------------------- keygen/run/gpg_copr.sh | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/keygen/run/gpg_copr.sh b/keygen/run/gpg_copr.sh index b581f5b..8ff9db1 100644 --- a/keygen/run/gpg_copr.sh +++ b/keygen/run/gpg_copr.sh @@ -1,3 +1,3 @@ #!/bin/bash -runuser -u copr-signer /usr/bin/gpg2 -- --homedir /var/lib/copr-keygen/gnupg $@ +runuser -u copr-signer /usr/bin/gpg2 -- --homedir /var/lib/copr-keygen/gnupg --no-auto-check-trustdb $@

9 years, 6 months

1
0
0 / 0

[copr] master: [frontent] [WIP]fixing unittest, better isolation during test run (02ca85b)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 02ca85ba160d13ef81881abc74cbfbd3b1adb943 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Oct 22 17:34:37 2014 +0200 [frontent] [WIP]fixing unittest, better isolation during test run >--------------------------------------------------------------- frontend/coprs_frontend/config/copr_unit_test.conf | 18 ++++++-- .../coprs/views/coprs_ns/coprs_general.py | 9 ++-- frontend/coprs_frontend/tests/coprs_test_case.py | 25 ++++++++++-- .../test_views/test_coprs_ns/test_coprs_general.py | 42 ++++++++++++++++++++ test_suite.sh | 2 +- 5 files changed, 82 insertions(+), 14 deletions(-) diff --git a/frontend/coprs_frontend/config/copr_unit_test.conf b/frontend/coprs_frontend/config/copr_unit_test.conf index 2f29680..ab6e1ab 100644 --- a/frontend/coprs_frontend/config/copr_unit_test.conf +++ b/frontend/coprs_frontend/config/copr_unit_test.conf @@ -2,11 +2,19 @@ DATA_DIR = '/tmp' import time -#DATABASE = '/tmp/copr.db' # when executing >1 test instances use different db -DATABASE = '/tmp/copr_{}.db'.format(time.time()) +import os -OPENID_STORE = '/tmp/openid_store' -WHOOSHEE_DIR = '/tmp/whooshee' +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', '..', '..')) +LOCAL_TMP_DIR = os.path.join(PROJECT_ROOT, '_tmp', str(int(time.time())) ) + +print(LOCAL_TMP_DIR) + +#DATABASE = './tmp/copr.db' # when executing >1 test instances use different db +DATABASE = os.path.join(LOCAL_TMP_DIR, 'copr.db') + + +OPENID_STORE = os.path.join(LOCAL_TMP_DIR, 'openid_store') +WHOOSHEE_DIR = os.path.join(LOCAL_TMP_DIR, 'whooshee') # salt for CSRF codes #SECRET_KEY = 'put_some_secret_here' @@ -30,7 +38,7 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:///' + DATABASE #LOGGING_LEVEL = logging.ERROR #DEBUG = False -#SQLALCHEMY_ECHO = False +SQLALCHEMY_ECHO = False CSRF_ENABLED = False # as of Flask-WTF 0.9+ diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 1789453..dc14a90 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -2,6 +2,7 @@ import time import re import flask +from flask import render_template import platform import smtplib import sqlalchemy @@ -105,10 +106,10 @@ def coprs_fulltext_search(page=1): paginator = helpers.Paginator(query, query.count(), page) coprs = paginator.sliced_query - return flask.render_template("coprs/show.html", - coprs=coprs, - paginator=paginator, - fulltext=fulltext) + return render_template("coprs/show.html", + coprs=coprs, + paginator=paginator, + fulltext=fulltext) @coprs_ns.route("/<username>/add/") diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py index 15fb0b2..73daaf1 100644 --- a/frontend/coprs_frontend/tests/coprs_test_case.py +++ b/frontend/coprs_frontend/tests/coprs_test_case.py @@ -2,10 +2,12 @@ import base64 from collections import defaultdict import os import time +import glob from functools import wraps import pytest import decorator +import shutil import coprs @@ -15,6 +17,21 @@ from coprs import models class CoprsTestCase(object): + @classmethod + def setup_class(cls): + config = coprs.app.config + if "LOCAL_TMP_DIR" in config: + path = os.path.abspath(config["LOCAL_TMP_DIR"]) + if not os.path.exists(path): + os.makedirs(path) + + @classmethod + def teardown_class(cls): + config = coprs.app.config + # TODO: some tests fails with this cleanup - investigate and fix + #if "LOCAL_TMP_DIR" in config: + # shutil.rmtree(os.path.abspath(config["LOCAL_TMP_DIR"])) + def setup_method(self, method): self.tc = coprs.app.test_client() self.app = coprs.app @@ -25,10 +42,10 @@ class CoprsTestCase(object): self.helpers = helpers self.backend_passwd = coprs.app.config["BACKEND_PASSWORD"] # create datadir if it doesn't exist - datadir = os.path.commonprefix( - [self.app.config["DATABASE"], self.app.config["OPENID_STORE"]]) - if not os.path.exists(datadir): - os.makedirs(datadir) + #datadir = os.path.commonprefix( + # [self.app.config["DATABASE"], self.app.config["OPENID_STORE"]]) + #if not os.path.exists(datadir): + # os.makedirs(datadir) coprs.db.create_all() self.db.session.commit() diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py index 07808ed..94bc869 100644 --- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py +++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py @@ -1,7 +1,11 @@ +import flask import pytest from flexmock import flexmock +import mock +import time +from coprs import models from coprs.signals import copr_created from tests.coprs_test_case import CoprsTestCase, TransactionDecorator @@ -632,3 +636,41 @@ class TestCoprRepoGeneration(CoprsTestCase): app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig raise e app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig + + +class TestSearch(CoprsTestCase): + + @mock.patch("coprs.views.coprs_ns.coprs_general.render_template") + def test_search_basic(self, mc_render_template, f_users, f_db): + #mc_flask.render_template.return_value = mock.MagicMock() + #self.prefix = u"prefix_{}_".format(int(time.time())) + self.prefix = u"prefix" + + self.s_coprs = [] + + for x in range(5): + self.s_coprs.append(models.Copr(name=self.prefix + str(x), owner=self.u1)) + + for x in range(7): + self.s_coprs.append(models.Copr(name=self.prefix + str(x), owner=self.u2)) + + self.db.session.add_all(self.s_coprs) + self.db.session.commit() + + mc_render_template.side_effect = lambda *args, **kwargs: flask.render_template(*args, **kwargs) + + self.tc.get("/coprs/fulltext/?fulltext={}".format(self.prefix)) + qargs, qkwargs = mc_render_template.call_args + assert qkwargs["paginator"].total_count == 5+7 + + self.tc.get("/coprs/fulltext/?fulltext={}".format("user1/prefix")) + qargs, qkwargs = mc_render_template.call_args + assert qkwargs["paginator"].total_count == 5 + + self.tc.get("/coprs/fulltext/?fulltext={}".format("user1")) + qargs, qkwargs = mc_render_template.call_args + assert qkwargs["paginator"].total_count == 5 + + self.tc.get("/coprs/fulltext/?fulltext={}".format("user1/")) + qargs, qkwargs = mc_render_template.call_args + assert qkwargs["paginator"].total_count == 5 diff --git a/test_suite.sh b/test_suite.sh index 5161232..d98e3b4 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -39,5 +39,5 @@ PYTHONPATH=keygen/src:$PYTHONPATH python -B -m pytest keygen/tests --junitxml=_ mv {,_report/keygen.}coverage.xml deactivate - +rm -rf _tmp/*

9 years, 6 months

1
0
0 / 0

[copr] tag 'copr-selinux-1.35-1' created

by Miroslav Suchý

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-selinux-1.35-1 Referencing: f46182a296d7861d54dee361575e0ded77a32c86

9 years, 6 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits October 2014