[copr] master: [frontend][dist-git] Building from PyPI - initial commit (ebb35fb)

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master commit ebb35fb78f22930fd59533c8471e581a966e05e9 Author: clime <clime(a)redhat.com&gt; Date: Mon Feb 15 17:56:25 2016 +0100 [frontend][dist-git] Building from PyPI - initial commit dist-git/dist_git/dist_git_importer.py | 41 ++++++++++- dist-git/dist_git/exceptions.py | 4 + frontend/coprs_frontend/coprs/forms.py | 26 +++++++- frontend/coprs_frontend/coprs/helpers.py | 4 +- .../coprs_frontend/coprs/logic/builds_logic.py | 43 +++++++++++ .../templates/coprs/detail/_builds_forms.html | 13 +++- .../coprs/templates/coprs/detail/add_build.html | 1 + .../detail/add_build/{mock.html => pypi.html} | 6 +- .../coprs/views/coprs_ns/coprs_builds.py | 76 ++++++++++++++++++++ 9 files changed, 207 insertions(+), 7 deletions(-) diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py index 071a73f..eca9c29 100755 --- a/dist-git/dist_git/dist_git_importer.py +++ b/dist-git/dist_git/dist_git_importer.py @@ -24,6 +24,7 @@ class SourceType: SRPM_UPLOAD = 2 GIT_AND_TITO = 3 MOCK_SCM = 4 + PYPI = 5 class ImportTask(object): @@ -59,6 +60,11 @@ class ImportTask(object): self.mock_scm_branch = None self.mock_spec = None + # For PyPI + self.pypi_package_name = None + self.pypi_package_version = None + self.pypi_python_version = None + @property def reponame(self): if any(x is None for x in [self.user, self.project, self.package_name]): @@ -99,6 +105,11 @@ class ImportTask(object): task.mock_scm_branch = task.source_data["scm_branch"] task.mock_spec = task.source_data["spec"] + elif task.source_type == SourceType.PYPI: + task.pypi_package_name = task.source_data["package_name"] + task.pypi_package_version = task.source_data["package_version"] + task.pypi_python_version = task.source_data["python_version"] + else: raise PackageImportException("Got unknown source type: {}".format(task.source_type)) @@ -138,6 +149,9 @@ class SourceProvider(object): elif task.source_type == SourceType.MOCK_SCM: self.provider = MockScmProvider + elif task.source_type == SourceType.PYPI: + self.provider = PyPIProvider + else: raise PackageImportException("Got unknown source type: {}".format(task.source_type)) @@ -166,7 +180,7 @@ class SrpmBuilderProvider(BaseSourceProvider): srpm_name = dest_srpms[0] else: log.debug("ERROR :( :( :(") - log.debug("git_dir: {}".format(self.git_dir)) + #log.debug("git_dir: {}".format(self.git_dir)) log.debug("dest_files: {}".format(dest_files)) log.debug("dest_srpms: {}".format(dest_srpms)) log.debug("") @@ -313,6 +327,31 @@ class MockScmProvider(SrpmBuilderProvider): }[self.task.mock_scm_type].format(self.task.mock_scm_url) +class PyPIProvider(SrpmBuilderProvider): + """ + Used for PyPI + """ + def get_srpm(self): + log.debug("GIT_BUILDER: 3. build via pyp2rpm") + cmd = ['pyp2rpm', self.task.pypi_package_name, '--srpm', '-d', self.tmp_dest, '-b', self.task.pypi_python_version] + if self.task.pypi_package_version: + cmd += ['-v', self.task.pypi_package_version] + + log.info(cmd) + + try: + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, error = proc.communicate() + except OSError as e: + log.error(str(e)) + raise PyPIException(FailTypeEnum("srpm_build_error")) + if proc.returncode != 0: + log.error(error) + raise PyPIException(FailTypeEnum("srpm_build_error")) # pass error message somehow? + + self.copy() + + class SrpmUrlProvider(BaseSourceProvider): def get_srpm(self): """ diff --git a/dist-git/dist_git/exceptions.py b/dist-git/dist_git/exceptions.py index 47e6db4..673c5be 100644 --- a/dist-git/dist_git/exceptions.py +++ b/dist-git/dist_git/exceptions.py @@ -28,6 +28,10 @@ class GitException(SrpmBuilderException): pass +class PyPIException(SrpmBuilderException): + pass + + class GitAndTitoException(GitException): def __init__(self, error_code=None): super(GitAndTitoException, self).__init__(error_code) diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index b42f030..bd87e74 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -316,7 +316,7 @@ class BuildFormRebuildFactory(object): class BasePackageForm(wtf.Form): package_name = wtforms.StringField( - "Package Name", + "Package name", validators=[wtforms.validators.DataRequired()]) webhook_rebuild = wtforms.BooleanField(default=False) @@ -373,6 +373,25 @@ class PackageFormMock(BasePackageForm): message="RPM spec file must end with .spec")]) +class PackageFormPyPI(BasePackageForm): + package_version = wtforms.StringField( + "Package version", + validators=[ + wtforms.validators.Optional(), + ]) + + python_version = wtforms.RadioField( + "Base Python version", + choices=[ + ('2', '2'), + ('3', '3'), + ], + default='3', + validators=[ + wtforms.validators.DataRequired(), + ]) + + class BaseBuildFormFactory(object): def __new__(cls, active_chroots, form): class F(form): @@ -428,6 +447,11 @@ class BuildFormMockFactory(object): return BaseBuildFormFactory(active_chroots, PackageFormMock) +class BuildFormPyPIFactory(object): + def __new__(cls, active_chroots): + return BaseBuildFormFactory(active_chroots, PackageFormPyPI) + + class BuildFormUploadFactory(object): def __new__(cls, active_chroots): form = BaseBuildFormFactory(active_chroots, wtf.Form) diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index c47c5f1..6422866 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -94,7 +94,9 @@ class BuildSourceEnum(with_metaclass(EnumType, object)): "srpm_link": 1, # url "srpm_upload": 2, # pkg, tmp "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test - "mock_scm": 4} # scm_type, scm_url, spec, scm_branch + "mock_scm": 4, # scm_type, scm_url, spec, scm_branch + "pypi": 5, # package_name, version, python_version + } # The same enum is also in distgit's helpers.py diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 4009e74..4b23ad2 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -409,6 +409,49 @@ GROUP BY return build + + @classmethod + def create_new_from_pypi(cls, user, copr, package_name, package_version, python_version, + chroot_names=None, **build_options): + """ + :type user: models.User + :type copr: models.Copr + :type package_name: str + :type version: str + :type python_version: str + + :type chroot_names: List[str] + + :rtype: models.Build + """ + if chroot_names is None: + chroots = [c for c in copr.active_chroots] + else: + chroots = [] + for chroot in copr.active_chroots: + if chroot.name in chroot_names: + chroots.append(chroot) + + source_type = helpers.BuildSourceEnum("pypi") + source_json = json.dumps({"package_name": package_name, + "package_version": package_version, + "python_version": python_version}) + + build = cls.add( + user=user, + pkgs="", + copr=copr, + chroots=chroots, + source_type=source_type, + source_json=source_json, + enable_net=build_options.get("enable_net", copr.build_enable_net)) + + if user.proven: + if "timeout" in build_options: + build.timeout = build_options["timeout"] + + return build + @classmethod def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, chroot_names=None, **build_options): diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html index dd4c152..8011eeb 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html @@ -184,13 +184,24 @@ {% endmacro %} +{% macro copr_build_form_pypi(form, view, copr) %} + {{ copr_build_form_begin(form, view, copr) }} + + {{ source_description('This method uses <a href="https://github.com/fedora-python/pyp2rpm">pyp2rpm</... to create the RPM for you automatically from PyPI - the Python Package Index. Please provide the package name.')}} + + {{ render_field(form.package_name, placeholder="Package name in the Python Package Index.") }} + {{ render_field(form.package_version, placeholder="Optional - Version of the package") }} + {{ render_field(form.python_version, placeholder="Base python version that you want to build for") }} + {{ copr_build_form_end(form, view, copr) }} +{% endmacro %} + + {% macro copr_build_form_rebuild(form, view, copr, build) %} {{ copr_build_form_begin(form, view, copr, build, hide_panels=True) }} {{ copr_build_form_end(form, view, copr, hide_panels=True) }} {% endmacro %} - {% macro copr_build_cancel_form(build, page, class="") %} {% if build.cancelable %} <form class="{{class}}" action="{{ copr_url('coprs_ns.copr_cancel_build', build.copr, build_id=build.id) }}" method="post"> diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build.html index af02644..3715af9 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build.html @@ -37,6 +37,7 @@ {{ nav_element("upload", "Upload SRPM", copr_url('coprs_ns.copr_add_build_upload', copr)) }} {{ nav_element("tito", "Tito", copr_url('coprs_ns.copr_add_build_tito', copr)) }} {{ nav_element("mock", "Mock SCM", copr_url('coprs_ns.copr_add_build_mock', copr)) }} + {{ nav_element("pypi", "PyPI", copr_url('coprs_ns.copr_add_build_pypi', copr)) }} </ul> </div> </div> diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/mock.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/pypi.html similarity index 59% copy from frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/mock.html copy to frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/pypi.html index ce85e52..707a4b9 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/mock.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/add_build/pypi.html @@ -1,10 +1,10 @@ {% extends "coprs/detail/add_build.html" %} -{% from "coprs/detail/_builds_forms.html" import copr_build_form_mock with context %} +{% from "coprs/detail/_builds_forms.html" import copr_build_form_pypi with context %} -{% set add_build_tab = "mock" %} +{% set add_build_tab = "PyPI" %} {% block build_form %} - {{ copr_build_form_mock(form, view, copr) }} + {{ copr_build_form_pypi(form, view, copr) }} {% endblock %} diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py index 686182f..38b9517 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py @@ -336,6 +336,82 @@ def process_new_build_mock(copr, add_view, url_on_success): return render_add_build_mock(copr, form, add_view) +################################ PyPI builds ################################ + +@coprs_ns.route("/<username>/<coprname>/add_build_pypi/") +@login_required +@req_with_copr +def copr_add_build_pypi(copr, form=None): + return render_add_build_pypi( + copr, form, view='coprs_ns.copr_new_build_pypi') + + +@coprs_ns.route("/g/<group_name>/<coprname>/add_build_pypi/") +@login_required +@req_with_copr +def group_copr_add_build_pypi(copr, form=None): + return render_add_build_pypi( + copr, form, view='coprs_ns.copr_new_build_pypi') + + +def render_add_build_pypi(copr, form, view, package=None): + if not form: + form = forms.BuildFormPyPIFactory(copr.active_chroots)() + return flask.render_template("coprs/detail/add_build/pypi.html", + copr=copr, form=form, view=view, package=package) + + +@coprs_ns.route("/<username>/<coprname>/new_build_pypi/", methods=["POST"]) +@login_required +@req_with_copr +def copr_new_build_pypi(copr): + view = 'coprs_ns.copr_new_build_pypi' + url_on_success = url_for("coprs_ns.copr_builds", + username=copr.owner.username, coprname=copr.name) + return process_new_build_pypi(copr, view, url_on_success) + + +@coprs_ns.route("/g/<group_name>/<coprname>/new_build_pypi/", methods=["POST"]) +@login_required +@req_with_copr +def group_copr_new_build_pypi(copr): + view = 'coprs_ns.copr_new_build_pypi' + url_on_success = url_for("coprs_ns.group_copr_builds", + group_name=copr.group.name, coprname=copr.name) + return process_new_build_pypi(copr, view, url_on_success) + + +def process_new_build_pypi(copr, add_view, url_on_success): + form = forms.BuildFormPyPIFactory(copr.active_chroots)() + + if form.validate_on_submit(): + build_options = { + "enable_net": form.enable_net.data, + "timeout": form.timeout.data, + } + + try: + BuildsLogic.create_new_from_pypi( + flask.g.user, + copr, + form.package_name.data, + form.package_version.data, + form.python_version.data, + form.selected_chroots, + **build_options + ) + db.session.commit() + except (ActionInProgressException, InsufficientRightsException) as e: + db.session.rollback() + flask.flash(str(e), "error") + else: + flask.flash("New build has been created.") + + return flask.redirect(url_on_success) + else: + return render_add_build_pypi(copr, form, add_view) + + ################################ Upload builds ################################ @coprs_ns.route("/<username>/<coprname>/add_build_upload/")