July 2015 - copr-commits - Fedora Mailing-Lists

[copr] master: [frontend] forbid resubmit build for srpm_upload source type (57b20d0)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 57b20d0e08e0da5ff0e0efdd2cde2fd8bd6b25b3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 30 12:05:16 2015 +0200 [frontend] forbid resubmit build for srpm_upload source type >--------------------------------------------------------------- frontend/coprs_frontend/coprs/helpers.py | 5 ++- frontend/coprs_frontend/coprs/models.py | 46 +++++++++++++++-------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index d5a4474..b9a4c8d 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -92,8 +92,9 @@ class StatusEnum(object): class BuildSourceEnum(object): __metaclass__ = EnumType vals = {"unset": 0, - "srpm_link": 1, # url - "srpm_upload": 2} # pkg, tmp + "srpm_link": 1, # url + "srpm_upload": 2} # pkg, tmp + class Paginator(object): diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index df8a44e..47385ec 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -10,6 +10,7 @@ from coprs import helpers import itertools import operator +from coprs.helpers import BuildSourceEnum, StatusEnum, ActionTypeEnum class User(db.Model, helpers.Serializer): @@ -415,18 +416,18 @@ class Build(db.Model, helpers.Serializer): # FIXME bad name # used when checking if the repo is initialized and results can be set # i think this is the only purpose - check - return helpers.StatusEnum("pending") in self.chroot_states or \ - helpers.StatusEnum("starting") in self.chroot_states + return StatusEnum("pending") in self.chroot_states or \ + StatusEnum("starting") in self.chroot_states @property def has_unfinished_chroot(self): - return helpers.StatusEnum("pending") in self.chroot_states or \ - helpers.StatusEnum("starting") in self.chroot_states or \ - helpers.StatusEnum("running") in self.chroot_states + return StatusEnum("pending") in self.chroot_states or \ + StatusEnum("starting") in self.chroot_states or \ + StatusEnum("running") in self.chroot_states @property def has_importing_chroot(self): - return helpers.StatusEnum("importing") in self.chroot_states + return StatusEnum("importing") in self.chroot_states @property def status(self): @@ -434,11 +435,11 @@ class Build(db.Model, helpers.Serializer): Return build status according to build status of its chroots """ if self.canceled: - return helpers.StatusEnum("canceled") + return StatusEnum("canceled") for state in ["failed", "running", "starting", "importing", "pending", "succeeded", "skipped"]: - if helpers.StatusEnum(state) in self.chroot_states: - return helpers.StatusEnum(state) + if StatusEnum(state) in self.chroot_states: + return StatusEnum(state) @property def state(self): @@ -447,7 +448,7 @@ class Build(db.Model, helpers.Serializer): """ if self.status is not None: - return helpers.StatusEnum(self.status) + return StatusEnum(self.status) return "unknown" @@ -459,8 +460,8 @@ class Build(db.Model, helpers.Serializer): Build is cancelabel only when it's pending (not started) """ - return self.status == helpers.StatusEnum("pending") or \ - self.status == helpers.StatusEnum("importing") + return self.status == StatusEnum("pending") or \ + self.status == StatusEnum("importing") @property def repeatable(self): @@ -469,10 +470,11 @@ class Build(db.Model, helpers.Serializer): Build is repeatable only if it's not pending, starting or running """ - - return self.status not in [helpers.StatusEnum("pending"), - helpers.StatusEnum("starting"), - helpers.StatusEnum("running"), ] + if self.source_type == BuildSourceEnum("srpm_upload"): + return False + return self.status not in [StatusEnum("pending"), + StatusEnum("starting"), + StatusEnum("running"), ] @property def deletable(self): @@ -578,7 +580,7 @@ class BuildChroot(db.Model, helpers.Serializer): primary_key=True) build = db.relationship("Build", backref=db.backref("build_chroots")) git_hash = db.Column(db.String(40)) - status = db.Column(db.Integer, default=helpers.StatusEnum("importing")) + status = db.Column(db.Integer, default=StatusEnum("importing")) started_on = db.Column(db.Integer) ended_on = db.Column(db.Integer) @@ -598,7 +600,7 @@ class BuildChroot(db.Model, helpers.Serializer): """ if self.status is not None: - return helpers.StatusEnum(self.status) + return StatusEnum(self.status) return "unknown" @@ -643,7 +645,7 @@ class Action(db.Model, helpers.Serializer): """ id = db.Column(db.Integer, primary_key=True) - # delete, rename, ...; see helpers.ActionTypeEnum + # delete, rename, ...; see ActionTypeEnum action_type = db.Column(db.Integer, nullable=False) # copr, ...; downcase name of class of modified object object_type = db.Column(db.String(20)) @@ -668,13 +670,13 @@ class Action(db.Model, helpers.Serializer): return self.__unicode__() def __unicode__(self): - if self.action_type == helpers.ActionTypeEnum("delete"): + if self.action_type == ActionTypeEnum("delete"): return "Deleting {0} {1}".format(self.object_type, self.old_value) - elif self.action_type == helpers.ActionTypeEnum("rename"): + elif self.action_type == ActionTypeEnum("rename"): return "Renaming {0} from {1} to {2}.".format(self.object_type, self.old_value, self.new_value) - elif self.action_type == helpers.ActionTypeEnum("legal-flag"): + elif self.action_type == ActionTypeEnum("legal-flag"): return "Legal flag on copr {0}.".format(self.old_value) return "Action {0} on {1}, old value: {2}, new value: {3}.".format(

8 years, 9 months

1
0
0 / 0

[copr] master: typo (b75593c)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit b75593cc0a6a6a3a474c0b332d51ac0fe8e24c49 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Jul 29 17:42:16 2015 +0200 typo >--------------------------------------------------------------- backend/backend/mockremote/builder.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index fa17bc4..e1102af 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -213,7 +213,7 @@ class Builder(object): self.log.exception("Failed to obtain srpm from dist-git") raise BuilderError("Failed to obtain srpm from dist-git: ansible results {}".format(results)) - self.log.info("Gor srpm to build: {}".format(self.remote_pkg_path)) + self.log.info("Got srpm to build: {}".format(self.remote_pkg_path)) def pre_process_repo_url(self, repo_url): """

8 years, 9 months

1
0
0 / 0

[copr] master: [backend] using package name and versiong given in the build task; cleanup; updated tests; (38d6365)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 38d636550fd51334af10bd281593270857f0532c Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Jul 29 17:29:52 2015 +0200 [backend] using package name and versiong given in the build task; cleanup; updated tests; >--------------------------------------------------------------- backend/backend/daemons/dispatcher.py | 66 +++++++-------------- backend/backend/frontend.py | 4 +- backend/backend/job.py | 44 ++++++++------ backend/backend/mockremote/__init__.py | 15 ++--- backend/backend/mockremote/builder.py | 66 ++++++---------------- backend/tests/deamons/test_dispatcher.py | 33 ++--------- backend/tests/mockremote/test_builder.py | 83 +++++--------------------- backend/tests/mockremote/test_mockremote.py | 14 ++--- 8 files changed, 100 insertions(+), 225 deletions(-) diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index d2bcefe..58076c7 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -141,7 +141,7 @@ class Worker(multiprocessing.Process): content = dict(user=job.submitter, copr=job.project_name, owner=job.project_owner, - pkg=job.package_name, version=job.pkg_version, + pkg=job.package_name, version=job.package_version, build=job.build_id, ip=self.vm_ip, pid=self.pid, status=job.status, chroot=job.chroot) self.fedmsg_notify("build.end", template, content) @@ -182,20 +182,18 @@ class Worker(multiprocessing.Process): def starting_build(self, job): """ Announce to the frontend that a build is starting. + Checks if we can and/or should start job :return True: if the build can start :return False: if the build can not start (build is cancelled) """ try: - can_start = self.frontend_client.starting_build(job.build_id, job.chroot, job.pkg_version) + return self.frontend_client.starting_build(job.build_id, job.chroot) except Exception as err: - raise CoprWorkerError( - "Could not communicate to front end to submit results: {}" - .format(err) - ) - - return can_start + msg = "Could not communicate to front end to confirm build start" + self.log.exception(msg) + raise CoprWorkerError(msg) @classmethod def pkg_built_before(cls, pkg, chroot, destdir): @@ -235,24 +233,6 @@ class Worker(multiprocessing.Process): # self.notify_job_grab_about_task_end(job) # self._announce_end(job) - def can_start_job(self, job): - """ - Checks if we can and/or should start job - :type job: BuildJob - :rtype: bool - """ - # Checking whether the build is not cancelled - if not self.starting_build(job): - self.log.info("Couldn't start job: {}".format(job)) - return False - - # Checking whether to build or skip - # if self.pkg_built_before(job.pkg, job.chroot, job.destdir): - # self.on_pkg_skip(job) - # return False - - return True - def obtain_job(self): """ Retrieves new build task from queue. @@ -307,24 +287,20 @@ class Worker(multiprocessing.Process): # and run a series of checks on the package before we # start the build - most importantly license checks. - self.log.info( - "Starting build: id={} builder={} job: {}" - .format(job.build_id, self.vm_ip, job)) - - chroot_repos = list(job.repos) - chroot_repos.append(job.results + job.chroot + '/') - chroot_repos.append(job.results + job.chroot + '/devel/') + self.log.info("Starting build: id={} builder={} job: {}" + .format(job.build_id, self.vm_ip, job)) - chroot_logfile = os.path.join(job.chroot_dir, job.chroot_log_name) + build_logger = create_file_logger( + "{}.builder.mr".format(self.logger_name), + job.chroot_log_path, fmt=build_log_format) - build_logger = create_file_logger("{}.builder.mr".format(self.logger_name), - chroot_logfile, fmt=build_log_format) try: mr = MockRemote( - builder_host=self.vm_ip, job=job, + builder_host=self.vm_ip, + job=job, logger=build_logger, - repos=chroot_repos, - opts=self.opts, lock=self.lock, + opts=self.opts, + lock=self.lock, ) mr.check() @@ -368,12 +344,12 @@ class Worker(multiprocessing.Process): if not os.path.isdir(job.results_dir): return - logs = [(job.chroot_log_name, "mockchain.log.gz"), - (job.rsync_log_name, "rsync.log.gz")] + log_names = [(job.chroot_log_name, "mockchain.log.gz"), + (job.rsync_log_name, "rsync.log.gz")] - for log in logs: - src = os.path.join(job.chroot_dir, log[0]) - dst = os.path.join(job.results_dir, log[1]) + for src_name, dst_name in log_names: + src = os.path.join(job.chroot_dir, src_name) + dst = os.path.join(job.results_dir, dst_name) try: with open(src, "rb") as f_src, gzip.open(dst, "wb") as f_dst: f_dst.writelines(f_src) @@ -460,7 +436,7 @@ class Worker(multiprocessing.Process): return try: - if not self.can_start_job(job): + if not self.starting_build(job): self.notify_job_grab_about_task_end(job) return except Exception: diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py index ef183ed..52c91b6 100644 --- a/backend/backend/frontend.py +++ b/backend/backend/frontend.py @@ -55,13 +55,13 @@ class FrontendClient(object): """ self._post_to_frontend_repeatedly(data, "update") - def starting_build(self, build_id, chroot_name, version=None): + def starting_build(self, build_id, chroot_name): """ Announce to the frontend that a build is starting. Return: True if the build can start False if the build can not start (can be cancelled or deleted) """ - data = {"build_id": build_id, "chroot": chroot_name, "version": version or None} + data = {"build_id": build_id, "chroot": chroot_name} response = self._post_to_frontend_repeatedly(data, "starting_build") if "can_start" not in response.json(): raise RequestException("Bad respond from the frontend") diff --git a/backend/backend/job.py b/backend/backend/job.py index ace4e37..3a62678 100644 --- a/backend/backend/job.py +++ b/backend/backend/job.py @@ -43,24 +43,21 @@ class BuildJob(object): self.buildroot_pkgs = None self.task_id = None + self.build_id = None - # TODO: validate update data + self.package_name = None + self.package_version = None + + self.git_repo = None + self.git_hash = None + self.git_branch = None + + # TODO: validate update data, user marshmallow for key, val in task_data.items(): key = str(key) setattr(self, key, val) - self.pkg_path = task_data["git_repo"] - self.git_hash = task_data["git_hash"] - self.git_branch = task_data["git_branch"] - self.package_name = task_data["git_repo"].split("/")[2] - - # self.pkg = "{:08d}-{}".format(task_data["build_id"], package_name) - self.target_dir_name = "{:08d}-{}".format(task_data["build_id"], self.package_name) - - del self.pkgs # better to produce error, than use it blindly - self.repos = [r for r in task_data["repos"].split(" ") if r.strip()] - self.build_id = task_data["build_id"] self.destdir = os.path.normpath(os.path.join( worker_opts.destdir, @@ -71,16 +68,20 @@ class BuildJob(object): self.results = u"/".join([ worker_opts.results_baseurl, task_data["project_owner"], - task_data["project_name"] + "/" + task_data["project_name"] ]) - - self.pkg_main_version = "" - self.pkg_epoch = None - self.pkg_release = None + self.results_repo_url = self.results self.built_packages = "" @property + def chroot_repos_extended(self): + repos = list(self.repos) + repos.append("{}/{}".format(self.results_repo_url, self.chroot)) + repos.append("{}/{}/devel".format(self.results_repo_url, self.chroot)) + return repos + + @property def chroot_dir(self): return os.path.normpath("{}/{}".format(self.destdir, self.chroot)) @@ -89,10 +90,18 @@ class BuildJob(object): return os.path.join(self.chroot_dir, self.package_name) @property + def target_dir_name(self): + return "{:08d}-{}".format(self.build_id, self.package_name) + + @property def chroot_log_name(self): return "build-{:08d}.log".format(self.build_id) @property + def chroot_log_path(self): + return os.path.join(self.chroot_dir, self.chroot_log_name) + + @property def rsync_log_name(self): return "build-{:08d}.rsync.log".format(self.build_id) @@ -111,7 +120,6 @@ class BuildJob(object): """ result = copy.deepcopy(self.__dict__) result["id"] = self.build_id - result["pkg_version"] = self.pkg_version result["mockchain_macros"] = self.mockchain_macros return result diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index 3f2eee9..0b2e0a4 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -130,9 +130,7 @@ class MockRemote(object): self.opts.update(opts) self.log = logger - self.job = job - self.repos = repos or DEF_REPOS # TODO: remove or re-implement # self.cont = cont # unused since we build only one pkg at time @@ -149,7 +147,6 @@ class MockRemote(object): hostname=builder_host, job=self.job, logger=logger, - repos=self.repos ) self.failed = [] @@ -308,16 +305,16 @@ class MockRemote(object): self.log.info("Start build: {}".format(self.job)) try: - build_details, build_stdout = self.builder.build(self.job.git_repo, - self.job.git_hash, - self.job.git_branch) - self.log.info("builder.build finished; details: {}\n stdout: {}".format(build_details, build_stdout)) + build_stdout = self.builder.build() + build_details = {"built_packages": self.builder.collect_built_packages()} + + self.log.info("builder.build finished; details: {}\n stdout: {}" + .format(build_details, build_stdout)) except BuilderError as error: self.log.exception("builder.build error building pkg `{}`: {}" .format(self.job.package_name, error)) - build_error = error raise MockRemoteError("Error occurred during build {}: {}" - .format(self.job, build_error)) + .format(self.job, error)) finally: self.builder.download(self.pkg_dest_path) # self.add_log_symlinks() # todo: add config option, need this for nginx diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index bfc65a4..fa17bc4 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -17,13 +17,13 @@ from ..constants import mockchain, rsync, DEF_BUILD_TIMEOUT class Builder(object): - def __init__(self, opts, hostname, job, logger, repos=None): + def __init__(self, opts, hostname, job, logger): self.opts = opts self.hostname = hostname self.job = job self.timeout = self.job.timeout or DEF_BUILD_TIMEOUT - self.repos = repos or [] + self.repos = [] self.log = logger self.buildroot_pkgs = self.job.buildroot_pkgs or "" @@ -167,10 +167,8 @@ class Builder(object): self.log.exception(err) raise - def collect_built_packages(self, build_details): + def collect_built_packages(self): self.log.info("Listing built binary packages") - # self.conn.module_name = "shell" - results = self._run_ansible( "cd {0} && " "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" @@ -178,18 +176,19 @@ class Builder(object): "done".format(pipes.quote(self._get_remote_results_dir())) ) - build_details["built_packages"] = list(results["contacted"].values())[0][u"stdout"] - self.log.info("Packages:\n{}".format(build_details["built_packages"])) + built_packages = list(results["contacted"].values())[0][u"stdout"] + self.log.info("Built packages:\n{}".format(built_packages)) + return built_packages def check_build_success(self): successfile = os.path.join(self._get_remote_results_dir(), "success") ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile)) check_for_ans_error(ansible_test_results, self.hostname) - def download_job_pkg_to_builder(self, git_repo, git_hash, git_branch): - pkg_name = git_repo.split("/")[2] - repo_url = "{}/{}.git".format(self.opts.dist_git_url, git_repo) - self.log.info("Cloning Dist Git repo {}, branch {}, hash {}".format(git_repo, git_hash, git_branch)) + def download_job_pkg_to_builder(self): + repo_url = "{}/{}.git".format(self.opts.dist_git_url, self.job.git_repo) + self.log.info("Cloning Dist Git repo {}, branch {}, hash {}".format( + self.job.git_repo, self.job.git_hash, self.job.git_branch)) results = self._run_ansible( "rm -rf /tmp/build_package_repo && " "mkdir /tmp/build_package_repo && " @@ -199,9 +198,9 @@ class Builder(object): "git checkout {git_hash} && " "fedpkg-copr --dist {branch} srpm" .format(repo_url=repo_url, - pkg_name=pkg_name, - git_hash=git_hash, - branch=git_branch)) + pkg_name=self.job.package_name, + git_hash=self.job.git_hash, + branch=self.job.git_branch)) # expected output: # ... @@ -216,29 +215,6 @@ class Builder(object): self.log.info("Gor srpm to build: {}".format(self.remote_pkg_path)) - def update_job_pkg_version(self): - self.log.info("Getting package information: version") - results = self._run_ansible( - "rpm -qp --qf \"%{{EPOCH}}\$\$%{{VERSION}}\$\$%{{RELEASE}}\" {}" - .format(self.remote_pkg_path)) - - if "contacted" in results: - # TODO: do more sane - raw = list(results["contacted"].values())[0][u"stdout"] - try: - epoch, version, release = raw.split("$$") - - if epoch == "(none)" or epoch == "0": - epoch = None - if release == "(none)": - release = None - - self.job.pkg_main_version = version - self.job.pkg_epoch = epoch - self.job.pkg_release = release - except ValueError: - pass - def pre_process_repo_url(self, repo_url): """ Expands variables and sanitize repo url to be used for mock config @@ -266,7 +242,7 @@ class Builder(object): buildcmd = "{} -r {} -l {} ".format( mockchain, pipes.quote(self.job.chroot), pipes.quote(self.remote_build_dir)) - for repo in self.repos: + for repo in self.job.chroot_repos_extended: repo = self.pre_process_repo_url(repo) if repo is not None: buildcmd += "-a {0} ".format(repo) @@ -335,14 +311,11 @@ class Builder(object): # buildcmd = self.gen_mockchain_command(dest) # - def build(self, git_repo, git_hash, git_branch): + def build(self): self.modify_mock_chroot_config() # download the package to the builder - self.download_job_pkg_to_builder(git_repo, git_hash, git_branch) - - # srpm version - self.update_job_pkg_version() + self.download_job_pkg_to_builder() # construct the mockchain command buildcmd = self.gen_mockchain_command() @@ -353,12 +326,7 @@ class Builder(object): # we know the command ended successfully but not if the pkg built # successfully self.check_build_success() - build_out = get_ans_results(ansible_build_results, self.hostname).get("stdout", "") - - build_details = {"pkg_version": self.job.pkg_version} - self.collect_built_packages(build_details) - - return build_details, build_out + return get_ans_results(ansible_build_results, self.hostname).get("stdout", "") def download(self, target_dir): if self._get_remote_results_dir(): diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py index cd9744f..1364100 100644 --- a/backend/tests/deamons/test_dispatcher.py +++ b/backend/tests/deamons/test_dispatcher.py @@ -118,6 +118,9 @@ class TestDispatcher(object): "git_repo": self.GIT_REPO, "git_hash": self.GIT_HASH, "git_branch": self.GIT_BRANCH, + + "package_name": self.PKG_NAME, + "package_version": self.PKG_VERSION } self.spawn_pb = "/spawn.yml" @@ -359,30 +362,6 @@ class TestDispatcher(object): obtained_job = self.worker.obtain_job() assert obtained_job.__dict__ == self.job.__dict__ - def test_can_start_job_skip_pkg(self, init_worker): - self.worker.starting_build = MagicMock() - self.worker.starting_build.return_value = False - self.worker.pkg_built_before = MagicMock() - self.worker.pkg_built_before.return_value = True - - assert self.worker.can_start_job(self.job) is False - - def test_can_start_job_not_starting(self, init_worker): - self.worker.starting_build = MagicMock() - self.worker.starting_build.return_value = False - self.worker.pkg_built_before = MagicMock() - self.worker.pkg_built_before.return_value = False - - assert self.worker.can_start_job(self.job) is False - - def test_can_start_job(self, init_worker): - self.worker.starting_build = MagicMock() - self.worker.starting_build.return_value = True - self.worker.pkg_built_before = MagicMock() - self.worker.pkg_built_before.return_value = False - - assert self.worker.can_start_job(self.job) is True - def test_obtain_job_dequeue_type_error(self, init_worker): mc_tq = MagicMock() self.worker.task_queue = mc_tq @@ -534,10 +513,10 @@ class TestDispatcher(object): self.worker.notify_job_grab_about_task_end = MagicMock() self.worker.obtain_job = MagicMock() self.worker.obtain_job.return_value = self.job - self.worker.can_start_job = MagicMock() - self.worker.can_start_job.return_value = False + self.worker.starting_build = MagicMock() + self.worker.starting_build.return_value = False self.worker.acquire_vm_for_job = MagicMock() self.worker.run_cycle() - assert self.worker.can_start_job.called + assert self.worker.starting_build.called assert not self.worker.acquire_vm_for_job.called diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py index f710f06..fd1e1cf 100644 --- a/backend/tests/mockremote/test_builder.py +++ b/backend/tests/mockremote/test_builder.py @@ -67,7 +67,9 @@ class TestBuilder(object): BUILDER_USER = "copr_builder" BUILDER_REMOTE_BASEDIR = "/tmp/copr-backend-test" BUILDER_REMOTE_TMPDIR = "/tmp/copr-backend-test-tmp" + BUILDER_PKG_NAME = "foovar" BUILDER_PKG_BASE = "foovar-2.41.f21" + BUILDER_PKG_VERSION = "2.41.f21" BUILDER_PKG = "http://example.com/foovar-2.41.f21.src.rpm" BUILD_REMOTE_TARGET = "/tmp/copr-backend-test/foovar-2.41.f21.src.rpm" @@ -107,10 +109,13 @@ class TestBuilder(object): "git_repo": self.GIT_REPO, "git_hash": self.GIT_HASH, "git_branch": self.GIT_BRANCH, + + "package_name": self.BUILDER_PKG_NAME, + "package_version": self.BUILDER_PKG_VERSION }, Bunch({ "timeout": 1800, "destdir": self.test_root_path, - "results_baseurl": "/tmp/", + "results_baseurl": "/tmp", })) self.mc_logger = MagicMock() @@ -524,15 +529,13 @@ class TestBuilder(object): def test_collect_build_packages(self): builder = self.get_test_builder() - - bd = {} stdout = "stdout" builder.conn.run.return_value = { "contacted": {self.BUILDER_HOSTNAME: {"rc": 0, "stdout": stdout}}, "dark": {} } - builder.collect_built_packages(bd) + builder.collect_built_packages() expected = ( "cd {} && " "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" @@ -610,7 +613,7 @@ class TestBuilder(object): def test_get_mockchain_command(self): builder = self.get_test_builder() - builder.repos = [ + builder.job.repos = [ "http://example.com/rhel7", "http://example.com/fedora-20; rm -rf", "http://example.com/fedora-$releasever", @@ -621,13 +624,15 @@ class TestBuilder(object): "/usr/bin/mockchain -r {chroot} -l /tmp/copr-backend-test-tmp/build/" " -a http://example.com/rhel7 -a 'http://example.com/fedora-20; rm -rf' " "-a 'http://example.com/fedora-$releasever' -a http://example.com/fedora-rawhide " + "-a {results_baseurl}/{owner}/{copr}/{chroot} -a {results_baseurl}/{owner}/{copr}/{chroot}/devel " "-m '--define=copr_username {owner}' -m '--define=copr_projectname {copr}'" " -m '--define=vendor Fedora Project COPR ({owner}/{copr})'" " {build_target}").format( owner=self.job.project_owner, copr=self.job.project_name, chroot=self.job.chroot, - build_target=self.BUILD_REMOTE_TARGET + build_target=self.BUILD_REMOTE_TARGET, + results_baseurl=self.RESULT_DIR ) assert result_cmd == expected @@ -724,41 +729,6 @@ class TestBuilder(object): with pytest.raises(BuilderError): builder.download(self.RESULT_DIR) - def test_update_package_version(self): - builder = self.get_test_builder() - - # (response, expected) - test_plan = [ - ("$$1.34$$", "1.34"), - ("$$1.34$$(none)", "1.34"), - ("(none)$$1.34$$(none)", "1.34"), - ("(none)$$1.34$$", "1.34"), - ("(none)$$1.34$$435", "1.34-435"), - ("2$$1.34$$435", "2:1.34-435"), - ("2$$1.34$$", "2:1.34"), - ] - - self._response = "" - - def fake_run_ansible(self_, pkg): - return { - "contacted": { - self.BUILDER_HOSTNAME: { - "rc": "0", "stdout": self._response - } - } - } - - builder._run_ansible = MethodType(fake_run_ansible, builder) - builder.update_job_pkg_version() - - assert builder.job.pkg_version == "" - - for resp, expected in test_plan: - self._response = resp - builder.update_job_pkg_version() - assert builder.job.pkg_version == expected - def test_build(self): builder = self.get_test_builder() builder.modify_mock_chroot_config = MagicMock() @@ -767,8 +737,6 @@ class TestBuilder(object): builder.download_job_pkg_to_builder.return_value = "foobar" builder.check_if_pkg_local_or_http.return_value = self.BUILDER_PKG - builder.update_job_pkg_version = MagicMock() - builder.run_build_and_wait = MagicMock() successful_wait_result = { "contacted": {self.BUILDER_HOSTNAME: { @@ -783,7 +751,7 @@ class TestBuilder(object): builder.collect_built_packages = MagicMock() - build_details, stdout = builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) + stdout = builder.build() assert stdout == self.STDOUT assert builder.modify_mock_chroot_config.called @@ -792,19 +760,19 @@ class TestBuilder(object): assert builder.collect_built_packages # test providing version / obsolete - builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) + builder.build() # test timeout handle builder.run_build_and_wait.side_effect = BuilderTimeOutError("msg") with pytest.raises(BuilderError) as error: - builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) + builder.build() assert error.value.msg == "msg" # remove timeout builder.run_build_and_wait.side_effect = None - builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) + builder.build() # error inside wait result unsuccessful_wait_result = { @@ -815,26 +783,7 @@ class TestBuilder(object): } builder.run_build_and_wait.return_value = unsuccessful_wait_result with pytest.raises(BuilderError): - builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) - - # make wait result successful again - builder.run_build_and_wait.return_value = successful_wait_result - # # error during build check - # builder.check_build_success.return_value = (self.STDERR, True, self.STDOUT) - # assert not success - # assert stdout == self.STDOUT - # assert stderr == self.STDERR - # - # # revert to successful check build - # builder.check_build_success.return_value = (self.STDERR, False, self.STDOUT) - - # check update build details - def upd(bd): - bd["foo"] = "bar" - - builder.collect_built_packages.side_effect = upd - build_details, stdout = builder.build(self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) - assert build_details["foo"] == "bar" + builder.build() def test_pre_process_repo_url(self): builder = self.get_test_builder() diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py index 1d2efbe..d1b17a8 100644 --- a/backend/tests/mockremote/test_mockremote.py +++ b/backend/tests/mockremote/test_mockremote.py @@ -74,6 +74,9 @@ class TestMockRemote(object): "git_repo": self.GIT_REPO, "git_hash": self.GIT_HASH, "git_branch": self.GIT_BRANCH, + + "package_name": self.PKG_NAME, + "package_version": self.PKG_VERSION }, Bunch({ "timeout": 1800, "destdir": self.test_root_path, @@ -204,20 +207,15 @@ class TestMockRemote(object): self.mr.mark_dir_with_build_id = MagicMock() build_details = MagicMock() - self.mr.builder.build.return_value = (build_details, STDOUT) + self.mr.builder.build.return_value = STDOUT + self.mr.builder.collect_built_packages.return_value = "foo bar" result = self.mr.build_pkg_and_process_results() - assert id(result) == id(build_details) + assert result["built_packages"] == "foo bar" assert self.mr.builder.build.called - assert self.mr.builder.build.call_args == mock.call( - self.GIT_REPO, self.GIT_HASH, self.GIT_BRANCH) - assert self.mr.builder.download.called - assert self.mr.builder.download.call_args == \ - mock.call(self.mr.pkg_dest_path) - assert self.mr.mark_dir_with_build_id.called assert self.mr.on_success_build.called

8 years, 9 months

1
0
0 / 0

[copr] master: [frontend] removed code for skip build from backend_ns/starting_build() (32b2b9b)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 32b2b9ba48d62887cd73690a12e1f6375ac99dd7 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Jul 29 16:39:37 2015 +0200 [frontend] removed code for skip build from backend_ns/starting_build() >--------------------------------------------------------------- .../coprs/views/backend_ns/backend_general.py | 25 +++++++------------ 1 files changed, 9 insertions(+), 16 deletions(-) diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index 2b8ce1f..dfcf642 100644 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -200,22 +200,15 @@ def starting_build(): if "build_id" in flask.request.json and "chroot" in flask.request.json: build = BuildsLogic.get_by_id(flask.request.json["build_id"]) chroot = flask.request.json.get("chroot") - version = flask.request.json.get("version") - if build and chroot: - if version and BackendLogic.build_version_already_done(build, chroot, version): - log.info("mark build {} chroot {} as skipped".format(build.id, chroot)) - BuildsLogic.update_state_from_dict(build, { - "chroot": chroot, - "status": StatusEnum("skipped") - }) - elif not build.canceled: - log.info("mark build {} chroot {} as starting".format(build.id, chroot)) - BuildsLogic.update_state_from_dict(build, { - "chroot": chroot, - "status": StatusEnum("starting") - }) - db.session.commit() - result["can_start"] = True + + if build and chroot and not build.canceled: + log.info("mark build {} chroot {} as starting".format(build.id, chroot)) + BuildsLogic.update_state_from_dict(build, { + "chroot": chroot, + "status": StatusEnum("starting") + }) + db.session.commit() + result["can_start"] = True return flask.jsonify(result)

8 years, 9 months

1
0
0 / 0

[copr] rest_api_2: [frontend] rest api broken ... (f3bb919)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : rest_api_2 >--------------------------------------------------------------- commit f3bb91936bea528256bf585670e3e368436c4217 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jul 17 16:36:38 2015 +0200 [frontend] rest api broken ... >--------------------------------------------------------------- frontend/coprs_frontend/coprs/views/misc.py | 5 +++++ frontend/requirements.txt | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-) diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index d3bb3e4..7ae809e 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -50,6 +50,11 @@ def krb_strip_realm(fullname): return re.sub(r'@.*', '', fullname) +(a)app.before_app_request +def set_empty_user(): + flask.g.user = None + + @app.before_request def lookup_current_user(): flask.g.user = username = None diff --git a/frontend/requirements.txt b/frontend/requirements.txt index af1a89b..8c3653f 100644 --- a/frontend/requirements.txt +++ b/frontend/requirements.txt @@ -17,5 +17,6 @@ decorator python-dateutil netaddr alembic -# flask-restful-swagger -- nice to have, but probably later +flask-restful +# flask-restful-swagger # -- nice to have, but probably later marshmallow

8 years, 9 months

1
0
0 / 0

[copr] rest_api_2: [frontend][api2] rebased to master, small updates (4d8e73f)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : rest_api_2 >--------------------------------------------------------------- commit 4d8e73f8859e60d3d9bb51d97c2a2e2cdef33790 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 16 17:37:16 2015 +0200 [frontend][api2] rebased to master, small updates >--------------------------------------------------------------- frontend/coprs_frontend/coprs/logic/coprs_logic.py | 2 + frontend/coprs_frontend/coprs/models.py | 13 +++ frontend/coprs_frontend/coprs/rest_api/__init__.py | 41 ++------ .../coprs/rest_api/resources/build.py | 14 ++- .../coprs/rest_api/resources/chroot.py | 26 ++++-- .../coprs/rest_api/resources/copr.py | 101 +++++++++----------- frontend/coprs_frontend/coprs/rest_api/schemas.py | 49 ++++++++++ frontend/coprs_frontend/coprs/rest_api/util.py | 16 +++- frontend/requirements.txt | 3 +- 9 files changed, 162 insertions(+), 103 deletions(-) diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 09f4f54..5835c7c 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -341,7 +341,9 @@ listen(models.Copr.auto_createrepo, 'set', on_auto_createrepo_change, class CoprChrootsLogic(object): @classmethod + def mock_chroots_from_names(cls, names): + db_chroots = models.MockChroot.query.all() mock_chroots = [] for ch in db_chroots: diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index d062bad..b0415e9 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -167,6 +167,19 @@ class Copr(db.Model, helpers.Serializer): } @property + def yum_repos(self): + return {} + # todo: add config option to set backen results base url + # release_tmpl = "{chroot.os_release}-{chroot.os_version}-{chroot.arch}" + # result = {} + # for chroot in repo.active_chroots: + # release = release_tmpl.format(chroot=chroot) + # url = fix_protocol_for_backend(urlparse.urljoin( + # app.config["BACKEND_RESULTS_BASE_URL"], release + '/')) + # result[release] = url + # return result + + @property def repos_list(self): """ Return repos of this copr as a list of strings diff --git a/frontend/coprs_frontend/coprs/rest_api/__init__.py b/frontend/coprs_frontend/coprs/rest_api/__init__.py index 167b7ee..309ba44 100644 --- a/frontend/coprs_frontend/coprs/rest_api/__init__.py +++ b/frontend/coprs_frontend/coprs/rest_api/__init__.py @@ -18,26 +18,6 @@ class RootR(Resource): @swagger.operation( notes='List main API endpoints', nickname='get', - # Parameters can be automatically extracted from URLs (e.g. <string:id>) - # but you could also override them here, or add other parameters. - # parameters=[ - # { - # "name": "todo_id_x", - # "description": "The ID of the TODO item", - # "required": True, - # "allowMultiple": False, - # "dataType": 'string', - # "paramType": "path" - # }, - # { - # "name": "a_bool", - # "description": "The ID of the TODO item", - # "required": True, - # "allowMultiple": False, - # "dataType": 'boolean', - # "paramType": "path" - # } - # ] ) def get(self): return { @@ -81,17 +61,18 @@ api = MyApi( ) ################################### +# todo: maybe add later # Wrap the Api with swagger.docs. It is a thin wrapper around the Api class that adds some swagger smarts -api = swagger.docs( - api, - # apiVersion='0.1', - # basePath=URL_PREFIX, - # resourcePath=URL_PREFIX, - # api_spec_url='{}/spec'.format(URL_PREFIX) - # api_spec_url='/spec', - # api_spec_url='{}/spec'.format(URL_PREFIX) - api_spec_url='/spec' -) +# api = swagger.docs( +# api, +# # apiVersion='0.1', +# # basePath=URL_PREFIX, +# # resourcePath=URL_PREFIX, +# # api_spec_url='{}/spec'.format(URL_PREFIX) +# # api_spec_url='/spec', +# # api_spec_url='{}/spec'.format(URL_PREFIX) +# api_spec_url='/spec' +# ) ################################### api.add_resource(RootR, "/") diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/build.py b/frontend/coprs_frontend/coprs/rest_api/resources/build.py index db4a82e..3378641 100644 --- a/frontend/coprs_frontend/coprs/rest_api/resources/build.py +++ b/frontend/coprs_frontend/coprs/rest_api/resources/build.py @@ -1,8 +1,9 @@ # coding: utf-8 import flask -from flask import url_for -from flask_restful_swagger import swagger + +# from flask_restful_swagger import swagger + from coprs.logic.coprs_logic import CoprsLogic from coprs.logic.builds_logic import BuildsLogic @@ -55,13 +56,14 @@ class BuildListR(Resource): } -(a)swagger.model -class BuildItem(object): - def __init__(self, build_id): - pass +# @swagger.model +# class BuildItem(object): +# def __init__(self, build_id): +# pass class BuildR(Resource): + def get(self, build_id): """ Get single build by id diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py b/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py index 9e4652a..61d6e5a 100644 --- a/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py +++ b/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py @@ -8,18 +8,17 @@ from flask_restful import Resource, reqparse from marshmallow import Schema, fields, pprint from coprs.views.misc import api_login_required -from coprs.logic.coprs_logic import MockChrootsLogic +from coprs.logic.coprs_logic import MockChrootsLogic, CoprChrootsLogic, CoprsLogic -from ..util import get_one_safe, json_loads_safe, mm_deserialize +from ..util import get_one_safe, json_loads_safe, mm_deserialize, bp_url_for class ChrootListR(Resource): - def get(self): chroots = MockChrootsLogic.get_multiple(None).all() return { "links": { - "self": url_for(ChrootListR.endpoint), + "self": bp_url_for(ChrootListR.endpoint), }, "chroots": [ { @@ -31,12 +30,25 @@ class ChrootListR(Resource): class ChrootR(Resource): - def get(self, name): chroot = get_one_safe(MockChrootsLogic.get_from_name(name)) return { + "chroot": chroot.to_dict(), "links": { - "self": url_for(ChrootR.endpoint, name=chroot.name) + "self": bp_url_for(ChrootR.endpoint, name=chroot.name) }, - "chroot": chroot.to_dict() + } + + +class BuildChrootR(Resource): + def get(self, owner, project, name): + copr = get_one_safe(CoprsLogic.get(flask.g.user, owner, project), + "Copr {}/{} not found".format(owner, project)) + chroot = get_one_safe(CoprChrootsLogic.get(copr, name)) + + return { + "chroot": chroot.to_dict(), + "links": { + "self": bp_url_for(BuildChrootR.endpoint, owner=owner, project=project, name=name) + } } diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/copr.py b/frontend/coprs_frontend/coprs/rest_api/resources/copr.py index 5783a7f..22efc84 100644 --- a/frontend/coprs_frontend/coprs/rest_api/resources/copr.py +++ b/frontend/coprs_frontend/coprs/rest_api/resources/copr.py @@ -1,25 +1,22 @@ import base64 -import json import datetime -import flask -from flask import url_for -from flask_restful import Resource, reqparse import functools -from marshmallow import Schema, fields, pprint +import flask +from flask_restful import Resource, reqparse +from marshmallow import pprint from coprs import db from coprs.exceptions import DuplicateException from coprs.logic.complex_logic import ComplexLogic from coprs.logic.users_logic import UsersLogic - from coprs.logic.coprs_logic import CoprsLogic - from coprs.exceptions import ActionInProgressException, InsufficientRightsException from .build import BuildListR - +from coprs.rest_api.schemas import CoprSchema from ..exceptions import ObjectAlreadyExists, AuthFailed -from ..util import get_one_safe, json_loads_safe, mm_deserialize +from ..util import get_one_safe, json_loads_safe, mm_deserialize, bp_url_for, render_allowed_method + def rest_api_auth_required(f): @functools.wraps(f) @@ -50,37 +47,6 @@ def rest_api_auth_required(f): return decorated_function -class CoprSchema(Schema): - name = fields.Str(required=True) - description = fields.Str() - instructions = fields.Str() - - auto_createrepo = fields.Bool() - - chroots = fields.List(fields.Str, required=True) - repos = fields.List(fields.Str) - - _keys_to_make_object = [ - "description", - "instructions", - "auto_createrepo" - ] - - def make_object(self, data): - """ - Create kwargs for CoprsLogic.add - """ - kwargs = dict( - name=data["name"].strip(), - repos=" ".join(data.get("repos", [])), - selected_chroots=data["chroots"], - ) - for key in self._keys_to_make_object: - if key in data: - kwargs[key] = data[key] - return kwargs - - class CoprListR(Resource): @rest_api_auth_required @@ -89,9 +55,9 @@ class CoprListR(Resource): Creates new copr """ owner = flask.g.user - new_copr_dict = json_loads_safe(flask.request.data, "") + # new_copr_dict = json_loads_safe(flask.request.data, "") - result = mm_deserialize(CoprSchema(), new_copr_dict) + result = mm_deserialize(CoprSchema(), flask.request.data) # todo check that chroots are available pprint(result.data) try: @@ -100,7 +66,7 @@ class CoprListR(Resource): except DuplicateException as error: raise ObjectAlreadyExists(data=error) - return copr.to_dict(), 201 + return "New copr was created", 201 def get(self): """ @@ -108,7 +74,6 @@ class CoprListR(Resource): :return: """ parser = reqparse.RequestParser() - parser.add_argument('owner', dest='username', type=str) parser.add_argument('limit', type=int) parser.add_argument('offset', type=int) @@ -144,17 +109,29 @@ class CoprListR(Resource): return { "links": { - "self": url_for(CoprListR.endpoint, **req_args) + "self": bp_url_for(CoprListR.endpoint, **req_args) }, "coprs": [ { - "copr": copr.to_dict(), - "link": url_for(CoprR.endpoint, - owner=copr.owner.name, - project=copr.name), + "copr": CoprSchema().dump(copr)[0], + "link": bp_url_for( + CoprR.endpoint, + owner=copr.owner.name, + project=copr.name + ), } for copr in coprs_list - ] + ], + # TODO: show only if user provided ?help=true param + # "allowed_methods": [ + # render_allowed_method("GET", "Get list of coprs", require_auth=False, + # params=[ + # "username: filter coprs owned by user", + # "limit: show only the given number of coprs", + # "offset: skip given number of coprs", + # ]), + # render_allowed_method("POST", "Creates new copr, send dict with copr fields"), + # ] } @@ -176,24 +153,34 @@ class CoprR(Resource): return None, 204 def get(self, owner, project): - parser = reqparse.RequestParser() - parser.add_argument('show_builds', type=bool, default=True) - parser.add_argument('show_chroots', type=bool, default=True) - req_args = parser.parse_args() + # parser = reqparse.RequestParser() + # parser.add_argument('show_builds', type=bool, default=True) + # parser.add_argument('show_chroots', type=bool, default=True) + # req_args = parser.parse_args() copr = get_one_safe(CoprsLogic.get(flask.g.user, owner, project), "Copr {}/{} not found".format(owner, project)) return { + "copr": CoprSchema().dump(copr)[0], "links": { - "self": url_for(CoprR.endpoint, + "self": bp_url_for(CoprR.endpoint, owner=owner, project=project), # "chroots": - "builds": url_for(BuildListR.endpoint, + "builds": bp_url_for(BuildListR.endpoint, owner=copr.owner.name, project=copr.name) }, - "copr": copr.to_dict() + # "allowed_methods": [ + # render_allowed_method("GET", "Get single copr", require_auth=False), + # render_allowed_method("DELETE", "Delete current copr", require_auth=True), + # ] } + @rest_api_auth_required + def put(self, owner, project): + """ + Modifies project by replacment of provided fields + """ + pass diff --git a/frontend/coprs_frontend/coprs/rest_api/schemas.py b/frontend/coprs_frontend/coprs/rest_api/schemas.py new file mode 100644 index 0000000..aabdd80 --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/schemas.py @@ -0,0 +1,49 @@ +# coding: utf-8 +from marshmallow import Schema, fields + + +class AllowedMethodSchema(Schema): + method = fields.Str() + doc = fields.Str() + require_auth = fields.Bool() + params = fields.List(fields.Str()) + + +class MockChroot(Schema): + pass + + +class CoprSchema(Schema): + name = fields.Str(required=True) + description = fields.Str() + instructions = fields.Str() + + auto_createrepo = fields.Bool() + build_enable_net = fields.Bool() + + additional_repos = fields.List(fields.Str, dump_only=True, attribute="repos_list") + # yum_repos = fields.List() + + # used only for creation + chroots_to_enable = fields.List(fields.Str, load_only=True) + + _keys_to_make_object = [ + "description", + "instructions", + "auto_createrepo" + ] + + def make_object(self, data): + """ + Create kwargs for CoprsLogic.add + """ + kwargs = dict( + name=data["name"].strip(), + repos=" ".join(data.get("repos", [])), + selected_chroots=data["chroots"], + ) + for key in self._keys_to_make_object: + if key in data: + kwargs[key] = data[key] + return kwargs + diff --git a/frontend/coprs_frontend/coprs/rest_api/util.py b/frontend/coprs_frontend/coprs/rest_api/util.py index 87fab9a..ac9b0b6 100644 --- a/frontend/coprs_frontend/coprs/rest_api/util.py +++ b/frontend/coprs_frontend/coprs/rest_api/util.py @@ -3,10 +3,22 @@ import json import sqlalchemy.orm.exc from .exceptions import ObjectNotFoundError, MalformedRequest - +from schemas import AllowedMethodSchema from flask import Response, url_for, Blueprint +class AllowedMethod(object): + def __init__(self, method, doc, require_auth=True, params=None): + self.method = method + self.doc = doc + self.require_auth = require_auth + self.params = params or [] + + +def render_allowed_method(method, doc, require_auth=True, params=None): + return AllowedMethodSchema().dump(AllowedMethod(method, doc, require_auth, params))[0] + + def bp_url_for(endpoint, *args, **kwargs): """ Prepend endpoing with dot @@ -35,7 +47,7 @@ def json_loads_safe(raw, data_on_error=None): def mm_deserialize(schema, obj_dict): - result = schema.load(obj_dict) + result = schema.loads(obj_dict) if result.errors: raise MalformedRequest(data=result.errors) # import ipdb; ipdb.set_trace() diff --git a/frontend/requirements.txt b/frontend/requirements.txt index 9734ddc..af1a89b 100644 --- a/frontend/requirements.txt +++ b/frontend/requirements.txt @@ -17,4 +17,5 @@ decorator python-dateutil netaddr alembic -flask-restful-swagger +# flask-restful-swagger -- nice to have, but probably later +marshmallow

8 years, 9 months

1
0
0 / 0

[copr] rest_api_2: [frontend] adding more stuff to api 2 (17757a9)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : rest_api_2 >--------------------------------------------------------------- commit 17757a99d843d8e740a089ac058fd5aef938eefa Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jul 17 16:06:43 2015 +0200 [frontend] adding more stuff to api 2 >--------------------------------------------------------------- frontend/coprs_frontend/coprs/helpers.py | 5 ++- frontend/coprs_frontend/coprs/models.py | 7 ++++ .../coprs/rest_api/resources/build.py | 17 +++++---- frontend/coprs_frontend/coprs/rest_api/schemas.py | 38 ++++++++++++++++++++ 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index d5a4474..b9a4c8d 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -92,8 +92,9 @@ class StatusEnum(object): class BuildSourceEnum(object): __metaclass__ = EnumType vals = {"unset": 0, - "srpm_link": 1, # url - "srpm_upload": 2} # pkg, tmp + "srpm_link": 1, # url + "srpm_upload": 2} # pkg, tmp + class Paginator(object): diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index b0415e9..1babe71 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -372,6 +372,13 @@ class Build(db.Model, helpers.Serializer): chroots = association_proxy("build_chroots", "mock_chroot") @property + def repos_list(self): + if self.repos is None: + return list() + else: + return self.repos.split() + + @property def result_dir_name(self): return "{:08d}-{}".format(self.id, self.package.name) diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/build.py b/frontend/coprs_frontend/coprs/rest_api/resources/build.py index 3378641..1db3bf4 100644 --- a/frontend/coprs_frontend/coprs/rest_api/resources/build.py +++ b/frontend/coprs_frontend/coprs/rest_api/resources/build.py @@ -7,6 +7,8 @@ import flask from coprs.logic.coprs_logic import CoprsLogic from coprs.logic.builds_logic import BuildsLogic +from coprs.rest_api.schemas import BuildSchema + from coprs.rest_api.util import get_one_safe, bp_url_for from flask_restful import Resource, reqparse @@ -44,15 +46,16 @@ class BuildListR(Resource): builds = query.all() return { - "links": { - "self": bp_url_for(BuildListR.endpoint, **req_args), - }, + "builds": [ { - "build": build.to_dict(), + "build": BuildSchema().dump(build)[0], "link": bp_url_for(BuildR.endpoint, build_id=build.id), } for build in builds - ] + ], + "links": { + "self": bp_url_for(BuildListR.endpoint, **req_args), + }, } @@ -71,10 +74,10 @@ class BuildR(Resource): build = get_one_safe(BuildsLogic.get(build_id), "Not found build with id: {}".format(build_id)) return { - "build": build.to_dict(), + "build": BuildSchema().dump(build)[0], "links": { "self": bp_url_for(BuildR.endpoint, build_id=build_id), - # TODO: can't do it due to circular imports + # TODO: can't do this due to circular imports # "parent_copr": url_for(CoprR.endpoint, # owner=build.copr.owner.name, # project=build.copr.name), diff --git a/frontend/coprs_frontend/coprs/rest_api/schemas.py b/frontend/coprs_frontend/coprs/rest_api/schemas.py index aabdd80..9c243a7 100644 --- a/frontend/coprs_frontend/coprs/rest_api/schemas.py +++ b/frontend/coprs_frontend/coprs/rest_api/schemas.py @@ -20,6 +20,7 @@ class CoprSchema(Schema): auto_createrepo = fields.Bool() build_enable_net = fields.Bool() + last_modified = fields.DateTime() additional_repos = fields.List(fields.Str, dump_only=True, attribute="repos_list") # yum_repos = fields.List() @@ -47,3 +48,40 @@ class CoprSchema(Schema): kwargs[key] = data[key] return kwargs + +class BuildChrootSchema(Schema): + # used only for presentation + state = fields.Str() + started_on = fields.Int() + ended_on = fields.Int() + git_hash = fields.Str() + + +class BuildSchema(Schema): + + id = fields.Int() + pkgs = fields.Str() + build_packages = fields.Str() + pkg_version = fields.Str() + + repos_list = fields.List(fields.Str()) + repos = fields.Str() # legacy + + submitted_on = fields.Int() + started_on = fields.Int() + ended_on = fields.Int() + + results = fields.Str() + memory_reqs = fields.Int() + timeout = fields.Int() + + enable_net = fields.Bool() + + source_type = fields.Int() + source_json = fields.Str() + + # chroots = fields.List(fields.Nested(BuildChrootSchema)) + + + +

8 years, 9 months

1
0
0 / 0

[copr] rest_api_2: [frontend] REST API [WIP] (db65449)

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : rest_api_2 >--------------------------------------------------------------- commit db654492d1014890872c5bf442ad678b281053c3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Dec 16 19:26:49 2014 +0100 [frontend] REST API [WIP] >--------------------------------------------------------------- frontend/coprs_frontend/coprs/__init__.py | 5 + .../coprs_frontend/coprs/logic/complex_logic.py | 27 +++ frontend/coprs_frontend/coprs/logic/coprs_logic.py | 13 +- frontend/coprs_frontend/coprs/models.py | 24 +++ frontend/coprs_frontend/coprs/rest_api/__init__.py | 120 ++++++++++++ .../coprs_frontend/coprs/rest_api/exceptions.py | 45 +++++ .../coprs/rest_api/resources}/__init__.py | 0 .../coprs/rest_api/resources/build.py | 84 ++++++++ .../coprs/rest_api/resources/chroot.py | 42 ++++ .../coprs/rest_api/resources/copr.py | 199 ++++++++++++++++++++ frontend/coprs_frontend/coprs/rest_api/util.py | 42 ++++ .../coprs/views/api_ns/api_general.py | 6 +- .../coprs/views/coprs_ns/coprs_general.py | 7 +- frontend/coprs_frontend/coprs/views/misc.py | 1 + .../test_views/test_coprs_ns/test_coprs_general.py | 2 +- frontend/requirements.txt | 1 + 16 files changed, 604 insertions(+), 14 deletions(-) diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py index fa02421..dd0eb05 100644 --- a/frontend/coprs_frontend/coprs/__init__.py +++ b/frontend/coprs_frontend/coprs/__init__.py @@ -70,3 +70,8 @@ app.register_blueprint(stats_receiver.stats_rcv_ns) app.register_blueprint(tmp_ns.tmp_ns) app.add_url_rule("/", "coprs_ns.coprs_show", coprs_general.coprs_show) + +from rest_api import rest_api_bp, register_api_error_handler, URL_PREFIX +register_api_error_handler(app) +app.register_blueprint(rest_api_bp, url_prefix=URL_PREFIX) +# register_api(app, db) diff --git a/frontend/coprs_frontend/coprs/logic/complex_logic.py b/frontend/coprs_frontend/coprs/logic/complex_logic.py new file mode 100644 index 0000000..3f2bd53 --- /dev/null +++ b/frontend/coprs_frontend/coprs/logic/complex_logic.py @@ -0,0 +1,27 @@ +# coding: utf-8 + +import flask +from .builds_logic import BuildsLogic +from .coprs_logic import CoprsLogic + + +class ComplexLogic(object): + """ + Used for manipulation which affects multiply models + """ + + @classmethod + def delete_copr(cls, copr): + """ + Delete copr and all its builds. + + :param copr: + :raises ActionInProgressException: + :raises InsufficientRightsException: + """ + builds_query = BuildsLogic.get_multiple_by_copr(copr=copr) + + for build in builds_query: + BuildsLogic.delete_build(flask.g.user, build) + + CoprsLogic.delete_unsafe(flask.g.user, copr) diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 81923fe..09f4f54 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -129,13 +129,13 @@ class CoprsLogic(object): return query @classmethod - def add(cls, user, name, repos, selected_chroots, description, - instructions, check_for_duplicates=False, **kwargs): + def add(cls, user, name, repos, selected_chroots, description=None, + instructions=None, check_for_duplicates=False, **kwargs): copr = models.Copr(name=name, repos=repos, owner_id=user.id, - description=description, - instructions=instructions, + description=description or u"", + instructions=instructions or u"", created_on=int(time.time()), **kwargs) @@ -187,7 +187,10 @@ class CoprsLogic(object): db.session.add(copr) @classmethod - def delete(cls, user, copr, check_for_duplicates=True): + def delete_unsafe(cls, user, copr): + """ + Deletes copr without termination of ongoing builds. + """ cls.raise_if_cant_delete(user, copr) # TODO: do we want to dump the information somewhere, so that we can # search it in future? diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index df8a44e..d062bad 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1,3 +1,4 @@ +import copy import datetime import json @@ -256,6 +257,14 @@ class Copr(db.Model, helpers.Serializer): return True return False + def to_dict(self, private=False, show_builds=True, show_chroots=True): + result = {} + for key in ["id", "name", "description", "instructions"]: + result[key] = str(copy.copy(getattr(self, key))) + result["owner"] = self.owner.name + return result + + class CoprPermission(db.Model, helpers.Serializer): """ @@ -501,6 +510,15 @@ class Build(db.Model, helpers.Serializer): else: return src_rpm_name + def to_dict(self, options=None): + result = super(Build, self).to_dict(options) + result["src_pkg"] = result["pkgs"] + del result["pkgs"] + del result["copr_id"] + + result["state"] = self.state + return result + class MockChroot(db.Model, helpers.Serializer): @@ -545,6 +563,12 @@ class MockChroot(db.Model, helpers.Serializer): """ return "{0} {1}".format(self.os_release, self.os_version) + @property + def serializable_attributes(self): + attr_list = super(MockChroot, self).serializable_attributes + attr_list.extend(["name", "os"]) + return attr_list + class CoprChroot(db.Model, helpers.Serializer): diff --git a/frontend/coprs_frontend/coprs/rest_api/__init__.py b/frontend/coprs_frontend/coprs/rest_api/__init__.py new file mode 100644 index 0000000..167b7ee --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/__init__.py @@ -0,0 +1,120 @@ +# coding: utf-8 +from flask import Response, url_for, Blueprint + +from flask_restful import Resource, Api +from flask_restful_swagger import swagger + +from coprs.rest_api.exceptions import ApiError +from coprs.rest_api.resources.build import BuildListR, BuildR +from coprs.rest_api.resources.chroot import ChrootListR, ChrootR +from coprs.rest_api.resources.copr import CoprListR, CoprR +from coprs.rest_api.util import bp_url_for + + +URL_PREFIX = "/api_2.0" + + +class RootR(Resource): + @swagger.operation( + notes='List main API endpoints', + nickname='get', + # Parameters can be automatically extracted from URLs (e.g. <string:id>) + # but you could also override them here, or add other parameters. + # parameters=[ + # { + # "name": "todo_id_x", + # "description": "The ID of the TODO item", + # "required": True, + # "allowMultiple": False, + # "dataType": 'string', + # "paramType": "path" + # }, + # { + # "name": "a_bool", + # "description": "The ID of the TODO item", + # "required": True, + # "allowMultiple": False, + # "dataType": 'boolean', + # "paramType": "path" + # } + # ] + ) + def get(self): + return { + "links": { + "self": bp_url_for(RootR.endpoint), + "coprs": bp_url_for(CoprListR.endpoint), + "chroots": bp_url_for(ChrootListR.endpoint), + "builds": bp_url_for(BuildListR.endpoint), + } + } + + +class MyApi(Api): + # flask-restfull error handling quite buggy right now + def error_router(self, original_handler, e): + return original_handler(e) + # def handle_error(self, e): + # + # if isinstance(e, sqlalchemy.orm.exc.NoResultFound): + # return self.make_response(str(e), 404) + # + # + # super(MyApi, self).handle_error(e) + + +# def register_api(app, db): + + +rest_api_bp = Blueprint("rest_api_bp", + __name__, + # app.import_name +) + +api = MyApi( +# api = Api( + # app, + rest_api_bp, + # prefix=URL_PREFIX, + catch_all_404s=True, + +) + +################################### +# Wrap the Api with swagger.docs. It is a thin wrapper around the Api class that adds some swagger smarts +api = swagger.docs( + api, + # apiVersion='0.1', + # basePath=URL_PREFIX, + # resourcePath=URL_PREFIX, + # api_spec_url='{}/spec'.format(URL_PREFIX) + # api_spec_url='/spec', + # api_spec_url='{}/spec'.format(URL_PREFIX) + api_spec_url='/spec' +) +################################### + +api.add_resource(RootR, "/") +api.add_resource(CoprListR, "/coprs") +api.add_resource(CoprR, "/coprs/<owner>/<project>") + +api.add_resource(ChrootListR, "/chroots") +api.add_resource(ChrootR, "/chroots/<name>") + +api.add_resource(BuildListR, "/builds") +api.add_resource(BuildR, "/builds/<int:build_id>") + +# app.register_blueprint(rest_api_bp, url_prefix=URL_PREFIX) + + +# TODO: try: https://github.com/sloria/flask-marshmallow +def register_api_error_handler(app): + @app.errorhandler(ApiError) + def handle_api_error(error): + response = Response( + response="{}\n".format(error.data), + status=error.code, + mimetype="text/plain", + headers=error.headers, + ) + return response diff --git a/frontend/coprs_frontend/coprs/rest_api/exceptions.py b/frontend/coprs_frontend/coprs/rest_api/exceptions.py new file mode 100644 index 0000000..0fb14da --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/exceptions.py @@ -0,0 +1,45 @@ +# coding: utf-8 +import six + + +class ApiError(Exception): + def __init__(self, code, data, **kwargs): + super(ApiError, self).__init__(**kwargs) + + self.code = code + self.data = data + + self.headers = kwargs.get("headers", {}) + + def __str__(self): + return str(self.data) + + if six.PY2: + def __unicode__(self): + return unicode(self.data) + + +class AuthFailed(ApiError): + def __init__(self, data, **kwargs): + + super(AuthFailed, self).__init__( + 401, data, + # headers={"Authorization": "Basic"}, + **kwargs) + + self.headers["Authorization"] = "Basic" + + +class ObjectNotFoundError(ApiError): + def __init__(self, data, **kwargs): + super(ObjectNotFoundError, self).__init__(404, data, **kwargs) + + +class ObjectAlreadyExists(ApiError): + def __init__(self, data, **kwargs): + super(ObjectAlreadyExists, self).__init__(409, data, **kwargs) + + +class MalformedRequest(ApiError): + def __init__(self, data=None, **kwargs): + super(MalformedRequest, self).__init__(400, data, **kwargs) diff --git a/builder_image/src/__init__.py b/frontend/coprs_frontend/coprs/rest_api/resources/__init__.py similarity index 100% copy from builder_image/src/__init__.py copy to frontend/coprs_frontend/coprs/rest_api/resources/__init__.py diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/build.py b/frontend/coprs_frontend/coprs/rest_api/resources/build.py new file mode 100644 index 0000000..db4a82e --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/resources/build.py @@ -0,0 +1,84 @@ +# coding: utf-8 + +import flask +from flask import url_for +from flask_restful_swagger import swagger +from coprs.logic.coprs_logic import CoprsLogic +from coprs.logic.builds_logic import BuildsLogic + +from coprs.rest_api.util import get_one_safe, bp_url_for + +from flask_restful import Resource, reqparse + + +class BuildListR(Resource): + + def get(self): + + parser = reqparse.RequestParser() + + parser.add_argument('owner', type=str,) + parser.add_argument('project', type=str) + + parser.add_argument('limit', type=int) + parser.add_argument('offset', type=int) + + req_args = parser.parse_args() + + if "owner" and "project" in req_args: + query = BuildsLogic.get_multiple_by_name( + req_args["owner"], req_args["project"]) + else: + query = BuildsLogic.get_multiple() + + if "limit" in req_args: + limit = req_args["limit"] + else: + limit = 100 + + query = query.limit(limit) + + if "offset" in req_args: + query = query.offset(req_args["offset"]) + + builds = query.all() + return { + "links": { + "self": bp_url_for(BuildListR.endpoint, **req_args), + }, + "builds": [ + { + "build": build.to_dict(), + "link": bp_url_for(BuildR.endpoint, build_id=build.id), + } for build in builds + ] + } + + +(a)swagger.model +class BuildItem(object): + def __init__(self, build_id): + pass + + +class BuildR(Resource): + def get(self, build_id): + """ + Get single build by id + """ + build = get_one_safe(BuildsLogic.get(build_id), + "Not found build with id: {}".format(build_id)) + return { + "build": build.to_dict(), + "links": { + "self": bp_url_for(BuildR.endpoint, build_id=build_id), + # TODO: can't do it due to circular imports + # "parent_copr": url_for(CoprR.endpoint, + # owner=build.copr.owner.name, + # project=build.copr.name), + } + } + + + + diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py b/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py new file mode 100644 index 0000000..9e4652a --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/resources/chroot.py @@ -0,0 +1,42 @@ +# coding: utf-8 + +import json +import flask +from flask import url_for +from flask_restful import Resource, reqparse + +from marshmallow import Schema, fields, pprint + +from coprs.views.misc import api_login_required +from coprs.logic.coprs_logic import MockChrootsLogic + +from ..util import get_one_safe, json_loads_safe, mm_deserialize + + +class ChrootListR(Resource): + + def get(self): + chroots = MockChrootsLogic.get_multiple(None).all() + return { + "links": { + "self": url_for(ChrootListR.endpoint), + }, + "chroots": [ + { + "chroot": chroot.to_dict(), + "link": url_for(ChrootR.endpoint, name=chroot.name), + } for chroot in chroots + ] + } + + +class ChrootR(Resource): + + def get(self, name): + chroot = get_one_safe(MockChrootsLogic.get_from_name(name)) + return { + "links": { + "self": url_for(ChrootR.endpoint, name=chroot.name) + }, + "chroot": chroot.to_dict() + } diff --git a/frontend/coprs_frontend/coprs/rest_api/resources/copr.py b/frontend/coprs_frontend/coprs/rest_api/resources/copr.py new file mode 100644 index 0000000..5783a7f --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/resources/copr.py @@ -0,0 +1,199 @@ +import base64 +import json +import datetime +import flask +from flask import url_for +from flask_restful import Resource, reqparse +import functools + +from marshmallow import Schema, fields, pprint + +from coprs import db +from coprs.exceptions import DuplicateException +from coprs.logic.complex_logic import ComplexLogic +from coprs.logic.users_logic import UsersLogic + +from coprs.logic.coprs_logic import CoprsLogic + +from coprs.exceptions import ActionInProgressException, InsufficientRightsException +from .build import BuildListR + +from ..exceptions import ObjectAlreadyExists, AuthFailed +from ..util import get_one_safe, json_loads_safe, mm_deserialize + +def rest_api_auth_required(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + token = None + apt_login = None + if "Authorization" in flask.request.headers: + base64string = flask.request.headers["Authorization"] + base64string = base64string.split()[1].strip() + userstring = base64.b64decode(base64string) + (apt_login, token) = userstring.split(":") + token_auth = False + if token and apt_login: + user = UsersLogic.get_by_api_login(apt_login).first() + if (user and user.api_token == token and + user.api_token_expiration >= datetime.date.today()): + + token_auth = True + flask.g.user = user + if not token_auth: + message = ( + "Login invalid/expired. " + "Please visit https://copr.fedoraproject.org/api " + "get or renew your API token.") + + raise AuthFailed(message) + return f(*args, **kwargs) + return decorated_function + + +class CoprSchema(Schema): + name = fields.Str(required=True) + description = fields.Str() + instructions = fields.Str() + + auto_createrepo = fields.Bool() + + chroots = fields.List(fields.Str, required=True) + repos = fields.List(fields.Str) + + _keys_to_make_object = [ + "description", + "instructions", + "auto_createrepo" + ] + + def make_object(self, data): + """ + Create kwargs for CoprsLogic.add + """ + kwargs = dict( + name=data["name"].strip(), + repos=" ".join(data.get("repos", [])), + selected_chroots=data["chroots"], + ) + for key in self._keys_to_make_object: + if key in data: + kwargs[key] = data[key] + return kwargs + + +class CoprListR(Resource): + + @rest_api_auth_required + def post(self): + """ + Creates new copr + """ + owner = flask.g.user + new_copr_dict = json_loads_safe(flask.request.data, "") + + result = mm_deserialize(CoprSchema(), new_copr_dict) + # todo check that chroots are available + pprint(result.data) + try: + copr = CoprsLogic.add(user=owner, check_for_duplicates=True, **result.data) + db.session.commit() + except DuplicateException as error: + raise ObjectAlreadyExists(data=error) + + return copr.to_dict(), 201 + + def get(self): + """ + Get coprs collection + :return: + """ + parser = reqparse.RequestParser() + + parser.add_argument('owner', dest='username', type=str) + parser.add_argument('limit', type=int) + parser.add_argument('offset', type=int) + req_args = parser.parse_args() + + kwargs = {} + for key in ["username"]: + if req_args[key]: + kwargs[key] = req_args[key] + + if "username" in kwargs: + + kwargs["username"] = req_args["username"] + kwargs["user_relation"] = "owned" + + query = CoprsLogic.get_multiple( + flask.g.user, + with_builds=True, + **kwargs + ) + + if req_args["offset"]: + query = query.offset(req_args["offset"]) + + if req_args["limit"]: + query = query.limit(req_args["limit"]) + + # try: + coprs_list = query.all() + # except Exception as error: + # import ipdb; ipdb.set_trace() + # a = 2 + + return { + "links": { + "self": url_for(CoprListR.endpoint, **req_args) + }, + "coprs": [ + { + "copr": copr.to_dict(), + "link": url_for(CoprR.endpoint, + owner=copr.owner.name, + project=copr.name), + } + for copr in coprs_list + ] + } + + +class CoprR(Resource): + + @rest_api_auth_required + def delete(self, owner, project): + copr = get_one_safe(CoprsLogic.get(flask.g.user, owner, project), + "Copr {}/{} not found".format(owner, project)) + try: + ComplexLogic.delete_copr(copr) + except (ActionInProgressException, + InsufficientRightsException) as err: + db.session.rollback() + raise + else: + db.session.commit() + + return None, 204 + + def get(self, owner, project): + parser = reqparse.RequestParser() + parser.add_argument('show_builds', type=bool, default=True) + parser.add_argument('show_chroots', type=bool, default=True) + req_args = parser.parse_args() + + copr = get_one_safe(CoprsLogic.get(flask.g.user, owner, project), + "Copr {}/{} not found".format(owner, project)) + return { + "links": { + "self": url_for(CoprR.endpoint, + owner=owner, + project=project), + # "chroots": + "builds": url_for(BuildListR.endpoint, + owner=copr.owner.name, + project=copr.name) + }, + "copr": copr.to_dict() + } + + diff --git a/frontend/coprs_frontend/coprs/rest_api/util.py b/frontend/coprs_frontend/coprs/rest_api/util.py new file mode 100644 index 0000000..87fab9a --- /dev/null +++ b/frontend/coprs_frontend/coprs/rest_api/util.py @@ -0,0 +1,42 @@ +# coding: utf-8 +import json + +import sqlalchemy.orm.exc +from .exceptions import ObjectNotFoundError, MalformedRequest + +from flask import Response, url_for, Blueprint + + +def bp_url_for(endpoint, *args, **kwargs): + """ + Prepend endpoing with dot + :param endpoint: + :param args: + :param kwargs: + :return: + """ + + return url_for(".{}".format(endpoint), *args, **kwargs) + + +def get_one_safe(query, data_on_error=None): + try: + return query.one() + except sqlalchemy.orm.exc.NoResultFound: + raise ObjectNotFoundError(data_on_error) + + +def json_loads_safe(raw, data_on_error=None): + try: + return json.loads(raw) + except ValueError: + raise MalformedRequest(data_on_error or + "Failed to deserialize json string") + + +def mm_deserialize(schema, obj_dict): + result = schema.load(obj_dict) + if result.errors: + raise MalformedRequest(data=result.errors) + # import ipdb; ipdb.set_trace() + return result diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 222c155..b6bc838 100644 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -11,6 +11,7 @@ from coprs import forms from coprs import helpers from coprs.helpers import fix_protocol_for_backend from coprs.logic.api_logic import MonitorWrapper +from coprs.logic.complex_logic import ComplexLogic from coprs.views.misc import login_required, api_login_required @@ -140,11 +141,8 @@ def api_copr_delete(username, coprname): httpcode = 200 if form.validate_on_submit() and copr: - builds_query = builds_logic.BuildsLogic.get_multiple_by_copr(copr=copr) try: - for build in builds_query: - builds_logic.BuildsLogic.delete_build(flask.g.user, build) - CoprsLogic.delete(flask.g.user, copr) + ComplexLogic.delete_copr(copr) except (exceptions.ActionInProgressException, exceptions.InsufficientRightsException) as err: output = {"output": "notok", "error": err} diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 8d8e906..91cf8b4 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -20,6 +20,8 @@ from coprs import models from coprs.logic.stat_logic import CounterStatLogic from coprs.rmodels import TimedStatEvents +from coprs.logic.complex_logic import ComplexLogic + from coprs.views.misc import login_required, page_not_found from coprs.views.coprs_ns import coprs_ns @@ -499,12 +501,9 @@ def copr_delete(username, coprname): copr = coprs_logic.CoprsLogic.get(username, coprname).first() if form.validate_on_submit() and copr: - builds_query = builds_logic.BuildsLogic.get_multiple_by_copr(copr=copr) try: - for build in builds_query: - builds_logic.BuildsLogic.delete_build(flask.g.user, build) - coprs_logic.CoprsLogic.delete(flask.g.user, copr) + ComplexLogic.delete_copr(copr) except (exceptions.ActionInProgressException, exceptions.InsufficientRightsException) as e: diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index 38d0ab5..d3bb3e4 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -222,6 +222,7 @@ def api_login_required(f): return decorated_function + def login_required(role=helpers.RoleEnum("user")): def view_wrapper(f): @functools.wraps(f) diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py index da3f778..bd06cd9 100644 --- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py +++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py @@ -39,7 +39,7 @@ class TestMonitor(CoprsTestCase): res = self.tc.get("/coprs/{}/{}/monitor/".format(self.u1.name, copr_name)) assert res.status_code == 200 - self.db.session.add(CoprsLogic.delete(self.u1, tmp_copr)) + self.db.session.add(CoprsLogic.delete_unsafe(self.u1, tmp_copr)) self.db.session.commit() res = self.tc.get("/coprs/{}/{}/monitor/".format(self.u1.name, copr_name)) diff --git a/frontend/requirements.txt b/frontend/requirements.txt index 82354b3..9734ddc 100644 --- a/frontend/requirements.txt +++ b/frontend/requirements.txt @@ -17,3 +17,4 @@ decorator python-dateutil netaddr alembic +flask-restful-swagger

8 years, 9 months

1
0
0 / 0

[copr] branch 'rest_api_2' created

by vgologuz＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git New branch : rest_api_2 Referencing: f3bb91936bea528256bf585670e3e368436c4217

8 years, 9 months

1
0
0 / 0

[copr] master: get pkg name + version during import (fe8c77f)

by asamalik＠fedoraproject.org

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit fe8c77fe7d15fd95d8ab24d524cc35bae2a2f935 Author: Adam Samalik <asamalik(a)redhat.com> Date: Wed Jul 29 11:01:57 2015 +0200 get pkg name + version during import >--------------------------------------------------------------- dist-git/dist-git/dist_git_importer.py | 43 ++++++++++++++++++- .../coprs_frontend/coprs/logic/builds_logic.py | 17 +++++--- .../coprs/views/backend_ns/backend_general.py | 15 ++++++- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py index 6c9fc03..ef75d4d 100755 --- a/dist-git/dist-git/dist_git_importer.py +++ b/dist-git/dist-git/dist_git_importer.py @@ -8,6 +8,7 @@ import urllib import shutil import tempfile import logging +import subprocess from requests import get from requests import post @@ -48,6 +49,10 @@ class PackageDownloadException(Exception): pass +class PackageQueryException(Exception): + pass + + def _my_upload(repo_dir, reponame, filename, filehash): """ This is a replacement function for uploading sources. @@ -124,6 +129,35 @@ def import_srpm(user, project, pkg, branch, filepath): return git_hash +def pkg_name_evr(pkg): + """ + Queries a package for its name and evr (epoch:version-release) + """ + log.debug("Verifying packagage, getting name and version.") + cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg] + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except OSError as e: + raise PackageQueryException(e) + if error: + raise PackageQueryException('Error querying srpm: %s' % error) + + try: + name, epoch, version, release = output.split(" ") + except ValueError as e: + raise PackageQueryException(e) + + # Epoch is an integer or '(none)' if not set + if epoch.isdigit(): + evr = "{}:{}-{}".format(epoch, version, release) + else: + evr = "{}-{}".format(version, release) + + return name, evr + + class DistGitImporter(): def __init__(self, opts): self.opts = opts @@ -155,7 +189,7 @@ class DistGitImporter(): task_id = task["task_id"] user = task["user"] project = task["project"] - package = task["package"] + #package = task["package"] branch = task["branch"] source_type = task["source_type"] source_json = task["source_json"] @@ -191,14 +225,15 @@ class DistGitImporter(): # todo: check that obtained file is a REAL srpm # todo query package name & version and ise real name instead of task["package"] # if fetched file is not a proper srpm set error state + name, version = pkg_name_evr(fetched_srpm_path) - reponame = "{}/{}/{}".format(user, project, package) + reponame = "{}/{}/{}".format(user, project, name) log.debug("make sure repos exist: {}".format(reponame)) call(["/usr/share/dist-git/git_package.sh", reponame]) call(["/usr/share/dist-git/git_branch.sh", branch, reponame]) log.debug("import it and delete the srpm") - git_hash = import_srpm(user, project, package, branch, fetched_srpm_path) + git_hash = import_srpm(user, project, name, branch, fetched_srpm_path) log.debug("send a response - success") @@ -207,6 +242,8 @@ class DistGitImporter(): # send a response - success data = {"task_id": task_id, + "pkg_name": name, + "pkg_version": version, "repo_name": reponame, "git_hash": git_hash} post(upload_url, auth=auth, data=json.dumps(data), headers=headers) diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 6fd3fed..e6e0f11 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -182,14 +182,17 @@ class BuildsLogic(object): source_type = helpers.BuildSourceEnum("srpm_link") source_json = json.dumps({"url":pkgs}) - package_name = helpers.parse_package_name(os.path.basename(pkgs)) + # We no longer guess package name just from the filename + #package_name = helpers.parse_package_name(os.path.basename(pkgs)) - package = packages_logic.PackagesLogic.get(copr.id, package_name).first() + # And we no longer assign a package to the build (we don't know the package name, right...) + # This is done after package is imported. + #package = packages_logic.PackagesLogic.get(copr.id, package_name).first() - if not package: - package = packages_logic.PackagesLogic.add(user, copr, package_name) - db.session.add(package) - db.session.flush() + #if not package: + # package = packages_logic.PackagesLogic.add(user, copr, package_name) + # db.session.add(package) + # db.session.flush() build = models.Build( user=user, @@ -198,7 +201,7 @@ class BuildsLogic(object): repos=repos, source_type=source_type, source_json=source_json, - package_id=package.id, + #package_id=package.id, submitted_on=int(time.time()), enable_net=bool(enable_net), ) diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index dcc58d0..ca8a497 100644 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -33,7 +33,7 @@ def dist_git_importing_queue(): "task_id": "{}-{}".format(task.build.id, helpers.chroot_to_branch(task.mock_chroot.name)), "user": task.build.copr.owner.name, "project": task.build.copr.name, - "package": task.build.package.name, + #"package": task.build.package.name, "branch": helpers.chroot_to_branch(task.mock_chroot.name), "source_type": task.build.source_type, "source_json": task.build.source_json, @@ -68,6 +68,19 @@ def dist_git_upload_completed(): if "git_hash" in flask.request.json and "repo_name" in flask.request.json: git_hash = flask.request.json["git_hash"] repo_name = flask.request.json["repo_name"] + pkg_name = flask.request.json["pkg_name"] + pkg_version = flask.request.json["pkg_version"] + + # Now I need to assign a package to this build + package = PackagesLogic.get(build.copr.id, pkg_name).first() + if not package: + package = PackagesLogic.add(build.copr.owner, build.copr, pkg_name) + db.session.add(package) + db.session.flush() + + build.package_id = package.id + build.pkg_version = pkg_version + for ch in build_chroots: ch.status = helpers.StatusEnum("pending") ch.git_hash = git_hash

8 years, 9 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits July 2015