FBR: F28 onwards fedimg needs to parse AtomicHost and Cloud variants separately

Hi, F28 onwards fedimg needs to parse AtomicHost and Cloud variants separately. So, I have created a hotfix for the same. The same patch will be ported to fedimg for the issue[1] From c6dbcbcc2d105d0ce6976cbce676c311fe59991a Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; Date: Wed, 21 Mar 2018 15:00:59 +0530 Subject: [PATCH 1/2] fedimg: Put hotfix for add Atomic & Cloud variant Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; --- files/hotfix/fedimg/consumers.py | 88 ++++++++++++++++++++++++++++++++++++++++ roles/fedimg/tasks/main.yml | 8 ++++ 2 files changed, 96 insertions(+) create mode 100644 files/hotfix/fedimg/consumers.py diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py new file mode 100644 index 0000000..cad1495 --- /dev/null +++ b/files/hotfix/fedimg/consumers.py @@ -0,0 +1,88 @@ +# This file is part of fedimg. +# Copyright (C) 2014 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay <dgay(a)redhat.com&gt; +# + +import logging +log = logging.getLogger("fedmsg") + +import multiprocessing.pool + +import fedmsg.consumers +import fedmsg.encoding +import fedfind.release + +import fedimg.uploader +from fedimg.util import get_rawxz_urls, safeget + + +class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): + """ Listens for image Koji task completion and sends image files + produced by the child createImage tasks to the uploader. """ + + # It used to be that all *image* builds appeared as scratch builds on the + # task.state.change topic. However, with the switch to pungi4, some of + # them (and all of them in the future) appear as full builds under the + # build.state.change topic. That means we have to handle both cases like + # this, at least for now. + topic = [ + 'org.fedoraproject.prod.pungi.compose.status.change', + ] + + config_key = 'fedimgconsumer' + + def __init__(self, *args, **kwargs): + super(FedimgConsumer, self).__init__(*args, **kwargs) + + # threadpool for upload jobs + self.upload_pool = multiprocessing.pool.ThreadPool(processes=4) + + log.info("Super happy fedimg ready and reporting for duty.") + + def consume(self, msg): + """ This is called when we receive a message matching our topics. """ + + log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) + + STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) + + msg_info = msg['body']['msg'] + if msg_info['status'] not in STATUS_F: + return + + location = msg_info['location'] + compose_id = msg_info['compose_id'] + cmetadata = fedfind.release.get_release_cid(compose_id).metadata + + images_meta = safeget(cmetadata, 'images', 'payload', 'images', + 'CloudImages', 'x86_64') + + if images_meta is None: + return + + self.upload_urls = get_rawxz_urls(location, images_meta) + compose_meta = { + 'compose_id': compose_id, + } + + if len(self.upload_urls) > 0: + log.info("Processing compose id: %s" % compose_id) + fedimg.uploader.upload(self.upload_pool, + self.upload_urls, + compose_meta) diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index 6074903..f36ec72 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -136,3 +136,11 @@ tags: - cron - fedimg + +- name: hotfix - copy the consumers.py over to the site-packages + copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py + notify: + - restart fedmsg-hub + tags: + - fedimg + - hotfix -- 2.9.4 From 833da82dae0da1988e4345a1134120c981c84ba7 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; Date: Wed, 21 Mar 2018 15:14:31 +0530 Subject: [PATCH 2/2] fedimg: Add the hotfix patch to parse Cloud and Atomic Image variant Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; --- files/hotfix/fedimg/consumers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py index cad1495..3ebd4cb 100644 --- a/files/hotfix/fedimg/consumers.py +++ b/files/hotfix/fedimg/consumers.py @@ -70,8 +70,19 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): compose_id = msg_info['compose_id'] cmetadata = fedfind.release.get_release_cid(compose_id).metadata - images_meta = safeget(cmetadata, 'images', 'payload', 'images', - 'CloudImages', 'x86_64') + # Till F27, both cloud-base and atomic images were available + # under variant CloudImages. With F28 and onward releases, + # cloud-base image compose moved to cloud variant and atomic images + # moved under atomic variant. + prev_rel = ['26', '27'] + if msg_info['release_version'] in prev_rel: + images_meta = safeget(cmetadata, 'images', 'payload', 'images', + 'CloudImages', 'x86_64') + else: + images_meta = safeget(cmetadata, 'images', 'payload', 'images', + 'Cloud', 'x86_64') + images_meta.extend(safeget(cmetadata, 'images', 'payload', + 'images', 'AtomicHost', 'x86_64')) if images_meta is None: return -- 2.9.4 +1s? [1] https://github.com/fedora-infra/fedimg/issues/71