FBR: Fix for processing F28 Cloud & Atomic messages in Autocloud

Hi, This is similar to the fedimg patch sent to the list earlier. Autocloud right now is not processing the messages because of the change in the F28 messages. This patch would fix the issue. This also contains a commit to remove a earlier hotfix in autocloud. That hotfix was deployed during the last release but I forgot to remove the hotfix. Don't know if this is proper but if needed I can break this into two FBRs too. Here is the PR to the corresponding change[0] Thoughts? +1s? [0] https://github.com/kushaldas/autocloud/pull/62 From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; Date: Mon, 26 Mar 2018 18:42:33 +0530 Subject: [PATCH 1/3] autocloud, hotfix: Remove the autocloud hotfix for the PR#56 Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; --- files/hotfix/autocloud/consumer.py | 135 --------------------------------- roles/autocloud/backend/tasks/main.yml | 14 ---- 2 files changed, 149 deletions(-) delete mode 100644 files/hotfix/autocloud/consumer.py diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py deleted file mode 100644 index c216553..0000000 --- a/files/hotfix/autocloud/consumer.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime - -import requests -import fedmsg.consumers -import fedfind.release - -from sqlalchemy import exc - -import autocloud - -from autocloud.models import init_model, ComposeDetails, ComposeJobDetails -from autocloud.producer import publish_to_fedmsg -from autocloud.utils import is_valid_image, produce_jobs - -import logging -log = logging.getLogger("fedmsg") - -DEBUG = autocloud.DEBUG - - -class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): - """ - Fedmsg consumer for Autocloud - """ - - if DEBUG: - topic = [ - 'org.fedoraproject.dev.__main__.pungi.compose.status.change' - ] - - else: - topic = [ - 'org.fedoraproject.prod.pungi.compose.status.change' - ] - - config_key = 'autocloud.consumer.enabled' - - def __init__(self, *args, **kwargs): - self.supported_archs = [arch for arch, _ in ComposeJobDetails.ARCH_TYPES] - - log.info("Autocloud Consumer is ready for action.") - super(AutoCloudConsumer, self).__init__(*args, **kwargs) - - def consume(self, msg): - """ This is called when we receive a message matching the topic. """ - - log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) - - STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) - VARIANTS_F = ('CloudImages',) - - images = [] - compose_db_update = False - msg_body = msg['body'] - status = msg_body['msg']['status'] - compose_images_json = None - - if status in STATUS_F: - location = msg_body['msg']['location'] - json_metadata = '{}/metadata/images.json'.format(location) - resp = requests.get(json_metadata) - compose_images_json = getattr(resp, 'json', False) - - if compose_images_json is not None: - compose_images_json = compose_images_json() - compose_images = compose_images_json['payload']['images'] - compose_details = compose_images_json['payload']['compose'] - compose_images = dict((variant, compose_images[variant]) - for variant in VARIANTS_F - if variant in compose_images) - compose_id = compose_details['id'] - rel = fedfind.release.get_release(cid=compose_id) - release = rel.release - compose_details.update({'release': release}) - - compose_images_variants = [variant for variant in VARIANTS_F - if variant in compose_images] - - for variant in compose_images_variants: - compose_image = compose_images[variant] - for arch, payload in compose_image.iteritems(): - - if arch not in self.supported_archs: - continue - - for item in payload: - relative_path = item['path'] - if not is_valid_image(relative_path): - continue - absolute_path = '{}/{}'.format(location, relative_path) - item.update({ - 'compose': compose_details, - 'absolute_path': absolute_path, - }) - images.append(item) - compose_db_update = True - - if compose_db_update: - session = init_model() - compose_date = datetime.strptime(compose_details['date'], '%Y%m%d') - try: - cd = ComposeDetails( - date=compose_date, - compose_id=compose_details['id'], - respin=compose_details['respin'], - type=compose_details['type'], - status=u'q', - location=location, - ) - - session.add(cd) - session.commit() - - compose_details.update({ - 'status': 'queued', - 'compose_job_id': cd.id, - }) - publish_to_fedmsg(topic='compose.queued', - **compose_details) - except exc.IntegrityError: - session.rollback() - cd = session.query(ComposeDetails).filter_by( - compose_id=compose_details['id']).first() - log.info('Compose already exists %s: %s' % ( - compose_details['id'], - cd.id - )) - session.close() - - num_images = len(images) - for pos, image in enumerate(images): - image.update({'pos': (pos+1, num_images)}) - - produce_jobs(images) diff --git a/roles/autocloud/backend/tasks/main.yml b/roles/autocloud/backend/tasks/main.yml index bc50df1..7ff5811 100644 --- a/roles/autocloud/backend/tasks/main.yml +++ b/roles/autocloud/backend/tasks/main.yml @@ -139,17 +139,3 @@ tags: - autocloud - autocloud/backend - -# -# Install hotfix to ignore new architectures -# See PR - https://github.com/kushaldas/autocloud/pull/56/ -# -- name: hotfix - copy over consumer files - copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} - with_items: - - { src: 'hotfix/autocloud/consumer.py', dest: '/usr/lib/python2.7/site-packages/autocloud' } - notify: - - restart fedmsg-hub - tags: - - autocloud - - hotfix -- 2.9.4 From 7593e3dddaf85011b27c989a3f14c9794f4265bc Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; Date: Mon, 26 Mar 2018 18:50:42 +0530 Subject: [PATCH 2/3] autocloud, hotfix: Add the hotfix to process F28 Atomic, Cloud messages Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; --- files/hotfix/autocloud/consumer.py | 136 +++++++++++++++++++++++++++++++++ roles/autocloud/backend/tasks/main.yml | 14 ++++ 2 files changed, 150 insertions(+) create mode 100644 files/hotfix/autocloud/consumer.py diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py new file mode 100644 index 0000000..a7b2aa5 --- /dev/null +++ b/files/hotfix/autocloud/consumer.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +from datetime import datetime + +import requests +import fedmsg.consumers +import fedfind.release + +from sqlalchemy import exc + +import autocloud + +from autocloud.models import init_model, ComposeDetails, ComposeJobDetails +from autocloud.producer import publish_to_fedmsg +from autocloud.utils import is_valid_image, produce_jobs + +import logging +log = logging.getLogger("fedmsg") + +DEBUG = autocloud.DEBUG + + +class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): + """ + Fedmsg consumer for Autocloud + """ + + if DEBUG: + topic = [ + 'org.fedoraproject.dev.__main__.pungi.compose.status.change' + ] + + else: + topic = [ + 'org.fedoraproject.prod.pungi.compose.status.change' + ] + + config_key = 'autocloud.consumer.enabled' + + def __init__(self, *args, **kwargs): + self.supported_archs = [arch for arch, _ in ComposeJobDetails.ARCH_TYPES] + + log.info("Autocloud Consumer is ready for action.") + super(AutoCloudConsumer, self).__init__(*args, **kwargs) + + def consume(self, msg): + """ This is called when we receive a message matching the topic. """ + + log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) + + STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) + VARIANTS_F = ('CloudImages',) + + images = [] + compose_db_update = False + msg_body = msg['body'] + status = msg_body['msg']['status'] + compose_images_json = None + + if status in STATUS_F: + location = msg_body['msg']['location'] + json_metadata = '{}/metadata/images.json'.format(location) + resp = requests.get(json_metadata) + compose_images_json = getattr(resp, 'json', False) + + if compose_images_json is not None: + compose_images_json = compose_images_json() + compose_images = compose_images_json['payload']['images'] + compose_details = compose_images_json['payload']['compose'] + compose_images = dict((variant, compose_images[variant]) + for variant in VARIANTS_F + if variant in compose_images) + compose_id = compose_details['id'] + rel = fedfind.release.get_release(cid=compose_id) + release = rel.release + compose_details.update({'release': release}) + + compose_images_variants = [variant for variant in VARIANTS_F + if variant in compose_images] + + for variant in compose_images_variants: + compose_image = compose_images[variant] + for arch, payload in compose_image.iteritems(): + + if arch not in self.supported_archs: + continue + + for item in payload: + relative_path = item['path'] + if not is_valid_image(relative_path): + continue + absolute_path = '{}/{}'.format(location, relative_path) + item.update({ + 'compose': compose_details, + 'absolute_path': absolute_path, + }) + images.append(item) + compose_db_update = True + + if compose_db_update: + session = init_model() + compose_date = datetime.strptime(compose_details['date'], '%Y%m%d') + try: + cd = ComposeDetails( + date=compose_date, + compose_id=compose_details['id'], + respin=compose_details['respin'], + type=compose_details['type'], + status=u'q', + location=location, + ) + + session.add(cd) + session.commit() + + compose_details.update({ + 'status': 'queued', + 'compose_job_id': cd.id, + }) + publish_to_fedmsg(topic='compose.queued', + **compose_details) + except exc.IntegrityError: + session.rollback() + cd = session.query(ComposeDetails).filter_by( + compose_id=compose_details['id']).first() + log.info('Compose already exists %s: %s' % ( + compose_details['id'], + cd.id + )) + session.close() + + num_images = len(images) + for pos, image in enumerate(images): + image.update({'pos': (pos+1, num_images)}) + + produce_jobs(images) + diff --git a/roles/autocloud/backend/tasks/main.yml b/roles/autocloud/backend/tasks/main.yml index 7ff5811..e1024c3 100644 --- a/roles/autocloud/backend/tasks/main.yml +++ b/roles/autocloud/backend/tasks/main.yml @@ -139,3 +139,17 @@ tags: - autocloud - autocloud/backend + +# +# Install hotfix to ignore new architectures +# See PR - https://github.com/kushaldas/autocloud/pull/62/ +# +- name: hotfix - copy over consumer files + copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} + with_items: + - { src: 'hotfix/autocloud/consumer.py', dest: '/usr/lib/python2.7/site-packages/autocloud' } + notify: + - restart fedmsg-hub + tags: + - autocloud + - hotfix -- 2.9.4 From 1403d8118ab9483d0753f1995bec5ce6e78f0df2 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; Date: Mon, 26 Mar 2018 18:53:01 +0530 Subject: [PATCH 3/3] autocloud, hotfix: Add the patch for the hotfix (F28 messages) Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com&gt; --- files/hotfix/autocloud/consumer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py index a7b2aa5..0ff8317 100644 --- a/files/hotfix/autocloud/consumer.py +++ b/files/hotfix/autocloud/consumer.py @@ -48,7 +48,6 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) - VARIANTS_F = ('CloudImages',) images = [] compose_db_update = False @@ -56,6 +55,16 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): status = msg_body['msg']['status'] compose_images_json = None + # Till F27, both cloud-base and atomic images were available + # under variant CloudImages. With F28 and onward releases, + # cloud-base image compose moved to cloud variant and atomic images + # moved under atomic variant. + prev_rel = ['26', '27'] + if msg_body['msg']['release_version'] in prev_rel: + VARIANTS_F = ('CloudImages',) + else: + VARIANTS_F = ('AtomicHost', 'Cloud') + if status in STATUS_F: location = msg_body['msg']['location'] json_metadata = '{}/metadata/images.json'.format(location) @@ -133,4 +142,3 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): image.update({'pos': (pos+1, num_images)}) produce_jobs(images) - -- 2.9.4 -- Sayan Chowdhury <https://sayanchowdhury.dgplug.org/> Senior Software Engineer, Fedora Engineering - Emerging Platform GPG Fingerprint : 0F16 E841 E517 225C 7D13 AB3C B023 9931 9CD0 5C8B Proud to work at The Open Organization!