rel-eng October 2014

rel-eng@lists.fedoraproject.org

16 participants
143 discussions

[releng] find_unblocked_orphans: Cache complete pkgdb info
by Till Maas 18 Oct '14

18 Oct '14

commit e6f7b5f47d4019b095d23781fe251d618385544b Author: Till Maas <opensource(a)till.name> Date: Sat Oct 18 10:49:24 2014 +0200 find_unblocked_orphans: Cache complete pkgdb info scripts/find_unblocked_orphans.py | 111 ++++++++++++++++++++----------------- 1 files changed, 61 insertions(+), 50 deletions(-) --- diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py index d485d6d..bf94aea 100755 --- a/scripts/find_unblocked_orphans.py +++ b/scripts/find_unblocked_orphans.py @@ -179,35 +179,46 @@ def write_cache(data, filename, cachedir='~/.cache'): pickle.dump(data, pickle_file) -def get_people(package, branch=RAWHIDE_RELEASE["branch"]): - def associated(pkginfo, exclude=None): - """ - - :param exclude: People to exclude, e.g. the point of contact. - :type exclude: list - """ - other_people = set() - for acl in pkginfo.get("acls", []): - if acl["status"] == "Approved": - fas_name = acl["fas_name"] - if fas_name != "group::provenpackager" and \ - fas_name not in exclude: - other_people.add(fas_name) - return sorted(other_people) +class PKGDBInfo(object): + def __init__(self, package, branch=RAWHIDE_RELEASE["branch"]): + self.package = package + self.branch = branch - try: - pkginfo = pkgdb.get_package(package, branches=branch) - except Exception as e: - sys.stderr.write( - "Error getting maintainer of package {} on branch {}\n".format( - package, branch)) - # FIXME: Write proper traceback - sys.stderr.write(str(e)) - return [] - pkginfo = pkginfo["packages"][0] - people_ = [pkginfo["point_of_contact"]] - people_.extend(associated(pkginfo, exclude=people_)) - return people_ + try: + pkginfo = pkgdb.get_package(package, branches=branch) + except Exception as e: + sys.stderr.write( + "Error getting pkgdb info for {} on {}\n".format( + package, branch)) + # FIXME: Write proper traceback + sys.stderr.write(str(e)) + self.pkginfo = None + return + + self.pkginfo = pkginfo["packages"][0] + + def get_people(self): + def associated(pkginfo, exclude=None): + """ + + :param exclude: People to exclude, e.g. the point of contact. + :type exclude: list + """ + other_people = set() + for acl in pkginfo.get("acls", []): + if acl["status"] == "Approved": + fas_name = acl["fas_name"] + if fas_name != "group::provenpackager" and \ + fas_name not in exclude: + other_people.add(fas_name) + return sorted(other_people) + + if self.pkginfo is not None: + people_ = [self.pkginfo["point_of_contact"]] + people_.extend(associated(self.pkginfo, exclude=people_)) + return people_ + else: + return [] def setup_yum(repo=RAWHIDE_RELEASE["repo"], @@ -285,9 +296,9 @@ class DepChecker(object): yumbase = setup_yum(repo=RELEASES[release]["repo"], source_repo=RELEASES[release]["source_repo"]) self.yumbase = yumbase - self.people_queue = Queue() - self.people_cache = "orphans-people-{}.pickle".format(release) - self.people_dict = get_cache(self.people_cache, default={}) + self.pkgdbinfo_queue = Queue() + self.pkgdb_cache = "orphans-pkgdb-{}.pickle".format(release) + self.pkgdb_dict = get_cache(self.pkgdb_cache, default={}) def create_mapping(self): src_by_bin = {} # Dict of source pkg objects by binary package objects @@ -394,19 +405,19 @@ class DepChecker(object): prov) return OrderedDict(sorted(dependent_packages.items())) - def people_worker(self): + def pkgdb_worker(self): branch = RELEASES[self.release]["branch"] while True: - package = self.people_queue.get() - if package not in self.people_dict: - people_ = get_people(package, branch) - self.people_dict[package] = people_ - self.people_queue.task_done() + package = self.pkgdbinfo_queue.get() + if package not in self.pkgdb_dict: + pkginfo = PKGDBInfo(package, branch) + self.pkgdb_dict[package] = pkginfo + self.pkgdbinfo_queue.task_done() def recursive_deps(self, packages, max_deps=20): # Start threads to get information about (co)maintainers for packages for i in range(0, 2): - people_thread = Thread(target=self.people_worker) + people_thread = Thread(target=self.pkgdb_worker) people_thread.daemon = True people_thread.start() # keep pylint silent @@ -414,7 +425,7 @@ class DepChecker(object): # get a list of all rpm_pkgs that are to be removed rpm_pkg_names = [] for name in packages: - self.people_queue.put(name) + self.pkgdbinfo_queue.put(name) # Empty list if pkg is only for a different arch bin_pkgs = self.by_src.get(name, []) rpm_pkg_names.extend([p.name for p in bin_pkgs]) @@ -456,7 +467,7 @@ class DepChecker(object): ).setdefault(pkg, set()).add(dep) for srpm_name in new_srpm_names: - self.people_queue.put(srpm_name) + self.pkgdbinfo_queue.put(srpm_name) ignore.extend(new_names) if allow_more: @@ -473,9 +484,9 @@ class DepChecker(object): " completed\n".format(max_deps, name)) sys.stderr.write("Waiting for (co)maintainer information...") - self.people_queue.join() + self.pkgdbinfo_queue.join() sys.stderr.write("done\n") - write_cache(self.people_dict, self.people_cache) + write_cache(self.pkgdb_dict, self.pkgdb_cache) return dep_map # This function was stolen from pungi @@ -497,7 +508,7 @@ class DepChecker(object): sys.exit(1) -def maintainer_table(packages, people_dict): +def maintainer_table(packages, pkgdb_dict): affected_people = {} if with_texttable: @@ -509,7 +520,7 @@ def maintainer_table(packages, people_dict): table = "" for package_name in packages: - people = people_dict[package_name] + people = pkgdb_dict[package_name].get_people() for p in people: affected_people.setdefault(p, set()).add(package_name) p = ', '.join(people) @@ -524,14 +535,14 @@ def maintainer_table(packages, people_dict): return table, affected_people -def dependency_info(dep_map, affected_people, people_dict): +def dependency_info(dep_map, affected_people, pkgdb_dict): info = "" for package_name, subdict in dep_map.items(): if subdict: info += "Depending on: {} ({})\n".format(package_name, len(subdict.keys())) for fedora_package, dependent_packages in subdict.items(): - people = people_dict[fedora_package] + people = pkgdb_dict[fedora_package].get_people() for p in people: affected_people.setdefault(p, set()).add(package_name) p = ", ".join(people) @@ -555,13 +566,13 @@ def maintainer_info(affected_people): return info -def package_info(packages, dep_map, people_dict, orphans=None, failed=None): +def package_info(packages, dep_map, pkgdb_dict, orphans=None, failed=None): info = "" - table, affected_people = maintainer_table(packages, people_dict) + table, affected_people = maintainer_table(packages, pkgdb_dict) info += table info += "\n\nThe following packages require above mentioned packages:\n" - info += dependency_info(dep_map, affected_people, people_dict) + info += dependency_info(dep_map, affected_people, pkgdb_dict) info += "Affected (co)maintainers\n" info += maintainer_info(affected_people) @@ -656,7 +667,7 @@ def main(): # TODO: add app args to either depsolve or not dep_map = depchecker.recursive_deps(unblocked) sys.stderr.write('done\n') - info, addresses = package_info(unblocked, dep_map, depchecker.people_dict, + info, addresses = package_info(unblocked, dep_map, depchecker.pkgdb_dict, orphans=orphans, failed=failed) text += "\n" text += info

1 0

while true; do sleep 1h; rpm-ostree compose tree...; done
by Colin Walters 17 Oct '14

17 Oct '14

Hi, right now the Atomic host is kind of missing its most important part - the ability to receive updates. I am actually kind of confused as to what the blockers are, because it seems like it'd be basically a two line shell script in $SUBJECT run as a daemon. I've been doing this on the atomic01.qa.fedoraproject.org and pushing it out to alt.fp.org, but I'd like to figure out how we move forward on the official compose. I think Dennis said something about filing a ticket for a location? Where is that? Are there any other blockers? Storage? Tool documentation?

6 14

[PATCHES] system-release-product virtual provides and rename of -standard to -nonproduct
by Stephen Gallagher 17 Oct '14

17 Oct '14

Patches are for F21 branch, master will need a merge. We need to ensure that at least one of the fedora-release-[nonproduct| $PRODUCT] packages is always installed. Also, based on feedback on and off-list, we've just decided to run with -nonproduct rather than -standard (which everyone agreed was not good).

2 4

[releng] find_unblocked_orphans: Clarify instructions
by Till Maas 16 Oct '14

16 Oct '14

commit 7c7bc5bb25269e6e59a72bf51e1815f3d91d09d8 Author: Till Maas <opensource(a)till.name> Date: Thu Oct 16 18:30:26 2014 +0200 find_unblocked_orphans: Clarify instructions scripts/find_unblocked_orphans.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) --- diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py index b47ef2f..d485d6d 100755 --- a/scripts/find_unblocked_orphans.py +++ b/scripts/find_unblocked_orphans.py @@ -105,8 +105,8 @@ should be retired, please do so now with a proper reason: https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life Note: If you received this mail directly you (co)maintain one of the affected -packages or a package that depends on one. Please orphan the affected package -or retire your package to avoid broken dependencies. +packages or a package that depends on one. Please adopt the affected package or +retire your depending package to avoid broken dependencies. """ FOOTER = """-- \nThe script creating this output is run and developed by Fedora

1 0

Fedora Release Engineering meeting summary for 2014-10-13
by Dennis Gilmore 14 Oct '14

14 Oct '14

-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 ====================================== #fedora-meeting-1: RELENG (2014-10-13) ====================================== Meeting started by dgilmore-bne at 14:34:27 UTC. The full logs are available at http://meetbot.fedoraproject.org/fedora-meeting-1/2014-10-13/releng.2014-10… . Meeting summary - --------------- * init process (dgilmore-bne, 14:34:47) * Secondary Architectures updates (dgilmore-bne, 14:41:06) * Secondary Architectures update - ppc (dgilmore-bne, 14:41:11) * Secondary Architectures update - s390 (dgilmore-bne, 14:48:41) * Secondary Architectures update - arm (dgilmore-bne, 14:50:25) * #5931 [Proposal] Move new branch and unretire requests to pkgdb2 (dgilmore-bne, 14:58:59) * LINK: https://fedorahosted.org/rel-eng/ticket/5931 (dgilmore-bne, 14:59:20) * open floor (dgilmore-bne, 15:28:34) Meeting ended at 15:33:57 UTC. Action Items - ------------ Action Items, by person - ----------------------- * **UNASSIGNED** * (none) People Present (lines said) - --------------------------- * dgilmore-bne (66) * pingou (29) * pbrobinson (22) * nirik (8) * zodbot (4) * sharkcz (4) * pbabinca (3) * bochecha (1) * tyll (0) * masta (0) * dgilmore (0) Generated by `MeetBot`_ 0.1.4 .. _`MeetBot`: http://wiki.debian.org/MeetBot -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 iQIcBAEBAgAGBQJUPHcrAAoJEH7ltONmPFDR/cUP/28Yw2H4e80/NSGa2j+oAeMg BOqANdkt2aV98gz6Ut/p7a8SAYUpSbzD7+CILomSBfdEnLeswlwSzvKF2VizjvOD jNOK1teKmhCogyy5c3b4puuFZ5Kp1MwXtmjTotj7dBu/DSUL8VRQf6twJwpBwUn3 6Rtx4IVzqdgTestGeT0706HDDiLAE+sQbPa1w9/tfqoYp6L5ZgcGrqEe9DQdjIwF 0LukTjGyRrEC0PgTMYsQOd2sS5K55a/5Xvix+JqTn71psdNPU0HomlBf/QuWcuon KzosN6+/HxY5OAIXPiNEUJnq2bC8E+ClNQ+UMXRQ1rEp9MFSyTeTp5bk5g5HiisP 0vpNBdPUoPjn/fdnMlY+b1Ers6HD3aDNGjCAoEoqmsVG2tPMyhXmLthNKZX6MugP MPITb3sfSFuHHo1750LHIAhV+IQh5c93OtrypV/XeHKFLQ5R//9T5B3nEzKNdE8H I5jvx9yj4wwbUJQesqHHxiyDbNDadZdY7Slzf6yD6Cco4KekD/5klEp/ysoN3iQj zlr7phaJPz5Q4fwJgROBuSDGDnq64kA4FcS08WowNshlRxbf6+Ck7d9sbCsbe1kN KWGOkZ0SDBn9yynlUtLVxy1IWDOFuQCGUVlwEBHS56VzvV7sbaMcWhCo4hUVLIQd 8LVwX9E14Z5IxcF/+euP =YQmS -----END PGP SIGNATURE-----

1 0

[Fedocal] Reminder meeting : Fedora Release Engineering
by Dennis Gilmore 13 Oct '14

13 Oct '14

Dear all, You are kindly invited to the meeting: Fedora Release Engineering on 2014-10-13 from 15:30:00 to 16:30:00 UTC At fedora-meeting-1(a)irc.freenode.net The meeting will be about: Fedora Release Engineering weekly meeting Source: https://apps.fedoraproject.org/calendar//meeting/162/

1 0

[releng] Fix line widths
by Till Maas 10 Oct '14

10 Oct '14

commit 45be26a7d7308840abeaef942822591a202a8860 Author: Till Maas <opensource(a)till.name> Date: Fri Oct 10 20:18:54 2014 +0200 Fix line widths scripts/find_unblocked_orphans.py | 22 +++++++++++++--------- 1 files changed, 13 insertions(+), 9 deletions(-) --- diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py index 9687377..b47ef2f 100755 --- a/scripts/find_unblocked_orphans.py +++ b/scripts/find_unblocked_orphans.py @@ -71,7 +71,8 @@ RAWHIDE_RELEASE = dict( BRANCHED_RELEASE = dict( repo='https://kojipkgs.fedoraproject.org/mash/branched/i386/os', - source_repo='https://kojipkgs.fedoraproject.org/mash/branched/source/SRPMS', + source_repo='https://kojipkgs.fedoraproject.org/mash/branched/source/' + 'SRPMS', tag='f21', branch='f21', mailto='devel(a)lists.fedoraproject.org', @@ -98,14 +99,14 @@ According to https://fedoraproject.org/wiki/Schedule branching will occur not earlier than 2014-07-08. The packages will be retired shortly before. """ -HEADER = """The following packages are orphaned and might be retired eventually, -unless someone adopts them. If you know for sure that the package should be -retired, please do so now with a proper reason: +HEADER = """The following packages are orphaned and might be retired +eventually, unless someone adopts them. If you know for sure that the package +should be retired, please do so now with a proper reason: https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life Note: If you received this mail directly you (co)maintain one of the affected -packages or a package that depends on one. Please orphan the affected package or -retire your package to avoid broken dependencies. +packages or a package that depends on one. Please orphan the affected package +or retire your package to avoid broken dependencies. """ FOOTER = """-- \nThe script creating this output is run and developed by Fedora @@ -338,7 +339,8 @@ class DepChecker(object): rpms = self.by_src[srpmname] except KeyError: # If we don't have a package in the repo, there is nothing to do - sys.stderr.write("Package {0} not found in repo\n".format(srpmname)) + sys.stderr.write( + "Package {0} not found in repo\n".format(srpmname)) rpms = [] # provides of all packages built from ``srpmname`` @@ -490,7 +492,8 @@ class DepChecker(object): arch='src')[0] return srpmpo except IndexError: - print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm + sys.stderr.write( + "Error: Cannot find a source rpm for {}\n".format(srpm)) sys.exit(1) @@ -579,7 +582,8 @@ def package_info(packages, dep_map, people_dict, orphans=None, failed=None): orphans_breaking_deps = [o for o in orphans if o in dep_map and dep_map[o]] - info += wrap_and_format("Orphans (dependend on)", orphans_breaking_deps) + info += wrap_and_format("Orphans (dependend on)", + orphans_breaking_deps) orphans_not_breaking_deps = [o for o in orphans if o not in dep_map or not dep_map[o]]

1 0

discussion on Fedora Council Engineering representative
by Matthew Miller 10 Oct '14

10 Oct '14

Spamming many lists to make sure this doesn't go unnoticed. (In case my accidentally posting to both devel and devel-announce separately doesn't do it.) Please see the thread titled "Engineering Representiatve for the new Fedora Council" on the devel list — this affects and should have the input of everyone involved in the broad category of "building and shipping the distro" efforts in Fedora. Thanks! -- Matthew Miller <mattdm(a)fedoraproject.org> Fedora Project Leader

1 0

Pkgdb2 - Update on new branch/package process
by Pierre-Yves Chibon 10 Oct '14

10 Oct '14

Dear all, Sorry for the long silence about this. Here is where we are now (in git, nothing merged/in prod atm): * user can request new packages on pkgdb * user can request a new branch for a package on pkgdb * these actions are recorded and stored in pkgdb Available at: http://209.132.184.188/admin/actions/ and http://209.132.184.188/api/admin/actions/ * there is a new pkgdb-admin part of pkgdb-cli allowing to list actions and act of them (approve (ie: process)/deny) -> the pkgdb UI only updates the status of the action, while the cli pkgdb-admin actually process the request (as well as updating the status of the action at the end) -> pkgdb-admin would be the place where we would add all the checks we want to do before processing a package/branch request fedmsg messages will be broadcasted for new branch. This part is all there and basically ready :) The second part is about listening to fedmsg message about new branches and adjusting the git repos accordingly. This is where things are a little more hairy. Currently, we have fedmsg-pkgdb-sync-git [1] that basically listens for pkgdb messages on the bus and run the ansible playbook: run_pkgdb_sync_git.yml [2] (which in turn runs pkgdb_sync_git_branches.py) when needed. The problem which I am facing is in the case of new EPEL branch creation. These branches are created from another one. For example in the case of the epel7 branch we used (are we still?) to branch either from either el6 or f19. With that in mind, I allowed in pkgdb to specify from which branch one would like to get the new branch. The problem is that this information is not available to fedmsg-pkgdb-sync-git and even if it was, we would not be able to pass it along to the ansible playbook. So afaics the options are: 1/ We hard-code somewhere from which branch each branch should come (ie: f* comes from master, el6 comes from ?, epel7 comes from f19...) 2/ We manage to create the branch in such a way that they point to the first commit in the repo (suggestion from Till), from there the maintainers should be able to fast-forward to the branch they think should apply 3/ We manage to just create the branch in gitolite and just let the maintainer populate it by merging the branch they think should apply I have a preference for #3, but our tests with bochecha for it didn't quite succeed. But in all 3 options, I basically can drop the `branch_from` field when requesting a new branch in pkgdb. So what is our preference and how do we want to proceed from here? Thanks, Pierre [1] https://github.com/fedora-infra/fedmsg-pkgdb-sync-git [2] https://infrastructure.fedoraproject.org/infra/ansible/playbooks/run_pkgdb_…

1 1

[releng] find_unblocked_orphans: refactor a little
by Till Maas 09 Oct '14

09 Oct '14

commit ff5c84609f0d8f8b03850d64d08dc2323a0eecfc Author: Till Maas <opensource(a)till.name> Date: Thu Oct 9 19:17:41 2014 +0200 find_unblocked_orphans: refactor a little scripts/find_unblocked_orphans.py | 32 +++++++++++++++++--------------- 1 files changed, 17 insertions(+), 15 deletions(-) --- diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py index 2760397..9687377 100755 --- a/scripts/find_unblocked_orphans.py +++ b/scripts/find_unblocked_orphans.py @@ -552,27 +552,20 @@ def maintainer_info(affected_people): return info -def package_info(packages, release, orphans=None, failed=None): - sys.stderr.write("Setting up dependency checker...") - depchecker = DepChecker(release) - sys.stderr.write("done\n") +def package_info(packages, dep_map, people_dict, orphans=None, failed=None): info = "" - sys.stderr.write('Calculating dependencies...') - # Create yum object and depsolve out if requested. - # TODO: add app args to either depsolve or not - dep_map = depchecker.recursive_deps(packages) - sys.stderr.write('done\n') - table, affected_people = maintainer_table(packages, depchecker.people_dict) + table, affected_people = maintainer_table(packages, people_dict) info += table info += "\n\nThe following packages require above mentioned packages:\n" - info += dependency_info(dep_map, affected_people, depchecker.people_dict) + info += dependency_info(dep_map, affected_people, people_dict) info += "Affected (co)maintainers\n" info += maintainer_info(affected_people) wrapper = textwrap.TextWrapper( - break_long_words=False, subsequent_indent=" ", break_on_hyphens=False + break_long_words=False, subsequent_indent=" ", + break_on_hyphens=False ) def wrap_and_format(label, pkgs): @@ -624,7 +617,8 @@ def main(): parser.add_argument("--skip-orphans", dest="skip_orphans", help="Do not look for orphans", default=False, action="store_true") - parser.add_argument("--release", choices=RELEASES.keys(), default="rawhide") + parser.add_argument("--release", choices=RELEASES.keys(), + default="rawhide") parser.add_argument("--mailto", default=None, help="Send mail to this address (for testing)") parser.add_argument( @@ -650,8 +644,16 @@ def main(): sys.stderr.write('done\n') text = HEADER.format(RELEASES[args.release]["tag"].upper()) - info, addresses = package_info(unblocked, args.release, orphans=orphans, - failed=failed) + sys.stderr.write("Setting up dependency checker...") + depchecker = DepChecker(args.release) + sys.stderr.write("done\n") + sys.stderr.write('Calculating dependencies...') + # Create yum object and depsolve out if requested. + # TODO: add app args to either depsolve or not + dep_map = depchecker.recursive_deps(unblocked) + sys.stderr.write('done\n') + info, addresses = package_info(unblocked, dep_map, depchecker.people_dict, + orphans=orphans, failed=failed) text += "\n" text += info text += FOOTER

1 0

← Newer
1
...
8
9
10
11
12
13
14
15
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

rel-eng October 2014