commit ce29f0dfdd4cdc4f80a80422c8c7cd7a729bf492 Author: Till Maas opensource@till.name Date: Mon Oct 6 22:23:39 2014 +0200
find_unblocked_orphans: Introduce DepChecker()
Make it easier to check for other releases than Rawhide.
scripts/find_unblocked_orphans.py | 375 +++++++++++++++++++------------------ 1 files changed, 191 insertions(+), 184 deletions(-) --- diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py index b1ee20a..f588cb4 100755 --- a/scripts/find_unblocked_orphans.py +++ b/scripts/find_unblocked_orphans.py @@ -30,24 +30,29 @@ try: except ImportError: with_texttable = False
-# Set some variables -# Some of these could arguably be passed in as args. -# If this is pre-branch, these repos should be rawhide; otherwise, -# they should be branched. -DEFAULT_REPO = 'http://kojipkgs.fedoraproject.org/mash/rawhide/i386/os' -DEFAULT_SOURCE_REPO = \ - 'http://kojipkgs.fedoraproject.org/mash/rawhide/source/SRPMS' -TAG = 'f21' # tag to check in koji - -# pre-branch, this should be master'. Post-branch, it will be something like -# fXY -RAWHIDE_BRANCHNAME = 'master' # pkgdb name for the devel branch + +RAWHIDE_RELEASE = dict( + repo='https://kojipkgs.fedoraproject.org/mash/rawhide/i386/os', + source_repo='https://kojipkgs.fedoraproject.org/mash/rawhide/source/SRPMS', + tag='f22', + branch='master') + +BRANCHED_RELEASE = dict( + repo='https://kojipkgs.fedoraproject.org/mash/branched/i386/os', + source_repo='https://kojipkgs.fedoraproject.org/mash/branched/source/SRPMS', + tag='f21', + branch='f21') + +RELEASES = { + "rawhide": RAWHIDE_RELEASE, + "branched": BRANCHED_RELEASE, +}
# pkgdb uid for orphan ORPHAN_UID = 'orphan'
HEADER = """The following packages are orphaned or did not build for two -releases and will be retired when Fedora ({0}) is branched, unless someone +releases and will be retired when Fedora ({}) is branched, unless someone adopts them. If you know for sure that the package should be retired, please do so now with a proper reason: https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life @@ -57,7 +62,7 @@ occur not earlier than 2014-07-08. The packages will be retired shortly before.
Note: If you received this mail directly you (co)maintain one of the affected packages or a package that depends on one. -""".format(TAG.upper()) +"""
FOOTER = """The script creating this output is run and developed by Fedora Release Engineering. Please report issues at its trac instance: @@ -115,7 +120,7 @@ people_queue = Queue() people_dict = get_cache("orphans-people.pickle", default={})
-def get_people(package, branch=RAWHIDE_BRANCHNAME): +def get_people(package, branch=RAWHIDE_RELEASE["tag"]): def associated(pkginfo, exclude=None): """
@@ -138,16 +143,17 @@ def get_people(package, branch=RAWHIDE_BRANCHNAME): return people_
-def people_worker(): +def people_worker(tag): while True: package = people_queue.get() if package not in people_dict: - people_ = get_people(package) + people_ = get_people(package, tag) people_dict[package] = people_ people_queue.task_done()
-def setup_yum(repo=DEFAULT_REPO, source_repo=DEFAULT_SOURCE_REPO): +def setup_yum(repo=RAWHIDE_RELEASE["repo"], + source_repo=RAWHIDE_RELEASE["source_repo"]): """ Setup YumBase with two repos This code was mostly stolen from http://yum.baseurl.org/wiki/YumCodeSnippet/SetupArbitraryRepo @@ -168,28 +174,6 @@ def setup_yum(repo=DEFAULT_REPO, source_repo=DEFAULT_SOURCE_REPO): yb.arch.archlist.append('src') return yb
-sys.stderr.write("Setting up yum...") -yb = setup_yum() -sys.stderr.write("done\n") - - -# This function was stolen from pungi -def SRPM(package): - """Given a package object, get a package object for the - corresponding source rpm. Requires yum still configured - and a valid package object.""" - srpm = package.sourcerpm.split('.src.rpm')[0] - (sname, sver, srel) = srpm.rsplit('-', 2) - try: - srpmpo = yb.pkgSack.searchNevra(name=sname, - ver=sver, - rel=srel, - arch='src')[0] - return srpmpo - except IndexError: - print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm - sys.exit(1) -
def orphan_packages(cache_filename='orphans.pickle'): orphans = get_cache(cache_filename, default={}) @@ -209,7 +193,7 @@ def orphan_packages(cache_filename='orphans.pickle'): return orphans
-def unblocked_packages(packages, tagID=TAG): +def unblocked_packages(packages, tagID=RAWHIDE_RELEASE["tag"]): unblocked = [] kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
@@ -233,21 +217,22 @@ def unblocked_packages(packages, tagID=TAG): return unblocked
-class BinSrcMapper(object): - def __init__(self): +class DepChecker(object): + def __init__(self, yumbase): self._src_by_bin = None self._bin_by_src = None + self.yumbase = yumbase
def create_mapping(self): src_by_bin = {} # Dict of source pkg objects by binary package objects bin_by_src = {} # Dict of binary pkgobjects by srpm name - all_packages = yb.pkgSack.returnPackages() + all_packages = self.yumbase.pkgSack.returnPackages()
# Populate the dicts for rpm_package in all_packages: if rpm_package.arch == 'src': continue - srpm = SRPM(rpm_package) + srpm = self.SRPM(rpm_package) src_by_bin[rpm_package] = srpm if srpm.name in bin_by_src: bin_by_src[srpm.name].append(rpm_package) @@ -269,142 +254,155 @@ class BinSrcMapper(object): self.create_mapping() return self._src_by_bin
-sys.stderr.write("Setting up packager mapper...") -package_mapper = BinSrcMapper() -sys.stderr.write("done\n") - + def find_dependent_packages(self, srpmname, ignore): + """ Return packages depending on packages built from SRPM ``srpmname`` + that are built from different SRPMS not specified in ``ignore``.
-def find_dependent_packages(srpmname, ignore): - """ Return packages depending on packages built from SRPM ``srpmname`` - that are built from different SRPMS not specified in ``ignore``. + :param ignore: list of SRPMs of packages that will not be returned + as dependent packages. + :type ignore: list() of str()
- :param ignore: list of SRPMs of packages that will not be returned as - dependent packages. - :type ignore: list() of str() - - :returns: OrderedDict dependent_package: list of requires only provided - by package ``srpmname`` - {dep_pkg: [prov, ...]} - """ - # Some of this code was stolen from repoquery - dependent_packages = {} + :returns: OrderedDict dependent_package: list of requires only + provided by package ``srpmname`` {dep_pkg: [prov, ...]} + """ + # Some of this code was stolen from repoquery + dependent_packages = {}
- # Handle packags not found in the repo - try: - rpms = package_mapper.by_src[srpmname] - except KeyError: - # If we don't have a package in the repo, there is nothing to do - sys.stderr.write("Package {0} not found in repo\n".format(srpmname)) - rpms = [] - - # provides of all packages built from ``srpmname`` - provides = [] - for pkg in rpms: - # add all the provides from the package as strings - string_provides = [yum.misc.prco_tuple_to_string(prov) - for prov in pkg.provides] - provides.extend(string_provides) - - # add all files as provides - # pkg.files is a dict with keys like "file" and "dir" - # values are a list of file/dir paths - for paths in pkg.files.itervalues(): - # sometimes paths start with "//" instead of "/" - # normalise "//" to "/": - # os.path.normpath("//") == "//", but - # os.path.normpath("///") == "/" - file_provides = [os.path.normpath('//%s' % fn) - for fn in paths] - provides.extend(file_provides) - - # Zip through the provides and find what's needed - for prov in provides: - # check only base provide, ignore specific versions - # "foo = 1.fc20" -> "foo" - base_provide = prov.split()[0] - - # Elide provide if also provided by another package - for pkg in yb.pkgSack.searchProvides(base_provide): - # FIXME: might miss broken dependencies in case the other provider - # depends on a to-be-removed package as well - if pkg.sourcerpm.rsplit('-', 2)[0] not in ignore: - break - else: - for dependent_pkg in yb.pkgSack.searchRequires(base_provide): - # skip if the dependent rpm package belongs to the - # to-be-removed Fedora package - if dependent_pkg in package_mapper.by_src[srpmname]: - continue - - # skip if the dependent rpm package is also a - # package that should be removed - if dependent_pkg.name in ignore: - continue - - # use setdefault to either create an entry for the - # dependent package or add the required prov - dependent_packages.setdefault(dependent_pkg, set()).add(prov) - return OrderedDict(sorted(dependent_packages.items())) - - -def recursive_deps(packages, max_deps=20): - # get a list of all rpm_pkgs that are to be removed - rpm_pkg_names = [] - for name in packages: - # Empty list if pkg is only for a different arch - bin_pkgs = package_mapper.by_src.get(name, []) - rpm_pkg_names.extend([p.name for p in bin_pkgs]) - - # dict for all dependent package for each to-be-removed package - dep_map = OrderedDict() - for name in packages: - ignore = rpm_pkg_names - dep_map[name] = OrderedDict() - to_check = [name] - allow_more = True - seen = [] - while True: - sys.stderr.write("to_check: {0}\n".format(repr(to_check))) - check_next = to_check.pop() - seen.append(check_next) - dependent_packages = find_dependent_packages(check_next, ignore) - if dependent_packages: - new_names = [] - new_srpm_names = set() - for pkg, dependencies in dependent_packages.items(): - if pkg.arch != "src": - srpm_name = package_mapper.by_bin[pkg].name - else: - srpm_name = pkg.name - if srpm_name not in to_check and \ - srpm_name not in new_names and \ - srpm_name not in seen: - new_names.append(srpm_name) - new_srpm_names.add(srpm_name) - - for dep in dependencies: - dep_map[name].setdefault( - srpm_name, - OrderedDict() - ).setdefault(pkg, set()).add(dep) - - for srpm_name in new_srpm_names: - people_queue.put(srpm_name) - - ignore.extend(new_names) - if allow_more: - to_check.extend(new_names) - dep_count = len(set(dep_map[name].keys() + to_check)) - if dep_count > max_deps: - allow_more = False - to_check = to_check[0:max_deps] - if not to_check: - break - if not allow_more: - sys.stderr.write("More than {0} broken deps for package" - "'{1}', dependency check not" - " completed\n".format(max_deps, name)) - return dep_map + # Handle packags not found in the repo + try: + rpms = self.by_src[srpmname] + except KeyError: + # If we don't have a package in the repo, there is nothing to do + sys.stderr.write("Package {0} not found in repo\n".format(srpmname)) + rpms = [] + + # provides of all packages built from ``srpmname`` + provides = [] + for pkg in rpms: + # add all the provides from the package as strings + string_provides = [yum.misc.prco_tuple_to_string(prov) + for prov in pkg.provides] + provides.extend(string_provides) + + # add all files as provides + # pkg.files is a dict with keys like "file" and "dir" + # values are a list of file/dir paths + for paths in pkg.files.itervalues(): + # sometimes paths start with "//" instead of "/" + # normalise "//" to "/": + # os.path.normpath("//") == "//", but + # os.path.normpath("///") == "/" + file_provides = [os.path.normpath('//%s' % fn) + for fn in paths] + provides.extend(file_provides) + + # Zip through the provides and find what's needed + for prov in provides: + # check only base provide, ignore specific versions + # "foo = 1.fc20" -> "foo" + base_provide = prov.split()[0] + + # Elide provide if also provided by another package + for pkg in self.yumbase.pkgSack.searchProvides(base_provide): + # FIXME: might miss broken dependencies in case the other + # provider depends on a to-be-removed package as well + if pkg.sourcerpm.rsplit('-', 2)[0] not in ignore: + break + else: + for dependent_pkg in self.yumbase.pkgSack.searchRequires( + base_provide): + # skip if the dependent rpm package belongs to the + # to-be-removed Fedora package + if dependent_pkg in self.by_src[srpmname]: + continue + + # skip if the dependent rpm package is also a + # package that should be removed + if dependent_pkg.name in ignore: + continue + + # use setdefault to either create an entry for the + # dependent package or add the required prov + dependent_packages.setdefault(dependent_pkg, set()).add( + prov) + return OrderedDict(sorted(dependent_packages.items())) + + def recursive_deps(self, packages, max_deps=20): + # get a list of all rpm_pkgs that are to be removed + rpm_pkg_names = [] + for name in packages: + # Empty list if pkg is only for a different arch + bin_pkgs = self.by_src.get(name, []) + rpm_pkg_names.extend([p.name for p in bin_pkgs]) + + # dict for all dependent package for each to-be-removed package + dep_map = OrderedDict() + for name in packages: + ignore = rpm_pkg_names + dep_map[name] = OrderedDict() + to_check = [name] + allow_more = True + seen = [] + while True: + sys.stderr.write("to_check: {0}\n".format(repr(to_check))) + check_next = to_check.pop() + seen.append(check_next) + dependent_packages = self.find_dependent_packages(check_next, + ignore) + if dependent_packages: + new_names = [] + new_srpm_names = set() + for pkg, dependencies in dependent_packages.items(): + if pkg.arch != "src": + srpm_name = self.by_bin[pkg].name + else: + srpm_name = pkg.name + if srpm_name not in to_check and \ + srpm_name not in new_names and \ + srpm_name not in seen: + new_names.append(srpm_name) + new_srpm_names.add(srpm_name) + + for dep in dependencies: + dep_map[name].setdefault( + srpm_name, + OrderedDict() + ).setdefault(pkg, set()).add(dep) + + for srpm_name in new_srpm_names: + people_queue.put(srpm_name) + + ignore.extend(new_names) + if allow_more: + to_check.extend(new_names) + dep_count = len(set(dep_map[name].keys() + to_check)) + if dep_count > max_deps: + allow_more = False + to_check = to_check[0:max_deps] + if not to_check: + break + if not allow_more: + sys.stderr.write("More than {0} broken deps for package" + "'{1}', dependency check not" + " completed\n".format(max_deps, name)) + return dep_map + + # This function was stolen from pungi + def SRPM(self, package): + """Given a package object, get a package object for the + corresponding source rpm. Requires yum still configured + and a valid package object.""" + srpm = package.sourcerpm.split('.src.rpm')[0] + (sname, sver, srel) = srpm.rsplit('-', 2) + try: + srpmpo = self.yumbase.pkgSack.searchNevra(name=sname, + ver=sver, + rel=srel, + arch='src')[0] + return srpmpo + except IndexError: + print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm + sys.exit(1)
def maintainer_table(packages): @@ -464,12 +462,19 @@ def maintainer_info(affected_people): return info
-def package_info(packages, orphans=None, failed=None): +def package_info(packages, release, orphans=None, failed=None): + sys.stderr.write("Setting up yum...") + yumbase = setup_yum(repo=RELEASES[release]["repo"], + source_repo=RELEASES[release]["source_repo"]) + sys.stderr.write("done\n") + sys.stderr.write("Setting up dependency checker...") + depchecker = DepChecker(yumbase) + sys.stderr.write("done\n") info = "" sys.stderr.write('Calculating dependencies...') # Create yum object and depsolve out if requested. # TODO: add app args to either depsolve or not - dep_map = recursive_deps(packages) + dep_map = depchecker.recursive_deps(packages) sys.stderr.write('done\n')
sys.stderr.write("Waiting for (co)maintainer information...") @@ -515,7 +520,7 @@ def package_info(packages, orphans=None, failed=None): return info, addresses
-def main(): +def main(release="rawhide"): parser = argparse.ArgumentParser() parser.add_argument("--skip-orphans", dest="skip_orphans", help="Do not look for orphans", @@ -535,7 +540,8 @@ def main():
# Start threads to get information about (co)maintainers for packages for i in range(0, 2): - people_thread = Thread(target=people_worker) + people_thread = Thread(target=people_worker, + args=(RELEASES[release]["tag"],)) people_thread.daemon = True people_thread.start() # keep pylint silent @@ -545,8 +551,9 @@ def main(): unblocked = unblocked_packages(sorted(list(set(list(orphans) + failed)))) sys.stderr.write('done\n')
- print HEADER - info, addresses = package_info(unblocked, orphans=orphans, failed=failed) + print HEADER.format(RELEASES[release]["tag"].upper()) + info, addresses = package_info(unblocked, release, orphans=orphans, + failed=failed) print info print FOOTER
rel-eng@lists.fedoraproject.org