If a package from older release is used (as there is no build for the current release if the package was not updated for the current release), store it in a separate folder called for_<release id>. Otherwise the package would be downloaded again each time, making the updates image creation slow. Without the subfolder makeupdates would not be enable to distinguish which cached packages are outdated and should not be used and which packages just don't have a new build and should be used. Signed-off-by: Martin Kolman <mkolman(a)redhat.com&gt; --- scripts/makeupdates | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/scripts/makeupdates b/scripts/makeupdates index 3b4fe63..8d36d79 100755 --- a/scripts/makeupdates +++ b/scripts/makeupdates @@ -39,6 +39,7 @@ except ImportError: exit(1) RPM_FOLDER_NAME = os.path.expanduser("~/.anaconda_updates_rpm_cache") +RPM_RELEASE_DIR_TEMPLATE = "for_%s" KOJI_BASE_URL = "http://kojipkgs.fedoraproject.org//packages/" \ "%(toplevel_name)s/%(toplevel_version)s/%(release)s/%(arch)s/%(rpm_name)s" @@ -190,6 +191,10 @@ def download_to_file(url, path): or None if the download fails for some reason """ try: + # try to make sure the folder for the download exists + download_folder = os.path.split(path)[0] + if not os.access(download_folder, os.W_OK): + os.makedirs(download_folder) result = urllib.urlretrieve(url, path) # return the storage path return result[0] @@ -577,9 +582,13 @@ def remove_local_packages(packages, arch, release_id, added_rpms): """Remove locally available RPMs from the list of needed packages, return locally unavailable packages and paths to relevant locally available RPMs for inclusion""" + current_release_dir = RPM_RELEASE_DIR_TEMPLATE % release_id # list all package names and version for the RPMs already in cache folder_glob = os.path.join(RPM_FOLDER_NAME, "*.rpm") folder_glob = os.path.abspath(folder_glob) + release_folder_glob = os.path.join(RPM_FOLDER_NAME, current_release_dir, "*.rpm") + release_folder_glob = os.path.abspath(release_folder_glob) + include_rpms = [] skipped_packages = [] @@ -603,7 +612,11 @@ def remove_local_packages(packages, arch, release_id, added_rpms): # currently specified architecture allowed = ("noarch.rpm", "%s.rpm" % arch) relevant_rpms = [x for x in glob.glob(folder_glob) if x.endswith(allowed)] - + # also add any RPMS from the current release folder + # (has RPMs from older releases that were not yet rebuilt + # for the current release) + relevant_rpms.extend([x for x in glob.glob(release_folder_glob) + if x.endswith(allowed)]) # iterate over all relevant cached RPMs and check if they are needed for rpm_path in relevant_rpms: proc = subprocess.Popen(['rpm', '-qp', '--queryformat', @@ -616,9 +629,16 @@ def remove_local_packages(packages, arch, release_id, added_rpms): name, version, release = proc_output[0].split() # get the build number and release id build_id, package_release_id = release.split(".", 1) - # skip cached RPMs that have different release id - if package_release_id != release_id: - continue + + # If a package is stored in the for_<release id> + # subfolder, we don't check its release id, + # because it as package that has not been rebuilt + # for a new release but it still the latest version. + # If a package is not stored in a for_<release id> subfolder, + # we check the release id to filter out old cached packages. + if not os.path.split(rpm_path)[0].endswith(current_release_dir): + if package_release_id != release_id: + continue # add the build id to the version string version_build = "%s-%s" % (version, build_id) # check if the package is needed @@ -792,8 +812,17 @@ def get_rpm_from_Koji_thread(package, fedora_number, arch, url = KOJI_BASE_URL % package_metadata # simple, isn't it ? :) - # append the RPM cache folder to the filename - download_path = os.path.join(RPM_FOLDER_NAME, rpm_name) + # build RPM storage path + release_dir = "" + if release_number_override: + # Using package from older release, store it in a sub-folder + # so that it is not downloaded again each time. + release_id = "fc%d" % fedora_number + release_dir = RPM_RELEASE_DIR_TEMPLATE % release_id + # if a package from and older release is used, the release subfolder is + # added to the storage path, otherwise the package is downloaded to the + # main folder + download_path = os.path.join(RPM_FOLDER_NAME, release_dir, rpm_name) # check if the download was successful storage_path = download_to_file(url, download_path) if storage_path is not None: -- 1.8.4.2

On Thu, 2013-11-28 at 20:06 +0100, Martin Kolman wrote: ^^ is a -- Vratislav Podzimek Anaconda Rider | Red Hat, Inc. | Brno - Czech Republic

Some packages, as for example NetworkManager, use long multi-component release ids, such as: 19.git20131003.fc21 To handle such cases, we need to split the string from the right and get the rightmost component. Signed-off-by: Martin Kolman <mkolman(a)redhat.com&gt; --- scripts/makeupdates | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/makeupdates b/scripts/makeupdates index 8d36d79..b1850e3 100755 --- a/scripts/makeupdates +++ b/scripts/makeupdates @@ -158,8 +158,9 @@ def check_package_version(filename, package, check_release_id=True): pkg_tuple = get_pkg_tuple(filename) if check_release_id: # get release ids for request and the package - request_release = package.req_tuple[2][2] - package_release = pkg_tuple[4] + # and strip it from any build/git garbage + request_release = package.req_tuple[2][2].rsplit(".", 1).pop() + package_release = pkg_tuple[4].rsplit(".", 1).pop() # rangeCheck actually ignores different release ids, # so we need to do it here if request_release != package_release: @@ -628,7 +629,7 @@ def remove_local_packages(packages, arch, release_id, added_rpms): continue name, version, release = proc_output[0].split() # get the build number and release id - build_id, package_release_id = release.split(".", 1) + build_id, package_release_id = release.rsplit(".", 1) # If a package is stored in the for_<release id> # subfolder, we don't check its release id, -- 1.8.4.2