From abda000979aa19361f701171dee9fcf5743d4c5f Mon Sep 17 00:00:00 2001
From: Carlos O'Donell <carlos@systemhalted.org>
Date: Thu, 1 Mar 2018 12:02:23 -0800
Subject: [PATCH 3/3] Add formal ABI verification for all shared objects.

The spec file is enhanced to allow a developer to save (save_abi=1)
the ABI of the package, and to turn on verification of the ABI
(verify_abi=1). The saved and manually inspected ABI for aarch64,
armv7hl, i686, ppc64, ppc64le, s390x, and x86_64 are also checked
in as compressed archives. When a release branch is in effect
(without the .9000 subrevision) the spec file automatically turns
ABI verification on and compares, during the install phase, the
generated ABI of the installed shared objects before stripping.
Carrying out the ABI verification at this point yields the highest
quality debug information, and least likelihood of problems. It is
expected that the distribution will do it's own rpm to rpm checking.
This ABI checking is meant as a glibc developer tool to assist in
catching ABI changes before deployment.

All ABI checks are against SONAMEs with the exception of multilibs
which we still treat as a distinct ABI for now.  We integrate the
upstream glibc-abi project data for the ABI baseline.
---
 glibc.spec | 277 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 sources    |   1 +
 2 files changed, 277 insertions(+), 1 deletion(-)

diff --git a/glibc.spec b/glibc.spec
index 0a17bf6..c8c97f1 100644
--- a/glibc.spec
+++ b/glibc.spec
@@ -1,6 +1,6 @@
 %define glibcsrcdir glibc-2.27.9000-188-gda6d4404ec
 %define glibcversion 2.27.9000
-%define glibcrelease 9%{?dist}
+%define glibcrelease 10%{?dist}
 # Pre-release tarballs are pulled in from git using a command that is
 # effectively:
 #
@@ -11,6 +11,16 @@
 # glibc_release_url is only defined when we have a release tarball.
 %{lua: if string.match(rpm.expand("%glibcsrcdir"), "^glibc%-[0-9.]+$") then
   rpm.define("glibc_release_url https://ftp.gnu.org/gnu/glibc/") end}
+
+# If we are on a released branch e.g. 2.27 (without a 9000 in the version),
+# then we must verify the built ABI against the frozen ABI specification.
+# This verification is carried out as part of the install phase since it
+# simplifies the process e.g. before stripping and debug splitting.
+%{lua: if not string.match(rpm.expand("%glibcversion"), "^.*9000$") then
+  rpm.define("verify_abi 1") end}
+# If warn_abi is set to 1 then ABI verification errors are downgraded to
+# warnings. Must be set to 0 for a release.
+%define warn_abi 0
 ##############################################################################
 # We support the following options:
 # --with/--without,
@@ -134,6 +144,18 @@ Source11: SUPPORTED
 # Include in the source RPM for reference.
 Source12: ChangeLog.old
 
+# Include the serialized frozen ABI specification from the glibc-abi project.
+%if %{without bootstrap}
+Source13: glibc-abi-glibc-2.27.9000.tar.gz
+%endif
+
+# When a final release is about to be uploaded we do the following:
+# Set warn_abi to 1, set save_abi to 1, do a scratch build, download all the
+# glibc rpms, extract all frozen ABIs, examine them for errors, examine the
+# logs for ABI warnings and make sure we understand them, and lastly update the
+# existing frozen ABIs. Must be set to 0 for a release.
+%define save_abi 0
+
 ##############################################################################
 # Start of glibc patches
 ##############################################################################
@@ -297,6 +319,14 @@ BuildRequires: bison >= 2.7
 # binutils 2.29 is needed for static PIE support in i386/x86_64.
 BuildRequires: binutils >= 2.29
 
+# For a released glibc we must compare against the frozen ABI.
+# We need libabigail if we are verifying or saving an ABI.
+%if %{without bootstrap}
+%if 0%{?verify_abi}%{?save_abi}
+BuildRequires: libabigail
+%endif
+%endif
+
 # Earlier releases have broken support for IRELATIVE relocations
 Conflicts: prelink < 0.4.2
 
@@ -1277,6 +1307,239 @@ mkdir -p %{glibc_sysroot}%{_libdir}
 mv -f %{glibc_sysroot}/%{_lib}/lib{pcprofile,memusage}.so \
 	%{glibc_sysroot}%{_libdir}
 
+###############################################################################
+# At this point all files, except libpthread.a are installed, and ready for
+# ABI analysis. We only analyze shared objects.
+###############################################################################
+
+# Read file $1, which is a null-separated list of paths, and process each entry
+# looking for valid ELF files with SONAMEs, and for each one that is valid use
+# abidw to dump the ABI specification with suffix of .$2 in the same directory,
+# and for each file dumped list it in $3 as a null-separated list. Lastly,
+# count is kept in $4 (variable by reference).
+function generate_abi_files ()
+{
+    local so_files="$1"
+    local dso_files="$2"
+    local abi_suffix="$3"
+    local -n abi_count="$4"
+
+    while IFS="" read -r -d $'\0' file ; do
+	echo "Processing $file"
+	if file "$file" | grep -q ' ELF '; then
+
+	    # Find object SONAME. We want comparisons against SONAMEs
+	    # to avoid versioned name changes breaking the ABI checking.
+	    # We are really storing and comparing SONAME ABIs.
+	    soname=$(readelf -d -W "$file" | grep SONAME | sed -e 's,^.*\[\(.*\)\]$,\1,g')
+	    is_gconv=$(echo "$file" | grep "/gconv/" || true)
+	    is_audit=$(echo "$file" | grep "/audit/" || true)
+
+	    # All objects except gconv modules and audit must have a SONAME.
+	    if [ -z "$soname" -a -z "$is_gconv" -a -z "$is_audit" ]; then
+		echo "File ${file} lacks dynamic SONAME entry! Error!"
+		exit 1
+	    fi
+
+	    # Without a SONAME we just check the ABI for the file.
+	    if [ -z "$soname" ]; then
+		file_abi="${file}"
+	    else
+		# Assume that the SONAME symlink is in the same directory as the
+		# installed DSO. While this need not be true, it is true for glibc and
+		# ldconfig. The SONAME named DSO can exist anywhere on any of the paths
+		# searched by the dynamic loader including ld.so.conf configured paths.
+		file_dir=$(dirname "${file}")
+		file_abi="${file_dir}/${soname}"
+	    fi
+
+	    # Final file to dump ABI for must exist.
+	    if [ ! -e "$file_abi" ]; then
+		echo "File ${file_abi} does not exist! Error!"
+		exit 1
+	    fi
+
+	    echo "Generating ${file_abi}.${abi_suffix}"
+	    abidw "$file" --out-file "${file_abi}.${abi_suffix}"
+	    echo -e -n "${file_abi}\x0" >> "$dso_files"
+	    abi_count=$(( $abi_count + 1 ))
+	fi
+    done < "$so_files"
+}
+
+%if %{without bootstrap}
+%if 0%{?save_abi}
+# Enter the build root, look for valid ELF DSOs, serialize their ABI, and store
+# it in a machine specific frozen-abi-ARCH.tar.xz archive.
+pushd %{glibc_sysroot}
+rm -f abi_files.list
+rm -f shared_objects.list
+# For now we treat the default multilib, and other non-default multilib as-if
+# they had distinct ABIs. This is not true, they should have compatible ABIs,
+# but we manage them with distinct files. It would be an enhancement to compare
+# all files with the same SONAME against a singular ABI definition for that
+# SONAME. We don't do that right now, instead we duplicate the definition.
+find . -name '*.so' -type f -print0 > so_files.list
+count=0
+# Generate the ABI files. We don't use the shared_objects.list, we just
+# regenerate the list with find because we want .abi.xml at the end.
+generate_abi_files "so_files.list" "shared_objects.list" "abi.xml" "count"
+echo "Genreated $count ABI files."
+find . -name '*.abi.xml' -print0 > abi_files.list
+tar --null -Jcvf frozen-abi.tar.xz --verbatim-files-from \
+	-T abi_files.list
+mv frozen-abi.tar.xz frozen-abi-%{_target_cpu}.tar.xz
+cat abi_files.list | xargs --null rm -f
+rm so_files.list
+rm shared_objects.list
+rm abi_files.list
+popd
+# save_abi == 1
+%endif
+
+%if 0%{?verify_abi}
+# If the glibc ABI is frozen then we must carry out release ABI verification
+# using libabigail. We embedded this into the package and do not rely on any
+# external verification. We want the package install phase to fail if there is
+# any problems with the comparison. Failing at later packaging stages is too
+# late. We expect external distribution tooling to do a rpm to rpm comparison.
+#
+# 1. Unpack frozen ABI representations.
+pushd %{glibc_sysroot}
+tar zxvf %{SOURCE13}
+
+# Map fedora rpm target arches to upstream glibc triplets and their ABIs
+# as used by the glibc-abi project.
+function arch_to_triplet ()
+{
+    local -n arch=$1
+    case $arch in
+	aarch64) arch="aarch64-linux-gnu" ;;
+	armv7hl) arch="arm-linux-gnueabihf" ;;
+	i686) arch="i686-linux-gnu" ;;
+	ppc64) arch="powerpc64-linux-gnu" ;;
+	ppc64le) arch="powerpc64le-linux-gnu" ;;
+	s390x) arch="s390x-linux-gnu" ;;
+	x86_64) arch="x86_64-linux-gnu" ;;
+	*) echo "Unknown target! Error!"; exit 1; ;;
+    esac
+}
+
+# Generate the list of frozen ABI files and count them.
+abi_dir=$(basename %{SOURCE13} .tar.gz)
+
+# Translate the RPM target architectures to a triplet.
+arch_dir=%{_target_cpu}
+arch_to_triplet arch_dir
+
+pushd "%{glibc_sysroot}/${abi_dir}/${arch_dir}"
+find . -name '*.abi.xml' -type f -print0 > abi_files.list
+popd
+mv "%{glibc_sysroot}/${abi_dir}/${arch_dir}/abi_files.list" .
+
+# Count the number of frozen ABI files.
+frozen_abi_count=0
+while IFS="" read -r -d $'\0' file ; do
+    frozen_abi_count=$(( $frozen_abi_count + 1 ))
+done < abi_files.list
+
+# 2. Generate shared object list.
+rm -f shared_objects.list
+find . -name '*.so' -type f -print0 > so_files.list
+
+# 3. Generate serialized ABI information. We do this step distinctly from the
+#    next step because it allows us to filter the SO file list for real ELF
+#    files and skip linker scripts.
+new_abi_count=0
+generate_abi_files "so_files.list" "shared_objects.list" "abi.xml.new" "new_abi_count"
+
+# No ABI-related tests have failed.
+failed=0
+warned=0
+passed=0
+
+# In warning mode we just print the message with a WARN: prefix, but in
+# verification mode we print the FAIL: prefix and increment the global
+# failed count.
+function record_fail()
+{
+%if 0%{?warn_abi}
+    echo "WARN: $1"
+    warned=$(( $warned + 1 ))
+%else
+    echo "FAIL: $1"
+    failed=$(( $failed + 1 ))
+%endif
+}
+
+function record_pass()
+{
+    echo "PASS: $1"
+    passed=$(( $passed + 1 ))
+}
+
+# Sanity check that we have a matching number of ABI files.
+# Could still be a disjoint set.
+if [ $frozen_abi_count -ne $new_abi_count ]; then
+    record_fail "Not the same number of shared objects for comparison!"
+fi
+
+# Compare $1 using the existing ABI baseline and the new ABI for the build.
+# Store error into failed global.
+function compare_results()
+{
+    local file="$1"
+    local frozen_abi="${abi_dir}/${arch_dir}/${file}.abi.xml"
+    local new_abi="${file}.abi.xml.new"
+    local abi_log="${file}.abi.log"
+    echo "Comparing ${file} ABI"
+    result=$(abidiff "${frozen_abi}" "${new_abi}" > "${abi_log}" || echo $?)
+    if [ -n "$result" ]; then
+	record_fail "ABI verification for ${new_abi} failed ($result)."
+	echo "Failure details:"
+	cat "${abi_log}"
+    else
+	record_pass "ABI verified for ${new_abi}."
+    fi
+}
+
+# 4. Compare the generated ABI to the saved ABI.
+#    Catches cases where we added new files.
+while IFS="" read -r -d $'\0' file ; do
+    compare_results "$file"
+done < shared_objects.list
+
+# 4. Compare the saved ABI to the generated ABI.
+#    Catches cases where we lost a file.
+while IFS="" read -r -d $'\0' file ; do
+    file=${file%.abi.xml}
+    compare_results "$file"
+done < abi_files.list
+
+total_abi_count=$(( $frozen_abi_count + $new_abi_count ))
+echo "Total test count is $total_abi_count"
+echo "$passed ABI tests passed."
+echo "$warned ABI tests with warnings."
+echo "$failed ABI tests failed."
+
+if [ $failed -gt 0 ]; then
+    echo "Failures detected. Aborting build!"
+    exit 1;
+fi
+
+# 5. Everything passed, so remove ABI files and list files.
+rm -rf "${abi_dir}"
+find . -name '*.abi.xml.new' -print0 | xargs --null rm -f
+find . -name '*.abi.log' -print0 | xargs --null rm -f
+rm so_files.list
+rm shared_objects.list
+rm abi_files.list
+popd
+# verify_abi == 1
+%endif
+# without bootstrap
+%endif
+
 # Strip all of the installed object files.
 strip -g %{glibc_sysroot}%{_libdir}/*.o
 
@@ -1906,6 +2169,13 @@ if test $1 = 0; then
 fi
 %systemd_postun_with_restart nscd.service
 
+%if %{without bootstrap}
+# If instructed, save the current package ABI for later use.
+%if 0%{?save_abi}
+frozen-abi-%{_target_cpu}.tar.xz
+%endif
+%endif
+
 %files -f rpm.filelist
 %dir %{_prefix}/%{_lib}/audit
 %if %{buildpower6}
@@ -2005,6 +2275,11 @@ fi
 %endif
 
 %changelog
+* Tue Mar 13 2018 Carlos O'Donell <codonell@redhat.com> - 2.27.9000-10
+- Enhance the package build process to verify the ABI on release branches.
+- Rebase the Fedora 29 ABI from glibc-abi for upstream branch master,
+  commit da6d4404ecfd7eacba8c096b0761a5758a59da4b.
+
 * Mon Mar 12 2018 Florian Weimer <fweimer@redhat.com> - 2.27.9000-9
 - Auto-sync with upstream branch master,
   commit da6d4404ecfd7eacba8c096b0761a5758a59da4b.
diff --git a/sources b/sources
index 1a5d96f..ebdd885 100644
--- a/sources
+++ b/sources
@@ -1 +1,2 @@
 SHA512 (glibc-2.27.9000-188-gda6d4404ec.tar.gz) = 88ca1f3699eed054887aff6eb5a77685db05b8082e7980632de66672d4ad9dd8e5a1a7389698c84d7b3c60477ee4b88c2e67df4784980d6e58566fbe0663363f
+SHA512 (glibc-abi-glibc-2.27.9000.tar.gz) = 9a230263fd84777881f79bbd3a5bd6fd8af5ac5706e6bf28dc2e7cd9b89417871ecc402ff9a8cca0b8d013e94de57ddb0bc8a9edd21e878176dd1ff605611efb
-- 
2.14.3

