stage2: 99-elf-stats for squashfs tuning

This kind of test was proposed by led@ to gather statistics on chroot's contents going to become squashfs (the script optimizations lowering added overhead from ~10 sec down to a subsecond range were also proposed by him). Intentionally not documented in doc/variables.txt due to the rather lowlevel nature of the probe (at least so far). The knobs involved are SQUASHFS (the additional effort kicks in only for "tight" case) and GLOBAL_SQUASHFS_SORT (must be non-empty for this extra overhead to occur). Additional experimentation is needed to find out whether the difference in squashfs size and performance is worth the trouble (seems the impact is non-zero but pretty minor).

stage2: 99-elf-stats for squashfs tuning
c2996cc4 · Michael Shigorin · 94b2052b · c2996cc4 · c2996cc4
Commit c2996cc4 authored Apr 06, 2012 by Michael Shigorin
Hide whitespace changes
Inline Side-by-side

Showing with 62 additions and 2 deletions

Makefile sub.in/stage2/Makefile +14 -2

99-elf-stats sub.in/stage2/scripts.d/99-elf-stats +48 -0

No files found.
--- a/sub.in/stage2/Makefile
+++ b/sub.in/stage2/Makefile
@@ -31,17 +31,29 @@ $(error STAGE1_KFLAVOUR is utterly empty; cannot guess either)
 endif

 # need kernel modules only (which require corresponding kernel-image);
-# these go into work chroot; NB: no vmlinuz there
+# these go into work chroot; NB: no vmlinuz there (unless live-install)
 IMAGE_PACKAGES_REGEXP = $(call kpackages, \
 			$(STAGE1_KMODULES) $(STAGE2_KMODULES), \
 			$(STAGE1_KFLAVOUR))

+# see also scripts.d/99-elf-stats
+ifdef GLOBAL_SQUASHFS_SORT
+ifeq (tight,$(SQUASHFS))
+CHROOT_PACKAGES += file
+pack-image: PACK_SQUASHFS_OPTS += -sort /.our/elf.lst
+pack-image: CLEANUP_OUTDIR=
+endif
+endif
+
+GLOBAL_SQUASHFS := $(SQUASHFS)
+
 include $(MKIMAGE_PREFIX)/targets.mk

 # NB: it's a collector variable, add()ing to a GLOBAL_ results in lost hair
 GLOBAL_CLEANUP_PACKAGES := $(CLEANUP_PACKAGES)

-all: | $(GLOBAL_DEBUG) build-image run-image-scripts pack-image \
+all: |  $(GLOBAL_DEBUG) \
+	build-image run-image-scripts run-scripts pack-image \
 	$(GLOBAL_CLEAN_WORKDIR)

 # dummy

--- a/sub.in/stage2/scripts.d/99-elf-stats
+++ b/sub.in/stage2/scripts.d/99-elf-stats
+#!/bin/sh -e
+# tally up sizes of chroot's ELF binaries for squashfs tuning
+
+# NB: it's unclear so far whether mksquashfs -sort is beneficial
+if [ -z "$GLOBAL_SQUASHFS_SORT" -o "$GLOBAL_SQUASHFS" != "tight" ]; then
+	echo "SKIP elf-stats: looks like it's not needed" >&2
+	exit 0
+fi
+
+exit_handler()
+{
+	local rc=$1
+	rm -f -- "$TMPFILE"
+	exit $rc
+}
+
+# it's controlled environment
+TMPFILE="$(mktemp)"
+
+trap 'exit_handler $?' EXIT HUP PIPE INT QUIT TERM
+
+# a list of ELF binaries (both executables and shared libraries)
+ELFLIST=/.our/elf.lst
+
+cd "$WORKDIR"
+
+# let's parallelize a bit, chroot should be on tmpfs or just-cached
+du -bsx | cut -f1 > "$TMPFILE" &
+
+elf="$(find {,usr/}{lib*,{,s}bin} \
+		-path lib/modules -prune -o \
+		-path usr/lib/locale -prune -o \
+		-type f \
+	| file -nN -e ascii -e compress -e tar -e elf -f - \
+	| sed -n "/: ELF /s/: .*$//p" \
+	| tee "$ELFLIST.in" \
+	| tr '\n' '\0' \
+	| du -cb --files0-from=- \
+	| tail -1 \
+	| cut -f1)"
+
+wait
+read root < "$TMPFILE"
+
+echo "** live chroot ELF ratio: $((100*$elf/$root))% ($elf/$root)" >&2
+
+# add priorities
+sed 's,^.*$,& 1,' < "$ELFLIST.in" > "$ELFLIST"