add warmup bases support and use it

bin/epm-filelist

@@ -19,6 +19,7 @@
 
 load_helper epm-query
 load_helper epm-print
+load_helper epm-sh-warmup
 
 # TODO: port or rewrite apt-file
 # https://bugzilla.altlinux.org/show_bug.cgi?id=14449
@@ -110,6 +111,8 @@ __epm_filelist_name()
 
 	[ -z "$*" ] && return
 
+	warmup_lowbase
+
 	case $PMTYPE in
 		*-rpm)
 			CMD="rpm -ql"

bin/epm-install

@@ -21,6 +21,7 @@ load_helper epm-sh-altlinux
 load_helper epm-query
 load_helper epm-assure
 load_helper epm-check_updated_repo
+load_helper epm-sh-warmup
 
 # TODO: use when run install with epm --skip-installed install
 filter_out_installed_packages()
@@ -93,6 +94,7 @@ epm_install_names()
 	fi
 
 	[ -z "$1" ] && return
+
 	case $PMTYPE in
 		apt-rpm|apt-dpkg)
 			APTOPTIONS="$APTOPTIONS $(subst_option verbose "-o Debug::pkgMarkInstall=1 -o Debug::pkgProblemResolver=1")"
@@ -186,6 +188,7 @@ epm_install_names()
 epm_ni_install_names()
 {
 	[ -z "$1" ] && return
+
 	case $PMTYPE in
 		apt-rpm|apt-dpkg)
 			export DEBIAN_FRONTEND=noninteractive
@@ -590,6 +593,9 @@ epm_install()
 
     [ -z "$pkg_files$pkg_names" ] && info "Skip empty install list" && return 22
 
+    # to be filter happy
+    warmup_lowbase
+
     local names="$(echo $pkg_names | filter_out_installed_packages)"
     local files="$(echo $pkg_files | filter_out_installed_packages)"
 
@@ -600,6 +606,8 @@ epm_install()
         update_repo_if_needed
     fi
 
+    warmup_hibase
+
     epm_install_names $names || return
     epm_install_files $files
     local RETVAL=$?

bin/epm-kernel_update

@@ -18,9 +18,12 @@
 #
 
 load_helper epm-check_updated_repo
+load_helper epm-sh-warmup
 
 epm_kernel_update()
 {
+	warmup_bases
+
 	info "Updating system kernel to the latest version..."
 
 	case $DISTRNAME in

bin/epm-packages

 #!/bin/sh
 #
-# Copyright (C) 2012, 2016  Etersoft
-# Copyright (C) 2012, 2016  Vitaly Lipatov <lav@etersoft.ru>
+# Copyright (C) 2012, 2016, 2017  Etersoft
+# Copyright (C) 2012, 2016, 2017  Vitaly Lipatov <lav@etersoft.ru>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,15 +17,19 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+load_helper epm-sh-warmup
+
 __epm_packages_sort()
 {
 # FIXME: sort depends on --sort value
 case $PMTYPE in
 	apt-rpm|yum-rpm|urpm-rpm|zypper-rpm|dnf-rpm)
 		# FIXME: space with quotes problems, use point instead
+		warmup_rpmbase
 		docmd rpm -qa --queryformat "%{size}@%{name}-%{version}-%{release}\n" $pkg_filenames | sed -e "s|@| |g" | sort -n -k1
 		;;
 	apt-dpkg)
+		warmup_dpkgbase
 		docmd dpkg-query -W --showformat="\${Installed-Size}@\${Package}-\${Version}\n" $pkg_filenames | sed -e "s|@| |g" | sort -n -k1
 		;;
 	*)
@@ -54,12 +58,14 @@ epm_packages()
 
 case $PMTYPE in
 	apt-rpm)
+		warmup_rpmbase
 		# FIXME: strong equal
 		CMD="rpm -qa $pkg_filenames"
 		[ -n "$short" ] && CMD="rpm -qa --queryformat %{name}\n $pkg_filenames"
 		docmd $CMD
 		return ;;
 	*-dpkg)
+		warmup_dpkgbase
 		# FIXME: strong equal
 		#CMD="dpkg -l $pkg_filenames"
 		CMD="dpkg-query -W --showformat=\${db:Status-Abbrev}\${Package}-\${Version}\n $pkg_filenames"
@@ -71,6 +77,7 @@ case $PMTYPE in
 		CMD="snappy info"
 		;;
 	yum-rpm|urpm-rpm|zypper-rpm|dnf-rpm)
+		warmup_rpmbase
 		# FIXME: strong equal
 		CMD="rpm -qa $pkg_filenames"
 		[ -n "$short" ] && CMD="rpm -qa --queryformat %{name}\n $pkg_filenames"

bin/epm-policy

@@ -18,12 +18,15 @@
 #
 
 load_helper epm-query
+load_helper epm-sh-warmup
 
 epm_policy()
 {
 
 [ -n "$pkg_names" ] || fatal "Info: missing package(s) name"
 
+warmup_bases
+
 # get package name for hi level package management command (with version if supported and if possible)
 pkg_names=$(__epm_get_hilevel_name $pkg_names)
 

bin/epm-reinstall

 #!/bin/sh
 #
-# Copyright (C) 2012,2014,2016  Etersoft
-# Copyright (C) 2012,2014,2016  Vitaly Lipatov <lav@etersoft.ru>
+# Copyright (C) 2012,2014,2016,2017  Etersoft
+# Copyright (C) 2012,2014,2016,2017  Vitaly Lipatov <lav@etersoft.ru>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,11 +18,13 @@
 #
 
 load_helper epm-install
+load_helper epm-sh-warmup
 
 # copied from etersoft-build-utils/share/eterbuild/functions/rpmpkg
 epm_reinstall_names()
 {
 	[ -n "$1" ] || return
+
 	case $PMTYPE in
 		apt-rpm|apt-dpkg)
 			local APTOPTIONS="$(subst_option non_interactive -y)"
@@ -75,9 +77,13 @@ epm_reinstall()
 {
     [ -n "$pkg_filenames" ] || fatal "Reinstall: missing package(s) name."
 
+    warmup_lowbase
+
     # get package name for hi level package management command (with version if supported and if possible)
     pkg_names=$(__epm_get_hilevel_name $pkg_names)
 
+    warmup_hibase
+
     epm_reinstall_names $pkg_names
     epm_reinstall_files $pkg_files
 }

bin/epm-remove

@@ -20,11 +20,15 @@
 load_helper epm-sh-altlinux
 load_helper epm-query
 load_helper epm-print
+load_helper epm-sh-warmup
 
 # Try remove with low level removing
 epm_remove_low()
 {
 	[ -z "$1" ] && return
+
+	warmup_lowbase
+
 	case $PMTYPE in
 		apt-rpm|yum-rpm|zypper-rpm|urpm-rpm|dnf-rpm)
 			sudocmd rpm -ev $nodeps $@
@@ -56,6 +60,8 @@ epm_remove_names()
 {
 	[ -z "$1" ] && return
 
+	warmup_bases
+
 	case $PMTYPE in
 		apt-dpkg)
 			sudocmd apt-get remove --purge $@
@@ -146,6 +152,8 @@ epm_remove_names()
 # TODO
 epm_remove_nonint()
 {
+	warmup_bases
+
 	case $PMTYPE in
 		apt-dpkg)
 			sudocmd apt-get -y --force-yes remove --purge $@

bin/epm-remove_old_kernels

 #!/bin/sh
 #
-# Copyright (C) 2016  Etersoft
-# Copyright (C) 2016  Vitaly Lipatov <lav@etersoft.ru>
+# Copyright (C) 2016-2017  Etersoft
+# Copyright (C) 2016-2017  Vitaly Lipatov <lav@etersoft.ru>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,8 +17,13 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+load_helper epm-sh-warmup
+
 epm_remove_old_kernels()
 {
+
+	warmup_bases
+
 	case $DISTRNAME in
 	ALTLinux)
 		load_helper epm-query_package

bin/epm-search

 #!/bin/sh
 #
-# Copyright (C) 2012, 2013, 2016  Etersoft
-# Copyright (C) 2012, 2013, 2016  Vitaly Lipatov <lav@etersoft.ru>
+# Copyright (C) 2012, 2013, 2016-2017  Etersoft
+# Copyright (C) 2012, 2013, 2016-2017  Vitaly Lipatov <lav@etersoft.ru>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,6 +18,7 @@
 #
 
 load_helper epm-check_updated_repo
+load_helper epm-sh-warmup
 
 __epm_search_output()
 {
@@ -176,6 +177,8 @@ epm_search()
 	# it is useful for first time running
 	update_repo_if_needed soft
 
+	warmup_bases
+
 	# FIXME: do it better
 	local MGS
 	MGS=$(eval __epm_search_make_grep $quoted_args)

bin/epm-sh-warmup

+#!/bin/sh
+#
+# Copyright (C) 2017  Etersoft
+# Copyright (C) 2017  Vitaly Lipatov <lav@etersoft.ru>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+is_warmup_allowed()
+{
+    local MEM
+    MEM=$($DISTRVENDOR -m)
+    # disable warm if have no enough memory
+    [ $MEM -le 1024 ] && return 1
+    return 0
+}
+
+__warmup_files()
+{
+    local D="$1"
+    shift
+    #showcmd "$*"
+    [ -n "$D" ] && info "Warming up $D ..."
+    # TODO: use progress, calc files size before
+    docmd cat $* >/dev/null 2>/dev/null
+}
+
+warmup_rpmbase()
+{
+    is_warmup_allowed || return
+    __warmup_files "rpm" "/var/lib/rpm/*"
+}
+
+warmup_dpkgbase()
+{
+    is_warmup_allowed || { warning "Skipping warmup bases due low memory size" ; return ; }
+    __warmup_files "dpkg" "/var/lib/dpkg/*"
+}
+
+warmup_lowbase()
+{
+    case $PKGFORMAT in
+        "rpm")
+            warmup_rpmbase "$@"
+            ;;
+        "dpkg")
+            warmup_dpkgbase "$@"
+            ;;
+        *)
+            ;;
+    esac
+}
+
+warmup_aptbase()
+{
+    is_warmup_allowed || return
+    __warmup_files "apt" "/var/lib/apt/lists/* /var/cache/apt/*.bin"
+}
+
+warmup_hibase()
+{
+    case $PMTYPE in
+        "apt-rpm"|"apt-dpkg")
+            warmup_aptbase "$@"
+            ;;
+        *)
+            ;;
+    esac
+}
+
+warmup_bases()
+{
+    DISquiet=1 warmup_lowbase
+    DISquiet=1 warmup_hibase
+}

bin/epm-update

 #!/bin/sh
 #
-# Copyright (C) 2012, 2014, 2016  Etersoft
-# Copyright (C) 2012, 2014, 2016  Vitaly Lipatov <lav@etersoft.ru>
+# Copyright (C) 2012, 2014, 2016-2017  Etersoft
+# Copyright (C) 2012, 2014, 2016-2017  Vitaly Lipatov <lav@etersoft.ru>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,11 +19,15 @@
 
 # copied from korinf/tools/run-script/scripts/update
 
+load_helper epm-sh-warmup
+
 epm_update()
 {
 	[ -z "$pkg_filenames" ] || fatal "No arguments are allowed here"
 	info "Running command for update remote package repository database"
 
+warmup_hibase
+
 case $PMTYPE in
 	apt-rpm)
 		sudocmd apt-get update || return

bin/epm-upgrade

@@ -18,6 +18,7 @@
 #
 
 load_helper epm-check_updated_repo
+load_helper epm-sh-warmup
 
 epm_upgrade()
 {
@@ -28,6 +29,7 @@ epm_upgrade()
 	# it is useful for first time running
 	update_repo_if_needed
 
+	warmup_bases
 	info "Running command for upgrade packages"
 
 	case $PMTYPE in