Skip to content

E
eepm
Commit d717e2d9 authored by Vitaly Lipatov's avatar Vitaly Lipatov
Browse files
Options

rewrite contents_index retrieving

parent 0a2866d3
Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 159 deletions
bin/epm-filelist
View file @ d717e2d9
......@@ -27,9 +27,10 @@ load_helper epm-sh-warmup
__alt_local_content_filelist()
{
load_helper epm-sh-altlinux
load_helper epm-sh-altlinux-contents-index
local CI="$(get_local_alt_contents_index_rsync)"
[ -n "$CI" ] || fatal "Have no local contents index. Check epm repo --help."
update_alt_contents_index
local CI="$(cat $ALT_CONTENTS_INDEX_LIST)"
# TODO: safe way to use less
#local OUTCMD="less"
......
bin/epm-search_file
View file @ d717e2d9
......@@ -29,16 +29,13 @@ __alt_search_file_output()
__alt_local_content_search()
{
load_helper epm-sh-altlinux
load_helper epm-sh-altlinux-contents-index
load_helper epm-search
info "Locate contents index file(s) ..."
local CI="$(get_local_alt_contents_index_rsync)"
# TODO use something like
[ -n "$CI" ] || fatal "Have no local contents index. Check epm repo --help."
update_alt_contents_index
local CI="$(cat $ALT_CONTENTS_INDEX_LIST)"
info "Searching in"
echo "$CI"
echo "for $1... "
info "Searching for $1 ... "
# FIXME: do it better
local MGS
......
bin/epm-sh-altlinux
View file @ d717e2d9
#!/bin/sh
#
# Copyright (C) 2014, 2017 Etersoft
# Copyright (C) 2014, 2017 Vitaly Lipatov <lav@etersoft.ru>
# Copyright (C) 2014, 2017, 2021 Etersoft
# Copyright (C) 2014, 2017, 2021 Vitaly Lipatov <lav@etersoft.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
......@@ -17,154 +17,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# apt-file like. See also
# https://bugzilla.altlinux.org/show_bug.cgi?id=14449
# convert "http://download.etersoft.ru/pub/ALTLinux/p8/branch x86_64" to ALTLinux/p8/branch/x86_64
get_alt_repo_path()
{
local DN1=$(dirname "$1")
local DN2=$(dirname $DN1)
local DN3=$(dirname $DN2)
local BN0=$(basename "$1") # arch
local BN1=$(basename $DN1) # branch/Sisyphus
local BN2=$(basename $DN2) # p8/ALTLinux
local BN3=$(basename $DN3) # ALTLinux/
[ "$BN1" = "branch" ] && echo "$BN3/$BN2/$BN1/$BN0" || echo "$BN2/$BN1/$BN0"
}
# convert "http://download.etersoft.ru/pub/ALTLinux/p8/branch x86_64" to /tmp/epm/ALTLinux/p8/branch/x86_64
get_local_alt_mirror_path()
{
echo "/tmp/eepm/$(get_alt_repo_path "$1")"
}
__local_ercat()
{
local i
for i in "$@" ; do
case "$i" in
*.xz)
a='' xzcat $i
;;
*.lz4)
a='' lz4cat $i
;;
*.failed)
# just ignore
;;
*)
cat $i
;;
esac
done
}
# TODO: obsoleted
# something like gzip
compress_file_inplace()
{
local OFILE="$1"
if epm assure lz4 </dev/null ; then
#docmd lz4 --rm "$OFILE" "$OFILE.lz4" || return
# due old lz4
docmd lz4 -f "$OFILE" "$OFILE.lz4" || return
rm -fv "$OFILE"
else
epm assure xz </dev/null || fatal "Can't install nor lz4, nor xz compressor"
docmd xz -f "$OFILE" || return
fi
return 0
}
# TODO: obsoleted
# args: url/path target_file
# result: will set FILE
download_alt_contents_index()
{
local URL="$1"
local TD="$2"
local OFILE="$TD/$(basename "$URL")"
local DONE=$(echo $OFILE*)
# TODO: check if too old
if [ -r "$DONE" ] ; then
return
fi
mkdir -p "$TD"
if echo "$URL" | grep -q "^file:/" ; then
URL=$(echo "$URL" | sed -e "s|^file:||")
[ -s "$URL" ] || { touch $OFILE.failed ; return 1; }
ln -sf "$URL" "$OFILE" || { touch $OFILE.failed ; return 1; }
# TODO rsync -z
# https://bugzilla.altlinux.org/show_bug.cgi?id=14449#c22
else
docmd eget -O "$OFILE" "$URL" || { rm -fv $OFILE ; touch $OFILE.failed ; return 1; }
fi
rm -f $OFILE.failed
compress_file_inplace "$OFILE"
}
# URL TARGETDIR
rsync_alt_contents_index()
{
local URL="$1"
local TD="$2"
assure_exists rsync
a= rsync --partial --inplace -z -av --progress "$URL" "$TD"
test -s "$TD"
}
# URL
get_url_to_etersoft_mirror()
{
local REPOPATH
local ETERSOFT_MIRROR="rsync://download.etersoft.ru/pub"
echo "$ETERSOFT_MIRROR/$(get_alt_repo_path "$1" | sed -e "s|^ALTLinux/|ALTLinux/contents_index/|")"
}
get_local_alt_contents_index_rsync()
{
load_helper epm-repolist
# TODO: fix for Etersoft/LINUX@Etersoft
epm_repolist | grep -v " task$" | grep -E "rpm.*(ftp://|http://|https://|file:/)" | sed -e "s@^rpm.*\(ftp://\|http://\|https://\|file:\)@\1@g" | while read -r URL ARCH other ; do
if echo "$URL" | grep -q "^file:/" ; then
# first check for local mirror
local LOCALPATH="$(echo "$URL" | sed -e "s|^file:||")/$ARCH/base"
local LOCALPATHGZIP="$(echo "$LOCALPATH" | sed -e "s|/ALTLinux/|/ALTLinux/contents_index/|")"
[ -s "$LOCALPATHGZIP/contents_index.gz" ] && echo "$LOCALPATHGZIP/contents_index.gz" && continue
[ -s "$LOCALPATH/contents_index" ] && echo "$LOCALPATH/contents_index"
else
local LOCALPATH="$(get_local_alt_mirror_path "$URL/$ARCH")"
local REMOTEURL="$(get_url_to_etersoft_mirror "$URL/$ARCH")/base"
rsync_alt_contents_index $REMOTEURL/contents_index.gz $LOCALPATH/contents_index.gz >/dev/null 2>/dev/null </dev/null && echo "$LOCALPATH/contents_index.gz" && continue
[ -n "$verbose" ] && info "Note: Can't retrieve $REMOTEURL/contents_index.gz, fallback to $URL/$ARCH/base/contents_index"
rsync_alt_contents_index $URL/$ARCH/base/contents_index $LOCALPATH/contents_index >/dev/null 2>/dev/null </dev/null && echo "$LOCALPATH/contents_index" && continue
fi
done
}
# TODO: obsoleted
get_local_alt_contents_index()
{
load_helper epm-repolist
local LOCALPATH
epm_repolist | grep -v " task$" | grep -E "rpm.*(ftp://|http://|https://|file:/)" | sed -e "s@^rpm.*\(ftp://\|http://\|https://\|file:\)@\1@g" | while read -r URL ARCH other ; do
LOCALPATH=$(get_local_alt_mirror_path "$URL/$ARCH")
download_alt_contents_index $URL/$ARCH/base/contents_index $LOCALPATH >&2 </dev/null || continue
echo "$LOCALPATH/contents_index*"
done
}
tasknumber()
{
local num="$(echo "$1" | sed -e "s| *#*||g")"
......
bin/epm-sh-altlinux-contents-index 0 → 100644
View file @ d717e2d9
#!/bin/sh
#
# Copyright (C) 2014, 2017, 2021 Etersoft
# Copyright (C) 2014, 2017, 2021 Vitaly Lipatov <lav@etersoft.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# apt-file like. See also
# https://bugzilla.altlinux.org/show_bug.cgi?id=14449
# convert "http://download.etersoft.ru/pub/ALTLinux/p8/branch x86_64" to ALTLinux/p8/branch/x86_64
get_alt_repo_path()
{
local DN1=$(dirname "$1")
local DN2=$(dirname $DN1)
local DN3=$(dirname $DN2)
local BN0=$(basename "$1") # arch
local BN1=$(basename $DN1) # branch/Sisyphus
local BN2=$(basename $DN2) # p8/ALTLinux
local BN3=$(basename $DN3) # ALTLinux/
[ "$BN1" = "branch" ] && echo "$BN3/$BN2/$BN1/$BN0" || echo "$BN2/$BN1/$BN0"
}
# convert "http://download.etersoft.ru/pub/ALTLinux/p8/branch x86_64" to /tmp/epm/ALTLinux/p8/branch/x86_64
get_local_alt_mirror_path()
{
# TODO: /var/cache/eepm
echo "$TMPDIR/eepm/$(get_alt_repo_path "$1")"
}
ALT_CONTENTS_INDEX_LIST=$TMPDIR/eepm/contents_index_list
__local_ercat()
{
local i
for i in "$@" ; do
case "$i" in
*.xz)
a='' xzcat $i
;;
*.lz4)
a='' lz4cat $i
;;
*.gz)
a='' zcat $i
;;
*.failed)
# just ignore
;;
*)
cat $i
;;
esac
done
}
# URL TARGETDIR
rsync_alt_contents_index()
{
local URL="$1"
local TD="$2"
assure_exists rsync
mkdir -p "$(dirname "$TD")"
if [ -n "$verbose" ] ; then
docmd rsync --partial --inplace -z -a --progress "$URL" "$TD"
else
a= rsync --partial --inplace -z -a --progress "$URL" "$TD" >/dev/null 2>/dev/null
fi
}
# URL
get_url_to_etersoft_mirror()
{
local REPOPATH
local ETERSOFT_MIRROR="rsync://download.etersoft.ru/pub"
local ALTREPO=$(get_alt_repo_path "$1")
echo "$ALTREPO" | grep -q "^ALTLinux" || return
echo "$ETERSOFT_MIRROR/$(get_alt_repo_path "$1" | sed -e "s|^ALTLinux/|ALTLinux/contents_index/|")"
}
__init_contents_index_list()
{
mkdir -p "$(dirname $ALT_CONTENTS_INDEX_LIST)"
truncate -s0 $ALT_CONTENTS_INDEX_LIST
}
__add_to_contents_index_list()
{
echo " $1"
echo "$1" >>$ALT_CONTENTS_INDEX_LIST
}
# file file2
__add_better_to_contents_index_list()
{
if [ -s "$1" ] && [ -s "$2" ] ; then
[ "$1" -ot "$2" ] && __add_to_contents_index_list "$2" && return
__add_to_contents_index_list "$1" && return
fi
[ -s "$1" ] && __add_to_contents_index_list "$1" && return
[ -s "$2" ] && __add_to_contents_index_list "$2" && return
}
# fills ALT_CONTENTS_INDEX_LIST
update_alt_contents_index()
{
load_helper epm-repolist
__init_contents_index_list
# TODO: fix for Etersoft/LINUX@Etersoft
# TODO: fix for rsync
info "Retrieving contents_index ..."
epm_repolist | grep -v " task$" | grep -E "rpm.*(ftp://|http://|https://|rsync://|file:/)" | sed -e "s@^rpm.*\(ftp://\|http://\|https://\)@rsync://@g" | sed -e "s@^rpm.*\(file:\)@@g" | while read -r URL1 URL2 component ; do
[ "$component" = "debuginfo" ] && continue
URL="$URL1/$URL2"
if echo "$URL" | grep -q "^/" ; then
# first check for local mirror
local LOCALPATH="$(echo "$URL/base")"
local LOCALPATHGZIP="$(echo "$LOCALPATH" | sed -e "s|/ALTLinux/|/ALTLinux/contents_index/|")"
__add_better_to_contents_index_list "$LOCALPATHGZIP/contents_index.gz" "$LOCALPATH/contents_index"
else
local LOCALPATH="$(get_local_alt_mirror_path "$URL")"
local REMOTEURL="$(get_url_to_etersoft_mirror "$URL")"
if [ -n "$REMOTEURL" ] ; then
rsync_alt_contents_index $REMOTEURL/base/contents_index.gz $LOCALPATH/contents_index.gz && __add_to_contents_index_list "$LOCALPATH/contents_index.gz" && continue
[ -n "$verbose" ] && info "Note: Can't retrieve $REMOTEURL/base/contents_index.gz, fallback to $URL/base/contents_index"
fi
# fix rsync URL firstly
local RSYNCURL="$(echo "$URL" | sed -e "s|rsync://\(ftp.basealt.ru\|basealt.org\|altlinux.ru\)/pub/distributions/ALTLinux|rsync://\1/ALTLinux|")" #"
rsync_alt_contents_index $RSYNCURL/base/contents_index $LOCALPATH/contents_index && __add_to_contents_index_list "$LOCALPATH/contents_index" && continue
__add_better_to_contents_index_list "$LOCALPATH/contents_index.gz" "$LOCALPATH/contents_index"
fi
done
if [ ! -s "$ALT_CONTENTS_INDEX_LIST" ] ; then
fatal "Have no local contents index. Check epm repo --help."
fi
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment