epm-restore

#!/bin/sh
#
# Copyright (C) 2020, 2021  Etersoft
# Copyright (C) 2020, 2021  Vitaly Lipatov <lav@etersoft.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

load_helper epm-sh-altlinux
load_helper epm-assure

# file prefix suffix
__epm_restore_print_comment()
{
    echo "#$2 generated by 'epm restore --dry-run' from $(basename $(dirname $(realpath "$1")))/$(basename "$1")$3"
}

# FIXME: python modules are packaged into python packages, but we have only module names and rpm package names instead of python packages
# enable python3dest(PEP-503 normalized name) provides
# https://bugzilla.altlinux.org/show_bug.cgi?id=39003
__epm_filter_pip_to_rpm()
{
    tr "A-Z" "a-z" | sed -e "s|_|-|g" -e "s|^python[-_]||" -e "s|python$||" \
        -e "s|bs4|beautifulsoup4|" \
        -e "s|pillow|Pillow|" \
        -e "s|sqlalchemy|SQLAlchemy|" \
        -e "s|flask-SQLAlchemy|flask_sqlalchemy|" \
        -e "s|redis|redis-py|" \
        -e "s|pyjwt|jwt|" \
        -e "s|pymonetdb|monetdb|" \
        -e "s|pyyaml|yaml|" \
        -e "s|flask-migrate|Flask-Migrate|" \
        -e "s|twisted|twisted-core|" \
        -e "s|pymacaroons|pymacaroons-pynacl|" \
        -e "s|pygments|Pygments|" \
        -e "s|memcached|memcache|" \
        -e "s|pyinstaller||" \
        -e "s|pyopenssl|OpenSSL|"
}

# TODO: remove me
fill_sign()
{
    local sign="$1"
    echo "$2" | grep -E -- "$sign[[:space:]]*[0-9.]+?" | sed -E -e "s|.*$sign[[:space:]]*([0-9.]+?).*|\1|"
}


# macro pkg caseline
__epm_pi_sign_to_rpm()
{
    local t="$1"
    local l="$2"
    local equal="$3"
    [ -n "$equal" ] || equal=">="

    local pi=''
    local sign ll
    for sign in "<=" "<" ">=" ">" "==" "!=" "~="; do
        ll=$(fill_sign "$sign" "$l")
        [ -n "$ll" ] || continue
        [ "$sign" = "==" ] && sign="$equal"
        [ "$sign" = "~=" ] && sign="$equal"
        [ "$sign" = "!=" ] && sign=">="
        [ -n "$pi" ] && pi="$pi
"
        pi="$pi$t $sign $ll"
    done
    [ -n "$pi" ] || pi="$t"
    echo "$pi"
}

__epm_get_array_name()
{
    echo "$*" | grep "=" | head -n1 | sed -e 's| *=.*||'
}

__epm_lineprint_python_array()
{
    local a="$*"
    [ -n "$a" ] || return
    local name="$(__epm_get_array_name "$a")"
    (echo "$a" | sed -E -e 's@(\]|\)).*@\1@' ; echo "print('\n'.join($name))" ) | ( a= python3 - || a= python - )
}

# translate pip requirement lines to rpm notation
# (compare signs and package names)
__epm_restore_convert_to_rpm_notation()
{
    local equal="$1"
    local l
    while read l ; do
        if echo "$l" | grep -q 'platform_python_implementation != "PyPy"' ; then
            [ -n "$verbose" ] && warning "    $t is not PyPi requirement, skipped"
            continue
        fi
        if echo "$l" | grep -q 'sys_platform == "darwin"' ; then
            [ -n "$verbose" ] && warning "    $t is darwin only requirement, skipped"
            continue
        fi
        if echo "$l" | grep -q 'sys_platform == "win32"' ; then
            [ -n "$verbose" ] && warning "    $t is win32 only requirement, skipped"
            continue
        fi
        if echo "$l" | grep -q "; *python_version *< *['\"]3" ; then
            [ -n "$verbose" ] && warning "    $t is python2 only requirement, skipped"
            continue
        fi
        if echo "$l" | grep -q "; *python_version *<= *['\"]2\." ; then
            [ -n "$verbose" ] && warning "    $t is python2 only requirement, skipped"
            continue
        fi
        # drop various "python_version > '3.5'"
        l="$(echo "$l" | sed -e "s| *;.*||")"
        if echo "$l" | grep -qE "^ *#" || [ -z "$l" ] ; then
            continue
        fi
        local t="$(echo "$l" | sed -E -e "s|[[:space:]]*[<>!=~]+.*||" -e "s| *#.*||" | __epm_filter_pip_to_rpm)"
        [ -n "$t" ] || continue
        # until new section
        if echo "$l" | grep -qE "^\[" ; then
            break
        fi
        # if dependency_links URLs, use egg name
        if echo "$l" | grep -qE "://" ; then
            if echo "$l" | grep -q "#egg=" ; then
                t="$(echo "$l" | sed -e "s|.*#egg=||" -e "s|\[.*||" | __epm_filter_pip_to_rpm)"
            else
                warning "    skipping URL $l ..."
                continue
            fi
        fi

        __epm_pi_sign_to_rpm "$t" "$l" "$equal"
    done
}

__epm_restore_pip()
{
    local req_file="$1"
    local reqmacro
    local ilist

    if [ -n "$dryrun" ] ; then
        reqmacro="%py3_use"
        basename "$req_file" | egrep -q "(dev|test|coverage)" && reqmacro="%py3_buildrequires"
        echo
        __epm_restore_print_comment "$req_file"
        cat $req_file | __epm_restore_convert_to_rpm_notation | sed -e "s|^|$reqmacro |"
        return
    else
        info "Install requirements from $req_file ..."
        ilist="$(cat $req_file | __epm_restore_convert_to_rpm_notation | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
    fi

    ilist="$(estrlist list $ilist)"
    docmd epm install $ilist
}

__eresection()
{
    rhas "$1" "[[:space:]]*$2[[:space:]]*=[[:space:]]*[\[(]"
}

__epm_restore_setup_py()
{
    local req_file="$1"
    if [ -z "$dryrun" ] ; then
        info "Install requirements from $req_file ..."
    fi

    local ar=''
    local ilist=''
    local reqmacro
    local section=''
    while read l ; do
        if rhas "$l" "^ *#" ; then
            continue
        fi
        # start of section
        if __eresection "$l" "REQUIREMENTS" ; then
            reqmacro="%py3_use"
            section="$l"
        fi
        if __eresection "$l" "install_requires" ; then
            reqmacro="%py3_use"
            section="$l"
        fi
        if __eresection "$l" "setup_requires" ; then
            reqmacro="%py3_buildrequires"
            section="$l"
        fi
        if __eresection "$l" "tests_require" ; then
            reqmacro="%py3_buildrequires"
            section="$l"
        fi
        if [ -n "$section" ] ; then
            ar="$ar
$l"
        fi

        # not end of section
        if [ -z "$section" ] || ! rhas "$l" "(\]|\)),*" ; then
            continue
        fi

        if [ -n "$dryrun" ] ; then
            echo
            __epm_restore_print_comment "$req_file" "" " $(__epm_get_array_name "$section")"
            __epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | sed -e "s|^|$reqmacro |"
        else
            ilist="$ilist $(__epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
        fi
        section=''
        ar=''
    done < $req_file

    if [ -n "$dryrun" ] ; then
        return
    fi

    ilist="$(estrlist list $ilist)"
    docmd epm install $ilist
}

__epm_print_npm_list()
{
    local reqmacro="$1"
    local req_file="$2"
    local l
    while read l ; do
        # "tap": "^14.10.7"
        echo "$l" | grep -q '"\(.*\)": "\(.*\)"' || continue
        local name="$(echo "$l" | sed -e 's|.*"\(.*\)": ".*|\1|')"
        [ -z "$name" ] && continue
        local ver="$(echo "$l" | sed -e 's|.*"\(.*\)": "\(.*\)".*|\2|')" #'
        [ -z "$name" ] && continue

        if [ -n "$dryrun" ] ; then
            local pi=''
            local sign
            if echo "$ver" | grep -q "^\^" ; then
                sign=">="
            else
                sign="="
            fi
            ll=$(echo "$ver" | sed -e 's|^[^~]||')
            pi="$pi$reqmacro node-$name $sign $ll"
#         [ -n "$pi" ] && pi="$pi
#"
#           [ -n "$pi" ] || pi="$pi$reqmacro: node-$t"
            echo "$pi"
            continue
        else
            local pi="node-$name"
            #echo "    $l -> $name -> $pi"
        fi
        [ -n "$name" ] || continue
        ilist="$ilist $pi"
    done < $req_file

    [ -n "$dryrun" ] || echo "$ilist"
}


__epm_print_perl_list()
{
    local reqmacro="$1"
    local req_file="$2"
    local l
    for l in $(cat) ; do
        # perl(Class::ErrorHandler)>=0
        echo "$l" | grep -q '^perl(' || continue
        local name="$(echo "$l" | sed -e 's|>=.*||' -e 's|::|/|g' -e 's|)|.pm)|')"
        [ "$name" = "perl(perl.pm)" ] && continue
        [ -z "$name" ] && continue
        local ver="$(echo "$l" | sed -e 's|.*>=||')"
        [ -z "$name" ] && continue

        if [ -n "$dryrun" ] ; then
            local pi=''
            local sign=''
            [ "$ver" = "0" ] || sign=" >= $ver"
            pi="$pi$reqmacro $name$sign"
            echo "$pi"
            continue
        else
            local pi="$name"
            #echo "    $l -> $name -> $pi"
        fi
        [ -n "$name" ] || continue
        ilist="$ilist $pi"
    done < $req_file

    [ -n "$dryrun" ] || echo "$ilist"
}

__epm_print_perl_list_shyaml()
{
    local reqmacro="$1"
    local req_file="$2"
    local l
    while read l ; do
        # Convert::ASN1: 0.10
        echo "$l" | grep -q '^ *\(.*\): \(.*\)' || continue
        local name="$(echo "$l" | sed -e 's| *\(.*\): \(.*\)|\1|' -e 's|::|/|g')".pm
        [ "$name" = "perl.pm" ] && continue
        [ -z "$name" ] && continue
        local ver="$(echo "$l" | sed -e 's| *\(.*\): \(.*\)|\2|')"
        [ -z "$name" ] && continue

        if [ -n "$dryrun" ] ; then
            local pi=''
            local sign=''
            [ "$ver" = "0" ] || sign=" >= $ver"
            pi="$pi$reqmacro perl($name)$sign"
            echo "$pi"
            continue
        else
            local pi="perl($name)"
            #echo "    $l -> $name -> $pi"
        fi
        [ -n "$name" ] || continue
        ilist="$ilist $pi"
    done < $req_file

    [ -n "$dryrun" ] || echo "$ilist"
}


__epm_print_nupkg_list()
{
    a= dotnet list $1 package | grep "^   > " | while read n name req other; do
        if [ -n "$dryrun" ] ; then
            echo "BuildRequires: nupkg($name) >= $req"
        else
            echo "nupkg($name)"
        fi
    done
}

__epm_restore_nupkg()
{
    local req_file="$1"
    if [ -n "$dryrun" ] ; then
        echo "# generated via dotnet list $(basename $(dirname $(realpath "$req_file")))/$(basename "$req_file") package"
        __epm_print_nupkg_list $req_file
        return
    fi

    info "Install requirements from $req_file ..."
    ilist=$(__epm_print_nupkg_list $req_file)
    ilist="$(estrlist list $ilist)"
    docmd epm install $ilist
}

__epm_restore_npm()
{
    local req_file="$1"

    assure_exists jq || fatal

    if [ -n "$dryrun" ] ; then
        local lt=$(mktemp)
        a= jq .dependencies <$req_file >$lt
        echo
        __epm_restore_print_comment "$req_file"
        __epm_print_npm_list "Requires:" $lt

        echo
        __epm_restore_print_comment "$req_file" " devDependencies"
        a= jq .devDependencies <$req_file >$lt
        __epm_print_npm_list "BuildRequires:" $lt
        rm -f $lt
        return
    fi

    info "Install requirements from $req_file ..."
    local lt=$(mktemp)
    a= jq .dependencies <$req_file >$lt
    ilist="$(__epm_print_npm_list "" $lt)"
    a= jq .devDependencies <$req_file >$lt
    ilist="$ilist $(__epm_print_npm_list "" $lt)"
    rm -f $lt
    docmd epm install $ilist
}

# TODO: check 
__epm_restore_perl()
{
    local req_file="$1"

    if [ -n "$dryrun" ] ; then
        local lt=$(mktemp)
        a= /usr/bin/perl $req_file PRINT_PREREQ=1 >$lt
        # all requirements will autodetected during packing, put it to the buildreq
        echo
        __epm_restore_print_comment "$req_file"
        __epm_print_perl_list "BuildRequires:" $lt
        rm -f $lt
        return
    fi

    info "Install requirements from $req_file ..."
    local lt=$(mktemp)
    a= /usr/bin/perl $req_file PRINT_PREREQ=1 >$lt
    ilist="$(__epm_print_perl_list "" $lt)"
    rm -f $lt
    docmd epm install $ilist
}

# disabled
__epm_restore_perl_shyaml()
{
    local req_file="$1"

    assure_exists shyaml || fatal

    if [ -n "$dryrun" ] ; then
        local lt=$(mktemp)
        a= shyaml get-value requires <$req_file >$lt
        # all requirements will autodetected during packing, put it to the buildreq
        echo
        __epm_restore_print_comment "$req_file"
        __epm_print_perl_list "BuildRequires:" $lt

        echo
        __epm_restore_print_comment "$req_file" " build_requires"
        a= shyaml get-value build_requires <$req_file >$lt
        __epm_print_perl_list "BuildRequires:" $lt
        rm -f $lt
        return
    fi

    info "Install requirements from $req_file ..."
    local lt=$(mktemp)
    a= shyaml get-value requires <$req_file >$lt
    ilist="$(__epm_print_perl_list "" $lt)"
    a= shyaml get-value build_requires <$req_file >$lt
    ilist="$ilist $(__epm_print_perl_list "" $lt)"
    rm -f $lt
    docmd epm install $ilist
}

__epm_restore_by()
{
    local req_file="$1"
    [ -s "$req_file" ] || return

    if file $req_file | grep -q "ELF [3264]*-bit LSB executable" ; then
        assure_exists ldd-requires
        showcmd ldd-requires $req_file
        local TOINSTALL="$(a= ldd-requires $req_file | grep "^apt-get install" | sed -e "s|^apt-get install ||")"
        if [ -n "$dryrun" ] ; then
            estrlist list $TOINSTALL
            return
        fi
        [ -n "$TOINSTALL" ] || { info "There are no missed packages is found for $req_file binary." ; return ; }
        docmd epm install $TOINSTALL
        return
    fi

    case $req_file in
        requirements/default.txt|requirements/dev.txt|requirements/test.txt|requirements/coverage.txt)
            [ -s "$req_file" ] && __epm_restore_pip "$req_file" && return
            ;;
    esac

    case $(basename $req_file) in
        requirements.txt|dev-requirements.txt|requirements-dev.txt|requirements_dev.txt|requirements_test.txt|requirements-test.txt|test-requirements.txt|requires.txt)
            [ -s "$req_file" ] && __epm_restore_pip "$req_file"
            ;;
        setup.py|python_dependencies.py)
            [ -s "$req_file" ] && __epm_restore_setup_py "$req_file"
            ;;
        package.json)
            [ -s "$req_file" ] && __epm_restore_npm "$req_file"
            ;;
#        META.yml)
#            [ -s "$req_file" ] && __epm_restore_perl "$req_file"
#            ;;
        Makefile.PL)
            [ -s "$req_file" ] && __epm_restore_perl "$req_file"
            ;;
        *.sln|*.csproj)
            local PROJ="$(echo $req_file)"
            [ -s "$PROJ" ] && __epm_restore_nupkg "$PROJ"
            ;;
        Gemfile|package.json)
            info "$req_file support is not implemented yet"
            ;;
    esac
}

epm_restore()
{
    req_file="$pkg_filenames"
    if [ -n "$pkg_urls" ] && echo "$pkg_urls" | grep -qE "^https?://" ; then
        req_file="$(basename "$pkg_urls")"
        #assure eget
        [ -r "$req_file" ] && fatal "File $req_file is already exists in $(pwd)"
        info "Downloading '$req_file' from '$pkg_urls' ..."
        eget "$pkg_urls"
        [ -s "$req_file" ] || fatal "Can't download $req_file from '$pkg_urls'"
    fi

    if [ -n "$req_file" ] ; then
        __epm_restore_by $req_file
        return
    fi

# TODO: nowhere works: python3 setup.py --requires

    # if run with empty args
    for i in requirements.txt requirements/default.txt requirements_dev.txt requirements-dev.txt requirements/dev.txt dev-requirements.txt \
             requirements-test.txt requirements_test.txt requirements/test.txt test-requirements.txt requirements/coverage.txt \
             Gemfile requires.txt package.json setup.py python_dependencies.py Makefile.PL \
             *.sln *.csproj ; do
        __epm_restore_by $i
    done

}