epm-restore: rewrite to more full support of python arrays, add python_dependencies.py support

bin/epm-restore

@@ -27,11 +27,14 @@ __epm_filter_pip_to_rpm()
 {
     tr "A-Z" "a-z" | sed -e "s|-|_|g" -e "s|^python_||" \
         -e "s|beautifulsoup4|bs4|" \
-        -e "s|pillow|PIL|" \
+        -e "s|pillow|Pillow|" \
         -e "s|redis|redis-py|" \
         -e "s|pyjwt|jwt|" \
+        -e "s|pymonetdb|monetdb|" \
         -e "s|pyyaml|yaml|" \
-        -e "s|attrs|attr|" \
+        -e "s|twisted|twisted-core|" \
+        -e "s|pymacaroons|pymacaroons-pynacl|" \
+        -e "s|pyasn1_modules|pyasn1-modules|" \
         -e "s|pygments|Pygments|" \
         -e "s|patch_ng|patch-ng|" \
         -e "s|memcached|memcache|" \
@@ -49,11 +52,10 @@ fill_sign()
 # macro pkg caseline
 __epm_pi_sign_to_rpm()
 {
-    local reqmacro="$1"
-    local t="$2"
-    local l="$3"
-    local equal="$4"
-    [ -n "$equal" ] || equal="="
+    local t="$1"
+    local l="$2"
+    local equal="$3"
+    [ -n "$equal" ] || equal=">="
 
     local pi=''
     local sign ll
@@ -64,28 +66,37 @@ __epm_pi_sign_to_rpm()
         [ "$sign" = "!=" ] && sign=">="
         [ -n "$pi" ] && pi="$pi
 "
-        pi="$pi$reqmacro $t $sign $ll"
+        pi="$pi$t $sign $ll"
     done
-    [ -n "$pi" ] || pi="$reqmacro $t"
+    [ -n "$pi" ] || pi="$t"
     echo "$pi"
 }
 
-__epm_restore_pip()
+__epm_get_array_name()
 {
-    local req_file="$1"
-    if [ -n "$dryrun" ] ; then
-        echo
-        echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file"
-    else
-        info "Install requirements from $req_file ..."
-    fi
+    echo "$*" | grep "=" | head -n1 | sed -e 's| *=.*||'
+}
 
-    local ilist=''
+__epm_lineprint_python_array()
+{
+    local a="$*"
+    [ -n "$a" ] || return
+    local name="$(__epm_get_array_name "$a")"
+    (echo "$a" | sed -e 's|\].*|]|' ; echo "print('\n'.join($name))" ) | ( a= python - || a= python3 )
+}
+
+# translate pip requirement lines to rpm notation
+# (compare signs and package names)
+__epm_restore_convert_to_rpm_notation()
+{
+    local equal="$1"
+    local l
     while read l ; do
         local t="$(echo "$l" | sed -e "s| *[<>!]*=.*||" -e "s| *#.*||" | __epm_filter_pip_to_rpm)"
         if echo "$l" | grep -qE "^ *#" || [ -z "$l" ] ; then
             continue
         fi
+        [ -n "$t" ] || continue
         # until new section
         if echo "$l" | grep -qE "^\[" ; then
             break
@@ -103,28 +114,37 @@ __epm_restore_pip()
             warning "    $t is python2 only requirement, skipped"
             continue
         fi
-        [ -n "$t" ] || continue
-        if [ -n "$dryrun" ] ; then
-            local reqmacro="%py3_use"
-            [ "$(basename "$req_file")" = "requirements_dev.txt" ] && reqmacro="%py3_buildrequires"
-            __epm_pi_sign_to_rpm $reqmacro $t $l
-            continue
-        else
-            # TODO: python3-egg-info($t)
-            local pi="python3($t)"
-            echo "    $l -> $t -> $pi"
-        fi
-        ilist="$ilist $pi"
-    done < $req_file
+        __epm_pi_sign_to_rpm "$t" "$l" "$equal"
+    done
+}
+
+__epm_restore_pip()
+{
+    local req_file="$1"
+    local reqmacro
+    local ilist
 
     if [ -n "$dryrun" ] ; then
-        echo "$ilist"
+        reqmacro="%py3_use"
+        [ "$(basename "$req_file")" = "requirements_dev.txt" ] && reqmacro="%py3_buildrequires"
+        echo
+        echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file"
+        cat $req_file | __epm_restore_convert_to_rpm_notation | sed -e "s|^|$reqmacro |"
         return
+    else
+        info "Install requirements from $req_file ..."
+        ilist="$(cat $req_file | __epm_restore_convert_to_rpm_notation | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
     fi
 
+    ilist="$(estrlist list $ilist)"
     docmd epm install $ilist
 }
 
+__eresection()
+{
+    rhas "$1" "[[:space:]]*$2[[:space:]]*=[[:space:]]*\["
+}
+
 __epm_restore_setup_py()
 {
     local req_file="$1"
@@ -132,55 +152,57 @@ __epm_restore_setup_py()
         info "Install requirements from $req_file ..."
     fi
 
+    local ar=''
     local ilist=''
-    local fr fd
     local reqmacro
+    local section=''
     while read l ; do
         if rhas "$l" "^ *#" ; then
             continue
         fi
-        if rhas "$l" "\]," ; then
-            fr=''
-            fd=''
-            continue
-        fi
-        if [ -n "$fr$fd" ] ; then
-            local ll="$(echo "$l" | sed -e "s|[[:space:]]*'||" -e "s|'.*||")"
-            local t="$(echo "$ll" | sed -e "s| *[<>!]*=.*||" | __epm_filter_pip_to_rpm)"
-            if [ -n "$dryrun" ] ; then
-                __epm_pi_sign_to_rpm $reqmacro $t $ll ">="
-            else
-                local pi="python3-module-$t"
-                ilist="$ilist $pi"
-                echo "    $l -> $t -> $pi"
-            fi
-            continue
+        # start of section
+        if __eresection "$l" "REQUIREMENTS" ; then
+            reqmacro="%py3_use"
+            section="$l"
         fi
-        if rhas "$l" "install_requires=\[" ; then
+        if __eresection "$l" "install_requires" ; then
             reqmacro="%py3_use"
-            if [ -n "$dryrun" ] ; then
-                echo
-                echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file $l"
-            fi
-            fr=1
-            continue
+            section="$l"
         fi
-        if rhas "$l" "tests_require=\[" ; then
+        if __eresection "$l" "setup_requires" ; then
             reqmacro="%py3_buildrequires"
-            if [ -n "$dryrun" ] ; then
-                echo
-                echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file $l"
-            fi
-            fd=1
+            section="$l"
+        fi
+        if __eresection "$l" "tests_require" ; then
+            reqmacro="%py3_buildrequires"
+            section="$l"
+        fi
+        if [ -n "$section" ] ; then
+            ar="$ar
+$l"
+        fi
+
+        # not end of section
+        if [ -z "$section" ] || ! rhas "$l" "\],*" ; then
             continue
         fi
+
+        if [ -n "$dryrun" ] ; then
+            echo
+            echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file $(__epm_get_array_name "$section")"
+            __epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | sed -e "s|^|$reqmacro |"
+        else
+            ilist="$ilist $(__epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
+        fi
+        section=''
+        ar=''
     done < $req_file
 
     if [ -n "$dryrun" ] ; then
-        echo "$ilist"
         return
     fi
 
+    ilist="$(estrlist list $ilist)"
     docmd epm install $ilist
 }
 
@@ -276,7 +298,7 @@ __epm_restore_by()
         requirements.txt|requirements_dev.txt|requires.txt)
             [ -s "$req_file" ] && __epm_restore_pip "$req_file"
             ;;
-        setup.py)
+        setup.py|python_dependencies.py)
             [ -s "$req_file" ] && __epm_restore_setup_py "$req_file"
             ;;
         package.json)
@@ -308,7 +330,7 @@ epm_restore()
 # TODO: nowhere works: python3 setup.py --requires
 
     # if run with empty args
-    for i in requirements.txt requirements_dev.txt Gemfile requires.txt package.json setup.py; do
+    for i in requirements.txt requirements_dev.txt Gemfile requires.txt package.json setup.py python_dependencies.py; do
         __epm_restore_by $i
     done