cmd: Handle special case tokens=* in for /f.

programs/cmd/builtins.c

@@ -1831,6 +1831,14 @@ static int WCMD_for_nexttoken(int lasttoken, WCHAR *tokenstr,
     int nextnumber1, nextnumber2 = -1;
     WCHAR *nextchar;
 
+    /* It is valid syntax tokens=* which just means get whole line */
+    if (*pos == '*') {
+      if (doall) *doall = TRUE;
+      if (totalfound) (*totalfound)++;
+      nexttoken = 0;
+      break;
+    }
+
     /* Get the next number */
     nextnumber1 = strtoulW(pos, &nextchar, 10);
 
@@ -1959,22 +1967,22 @@ static void WCMD_parse_line(CMD_LIST    *cmdStart,
    */
   lasttoken = -1;
   nexttoken = WCMD_for_nexttoken(lasttoken, forf_tokens, &totalfound,
-                                 NULL, &thisduplicate);
+                                 &starfound, &thisduplicate);
   varidx = FOR_VAR_IDX(variable);
 
   /* Empty out variables */
   for (varoffset=0;
-       varidx >= 0 && varoffset<totalfound && ((varidx+varoffset)%26);
+       varidx >= 0 && varoffset<totalfound && (((varidx%26) + varoffset) < 26);
        varoffset++) {
     forloopcontext.variable[varidx + varoffset] = (WCHAR *)nullW;
-    /* Stop if we walk beyond z or Z */
-    if (((varidx+varoffset) % 26) == 0) break;
   }
 
-  /* Loop extracting the tokens */
+  /* Loop extracting the tokens
+     Note: nexttoken of 0 means there were no tokens requested, to handle
+           the special case of tokens=*                                   */
   varoffset = 0;
   WINE_TRACE("Parsing buffer into tokens: '%s'\n", wine_dbgstr_w(buffer));
-  while (varidx >= 0 && (nexttoken > lasttoken)) {
+  while (varidx >= 0 && (nexttoken > 0 && (nexttoken > lasttoken))) {
     anyduplicates |= thisduplicate;
 
     /* Extract the token number requested and set into the next variable context */
@@ -1982,9 +1990,9 @@ static void WCMD_parse_line(CMD_LIST    *cmdStart,
     WINE_TRACE("Parsed token %d(%d) as parameter %s\n", nexttoken,
                varidx + varoffset, wine_dbgstr_w(parm));
     if (varidx >=0) {
-      forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+      if (parm) forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
       varoffset++;
-      if (((varidx + varoffset) %26) == 0) break;
+      if (((varidx%26)+varoffset) >= 26) break;
     }
 
     /* Find the next token */
@@ -1995,12 +2003,12 @@ static void WCMD_parse_line(CMD_LIST    *cmdStart,
 
   /* If all the rest of the tokens were requested, and there is still space in
      the variable range, write them now                                        */
-  if (!anyduplicates && starfound && varidx >= 0 && ((varidx+varoffset) % 26)) {
+  if (!anyduplicates && starfound && varidx >= 0 && (((varidx%26) + varoffset) < 26)) {
     nexttoken++;
     WCMD_parameter_with_delims(buffer, (nexttoken-1), &parm, FALSE, FALSE, forf_delims);
     WINE_TRACE("Parsed allremaining tokens (%d) as parameter %s\n",
                varidx + varoffset, wine_dbgstr_w(parm));
-    forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
+    if (parm) forloopcontext.variable[varidx + varoffset] = heap_strdupW(parm);
   }
 
   /* Execute the body of the foor loop with these values */

programs/cmd/tests/test_builtins.cmd

@@ -1744,6 +1744,12 @@ for /f "tokens=1,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k
 for /f "tokens=1,1,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
 for /f "tokens=2,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
 for /f "tokens=3,2,3*" %%i in ("a b c d e f g") do echo h=%%h i=%%i j=%%j k=%%k l=%%l m=%%m n=%%n o=%%o
+rem Special case tokens=*
+echo 3.14>testfile
+FOR /F "tokens=*"  %%A IN (testfile) DO @echo 1:%%A,%%B
+FOR /F "tokens=1*" %%A IN (testfile) DO @echo 2:%%A,%%B
+FOR /F "tokens=2*" %%A IN (testfile) DO @echo 3:%%A,%%B
+del testfile
 cd ..
 rd /s/q foobar
 echo ------ parameter splitting
@@ -1756,6 +1762,12 @@ goto :forFParameterSplittingEnd
 echo %~0 %~1 %~2 %~3 %~4 %~5
 goto :eof
 :forFParameterSplittingEnd
+echo 3.14>testfile
+FOR /F "delims=. tokens=*"  %%A IN (testfile) DO @echo 4:%%A,%%B
+FOR /F "delims=. tokens=1*" %%A IN (testfile) DO @echo 5:%%A,%%B
+FOR /F "delims=. tokens=2*" %%A IN (testfile) DO @echo 6:%%A,%%B
+FOR /F "delims=. tokens=3*" %%A IN (testfile) DO @echo 7:%%A,%%B
+del testfile
 
 echo ------------ Testing del ------------
 echo abc > file

programs/cmd/tests/test_builtins.cmd.exp

@@ -1234,9 +1234,14 @@ h=%h i=a j=b k=c l=d e f g m=%m n=%n o=%o@or_broken@h=%h i=a j=b k=c l=d e f g m
 h=%h i=a j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=a j=c k= l= m= n=%n o=%o
 h=%h i=b j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
 h=%h i=b j=c k= l= m=%m n=%n o=%o@or_broken@h=%h i=b j=c k= l= m= n=%n o=%o
+1:3.14,%B
+2:3.14,
 ------ parameter splitting
 :forFParameterSplittingFunc myparam1=myvalue1 myparam2=myparam2 mytest@space@@space@@space@
 :forFParameterSplittingFunc myparam1=myvalue1 myparam2=myparam2 mytest@space@@space@@space@
+4:3.14,%B
+5:3,14
+6:14,
 ------------ Testing del ------------
 deleting 'file'
 errorlevel is 0, good