tokenize.c 8.72 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
12
** A tokenizer for SQL
13 14 15 16 17 18 19
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
*/

#include <ctype.h>
20
#include <stdarg.h>
21
#include <stdlib.h>
22 23 24

#include "windef.h"
#include "winbase.h"
25
#include "wine/unicode.h"
26
#include "query.h"
27
#include "sql.tab.h"
28 29 30 31 32 33 34

/*
** All the keywords of the SQL language are stored as in a hash
** table composed of instances of the following structure.
*/
typedef struct Keyword Keyword;
struct Keyword {
35
  const WCHAR *zName;             /* The keyword name */
36 37 38
  int tokenType;           /* The token value for this keyword */
};

39 40
#define MAX_TOKEN_LEN 11

41
static const WCHAR ADD_W[] = { 'A','D','D',0 };
42
static const WCHAR ALTER_W[] = { 'A','L','T','E','R',0 };
43 44 45 46 47 48 49
static const WCHAR AND_W[] = { 'A','N','D',0 };
static const WCHAR BY_W[] = { 'B','Y',0 };
static const WCHAR CHAR_W[] = { 'C','H','A','R',0 };
static const WCHAR CHARACTER_W[] = { 'C','H','A','R','A','C','T','E','R',0 };
static const WCHAR CREATE_W[] = { 'C','R','E','A','T','E',0 };
static const WCHAR DELETE_W[] = { 'D','E','L','E','T','E',0 };
static const WCHAR DISTINCT_W[] = { 'D','I','S','T','I','N','C','T',0 };
50
static const WCHAR DROP_W[] = { 'D','R','O','P',0 };
51
static const WCHAR FREE_W[] = { 'F','R','E','E',0 };
52 53 54 55
static const WCHAR FROM_W[] = { 'F','R','O','M',0 };
static const WCHAR HOLD_W[] = { 'H','O','L','D',0 };
static const WCHAR INSERT_W[] = { 'I','N','S','E','R','T',0 };
static const WCHAR INT_W[] = { 'I','N','T',0 };
56
static const WCHAR INTEGER_W[] = { 'I','N','T','E','G','E','R',0 };
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
static const WCHAR INTO_W[] = { 'I','N','T','O',0 };
static const WCHAR IS_W[] = { 'I','S',0 };
static const WCHAR KEY_W[] = { 'K','E','Y',0 };
static const WCHAR LIKE_W[] = { 'L','I','K','E',0 };
static const WCHAR LOCALIZABLE_W[] = { 'L','O','C','A','L','I','Z','A','B','L','E',0 };
static const WCHAR LONG_W[] = { 'L','O','N','G',0 };
static const WCHAR LONGCHAR_W[] = { 'L','O','N','G','C','H','A','R',0 };
static const WCHAR NOT_W[] = { 'N','O','T',0 };
static const WCHAR NULL_W[] = { 'N','U','L','L',0 };
static const WCHAR OBJECT_W[] = { 'O','B','J','E','C','T',0 };
static const WCHAR OR_W[] = { 'O','R',0 };
static const WCHAR ORDER_W[] = { 'O','R','D','E','R',0 };
static const WCHAR PRIMARY_W[] = { 'P','R','I','M','A','R','Y',0 };
static const WCHAR SELECT_W[] = { 'S','E','L','E','C','T',0 };
static const WCHAR SET_W[] = { 'S','E','T',0 };
static const WCHAR SHORT_W[] = { 'S','H','O','R','T',0 };
static const WCHAR TABLE_W[] = { 'T','A','B','L','E',0 };
static const WCHAR TEMPORARY_W[] = { 'T','E','M','P','O','R','A','R','Y',0 };
static const WCHAR UPDATE_W[] = { 'U','P','D','A','T','E',0 };
static const WCHAR VALUES_W[] = { 'V','A','L','U','E','S',0 };
static const WCHAR WHERE_W[] = { 'W','H','E','R','E',0 };

79 80
/*
** These are the keywords
81
** They MUST be in alphabetical order
82 83
*/
static const Keyword aKeywordTable[] = {
84
  { ADD_W, TK_ADD },
85
  { ALTER_W, TK_ALTER },
86 87 88 89 90 91 92
  { AND_W, TK_AND },
  { BY_W, TK_BY },
  { CHAR_W, TK_CHAR },
  { CHARACTER_W, TK_CHAR },
  { CREATE_W, TK_CREATE },
  { DELETE_W, TK_DELETE },
  { DISTINCT_W, TK_DISTINCT },
93
  { DROP_W, TK_DROP },
94
  { FREE_W, TK_FREE },
95 96 97 98
  { FROM_W, TK_FROM },
  { HOLD_W, TK_HOLD },
  { INSERT_W, TK_INSERT },
  { INT_W, TK_INT },
99
  { INTEGER_W, TK_INT },
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
  { INTO_W, TK_INTO },
  { IS_W, TK_IS },
  { KEY_W, TK_KEY },
  { LIKE_W, TK_LIKE },
  { LOCALIZABLE_W, TK_LOCALIZABLE },
  { LONG_W, TK_LONG },
  { LONGCHAR_W, TK_LONGCHAR },
  { NOT_W, TK_NOT },
  { NULL_W, TK_NULL },
  { OBJECT_W, TK_OBJECT },
  { OR_W, TK_OR },
  { ORDER_W, TK_ORDER },
  { PRIMARY_W, TK_PRIMARY },
  { SELECT_W, TK_SELECT },
  { SET_W, TK_SET },
  { SHORT_W, TK_SHORT },
  { TABLE_W, TK_TABLE },
117
  { TEMPORARY_W, TK_TEMPORARY },
118 119 120
  { UPDATE_W, TK_UPDATE },
  { VALUES_W, TK_VALUES },
  { WHERE_W, TK_WHERE },
121 122 123 124
};

#define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )

125 126 127 128 129 130 131 132 133
/*
** Comparison function for binary search.
*/
static int compKeyword(const void *m1, const void *m2){
  const Keyword *k1 = m1, *k2 = m2;

  return strcmpiW( k1->zName, k2->zName );
}

134 135 136 137 138
/*
** This function looks up an identifier to determine if it is a
** keyword.  If it is a keyword, the token code of that keyword is 
** returned.  If the input is not a keyword, TK_ID is returned.
*/
139
static int sqliteKeywordCode(const WCHAR *z, int n){
140 141 142 143 144
  WCHAR str[MAX_TOKEN_LEN+1];
  Keyword key, *r;

  if( n>MAX_TOKEN_LEN )
    return TK_ID;
145

146 147 148 149 150 151 152
  memcpy( str, z, n*sizeof (WCHAR) );
  str[n] = 0;
  key.tokenType = 0;
  key.zName = str;
  r = bsearch( &key, aKeywordTable, KEYWORD_COUNT, sizeof (Keyword), compKeyword );
  if( r )
    return r->tokenType;
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
  return TK_ID;
}


/*
** If X is a character that can be used in an identifier then
** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
**
** In this implementation, an identifier can be a string of
** alphabetic characters, digits, and "_" plus any character
** with the high-order bit set.  The latter rule means that
** any sequence of UTF-8 characters or characters taken from
** an extended ISO8859 character set can form an identifier.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
169
    0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
170
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
171
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,  /* 2x */
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
};


/*
** Return the length of the token that begins at z[0].  Return
** -1 if the token is (or might be) incomplete.  Store the token
** type in *tokenType before returning.
*/
193
int sqliteGetToken(const WCHAR *z, int *tokenType, int *skip){
194
  int i;
195 196

  *skip = 0;
197
  switch( *z ){
198
    case ' ': case '\t': case '\n': case '\f':
199
      for(i=1; isspace(z[i]) && z[i] != '\r'; i++){}
200 201
      *tokenType = TK_SPACE;
      return i;
202
    case '-':
203 204 205
      if( z[1]==0 ) return -1;
      *tokenType = TK_MINUS;
      return 1;
206 207
    case '(':
      *tokenType = TK_LP;
208
      return 1;
209 210
    case ')':
      *tokenType = TK_RP;
211
      return 1;
212
    case '*':
213 214
      *tokenType = TK_STAR;
      return 1;
215
    case '=':
216
      *tokenType = TK_EQ;
217
      return 1;
218
    case '<':
219 220 221 222 223 224 225 226 227 228
      if( z[1]=='=' ){
        *tokenType = TK_LE;
        return 2;
      }else if( z[1]=='>' ){
        *tokenType = TK_NE;
        return 2;
      }else{
        *tokenType = TK_LT;
        return 1;
      }
229
    case '>':
230 231 232 233 234 235 236
      if( z[1]=='=' ){
        *tokenType = TK_GE;
        return 2;
      }else{
        *tokenType = TK_GT;
        return 1;
      }
237
    case '!':
238 239 240 241 242 243 244
      if( z[1]!='=' ){
        *tokenType = TK_ILLEGAL;
        return 2;
      }else{
        *tokenType = TK_NE;
        return 2;
      }
245
    case '?':
246 247
      *tokenType = TK_WILDCARD;
      return 1;
248
    case ',':
249 250
      *tokenType = TK_COMMA;
      return 1;
251
    case '`': case '\'': {
252 253
      int delim = z[0];
      for(i=1; z[i]; i++){
254 255
        if( z[i]==delim )
          break;
256 257
      }
      if( z[i] ) i++;
258 259 260 261
      if( delim == '`' )
        *tokenType = TK_ID;
      else
        *tokenType = TK_STRING;
262 263
      return i;
    }
264
    case '.':
265 266 267 268
      if( !isdigit(z[1]) ){
        *tokenType = TK_DOT;
        return 1;
      }
269
      /* Fall through */
270
    case '0': case '1': case '2': case '3': case '4':
271
    case '5': case '6': case '7': case '8': case '9':
272 273 274
      *tokenType = TK_INTEGER;
      for(i=1; isdigit(z[i]); i++){}
      return i;
275
    case '[':
276 277 278
      for(i=1; z[i] && z[i-1]!=']'; i++){}
      *tokenType = TK_ID;
      return i;
279
    default:
280 281 282 283 284
      if( !isIdChar[*z] ){
        break;
      }
      for(i=1; isIdChar[z[i]]; i++){}
      *tokenType = sqliteKeywordCode(z, i);
285
      if( *tokenType == TK_ID && z[i] == '`' ) *skip = 1;
286 287 288 289 290
      return i;
  }
  *tokenType = TK_ILLEGAL;
  return 1;
}