tokenize.c 9.95 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
12
** A tokenizer for SQL
13 14 15 16 17 18 19
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
*/

#include <ctype.h>
20
#include <stdarg.h>
21
#include <stdlib.h>
22 23 24

#include "windef.h"
#include "winbase.h"
25
#include "wine/unicode.h"
26
#include "query.h"
27
#include "sql.tab.h"
28 29 30 31 32 33 34

/*
** All the keywords of the SQL language are stored as in a hash
** table composed of instances of the following structure.
*/
typedef struct Keyword Keyword;
struct Keyword {
35 36
  const WCHAR *name;             /* The keyword name */
  unsigned int len;
37 38 39
  int tokenType;           /* The token value for this keyword */
};

40 41
#define MAX_TOKEN_LEN 11

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
static const WCHAR addW[] = {'A','D','D'};
static const WCHAR alterW[] = {'A','L','T','E','R'};
static const WCHAR andW[] = {'A','N','D'};
static const WCHAR byW[] = {'B','Y'};
static const WCHAR charW[] = {'C','H','A','R'};
static const WCHAR characterW[] = {'C','H','A','R','A','C','T','E','R'};
static const WCHAR createW[] = {'C','R','E','A','T','E'};
static const WCHAR deleteW[] = {'D','E','L','E','T','E'};
static const WCHAR distinctW[] = {'D','I','S','T','I','N','C','T'};
static const WCHAR dropW[] = {'D','R','O','P'};
static const WCHAR freeW[] = {'F','R','E','E'};
static const WCHAR fromW[] = {'F','R','O','M'};
static const WCHAR holdW[] = {'H','O','L','D'};
static const WCHAR insertW[] = {'I','N','S','E','R','T'};
static const WCHAR intW[] = {'I','N','T'};
static const WCHAR integerW[] = {'I','N','T','E','G','E','R'};
static const WCHAR intoW[] = {'I','N','T','O'};
static const WCHAR isW[] = {'I','S'};
static const WCHAR keyW[] = {'K','E','Y'};
static const WCHAR likeW[] = {'L','I','K','E'};
static const WCHAR localizableW[] = {'L','O','C','A','L','I','Z','A','B','L','E'};
static const WCHAR longW[] = {'L','O','N','G'};
static const WCHAR longcharW[] = {'L','O','N','G','C','H','A','R'};
static const WCHAR notW[] = {'N','O','T'};
static const WCHAR nullW[] = {'N','U','L','L'};
static const WCHAR objectW[] = {'O','B','J','E','C','T'};
static const WCHAR orW[] = {'O','R'};
static const WCHAR orderW[] = {'O','R','D','E','R'};
static const WCHAR primaryW[] = {'P','R','I','M','A','R','Y'};
static const WCHAR selectW[] = {'S','E','L','E','C','T'};
static const WCHAR setW[] = {'S','E','T'};
static const WCHAR shortW[] = {'S','H','O','R','T'};
static const WCHAR tableW[] = {'T','A','B','L','E'};
static const WCHAR temporaryW[] = {'T','E','M','P','O','R','A','R','Y'};
static const WCHAR updateW[] = {'U','P','D','A','T','E'};
static const WCHAR valuesW[] = {'V','A','L','U','E','S'};
static const WCHAR whereW[] = {'W','H','E','R','E'};

80 81
/*
** These are the keywords
82
** They MUST be in alphabetical order
83 84
*/
static const Keyword aKeywordTable[] = {
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  { addW,         ARRAY_SIZE(addW),         TK_ADD },
  { alterW,       ARRAY_SIZE(alterW),       TK_ALTER },
  { andW,         ARRAY_SIZE(andW),         TK_AND },
  { byW,          ARRAY_SIZE(byW),          TK_BY },
  { charW,        ARRAY_SIZE(charW),        TK_CHAR },
  { characterW,   ARRAY_SIZE(characterW),   TK_CHAR },
  { createW,      ARRAY_SIZE(createW),      TK_CREATE },
  { deleteW,      ARRAY_SIZE(deleteW),      TK_DELETE },
  { distinctW,    ARRAY_SIZE(distinctW),    TK_DISTINCT },
  { dropW,        ARRAY_SIZE(dropW),        TK_DROP },
  { freeW,        ARRAY_SIZE(freeW),        TK_FREE },
  { fromW,        ARRAY_SIZE(fromW),        TK_FROM },
  { holdW,        ARRAY_SIZE(holdW),        TK_HOLD },
  { insertW,      ARRAY_SIZE(insertW),      TK_INSERT },
  { intW,         ARRAY_SIZE(intW),         TK_INT },
  { integerW,     ARRAY_SIZE(integerW),     TK_INT },
  { intoW,        ARRAY_SIZE(intoW),        TK_INTO },
  { isW,          ARRAY_SIZE(isW),          TK_IS },
  { keyW,         ARRAY_SIZE(keyW),         TK_KEY },
  { likeW,        ARRAY_SIZE(likeW),        TK_LIKE },
  { localizableW, ARRAY_SIZE(localizableW), TK_LOCALIZABLE },
  { longW,        ARRAY_SIZE(longW),        TK_LONG },
  { longcharW,    ARRAY_SIZE(longcharW),    TK_LONGCHAR },
  { notW,         ARRAY_SIZE(notW),         TK_NOT },
  { nullW,        ARRAY_SIZE(nullW),        TK_NULL },
  { objectW,      ARRAY_SIZE(objectW),      TK_OBJECT },
  { orW,          ARRAY_SIZE(orW),          TK_OR },
  { orderW,       ARRAY_SIZE(orderW),       TK_ORDER },
  { primaryW,     ARRAY_SIZE(primaryW),     TK_PRIMARY },
  { selectW,      ARRAY_SIZE(selectW),      TK_SELECT },
  { setW,         ARRAY_SIZE(setW),         TK_SET },
  { shortW,       ARRAY_SIZE(shortW),       TK_SHORT },
  { tableW,       ARRAY_SIZE(tableW),       TK_TABLE },
  { temporaryW,   ARRAY_SIZE(temporaryW),   TK_TEMPORARY },
  { updateW,      ARRAY_SIZE(updateW),      TK_UPDATE },
  { valuesW,      ARRAY_SIZE(valuesW),      TK_VALUES },
  { whereW,       ARRAY_SIZE(whereW),       TK_WHERE },
122 123
};

124 125 126 127 128
/*
** Comparison function for binary search.
*/
static int compKeyword(const void *m1, const void *m2){
  const Keyword *k1 = m1, *k2 = m2;
129
  int ret, len = min( k1->len, k2->len );
130

131 132 133 134
  if ((ret = memicmpW( k1->name, k2->name, len ))) return ret;
  if (k1->len < k2->len) return -1;
  else if (k1->len > k2->len) return 1;
  return 0;
135 136
}

137 138 139 140 141
/*
** This function looks up an identifier to determine if it is a
** keyword.  If it is a keyword, the token code of that keyword is 
** returned.  If the input is not a keyword, TK_ID is returned.
*/
142
static int sqliteKeywordCode(const WCHAR *z, int n){
143 144 145 146
  Keyword key, *r;

  if( n>MAX_TOKEN_LEN )
    return TK_ID;
147

148
  key.tokenType = 0;
149 150 151
  key.name = z;
  key.len = n;
  r = bsearch( &key, aKeywordTable, ARRAY_SIZE(aKeywordTable), sizeof(Keyword), compKeyword );
152 153
  if( r )
    return r->tokenType;
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
  return TK_ID;
}


/*
** If X is a character that can be used in an identifier then
** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
**
** In this implementation, an identifier can be a string of
** alphabetic characters, digits, and "_" plus any character
** with the high-order bit set.  The latter rule means that
** any sequence of UTF-8 characters or characters taken from
** an extended ISO8859 character set can form an identifier.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
170
    0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
171
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
172
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,  /* 2x */
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
};

188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
/*
** WCHAR safe version of isdigit()
*/
static inline int isDigit(WCHAR c)
{
    return c >= '0' && c <= '9';
}

/*
** WCHAR safe version of isspace(), except '\r'
*/
static inline int isSpace(WCHAR c)
{
    return c == ' ' || c == '\t' || c == '\n' || c == '\f';
}
203 204 205 206 207 208

/*
** Return the length of the token that begins at z[0].  Return
** -1 if the token is (or might be) incomplete.  Store the token
** type in *tokenType before returning.
*/
209
int sqliteGetToken(const WCHAR *z, int *tokenType, int *skip){
210
  int i;
211 212

  *skip = 0;
213
  switch( *z ){
214
    case ' ': case '\t': case '\n': case '\f':
215
      for(i=1; isSpace(z[i]); i++){}
216 217
      *tokenType = TK_SPACE;
      return i;
218
    case '-':
219 220 221
      if( z[1]==0 ) return -1;
      *tokenType = TK_MINUS;
      return 1;
222 223
    case '(':
      *tokenType = TK_LP;
224
      return 1;
225 226
    case ')':
      *tokenType = TK_RP;
227
      return 1;
228
    case '*':
229 230
      *tokenType = TK_STAR;
      return 1;
231
    case '=':
232
      *tokenType = TK_EQ;
233
      return 1;
234
    case '<':
235 236 237 238 239 240 241 242 243 244
      if( z[1]=='=' ){
        *tokenType = TK_LE;
        return 2;
      }else if( z[1]=='>' ){
        *tokenType = TK_NE;
        return 2;
      }else{
        *tokenType = TK_LT;
        return 1;
      }
245
    case '>':
246 247 248 249 250 251 252
      if( z[1]=='=' ){
        *tokenType = TK_GE;
        return 2;
      }else{
        *tokenType = TK_GT;
        return 1;
      }
253
    case '!':
254 255 256 257 258 259 260
      if( z[1]!='=' ){
        *tokenType = TK_ILLEGAL;
        return 2;
      }else{
        *tokenType = TK_NE;
        return 2;
      }
261
    case '?':
262 263
      *tokenType = TK_WILDCARD;
      return 1;
264
    case ',':
265 266
      *tokenType = TK_COMMA;
      return 1;
267
    case '`': case '\'': {
268 269
      int delim = z[0];
      for(i=1; z[i]; i++){
270 271
        if( z[i]==delim )
          break;
272 273
      }
      if( z[i] ) i++;
274 275 276 277
      if( delim == '`' )
        *tokenType = TK_ID;
      else
        *tokenType = TK_STRING;
278 279
      return i;
    }
280
    case '.':
281
      if( !isDigit(z[1]) ){
282 283 284
        *tokenType = TK_DOT;
        return 1;
      }
285
      /* Fall through */
286
    case '0': case '1': case '2': case '3': case '4':
287
    case '5': case '6': case '7': case '8': case '9':
288
      *tokenType = TK_INTEGER;
289
      for(i=1; isDigit(z[i]); i++){}
290
      return i;
291
    case '[':
292 293 294
      for(i=1; z[i] && z[i-1]!=']'; i++){}
      *tokenType = TK_ID;
      return i;
295
    default:
296 297 298 299 300
      if( !isIdChar[*z] ){
        break;
      }
      for(i=1; isIdChar[z[i]]; i++){}
      *tokenType = sqliteKeywordCode(z, i);
301
      if( *tokenType == TK_ID && z[i] == '`' ) *skip = 1;
302 303 304 305 306
      return i;
  }
  *tokenType = TK_ILLEGAL;
  return 1;
}