tokenize.c 9.14 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
*/

#include <ctype.h>
20
#include <stdarg.h>
21
#include <stdlib.h>
22 23 24

#include "windef.h"
#include "winbase.h"
25
#include "wine/unicode.h"
26
#include "query.h"
27
#include "sql.tab.h"
28 29 30 31 32 33 34

/*
** All the keywords of the SQL language are stored as in a hash
** table composed of instances of the following structure.
*/
typedef struct Keyword Keyword;
struct Keyword {
35
  const WCHAR *zName;             /* The keyword name */
36 37 38
  int tokenType;           /* The token value for this keyword */
};

39 40
#define MAX_TOKEN_LEN 11

41
static const WCHAR ALTER_W[] = { 'A','L','T','E','R',0 };
42 43 44 45 46 47 48 49
static const WCHAR AND_W[] = { 'A','N','D',0 };
static const WCHAR BY_W[] = { 'B','Y',0 };
static const WCHAR CHAR_W[] = { 'C','H','A','R',0 };
static const WCHAR CHARACTER_W[] = { 'C','H','A','R','A','C','T','E','R',0 };
static const WCHAR CREATE_W[] = { 'C','R','E','A','T','E',0 };
static const WCHAR CROSS_W[] = { 'C','R','O','S','S',0 };
static const WCHAR DELETE_W[] = { 'D','E','L','E','T','E',0 };
static const WCHAR DISTINCT_W[] = { 'D','I','S','T','I','N','C','T',0 };
50
static const WCHAR FREE_W[] = { 'F','R','E','E',0 };
51 52 53 54 55 56
static const WCHAR FROM_W[] = { 'F','R','O','M',0 };
static const WCHAR FULL_W[] = { 'F','U','L','L',0 };
static const WCHAR HOLD_W[] = { 'H','O','L','D',0 };
static const WCHAR INNER_W[] = { 'I','N','N','E','R',0 };
static const WCHAR INSERT_W[] = { 'I','N','S','E','R','T',0 };
static const WCHAR INT_W[] = { 'I','N','T',0 };
57
static const WCHAR INTEGER_W[] = { 'I','N','T','E','G','E','R',0 };
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
static const WCHAR INTO_W[] = { 'I','N','T','O',0 };
static const WCHAR IS_W[] = { 'I','S',0 };
static const WCHAR JOIN_W[] = { 'J','O','I','N',0 };
static const WCHAR KEY_W[] = { 'K','E','Y',0 };
static const WCHAR LEFT_W[] = { 'L','E','F','T',0 };
static const WCHAR LIKE_W[] = { 'L','I','K','E',0 };
static const WCHAR LOCALIZABLE_W[] = { 'L','O','C','A','L','I','Z','A','B','L','E',0 };
static const WCHAR LONG_W[] = { 'L','O','N','G',0 };
static const WCHAR LONGCHAR_W[] = { 'L','O','N','G','C','H','A','R',0 };
static const WCHAR NATURAL_W[] = { 'N','A','T','U','R','A','L',0 };
static const WCHAR NOT_W[] = { 'N','O','T',0 };
static const WCHAR NULL_W[] = { 'N','U','L','L',0 };
static const WCHAR OBJECT_W[] = { 'O','B','J','E','C','T',0 };
static const WCHAR OR_W[] = { 'O','R',0 };
static const WCHAR ORDER_W[] = { 'O','R','D','E','R',0 };
static const WCHAR OUTER_W[] = { 'O','U','T','E','R',0 };
static const WCHAR PRIMARY_W[] = { 'P','R','I','M','A','R','Y',0 };
static const WCHAR RIGHT_W[] = { 'R','I','G','H','T',0 };
static const WCHAR SELECT_W[] = { 'S','E','L','E','C','T',0 };
static const WCHAR SET_W[] = { 'S','E','T',0 };
static const WCHAR SHORT_W[] = { 'S','H','O','R','T',0 };
static const WCHAR TABLE_W[] = { 'T','A','B','L','E',0 };
static const WCHAR TEMP_W[] = { 'T','E','M','P',0 };
static const WCHAR TEMPORARY_W[] = { 'T','E','M','P','O','R','A','R','Y',0 };
static const WCHAR UPDATE_W[] = { 'U','P','D','A','T','E',0 };
static const WCHAR VALUES_W[] = { 'V','A','L','U','E','S',0 };
static const WCHAR WHERE_W[] = { 'W','H','E','R','E',0 };

86 87 88 89
/*
** These are the keywords
*/
static const Keyword aKeywordTable[] = {
90
  { ALTER_W, TK_ALTER },
91 92 93 94 95 96 97
  { AND_W, TK_AND },
  { BY_W, TK_BY },
  { CHAR_W, TK_CHAR },
  { CHARACTER_W, TK_CHAR },
  { CREATE_W, TK_CREATE },
  { DELETE_W, TK_DELETE },
  { DISTINCT_W, TK_DISTINCT },
98
  { FREE_W, TK_FREE },
99 100 101 102
  { FROM_W, TK_FROM },
  { HOLD_W, TK_HOLD },
  { INSERT_W, TK_INSERT },
  { INT_W, TK_INT },
103
  { INTEGER_W, TK_INT },
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
  { INTO_W, TK_INTO },
  { IS_W, TK_IS },
  { KEY_W, TK_KEY },
  { LIKE_W, TK_LIKE },
  { LOCALIZABLE_W, TK_LOCALIZABLE },
  { LONG_W, TK_LONG },
  { LONGCHAR_W, TK_LONGCHAR },
  { NOT_W, TK_NOT },
  { NULL_W, TK_NULL },
  { OBJECT_W, TK_OBJECT },
  { OR_W, TK_OR },
  { ORDER_W, TK_ORDER },
  { PRIMARY_W, TK_PRIMARY },
  { SELECT_W, TK_SELECT },
  { SET_W, TK_SET },
  { SHORT_W, TK_SHORT },
  { TABLE_W, TK_TABLE },
121
  { TEMPORARY_W, TK_TEMPORARY },
122 123 124
  { UPDATE_W, TK_UPDATE },
  { VALUES_W, TK_VALUES },
  { WHERE_W, TK_WHERE },
125 126 127 128
};

#define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )

129 130 131 132 133 134 135 136 137
/*
** Comparison function for binary search.
*/
static int compKeyword(const void *m1, const void *m2){
  const Keyword *k1 = m1, *k2 = m2;

  return strcmpiW( k1->zName, k2->zName );
}

138 139 140 141 142
/*
** This function looks up an identifier to determine if it is a
** keyword.  If it is a keyword, the token code of that keyword is 
** returned.  If the input is not a keyword, TK_ID is returned.
*/
143
static int sqliteKeywordCode(const WCHAR *z, int n){
144 145 146 147 148
  WCHAR str[MAX_TOKEN_LEN+1];
  Keyword key, *r;

  if( n>MAX_TOKEN_LEN )
    return TK_ID;
149

150 151 152 153 154 155 156
  memcpy( str, z, n*sizeof (WCHAR) );
  str[n] = 0;
  key.tokenType = 0;
  key.zName = str;
  r = bsearch( &key, aKeywordTable, KEYWORD_COUNT, sizeof (Keyword), compKeyword );
  if( r )
    return r->tokenType;
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
  return TK_ID;
}


/*
** If X is a character that can be used in an identifier then
** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
**
** In this implementation, an identifier can be a string of
** alphabetic characters, digits, and "_" plus any character
** with the high-order bit set.  The latter rule means that
** any sequence of UTF-8 characters or characters taken from
** an extended ISO8859 character set can form an identifier.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
};


/*
** Return the length of the token that begins at z[0].  Return
** -1 if the token is (or might be) incomplete.  Store the token
** type in *tokenType before returning.
*/
int sqliteGetToken(const WCHAR *z, int *tokenType){
  int i;
  switch( *z ){
    case ' ': case '\t': case '\n': case '\f': case '\r': {
      for(i=1; isspace(z[i]); i++){}
      *tokenType = TK_SPACE;
      return i;
    }
    case '-': {
      if( z[1]==0 ) return -1;
      *tokenType = TK_MINUS;
      return 1;
    }
210 211
    case '(':
      *tokenType = TK_LP;
212
      return 1;
213 214
    case ')':
      *tokenType = TK_RP;
215
      return 1;
216
    case '*':
217 218
      *tokenType = TK_STAR;
      return 1;
219
    case '=':
220
      *tokenType = TK_EQ;
221
      return 1;
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
    case '<': {
      if( z[1]=='=' ){
        *tokenType = TK_LE;
        return 2;
      }else if( z[1]=='>' ){
        *tokenType = TK_NE;
        return 2;
      }else{
        *tokenType = TK_LT;
        return 1;
      }
    }
    case '>': {
      if( z[1]=='=' ){
        *tokenType = TK_GE;
        return 2;
      }else{
        *tokenType = TK_GT;
        return 1;
      }
    }
    case '!': {
      if( z[1]!='=' ){
        *tokenType = TK_ILLEGAL;
        return 2;
      }else{
        *tokenType = TK_NE;
        return 2;
      }
    }
252
    case '?':
253 254
      *tokenType = TK_WILDCARD;
      return 1;
255
    case ',':
256 257 258 259 260 261 262 263 264 265 266 267 268 269
      *tokenType = TK_COMMA;
      return 1;
    case '`': case '\'': case '"': {
      int delim = z[0];
      for(i=1; z[i]; i++){
        if( z[i]==delim ){
          if( z[i+1]==delim ){
            i++;
          }else{
            break;
          }
        }
      }
      if( z[i] ) i++;
270 271 272 273
      if( delim == '`' )
        *tokenType = TK_ID;
      else
        *tokenType = TK_STRING;
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
      return i;
    }
    case '.': {
      if( !isdigit(z[1]) ){
        *tokenType = TK_DOT;
        return 1;
      }
      /* Fall thru into the next case */
    }
    case '0': case '1': case '2': case '3': case '4':
    case '5': case '6': case '7': case '8': case '9': {
      *tokenType = TK_INTEGER;
      for(i=1; isdigit(z[i]); i++){}
      return i;
    }
    case '[': {
      for(i=1; z[i] && z[i-1]!=']'; i++){}
      *tokenType = TK_ID;
      return i;
    }
    default: {
      if( !isIdChar[*z] ){
        break;
      }
      for(i=1; isIdChar[z[i]]; i++){}
      *tokenType = sqliteKeywordCode(z, i);
      return i;
    }
  }
  *tokenType = TK_ILLEGAL;
  return 1;
}