Search
lxdream.org :: lxdream/src/tools/mdparse.c
lxdream 0.9.1
released Jun 29
Download Now
filename src/tools/mdparse.c
changeset 1298:d0eb2307b847
prev1296:30ecee61f811
author nkeynes
date Wed Feb 04 08:38:23 2015 +1000 (5 years ago)
permissions -rw-r--r--
last change Fix assorted compile warnings reported by Clang
view annotate diff log raw
     1 /**
     2  * $Id$
     3  *
     4  * genmmio I/O register definition parser.
     5  *
     6  * Copyright (c) 2010 Nathan Keynes.
     7  *
     8  * This program is free software; you can redistribute it and/or modify
     9  * it under the terms of the GNU General Public License as published by
    10  * the Free Software Foundation; either version 2 of the License, or
    11  * (at your option) any later version.
    12  *
    13  * This program is distributed in the hope that it will be useful,
    14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    16  * GNU General Public License for more details.
    17  */
    19 #include <assert.h>
    20 #include <stdlib.h>
    21 #include <stdio.h>
    22 #include <string.h>
    23 #include <unistd.h>
    24 #include <fcntl.h>
    25 #include <ctype.h>
    26 #include <errno.h>
    27 #include <sys/stat.h>
    28 #include <glib.h>
    30 #include "genmach.h"
    31 /*
    32  * Grammar:
    33  *
    34  * start: start register_block |  ;
    35  * register_block: 'registers' IDENTIFIER 'at' INTEGER opt_paramlist '{' register_list '}';
    36  * opt_paramlist: '(' param_list ')' | '(' ')' |  ;
    37  * param_list: param_list ',' param | param ;
    38  * param: access_param | fill_param | endian_param | mask_param ;
    39  * access_param: 'access' '=' 'any'  // Behaves as if it were memory
    40  *        | 'access' '=' 'nooffset' // Recognize any size access (zero-ext or trunc) as long as the address is exactly the same
    41  *        | 'access' '=' 'exact'    // Recognize only the exact access type at the exact address
    42  *        ;
    43  * fill_param: 'fill' '=' literal;
    44  * endian_param: 'endian' '=' 'little'
    45  *        | 'endian' '=' 'big'
    46  *        ;
    47  * mask_param: 'mask' '=' literal;
    48  *
    49  * register_list: register_list register_desc | ;
    50  * register_desc: INTEGER ':' mode_spec type_spec IDENTIFIER opt_paramlist [ '=' literal ] [ ACTION ]
    51  * mode_spec: 'const' | 'rw' | 'ro' | 'wo';
    52  * type_spec: 'i8' | 'i16' | 'i32' | 'i64' | 'f32' | 'f64' | 'string' | 'cstring';
    53  * literal: INTEGER | STRING | 'undefined' ;
    54  *
    55  * C-style comments are recognized as such.
    56  */
    58 #define TOK_IDENTIFIER 1
    59 #define TOK_INTEGER    2
    60 #define TOK_STRING     3
    61 #define TOK_ACTION     4
    62 #define TOK_EOF        5
    63 #define TOK_ERROR      6
    64 #define TOK_LPAREN     7
    65 #define TOK_RPAREN     8
    66 #define TOK_LBRACE     9
    67 #define TOK_RBRACE    10
    68 #define TOK_COMMA     11
    69 #define TOK_COLON     12
    70 #define TOK_SEMI      13
    71 #define TOK_PERIOD    14
    72 #define TOK_RANGE     15
    73 #define TOK_LSQUARE   16
    74 #define TOK_RSQUARE   17
    75 #define TOK_EQUALS    18
    76 #define TOK_ACCESS    19
    77 #define TOK_AT        20
    78 #define TOK_ENDIAN    21
    79 #define TOK_FILL      22
    80 #define TOK_INCLUDE   23
    81 #define TOK_MASK      24
    82 #define TOK_REGISTERS 25
    83 #define TOK_SPACE     26
    84 #define TOK_STRIDE    27
    85 #define TOK_TEST      28
    86 #define TOK_TRACE     29
    87 #define TOK_UNDEFINED 30
    90 #define FIRST_KEYWORD TOK_ACCESS
    91 #define LAST_KEYWORD  TOK_UNDEFINED
    92 static const char *TOKEN_NAMES[] = {"NULL", "IDENTIFIER", "INTEGER", "STRING", "ACTION", "EOF", "ERROR",
    93         "(", ")", "{", "}", ",", ":", ";", ".", "..", "[", "]", "=",
    94         "ACCESS", "AT", "ENDIAN", "FILL", "INCLUDE", "MASK", "REGISTERS", "SPACE", "STRIDE", "TEST", "TRACE", "UNDEFINED" };
    95 static const char *MODE_NAMES[] = {"const", "rw", "ro", "wo", "mirror"};
    96 static const char *TYPE_NAMES[] = {"i8", "i16", "i32", "i64", "f32", "f64", "string"};
    97 static int TYPE_SIZES[] = {1,2,4,8,4,8,0};
    98 static const char *ENDIAN_NAMES[] = {"default", "little", "big"};
    99 static const char *ACCESS_NAMES[] = {"default", "any", "nooffset", "exact"};
   100 static const char *TRACE_NAMES[] = {"default", "off", "on"};
   102 #define elementsof(x) (sizeof(x)/sizeof(x[0]))
   104 typedef struct token_data {
   105     char *yytext;
   106     int yylength;
   107     int yyline;
   108     int yycol;
   109     union {
   110         uint64_t i;
   111         char *s;
   112     } v;
   113     int slen;
   114 } token_data;
   116 struct yystate {
   117     char *yybuffer;
   118     char *yyfilename;
   119     char *yyposn, *yylineposn, *yyend;
   120     int yylineno;
   121 };
   123 static GList *yyfile_stack = NULL;
   124 static struct yystate yystate;
   125 static struct token_data yytok;
   127 #define YYPARSE_ERROR( msg, ... ) \
   128     do { \
   129         fprintf( stderr, "Parse error in %s:%d:%d: " msg "\n", yystate.yyfilename, yytok.yyline, yytok.yycol, ## __VA_ARGS__ ); \
   130         exit(2); \
   131     } while(0)
   133 #define READ(x) \
   134 {   int _tok = iolex(x); \
   135     if( _tok != (x) ) { \
   136         YYPARSE_ERROR( "Expected %s but got %s", TOKEN_NAMES[x], TOKEN_NAMES[_tok] ); \
   137     } \
   138 }
   140 static int iolex( int expectToken );
   141 static int iolex_open( const char *filename );
   142 static void iolex_close();
   143 static int iolex_push( const char *filename );
   144 static int iolex_pop( );
   146 static inline char *yystrdup()
   147 {
   148     char *str = g_malloc0(yytok.yylength+1);
   149     memcpy( str, yytok.yytext, yytok.yylength);
   150     return str;
   151 }
   153 static inline int yystrcasecmp(const char *cmp)
   154 {
   155     int len = strlen(cmp);
   156     if( len != yytok.yylength ) {
   157         return yytok.yylength - len;
   158     }
   159     return strncasecmp(yytok.yytext, cmp, yytok.yylength);
   160 }
   162 static int yymatch( const char *arr[], unsigned numOptions, const char *desc )
   163 {
   164     for( unsigned i=0; i<numOptions; i++ ) {
   165         if( yystrcasecmp( arr[i] ) == 0 ) {
   166             return i;
   167         }
   168     }
   169     YYPARSE_ERROR( "Unknown %s '%s'", desc, yystrdup() );
   170 }
   172 static gint register_block_sort_cb( gconstpointer a, gconstpointer b )
   173 {
   174     struct regblock *blocka = (struct regblock *)a;
   175     struct regblock *blockb = (struct regblock *)b;
   176     return blocka->address - blockb->address;
   177 }
   179 static int register_ptr_sort_cb( const void *a, const void *b )
   180 {
   181     regdef_t *ptra = (regdef_t *)a;
   182     regdef_t *ptrb = (regdef_t *)b;
   183     if( (*ptra)->offset != (*ptrb)->offset )
   184         return (*ptra)->offset - (*ptrb)->offset;
   185     return (*ptra)->type - (*ptrb)->type;
   186 }
   188 static void ioparse_apval(int tok, union apval *apv, unsigned *numBytes)
   189 {
   190     if( tok == TOK_INTEGER ) {
   191         apv->i = yytok.v.i;
   192         if( *numBytes == 0 ) {
   193             YYPARSE_ERROR( "Expected string initializer but was an integer (0x%llx)", yytok.v.i );
   194         }
   195     } else if( tok == TOK_STRING ) {
   196         if( *numBytes == 0 ) {
   197             *numBytes = yytok.slen;
   198             apv->s = yytok.v.s;
   199         } else {
   200             if( *numBytes != yytok.slen ) {
   201                 YYPARSE_ERROR( "Expected %d byte initializer but was %d", *numBytes, yytok.slen );
   202             }
   203             assert( *numBytes < sizeof(uint64_t) );
   204             apv->i = 0;
   205             /* FIXME: handle endian mismatches */
   206             memcpy( &apv->i, yytok.v.s, yytok.slen );
   207         }
   208     } else {
   209         YYPARSE_ERROR( "Expected literal (integer or string), but got %s", TOKEN_NAMES[tok] );
   210     }
   211 }
   213 static void ioparse_regflags( regflags_t flags, unsigned *numBytes )
   214 {
   215     do {
   216         int tok = iolex(TOK_RPAREN);
   217         switch(tok) {
   218         case TOK_FILL:
   219             READ(TOK_EQUALS);
   220             tok = iolex(TOK_INTEGER);
   221             flags->fillSizeBytes = 4;
   222             ioparse_apval( tok, &flags->fillValue, &flags->fillSizeBytes );
   223             break;
   224         case TOK_ACCESS:
   225             READ(TOK_EQUALS);
   226             READ(TOK_IDENTIFIER);
   227             flags->access = yymatch(ACCESS_NAMES,elementsof(ACCESS_NAMES), "access mode");
   228             break;
   229         case TOK_MASK:
   230             if( numBytes ) {
   231                 READ(TOK_EQUALS);
   232                 tok = iolex(TOK_INTEGER);
   233                 ioparse_apval( tok, &flags->maskValue, numBytes );
   234             } else {
   235                 YYPARSE_ERROR("mask is not valid on a register block");
   236             }
   237             break;
   238         case TOK_ENDIAN:
   239             READ(TOK_EQUALS);
   240             READ(TOK_IDENTIFIER);
   241             flags->endian = yymatch(ENDIAN_NAMES,elementsof(ENDIAN_NAMES), "endianness");
   242             break;
   243         case TOK_TRACE:
   244             READ(TOK_EQUALS);
   245             READ(TOK_IDENTIFIER);
   246             flags->traceFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES), "trace flags");
   247             break;
   248         case TOK_TEST:
   249             READ(TOK_EQUALS);
   250             READ(TOK_IDENTIFIER);
   251             flags->testFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES), "test flag");
   252             break;
   253         case TOK_COMMA:
   254             break;
   255         case TOK_RPAREN:
   256             return;
   257         default:
   258             YYPARSE_ERROR("Expected flag or ')' but was %s", TOKEN_NAMES[tok]);
   259         }
   260     } while(1);
   261 }
   263 static void ioparse_regdef( struct regdef *reg )
   264 {
   265     reg->offset = (uint32_t)yytok.v.i;
   266     reg->numElements = 1;
   267     reg->numBytes = 0;
   268     reg->initValue.i = 0;
   269     reg->flags.maskValue.i = -1;
   270     uint32_t rangeOffset = reg->offset;
   272     int tok = iolex(TOK_COLON);
   273     if( tok == TOK_RANGE ) {
   274         READ(TOK_INTEGER);
   275         rangeOffset = (uint32_t)yytok.v.i;
   276         if( rangeOffset < reg->offset ) {
   277             YYPARSE_ERROR( "Range end (0x%x) must be greater than the range start (0x%x)", rangeOffset, reg->offset );
   278         }
   279         READ(TOK_COLON);
   280     } else if( tok != TOK_COLON ) {
   281         YYPARSE_ERROR( "Expected ':' but was %s\n", TOKEN_NAMES[tok] );
   282     }
   283     READ(TOK_IDENTIFIER);
   284     reg->mode = yymatch(MODE_NAMES, elementsof(MODE_NAMES), "register mode");
   285     if( reg->mode == REG_MIRROR ) {
   286         /* Mirror regions have a target range only */
   287         READ(TOK_INTEGER);
   288         reg->initValue.i = yytok.v.i;
   289         reg->type = REG_I8;
   290         tok = iolex(TOK_RANGE);
   291         if( tok == TOK_RANGE ) {
   292             READ(TOK_INTEGER);
   293             if( yytok.v.i < reg->initValue.i ) {
   294                 YYPARSE_ERROR( "Invalid mirror target range 0x%llx..0x%llx", reg->initValue.i, yytok.v.i );
   295             }
   296             reg->numBytes = (uint32_t)(yytok.v.i - reg->initValue.i + 1);
   297             tok = iolex(TOK_STRIDE);
   298         }
   299         if( tok == TOK_STRIDE ) {
   300             READ(TOK_INTEGER);
   301             reg->stride = (uint32_t)yytok.v.i;
   302             tok = iolex(TOK_ACTION);
   303         } else {
   304             reg->stride = reg->numBytes;
   305         }
   307         if( tok != TOK_SEMI ) {
   308             YYPARSE_ERROR( "Expected ; but gut %s", TOKEN_NAMES[tok] );
   309         }
   311         if( reg->stride < reg->numBytes ) {
   312             YYPARSE_ERROR( "Invalid mirror stride: %x is less than block size %x\n", reg->stride, reg->numBytes );
   313         }
   315         if( rangeOffset != reg->offset ) {
   316             reg->numElements *= ((rangeOffset - reg->offset) + reg->stride - 1) / reg->stride;
   317         }
   319     } else {
   320         READ(TOK_IDENTIFIER);
   321         reg->type = yymatch(TYPE_NAMES, elementsof(TYPE_NAMES), "register type");
   322         reg->numBytes = TYPE_SIZES[reg->type];
   323         tok = iolex(TOK_IDENTIFIER);
   324         if( tok == TOK_IDENTIFIER ) {
   325             reg->name = yystrdup();
   326             tok = iolex(TOK_ACTION);
   327         }
   328         if( tok == TOK_STRING ) {
   329             reg->description = yytok.v.s;
   330             tok = iolex(TOK_ACTION);
   331         }
   332         if( tok == TOK_LPAREN ) {
   333             ioparse_regflags(&reg->flags, &reg->numBytes);
   334             tok = iolex(TOK_ACTION);
   335         }
   336         if( tok == TOK_EQUALS ) {
   337             tok = iolex(TOK_INTEGER);
   338             if( tok == TOK_UNDEFINED ) {
   339                 reg->initValue.i = 0xDEADBEEFDEADBEEF;
   340                 reg->initUndefined = TRUE;
   341             } else {
   342                 ioparse_apval(tok, &reg->initValue, &reg->numBytes);
   343             }
   344             tok = iolex(TOK_ACTION);
   345         } else if( reg->type == REG_STRING ) {
   346             YYPARSE_ERROR( "String declarations must have an initializer (ie = 'abcd')" );
   347         }
   348         if( tok == TOK_ACTION ) {
   349             // reg->action = yystrdup();
   350         } else if( tok != TOK_SEMI ) {
   351             YYPARSE_ERROR( "Expected ; or {, but got %s", TOKEN_NAMES[tok] );
   352         }
   354         if( rangeOffset != reg->offset ) {
   355             reg->numElements *= ((rangeOffset - reg->offset) + reg->numBytes - 1) / reg->numBytes;
   356         }
   357     }
   359 }
   361 static struct regblock *ioparse_regblock( )
   362 {
   363     unsigned regsAllocSize = 128;
   364     struct regblock *block = g_malloc0(sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);
   365     block->numRegs = 0;
   367     READ(TOK_IDENTIFIER);
   368     block->name = yystrdup();
   369     int tok = iolex(TOK_AT);
   370     if( tok == TOK_STRING ) {
   371         block->description = yytok.v.s;
   372         tok = iolex(TOK_AT);
   373     }
   374     if( tok != TOK_AT ) {
   375         YYPARSE_ERROR("Expected AT but got %s\n", TOKEN_NAMES[tok] );
   376     }
   377     READ(TOK_INTEGER);
   378     block->address = (uint32_t)yytok.v.i;
   380     tok = iolex(TOK_LBRACE);
   381     if( tok == TOK_LPAREN) {
   382         ioparse_regflags(&block->flags, NULL);
   383         READ(TOK_LBRACE);
   384     } else if( tok != TOK_LBRACE ) {
   385         YYPARSE_ERROR("Expected { but got %s\n", TOKEN_NAMES[tok] );
   386     }
   388     tok = iolex(TOK_INTEGER);
   389     while( tok != TOK_RBRACE ) {
   390         if( tok != TOK_INTEGER ) {
   391             YYPARSE_ERROR("Expected INTEGER or } but got %s\n", TOKEN_NAMES[tok]);
   392         }
   393         struct regdef *regdef = g_malloc0(sizeof(struct regdef));
   394         if( block->numRegs >= regsAllocSize ) {
   395             regsAllocSize *= 2;
   396             block = g_realloc(block, sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);
   397         }
   398         block->regs[block->numRegs++] = regdef;
   399         ioparse_regdef(regdef);
   401         tok = iolex(TOK_INTEGER);
   402     }
   404     qsort( &block->regs[0], block->numRegs, sizeof(block->regs[0]), register_ptr_sort_cb );
   406     return block;
   407 }
   409 GList *ioparse( const char *filename, GList *list )
   410 {
   411     GList *blocks = list;
   412     int count = 0;
   414     if( iolex_open(filename) != 0  )
   415         return blocks;
   417     int tok;
   418     while(1) {
   419         tok = iolex(TOK_REGISTERS);
   420         if( tok == TOK_EOF ) {
   421             int result = iolex_pop();
   422             if( result == -1 )
   423                 break;
   424         } else if( tok == TOK_INCLUDE) {
   425             READ(TOK_STRING);
   426             char *tmp = yystrdup();
   427             READ(TOK_SEMI);
   428             int result = iolex_push( tmp );
   429             if( result == -1 ) {
   430                 YYPARSE_ERROR("Unable to include file '%s'", tmp);
   431             }
   432             free(tmp);
   433         } else if( tok == TOK_SPACE ) {
   434         } else if( tok == TOK_REGISTERS ) {
   435             struct regblock *block = ioparse_regblock();
   436             count++;
   437             blocks = g_list_insert_sorted(blocks, block, register_block_sort_cb);
   438         } else {
   439             YYPARSE_ERROR("Expected REGISTERS but got %s\n", TOKEN_NAMES[tok] );
   440         }
   441     }
   442     return blocks;
   443 }
   445 /**************************** Lexical analyser ***************************/
   447 static int iolex_push( const char *filename )
   448 {
   449     struct yystate *save = g_malloc(sizeof(struct yystate));
   450     memcpy( save, &yystate, sizeof(struct yystate) );
   452     int result = iolex_open(filename);
   453     if( result == 0 ) {
   454         yyfile_stack = g_list_prepend(yyfile_stack, save);
   455     }
   456     return result;
   457 }
   459 static int iolex_pop( )
   460 {
   461     iolex_close();
   462     if( yyfile_stack == NULL )
   463         return -1;
   464     struct yystate *top = (struct yystate *)yyfile_stack->data;
   465     yyfile_stack = g_list_remove(yyfile_stack, top);
   466     memcpy( &yystate, top, sizeof(struct yystate) );
   467     g_free( top );
   468     return 0;
   469 }
   471 static int iolex_open( const char *filename )
   472 {
   473     struct stat st;
   474     int fd = open( filename, O_RDONLY );
   475     if( fd == -1 ) {
   476         fprintf( stderr, "Error opening file '%s': %s\n", filename, strerror(errno) );
   477         return -1;
   478     }
   480     if( fstat( fd, &st ) != 0 ) {
   481         fprintf( stderr, "Error statting file '%s': %s\n", filename, strerror(errno) );
   482         close(fd);
   483         return -1;
   484     }
   486     char *data = g_malloc( st.st_size + 1 );
   487     if( read(fd, data, st.st_size) != st.st_size ) {
   488         fprintf( stderr, "Error reading file '%s': %s\n", filename, strerror(errno) );
   489         close(fd);
   490         return -1;
   491     }
   492     close(fd);
   493     data[st.st_size] = 0;
   495     yystate.yybuffer = yystate.yyposn = data;
   496     yystate.yyend = data + st.st_size;
   497     yystate.yylineno = 1;
   498     yystate.yylineposn = yystate.yyposn;
   499     yystate.yyfilename = strdup(filename);
   500     return 0;
   501 }
   503 static void iolex_close()
   504 {
   505     g_free(yystate.yybuffer);
   506     free(yystate.yyfilename);
   507     memset(&yystate, 0, sizeof(struct yystate));
   508 }
   510 #define YYRETURN(x) do{ \
   511     yytok.yylength = yystate.yyposn - yystart; \
   512     return (x); \
   513 } while(0)
   515 static int iolex_readhex(char *p, int digits)
   516 {
   517     int result = 0;
   518     for( int i=0; i < digits; i++ ) {
   519         result <<= 4;
   520         if( isdigit(p[i]) ) {
   521             result += p[i]-'0';
   522         } else if( p[i] >= 'a' && p[i] <= 'f' ) {
   523             result += p[i]-'a'+10;
   524         } else if( p[i] >= 'A' && p[i] <= 'F' ) {
   525             result += p[i]-'A'+10;
   526         } else {
   527             return (result >> 4);
   528         }
   530     }
   531     return result;
   532 }
   534 static int iolex_readoctal(char *p, int digits)
   535 {
   536     int result = 0;
   537     for( int i=0; i < digits; i++ ) {
   538         result <<= 3;
   539         if( p[i] >= '0' && p[i] <= '7' ) {
   540             result += p[i]-'0';
   541         } else {
   542             return (result >> 4);
   543         }
   544     }
   545     return result;
   546 }
   548 /**
   549  * Copy and interpret the string segment as a C string, replacing escape
   550  * expressions with the corresponding value.
   551  */
   552 static char *iolex_getcstring( char *start, char *end, int *len )
   553 {
   554    char *result = g_malloc0(end-start+1); /* Allocate enough memory for the string as-is */
   555    char *q = result;
   557    for( char *p = start; p != end; p++ ) {
   558        if( *p == '\\' ) {
   559            if( ++p == end ) {
   560                *q++ = '\\';
   561                break;
   562            }
   563            if( p[0] >= '0' && p[0] <= '3' && p+3 <= end &&
   564                p[1] >= '0' && p[1] <= '7' &&
   565                p[2] >= '0' && p[2] <= '7' ) {
   566                *q++ = (char)iolex_readoctal(p,3);
   567                p+=2;
   568            } else {
   569                switch( *p ) {
   570                case 'n':
   571                    *q++ = '\n';
   572                    break;
   573                case 'r':
   574                    *q++ = '\r';
   575                    break;
   576                case 't':
   577                    *q++ = '\t';
   578                    break;
   579                case 'x': /* hex */
   580                    if( p + 3 > end || !isxdigit(p[1]) || !isxdigit(p[2]) ) {
   581                        *q++ = '\\';
   582                        *q++ = *p;
   583                    } else {
   584                        *q++ = (char)iolex_readhex(p+1, 2);
   585                        p+=2;
   586                    }
   587                    break;
   588                default:
   589                    *q++ = '\\';
   590                    *q++ = *p;
   591                }
   592            }
   593        } else {
   594            *q++ = *p;
   595        }
   596    }
   597    *len = q - result;
   598    return result;
   599 }
   601 int iolex( int expectToken )
   602 {
   603     int count = 0;
   604     while( yystate.yyposn < yystate.yyend ) {
   605         char *yystart = yytok.yytext = yystate.yyposn;
   606         yytok.yylength = 1;
   607         yytok.yyline = yystate.yylineno;
   608         yytok.yycol = yystate.yyposn - yystate.yylineposn+1;
   609         int ch = *yystate.yyposn++;
   610         if( isdigit(ch) ) {
   611             /* INTEGER */
   612             if( ch == '0' ) {
   613                 if( *yystate.yyposn == 'x' ) {
   614                     while( yystate.yyposn < yystate.yyend && isxdigit(*++yystate.yyposn) ) ;
   615                 } else {
   616                     while( yystate.yyposn < yystate.yyend && *yystate.yyposn >= '0' && *yystate.yyposn <= '7' )
   617                         yystate.yyposn++;
   618                 }
   619             } else {
   620                 while( yystate.yyposn < yystate.yyend && isdigit(*yystate.yyposn) )
   621                     yystate.yyposn++;
   622             }
   623             yytok.v.i = strtoll( yystart, NULL, 0 );
   624             YYRETURN(TOK_INTEGER);
   625         } else if( isalpha(ch) || ch == '_' ) {
   626             /* IDENTIFIER */
   627             while( yystate.yyposn < yystate.yyend && (isalnum(*yystate.yyposn) || *yystate.yyposn == '_') )
   628                 yystate.yyposn++;
   629             yytok.yylength = yystate.yyposn - yystart;
   630             if( expectToken == TOK_IDENTIFIER ) {
   631                 YYRETURN(TOK_IDENTIFIER);
   632             }
   633             /* Otherwise check for keywords */
   634             for( int i=FIRST_KEYWORD; i <= LAST_KEYWORD; i++ ) {
   635                 if( strlen(TOKEN_NAMES[i]) == yytok.yylength &&
   636                     strncasecmp(TOKEN_NAMES[i], yystart, yytok.yylength ) == 0 ) {
   637                     YYRETURN(i);
   638                 }
   639             }
   640             YYRETURN(TOK_IDENTIFIER);
   641         } else if( isspace(ch) ) {
   642             if( ch == '\n' ) {
   643                 yystate.yylineno++;
   644                 yystate.yylineposn = yystate.yyposn;
   645             }
   646             while( isspace(*yystate.yyposn) ) {
   647                 if( *yystate.yyposn == '\n' ) {
   648                     yystate.yylineno++;
   649                     yystate.yylineposn = yystate.yyposn+1;
   650                 }
   651                 yystate.yyposn++;
   652             }
   653         } else {
   654             switch( ch ) {
   655             case '(': YYRETURN(TOK_LPAREN);
   656             case ')': YYRETURN(TOK_RPAREN);
   657             case '[': YYRETURN(TOK_LSQUARE);
   658             case ']': YYRETURN(TOK_RSQUARE);
   659             case ',': YYRETURN(TOK_COMMA);
   660             case ':': YYRETURN(TOK_COLON);
   661             case ';': YYRETURN(TOK_SEMI);
   662             case '=': YYRETURN(TOK_EQUALS);
   663             case '/':
   664                 if( *yystate.yyposn == '/' ) { /* Line comment */
   665                     while( yystate.yyposn < yystate.yyend && *++yystate.yyposn != '\n' ) ;
   666                 } else if( *yystate.yyposn == '*' ) { /* C comment */
   667                     while( yystate.yyposn < yystate.yyend && (*++yystate.yyposn != '*' || *++yystate.yyposn != '/' ) ) {
   668                         if( *yystate.yyposn == '\n' ) {
   669                             yystate.yylineno++;
   670                             yystate.yylineposn = yystate.yyposn+1;
   671                         }
   672                     }
   673                 }
   674                 break;
   675             case '\'': /* STRING */
   676                 while( *yystate.yyposn != '\'' ) {
   677                     if( *yystate.yyposn == '\n' ) {
   678                         fprintf( stderr, "Unexpected newline in string constant!\n" );
   679                         YYRETURN(TOK_ERROR);
   680                     } else if( yystate.yyposn >= yystate.yyend ) {
   681                         fprintf( stderr, "Unexpected EOF in string constant!\n" );
   682                         YYRETURN(TOK_ERROR);
   683                     } else if( *yystate.yyposn == '\\' && yystate.yyposn[1] == '\'' ) {
   684                         yystate.yyposn++;
   685                     }
   686                     yystate.yyposn++;
   687                 }
   688                 yystate.yyposn++;
   689                 yytok.v.s = iolex_getcstring(yystart+1, yystate.yyposn-1, &yytok.slen);
   690                 YYRETURN(TOK_STRING);
   691             case '\"': /* STRING */
   692                 while( *yystate.yyposn != '\"' ) {
   693                     if( *yystate.yyposn == '\n' ) {
   694                         fprintf( stderr, "Unexpected newline in string constant!\n" );
   695                         YYRETURN(TOK_ERROR);
   696                     } else if( yystate.yyposn >= yystate.yyend ) {
   697                         fprintf( stderr, "Unexpected EOF in string constant!\n" );
   698                         YYRETURN(TOK_ERROR);
   699                     } else if( *yystate.yyposn == '\\' && yystate.yyposn[1] == '\"' ) {
   700                         yystate.yyposn++;
   701                     }
   702                     yystate.yyposn++;
   703                 }
   704                 yystate.yyposn++;
   705                 yytok.v.s = iolex_getcstring(yystart+1, yystate.yyposn-1, &yytok.slen);
   706                 YYRETURN(TOK_STRING);
   707             case '}':
   708                 YYRETURN(TOK_RBRACE);
   709             case '{': /* ACTION or LBRACE */
   710                 if( expectToken == TOK_LBRACE ) {
   711                     YYRETURN(TOK_LBRACE);
   712                 } else {
   713                     count++;
   714                     while( count > 0 && yystate.yyposn < yystate.yyend ) {
   715                         if( *yystate.yyposn == '{' )
   716                             count++;
   717                         if( *yystate.yyposn == '}' )
   718                             count--;
   719                         yystate.yyposn++;
   720                     }
   721                     YYRETURN(TOK_ACTION);
   722                 }
   723             case '.':
   724                 if( *yystate.yyposn == '.' ) {
   725                     yystate.yyposn++;
   726                     YYRETURN(TOK_RANGE);
   727                 } else {
   728                     YYRETURN(TOK_PERIOD);
   729                 }
   730             default:
   731                 fprintf( stderr, "Illegal character: '%c'\n", ch );
   732                 YYRETURN(TOK_ERROR);
   733             }
   734         }
   736     }
   737     return TOK_EOF;
   738 }
.