Search
lxdream.org :: lxdream/src/tools/mdparse.c
lxdream 0.9.1
released Jun 29
Download Now
filename src/tools/mdparse.c
changeset 1104:700e16c321e5
next1105:73da8fd129fb
author nkeynes
date Mon Mar 15 22:10:24 2010 +1000 (12 years ago)
permissions -rw-r--r--
last change Commit genmach work-in-progress
view annotate diff log raw
     1 /**
     2  * $Id$
     3  *
     4  * genmmio I/O register definition parser.
     5  *
     6  * Copyright (c) 2010 Nathan Keynes.
     7  *
     8  * This program is free software; you can redistribute it and/or modify
     9  * it under the terms of the GNU General Public License as published by
    10  * the Free Software Foundation; either version 2 of the License, or
    11  * (at your option) any later version.
    12  *
    13  * This program is distributed in the hope that it will be useful,
    14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    16  * GNU General Public License for more details.
    17  */
    19 #include <assert.h>
    20 #include <stdlib.h>
    21 #include <stdio.h>
    22 #include <string.h>
    23 #include <unistd.h>
    24 #include <fcntl.h>
    25 #include <ctype.h>
    26 #include <errno.h>
    27 #include <sys/stat.h>
    28 #include <glib/gmem.h>
    30 #include "genmach.h"
    31 /*
    32  * Grammar:
    33  *
    34  * start: start register_block |  ;
    35  * register_block: 'registers' IDENTIFIER 'at' INTEGER opt_paramlist '{' register_list '}';
    36  * opt_paramlist: '(' param_list ')' | '(' ')' |  ;
    37  * param_list: param_list ',' param | param ;
    38  * param: access_param | fill_param | endian_param | mask_param ;
    39  * access_param: 'access' '=' 'any'  // Behaves as if it were memory
    40  *        | 'access' '=' 'nooffset' // Recognize any size access (zero-ext or trunc) as long as the address is exactly the same
    41  *        | 'access' '=' 'exact'    // Recognize only the exact access type at the exact address
    42  *        ;
    43  * fill_param: 'fill' '=' literal;
    44  * endian_param: 'endian' '=' 'little'
    45  *        | 'endian' '=' 'big'
    46  *        ;
    47  * mask_param: 'mask' '=' literal;
    48  *
    49  * register_list: register_list register_desc | ;
    50  * register_desc: INTEGER ':' mode_spec type_spec IDENTIFIER opt_paramlist [ '=' literal ] [ ACTION ]
    51  * mode_spec: 'const' | 'rw' | 'ro' | 'wo';
    52  * type_spec: 'i8' | 'i16' | 'i32' | 'i64' | 'f32' | 'f64' | 'string' | 'cstring';
    53  * literal: INTEGER | STRING | 'undefined' ;
    54  *
    55  * C-style comments are recognized as such.
    56  */
    58 #define TOK_IDENTIFIER 1
    59 #define TOK_INTEGER    2
    60 #define TOK_STRING     3
    61 #define TOK_ACTION     4
    62 #define TOK_EOF        5
    63 #define TOK_ERROR      6
    64 #define TOK_LPAREN     7
    65 #define TOK_RPAREN     8
    66 #define TOK_LBRACE     9
    67 #define TOK_RBRACE    10
    68 #define TOK_COMMA     11
    69 #define TOK_COLON     12
    70 #define TOK_SEMI      13
    71 #define TOK_PERIOD    14
    72 #define TOK_RANGE     15
    73 #define TOK_LSQUARE   16
    74 #define TOK_RSQUARE   17
    75 #define TOK_EQUALS    18
    76 #define TOK_ACCESS    19
    77 #define TOK_AT        20
    78 #define TOK_ENDIAN    21
    79 #define TOK_FILL      22
    80 #define TOK_INCLUDE   23
    81 #define TOK_MASK      24
    82 #define TOK_REGISTERS 25
    83 #define TOK_SPACE     26
    84 #define TOK_STRIDE    27
    85 #define TOK_TEST      28
    86 #define TOK_TRACE     29
    87 #define TOK_UNDEFINED 30
    90 #define FIRST_KEYWORD TOK_ACCESS
    91 #define LAST_KEYWORD  TOK_UNDEFINED
    92 static const char *TOKEN_NAMES[] = {"NULL", "IDENTIFIER", "INTEGER", "STRING", "ACTION", "EOF", "ERROR",
    93         "(", ")", "{", "}", ",", ":", ";", ".", "..", "[", "]", "=",
    94         "ACCESS", "AT", "ENDIAN", "FILL", "INCLUDE", "MASK", "REGISTERS", "SPACE", "STRIDE", "TEST", "TRACE", "UNDEFINED" };
    95 static const char *MODE_NAMES[] = {"const", "rw", "ro", "wo", "mirror"};
    96 static const char *TYPE_NAMES[] = {"i8", "i16", "i32", "i64", "f32", "f64", "string"};
    97 static int TYPE_SIZES[] = {1,2,4,8,4,8,0};
    98 static const char *ENDIAN_NAMES[] = {"default", "little", "big"};
    99 static const char *ACCESS_NAMES[] = {"default", "any", "nooffset", "exact"};
   100 static const char *TRACE_NAMES[] = {"default", "off", "on"};
   102 #define elementsof(x) (sizeof(x)/sizeof(x[0]))
   104 typedef struct token_data {
   105     char *yytext;
   106     int yylength;
   107     int yyline;
   108     int yycol;
   109     union {
   110         long i;
   111         char *s;
   112     } v;
   113     int slen;
   114 } token_data;
   116 static char *yybuffer;
   117 static char *yyposn, *yylineposn;
   118 static char *yyend;
   119 static int yyline;
   120 static struct token_data yytok;
   122 #define YYPARSE_ERROR( msg, ... ) \
   123     do { \
   124         fprintf( stderr, "Parse error at %d:%d: " msg "\n", yytok.yyline, yytok.yycol, __VA_ARGS__ ); \
   125         exit(2); \
   126     } while(0)
   128 #define READ(x) \
   129 {   int _tok = iolex(x); \
   130     if( _tok != (x) ) { \
   131         YYPARSE_ERROR( "Expected %s but got %s", TOKEN_NAMES[x], TOKEN_NAMES[_tok] ); \
   132     } \
   133 }
   135 static int iolex( int expectToken );
   136 static int iolex_open( const char *filename );
   137 static void iolex_close();
   139 static inline char *yystrdup()
   140 {
   141     char *str = g_malloc0(yytok.yylength+1);
   142     memcpy( str, yytok.yytext, yytok.yylength);
   143     return str;
   144 }
   146 static inline int yystrcasecmp(const char *cmp)
   147 {
   148     int len = strlen(cmp);
   149     if( len != yytok.yylength ) {
   150         return yytok.yylength - len;
   151     }
   152     return strncasecmp(yytok.yytext, cmp, yytok.yylength);
   153 }
   155 static int yymatch( const char *arr[], unsigned numOptions )
   156 {
   157     for( unsigned i=0; i<numOptions; i++ ) {
   158         if( yystrcasecmp( arr[i] ) == 0 ) {
   159             return i;
   160         }
   161     }
   162     return -1;
   163 }
   165 static gint register_block_sort_cb( gconstpointer a, gconstpointer b )
   166 {
   167     struct regblock *blocka = (struct regblock *)a;
   168     struct regblock *blockb = (struct regblock *)b;
   169     return blocka->address - blockb->address;
   170 }
   172 static int register_ptr_sort_cb( const void *a, const void *b )
   173 {
   174     regdef_t *ptra = (regdef_t *)a;
   175     regdef_t *ptrb = (regdef_t *)b;
   176     if( (*ptra)->offset != (*ptrb)->offset )
   177         return (*ptra)->offset - (*ptrb)->offset;
   178     return (*ptra)->type - (*ptrb)->type;
   179 }
   181 static void ioparse_apval(int tok, union apval *apv, unsigned *numBytes)
   182 {
   183     if( tok == TOK_INTEGER ) {
   184         apv->i = yytok.v.i;
   185         if( *numBytes == 0 ) {
   186             YYPARSE_ERROR( "Expected string initializer but was an integer (0x%x)", yytok.v.i );
   187         }
   188     } else if( tok = TOK_STRING ) {
   189         if( *numBytes == 0 ) {
   190             *numBytes = yytok.slen;
   191             apv->s = yytok.v.s;
   192         } else {
   193             if( *numBytes != yytok.slen ) {
   194                 YYPARSE_ERROR( "Expected %d byte initializer but was %d", *numBytes, yytok.slen );
   195             }
   196             assert( *numBytes < sizeof(uint64_t) );
   197             apv->i = 0;
   198             /* FIXME: handle endian mismatches */
   199             memcpy( &apv->i, yytok.v.s, yytok.slen );
   200         }
   201     } else {
   202         YYPARSE_ERROR( "Expected literal (integer or string), but got %s", TOKEN_NAMES[tok] );
   203     }
   204 }
   206 static void ioparse_regflags( regflags_t flags, unsigned *numBytes )
   207 {
   208     do {
   209         int tok = iolex(TOK_RPAREN);
   210         switch(tok) {
   211         case TOK_FILL:
   212             READ(TOK_EQUALS);
   213             tok = iolex(TOK_INTEGER);
   214             flags->fillSizeBytes = 4;
   215             ioparse_apval( tok, &flags->fillValue, &flags->fillSizeBytes );
   216             break;
   217         case TOK_ACCESS:
   218             READ(TOK_EQUALS);
   219             READ(TOK_IDENTIFIER);
   220             flags->access = yymatch(ACCESS_NAMES,elementsof(ACCESS_NAMES));
   221             if( flags->access == -1 ) {
   222                 YYPARSE_ERROR("Unknown access mode '%s'", yystrdup());
   223             }
   224             break;
   225         case TOK_MASK:
   226             if( numBytes ) {
   227                 READ(TOK_EQUALS);
   228                 tok = iolex(TOK_INTEGER);
   229                 ioparse_apval( tok, &flags->maskValue, numBytes );
   230             } else {
   231                 YYPARSE_ERROR("mask is not valid on a register block",0);
   232             }
   233             break;
   234         case TOK_ENDIAN:
   235             READ(TOK_EQUALS);
   236             READ(TOK_IDENTIFIER);
   237             flags->endian = yymatch(ENDIAN_NAMES,elementsof(ENDIAN_NAMES));
   238             if( flags->endian == -1 ) {
   239                 YYPARSE_ERROR("Unknown endianness '%s'", yystrdup());
   240             }
   241             break;
   242         case TOK_TRACE:
   243             READ(TOK_EQUALS);
   244             READ(TOK_IDENTIFIER);
   245             flags->traceFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES));
   246             if( flags->traceFlag == -1 ) {
   247                 YYPARSE_ERROR("Unknown trace flag '%s'", yystrdup());
   248             }
   249             break;
   250         case TOK_TEST:
   251             READ(TOK_EQUALS);
   252             READ(TOK_IDENTIFIER);
   253             flags->testFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES));
   254             if( flags->testFlag == -1 ) {
   255                 YYPARSE_ERROR("Unknown test flag '%s'", yystrdup());
   256             }
   257             break;
   258         case TOK_COMMA:
   259             break;
   260         case TOK_RPAREN:
   261             return;
   262         default:
   263             YYPARSE_ERROR("Expected flag or ')' but was %s", TOKEN_NAMES[tok]);
   264         }
   265     } while(1);
   266 }
   268 static void ioparse_regdef( struct regdef *reg )
   269 {
   270     reg->offset = yytok.v.i;
   271     reg->numElements = 1;
   272     reg->numBytes = 0;
   273     reg->initValue.i = 0;
   274     reg->flags.maskValue.i = -1;
   275     unsigned rangeOffset = reg->offset;
   277     int tok = iolex(TOK_COLON);
   278     if( tok == TOK_RANGE ) {
   279         READ(TOK_INTEGER);
   280         rangeOffset = yytok.v.i;
   281         if( rangeOffset < reg->offset ) {
   282             YYPARSE_ERROR( "Range end (0x%x) must be greater than the range start (0x%x)", rangeOffset, reg->offset );
   283         }
   284         READ(TOK_COLON);
   285     } else if( tok != TOK_COLON ) {
   286         YYPARSE_ERROR( "Expected ':' but was %s\n", TOKEN_NAMES[tok] );
   287     }
   288     READ(TOK_IDENTIFIER);
   289     reg->mode = yymatch(MODE_NAMES, elementsof(MODE_NAMES));
   290     if( reg->mode == -1 ) {
   291         YYPARSE_ERROR( "Unknown register mode '%s'", yystrdup() );
   292     }
   293     if( reg->mode == REG_MIRROR ) {
   294         /* Mirror regions have a target range only */
   295         READ(TOK_INTEGER);
   296         reg->initValue.i = yytok.v.i;
   297         reg->type = REG_I8;
   298         tok = iolex(TOK_RANGE);
   299         if( tok == TOK_RANGE ) {
   300             READ(TOK_INTEGER);
   301             if( yytok.v.i < reg->initValue.i ) {
   302                 YYPARSE_ERROR( "Invalid mirror target range 0x%x..0x%x", reg->initValue.i, yytok.v.i );
   303             }
   304             reg->numBytes = yytok.v.i - reg->initValue.i + 1;
   305             tok = iolex(TOK_STRIDE);
   306         }
   307         if( tok == TOK_STRIDE ) {
   308             READ(TOK_INTEGER);
   309             reg->stride = yytok.v.i;
   310             tok = iolex(TOK_ACTION);
   311         } else {
   312             reg->stride = reg->numBytes;
   313         }
   315         if( tok != TOK_SEMI ) {
   316             YYPARSE_ERROR( "Expected ; but gut %s", TOKEN_NAMES[tok] );
   317         }
   319         if( reg->stride < reg->numBytes ) {
   320             YYPARSE_ERROR( "Invalid mirror stride: %x is less than block size %x\n", reg->stride, reg->numBytes );
   321         }
   323         if( rangeOffset != reg->offset ) {
   324             reg->numElements *= ((rangeOffset - reg->offset) + reg->stride - 1) / reg->stride;
   325         }
   327     } else {
   328         READ(TOK_IDENTIFIER);
   329         reg->type = yymatch(TYPE_NAMES, elementsof(TYPE_NAMES));
   330         if( reg->type == -1 ) {
   331             YYPARSE_ERROR( "Unknown register type '%s'", yystrdup() );
   332         }
   333         reg->numBytes = TYPE_SIZES[reg->type];
   334         tok = iolex(TOK_IDENTIFIER);
   335         if( tok == TOK_IDENTIFIER ) {
   336             reg->name = yystrdup();
   337             tok = iolex(TOK_ACTION);
   338         }
   339         if( tok == TOK_STRING ) {
   340             reg->description = yytok.v.s;
   341             tok = iolex(TOK_ACTION);
   342         }
   343         if( tok == TOK_LPAREN ) {
   344             ioparse_regflags(&reg->flags, &reg->numBytes);
   345             tok = iolex(TOK_ACTION);
   346         }
   347         if( tok == TOK_EQUALS ) {
   348             tok = iolex(TOK_INTEGER);
   349             if( tok == TOK_UNDEFINED ) {
   350                 reg->initValue.i = 0xDEADBEEFDEADBEEF;
   351                 reg->initUndefined = TRUE;
   352             } else {
   353                 ioparse_apval(tok, &reg->initValue, &reg->numBytes);
   354             }
   355             tok = iolex(TOK_ACTION);
   356         } else if( reg->type == REG_STRING ) {
   357             YYPARSE_ERROR( "String declarations must have an initializer (ie = 'abcd')",0 );
   358         }
   359         if( tok == TOK_ACTION ) {
   360             // reg->action = yystrdup();
   361         } else if( tok != TOK_SEMI ) {
   362             YYPARSE_ERROR( "Expected ; or {, but got %s", TOKEN_NAMES[tok] );
   363         }
   365         if( rangeOffset != reg->offset ) {
   366             reg->numElements *= ((rangeOffset - reg->offset) + reg->numBytes - 1) / reg->numBytes;
   367         }
   368     }
   370 }
   372 static struct regblock *ioparse_regblock( )
   373 {
   374     unsigned regsAllocSize = 128;
   375     struct regblock *block = g_malloc0(sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);
   376     block->numRegs = 0;
   378     READ(TOK_IDENTIFIER);
   379     block->name = yystrdup();
   380     int tok = iolex(TOK_AT);
   381     if( tok == TOK_STRING ) {
   382         block->description = yytok.v.s;
   383         tok = iolex(TOK_AT);
   384     }
   385     if( tok != TOK_AT ) {
   386         YYPARSE_ERROR("Expected AT but got %s\n", TOKEN_NAMES[tok] );
   387     }
   388     READ(TOK_INTEGER);
   389     block->address = yytok.v.i;
   391     tok = iolex(TOK_LBRACE);
   392     if( tok == TOK_LPAREN) {
   393         ioparse_regflags(&block->flags, NULL);
   394         READ(TOK_LBRACE);
   395     } else if( tok != TOK_LBRACE ) {
   396         YYPARSE_ERROR("Expected { but got %s\n", TOKEN_NAMES[tok] );
   397     }
   399     tok = iolex(TOK_INTEGER);
   400     while( tok != TOK_RBRACE ) {
   401         if( tok != TOK_INTEGER ) {
   402             YYPARSE_ERROR("Expected INTEGER or } but got %s\n", TOKEN_NAMES[tok]);
   403         }
   404         struct regdef *regdef = g_malloc0(sizeof(struct regdef));
   405         if( block->numRegs >= regsAllocSize ) {
   406             regsAllocSize *= 2;
   407             block = g_realloc(block, sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);
   408         }
   409         block->regs[block->numRegs++] = regdef;
   410         ioparse_regdef(regdef);
   412         tok = iolex(TOK_INTEGER);
   413     }
   415     qsort( &block->regs[0], block->numRegs, sizeof(block->regs[0]), register_ptr_sort_cb );
   417     return block;
   418 }
   420 GList *ioparse( const char *filename, GList *list )
   421 {
   422     GList *blocks = list;
   423     int count = 0;
   425     if( iolex_open(filename) != 0  )
   426         return blocks;
   428     int tok;
   429     do {
   430         tok = iolex(TOK_REGISTERS);
   431         if( tok == TOK_EOF ) {
   432             return blocks;
   433         } else if( tok == TOK_INCLUDE) {
   434             READ(TOK_STRING);
   436         } else if( tok == TOK_SPACE ) {
   437         } else if( tok == TOK_REGISTERS ) {
   438             struct regblock *block = ioparse_regblock(block);
   439             count++;
   440             blocks = g_list_insert_sorted(blocks, block, register_block_sort_cb);
   441         } else {
   442             YYPARSE_ERROR("Expected REGISTERS but got %s\n", TOKEN_NAMES[tok] );
   443         }
   444     } while( tok != TOK_EOF );
   446     iolex_close();
   447     if( count == 0 ) {
   448         fprintf( stderr, "Warning: input file '%s' did not contain any register definitions\n" );
   449     }
   451     return blocks;
   452 }
   454 /**************************** Lexical analyser ***************************/
   456 static int iolex_open( const char *filename )
   457 {
   458     struct stat st;
   459     int fd = open( filename, O_RDONLY );
   460     if( fd == -1 ) {
   461         fprintf( stderr, "Error opening file '%s': %s\n", filename, strerror(errno) );
   462         return -1;
   463     }
   465     if( fstat( fd, &st ) != 0 ) {
   466         fprintf( stderr, "Error statting file '%s': %s\n", filename, strerror(errno) );
   467         close(fd);
   468         return -1;
   469     }
   471     char *data = g_malloc( st.st_size + 1 );
   472     if( read(fd, data, st.st_size) != st.st_size ) {
   473         fprintf( stderr, "Error reading file '%s': %s\n", filename, strerror(errno) );
   474         close(fd);
   475         return -1;
   476     }
   477     close(fd);
   478     data[st.st_size] = 0;
   480     yybuffer = yyposn = data;
   481     yyend = data + st.st_size;
   482     yyline = 1;
   483     yylineposn = yyposn;
   484     return 0;
   485 }
   487 static void iolex_close()
   488 {
   489     g_free(yybuffer);
   490     yybuffer = yyend = NULL;
   491 }
   493 #define YYRETURN(x) do{ \
   494     yytok.yylength = yyposn - yystart; \
   495     return (x); \
   496 } while(0)
   498 static int iolex_readhex(char *p, int digits)
   499 {
   500     int result = 0;
   501     for( int i=0; i < digits; i++ ) {
   502         result <<= 4;
   503         if( isdigit(p[i]) ) {
   504             result += p[i]-'0';
   505         } else if( p[i] >= 'a' && p[i] <= 'f' ) {
   506             result += p[i]-'a'+10;
   507         } else if( p[i] >= 'A' && p[i] <= 'F' ) {
   508             result += p[i]-'A'+10;
   509         } else {
   510             return (result >> 4);
   511         }
   513     }
   514     return result;
   515 }
   517 static int iolex_readoctal(char *p, int digits)
   518 {
   519     int result = 0;
   520     for( int i=0; i < digits; i++ ) {
   521         result <<= 3;
   522         if( p[i] >= '0' && p[i] <= '7' ) {
   523             result += p[i]-'0';
   524         } else {
   525             return (result >> 4);
   526         }
   527     }
   528     return result;
   529 }
   531 /**
   532  * Copy and interpret the string segment as a C string, replacing escape
   533  * expressions with the corresponding value.
   534  */
   535 static char *iolex_getcstring( char *start, char *end, int *len )
   536 {
   537    char *result = g_malloc0(end-start+1); /* Allocate enough memory for the string as-is */
   538    char *q = result;
   540    for( char *p = start; p != end; p++ ) {
   541        if( *p == '\\' ) {
   542            if( ++p == end ) {
   543                *q++ = '\\';
   544                break;
   545            }
   546            if( p[0] >= '0' && p[0] <= '3' && p+3 <= end &&
   547                p[1] >= '0' && p[1] <= '7' &&
   548                p[2] >= '0' && p[2] <= '7' ) {
   549                *q++ = (char)iolex_readoctal(p,3);
   550                p+=2;
   551            } else {
   552                switch( *p ) {
   553                case 'n':
   554                    *q++ = '\n';
   555                    break;
   556                case 'r':
   557                    *q++ = '\r';
   558                    break;
   559                case 't':
   560                    *q++ = '\t';
   561                    break;
   562                case 'x': /* hex */
   563                    if( p + 3 > end || !isxdigit(p[1]) || !isxdigit(p[2]) ) {
   564                        *q++ = '\\';
   565                        *q++ = *p;
   566                    } else {
   567                        *q++ = (char)iolex_readhex(p+1, 2);
   568                        p+=2;
   569                    }
   570                    break;
   571                default:
   572                    *q++ = '\\';
   573                    *q++ = *p;
   574                }
   575            }
   576        } else {
   577            *q++ = *p;
   578        }
   579    }
   580    *len = q - result;
   581    return result;
   582 }
   584 int iolex( int expectToken )
   585 {
   586     int count = 0;
   587     while( yyposn < yyend ) {
   588         char *yystart = yytok.yytext = yyposn;
   589         yytok.yylength = 1;
   590         yytok.yyline = yyline;
   591         yytok.yycol = yyposn - yylineposn+1;
   592         int ch = *yyposn++;
   593         if( isdigit(ch) ) {
   594             /* INTEGER */
   595             if( ch == '0' ) {
   596                 if( *yyposn == 'x' ) {
   597                     while( yyposn < yyend && isxdigit(*++yyposn) ) ;
   598                 } else {
   599                     while( yyposn < yyend && *yyposn >= '0' && *yyposn <= '7' )
   600                         yyposn++;
   601                 }
   602             } else {
   603                 while( yyposn < yyend && isdigit(*yyposn) )
   604                     yyposn++;
   605             }
   606             yytok.v.i = strtol( yystart, NULL, 0 );
   607             YYRETURN(TOK_INTEGER);
   608         } else if( isalpha(ch) || ch == '_' ) {
   609             /* IDENTIFIER */
   610             while( yyposn < yyend && (isalnum(*yyposn) || *yyposn == '_') )
   611                 yyposn++;
   612             yytok.yylength = yyposn - yystart;
   613             if( expectToken == TOK_IDENTIFIER ) {
   614                 YYRETURN(TOK_IDENTIFIER);
   615             }
   616             /* Otherwise check for keywords */
   617             for( int i=FIRST_KEYWORD; i <= LAST_KEYWORD; i++ ) {
   618                 if( strlen(TOKEN_NAMES[i]) == yytok.yylength &&
   619                     strncasecmp(TOKEN_NAMES[i], yystart, yytok.yylength ) == 0 ) {
   620                     YYRETURN(i);
   621                 }
   622             }
   623             YYRETURN(TOK_IDENTIFIER);
   624         } else if( isspace(ch) ) {
   625             if( ch == '\n' ) {
   626                 yyline++;
   627                 yylineposn = yyposn;
   628             }
   629             while( isspace(*yyposn) ) {
   630                 if( *yyposn == '\n' ) {
   631                     yyline++;
   632                     yylineposn = yyposn+1;
   633                 }
   634                 yyposn++;
   635             }
   636         } else {
   637             switch( ch ) {
   638             case '(': YYRETURN(TOK_LPAREN);
   639             case ')': YYRETURN(TOK_RPAREN);
   640             case '[': YYRETURN(TOK_LSQUARE);
   641             case ']': YYRETURN(TOK_RSQUARE);
   642             case ',': YYRETURN(TOK_COMMA);
   643             case ':': YYRETURN(TOK_COLON);
   644             case ';': YYRETURN(TOK_SEMI);
   645             case '=': YYRETURN(TOK_EQUALS);
   646             case '/':
   647                 if( *yyposn == '/' ) { /* Line comment */
   648                     while( yyposn < yyend && *++yyposn != '\n' ) ;
   649                 } else if( *yyposn == '*' ) { /* C comment */
   650                     while( yyposn < yyend && (*++yyposn != '*' || *++yyposn != '/' ) ) {
   651                         if( *yyposn == '\n' ) {
   652                             yyline++;
   653                             yylineposn = yyposn+1;
   654                         }
   655                     }
   656                 }
   657                 break;
   658             case '\'': /* STRING */
   659                 while( *yyposn != '\'' ) {
   660                     if( *yyposn == '\n' ) {
   661                         fprintf( stderr, "Unexpected newline in string constant!\n" );
   662                         YYRETURN(TOK_ERROR);
   663                     } else if( yyposn >= yyend ) {
   664                         fprintf( stderr, "Unexpected EOF in string constant!\n" );
   665                         YYRETURN(TOK_ERROR);
   666                     } else if( *yyposn == '\\' && yyposn[1] == '\'' ) {
   667                         yyposn++;
   668                     }
   669                     yyposn++;
   670                 }
   671                 yyposn++;
   672                 yytok.v.s = iolex_getcstring(yystart+1, yyposn-1, &yytok.slen);
   673                 YYRETURN(TOK_STRING);
   674             case '\"': /* STRING */
   675                 while( *yyposn != '\"' ) {
   676                     if( *yyposn == '\n' ) {
   677                         fprintf( stderr, "Unexpected newline in string constant!\n" );
   678                         YYRETURN(TOK_ERROR);
   679                     } else if( yyposn >= yyend ) {
   680                         fprintf( stderr, "Unexpected EOF in string constant!\n" );
   681                         YYRETURN(TOK_ERROR);
   682                     } else if( *yyposn == '\\' && yyposn[1] == '\"' ) {
   683                         yyposn++;
   684                     }
   685                     yyposn++;
   686                 }
   687                 yyposn++;
   688                 yytok.v.s = iolex_getcstring(yystart+1, yyposn-1, &yytok.slen);
   689                 YYRETURN(TOK_STRING);
   690             case '}':
   691                 YYRETURN(TOK_RBRACE);
   692             case '{': /* ACTION or LBRACE */
   693                 if( expectToken == TOK_LBRACE ) {
   694                     YYRETURN(TOK_LBRACE);
   695                 } else {
   696                     count++;
   697                     while( count > 0 && yyposn < yyend ) {
   698                         if( *yyposn == '{' )
   699                             count++;
   700                         if( *yyposn == '}' )
   701                             count--;
   702                         yyposn++;
   703                     }
   704                     YYRETURN(TOK_ACTION);
   705                 }
   706             case '.':
   707                 if( *yyposn == '.' ) {
   708                     yyposn++;
   709                     YYRETURN(TOK_RANGE);
   710                 } else {
   711                     YYRETURN(TOK_PERIOD);
   712                 }
   713             default:
   714                 fprintf( stderr, "Illegal character: '%c'\n", ch );
   715                 YYRETURN(TOK_ERROR);
   716             }
   717         }
   719     }
   720     return TOK_EOF;
   721 }
.