lxdream.org :: lxdream/src/tools/mdparse.c r1104:700e16c321e5

filename	src/tools/mdparse.c
changeset	1104:700e16c321e5
next	1105:73da8fd129fb
author	nkeynes
date	Mon Mar 15 22:10:24 2010 +1000 (12 years ago)
permissions	-rw-r--r--
last change	Commit genmach work-in-progress

view

annotate

diff

log

raw

     1 /**

     2  * $Id$

     3  *

     4  * genmmio I/O register definition parser.

     5  *

     6  * Copyright (c) 2010 Nathan Keynes.

     7  *

     8  * This program is free software; you can redistribute it and/or modify

     9  * it under the terms of the GNU General Public License as published by

    10  * the Free Software Foundation; either version 2 of the License, or

    11  * (at your option) any later version.

    12  *

    13  * This program is distributed in the hope that it will be useful,

    14  * but WITHOUT ANY WARRANTY; without even the implied warranty of

    15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    16  * GNU General Public License for more details.

    17  */

    19 #include <assert.h>

    20 #include <stdlib.h>

    21 #include <stdio.h>

    22 #include <string.h>

    23 #include <unistd.h>

    24 #include <fcntl.h>

    25 #include <ctype.h>

    26 #include <errno.h>

    27 #include <sys/stat.h>

    28 #include <glib/gmem.h>

    30 #include "genmach.h"

    31 /*

    32  * Grammar:

    33  *

    34  * start: start register_block |  ;

    35  * register_block: 'registers' IDENTIFIER 'at' INTEGER opt_paramlist '{' register_list '}';

    36  * opt_paramlist: '(' param_list ')' | '(' ')' |  ;

    37  * param_list: param_list ',' param | param ;

    38  * param: access_param | fill_param | endian_param | mask_param ;

    39  * access_param: 'access' '=' 'any'  // Behaves as if it were memory

    40  *        | 'access' '=' 'nooffset' // Recognize any size access (zero-ext or trunc) as long as the address is exactly the same

    41  *        | 'access' '=' 'exact'    // Recognize only the exact access type at the exact address

    42  *        ;

    43  * fill_param: 'fill' '=' literal;

    44  * endian_param: 'endian' '=' 'little'

    45  *        | 'endian' '=' 'big'

    46  *        ;

    47  * mask_param: 'mask' '=' literal;

    48  *

    49  * register_list: register_list register_desc | ;

    50  * register_desc: INTEGER ':' mode_spec type_spec IDENTIFIER opt_paramlist [ '=' literal ] [ ACTION ]

    51  * mode_spec: 'const' | 'rw' | 'ro' | 'wo';

    52  * type_spec: 'i8' | 'i16' | 'i32' | 'i64' | 'f32' | 'f64' | 'string' | 'cstring';

    53  * literal: INTEGER | STRING | 'undefined' ;

    54  *

    55  * C-style comments are recognized as such.

    56  */

    58 #define TOK_IDENTIFIER 1

    59 #define TOK_INTEGER    2

    60 #define TOK_STRING     3

    61 #define TOK_ACTION     4

    62 #define TOK_EOF        5

    63 #define TOK_ERROR      6

    64 #define TOK_LPAREN     7

    65 #define TOK_RPAREN     8

    66 #define TOK_LBRACE     9

    67 #define TOK_RBRACE    10

    68 #define TOK_COMMA     11

    69 #define TOK_COLON     12

    70 #define TOK_SEMI      13

    71 #define TOK_PERIOD    14

    72 #define TOK_RANGE     15

    73 #define TOK_LSQUARE   16

    74 #define TOK_RSQUARE   17

    75 #define TOK_EQUALS    18

    76 #define TOK_ACCESS    19

    77 #define TOK_AT        20

    78 #define TOK_ENDIAN    21

    79 #define TOK_FILL      22

    80 #define TOK_INCLUDE   23

    81 #define TOK_MASK      24

    82 #define TOK_REGISTERS 25

    83 #define TOK_SPACE     26

    84 #define TOK_STRIDE    27

    85 #define TOK_TEST      28

    86 #define TOK_TRACE     29

    87 #define TOK_UNDEFINED 30

    90 #define FIRST_KEYWORD TOK_ACCESS

    91 #define LAST_KEYWORD  TOK_UNDEFINED

    92 static const char *TOKEN_NAMES[] = {"NULL", "IDENTIFIER", "INTEGER", "STRING", "ACTION", "EOF", "ERROR",

    93         "(", ")", "{", "}", ",", ":", ";", ".", "..", "[", "]", "=",

    94         "ACCESS", "AT", "ENDIAN", "FILL", "INCLUDE", "MASK", "REGISTERS", "SPACE", "STRIDE", "TEST", "TRACE", "UNDEFINED" };

    95 static const char *MODE_NAMES[] = {"const", "rw", "ro", "wo", "mirror"};

    96 static const char *TYPE_NAMES[] = {"i8", "i16", "i32", "i64", "f32", "f64", "string"};

    97 static int TYPE_SIZES[] = {1,2,4,8,4,8,0};

    98 static const char *ENDIAN_NAMES[] = {"default", "little", "big"};

    99 static const char *ACCESS_NAMES[] = {"default", "any", "nooffset", "exact"};

   100 static const char *TRACE_NAMES[] = {"default", "off", "on"};

   102 #define elementsof(x) (sizeof(x)/sizeof(x[0]))

   104 typedef struct token_data {

   105     char *yytext;

   106     int yylength;

   107     int yyline;

   108     int yycol;

   109     union {

   110         long i;

   111         char *s;

   112     } v;

   113     int slen;

   114 } token_data;

   116 static char *yybuffer;

   117 static char *yyposn, *yylineposn;

   118 static char *yyend;

   119 static int yyline;

   120 static struct token_data yytok;

   122 #define YYPARSE_ERROR( msg, ... ) \

   123     do { \

   124         fprintf( stderr, "Parse error at %d:%d: " msg "\n", yytok.yyline, yytok.yycol, __VA_ARGS__ ); \

   125         exit(2); \

   126     } while(0)

   128 #define READ(x) \

   129 {   int _tok = iolex(x); \

   130     if( _tok != (x) ) { \

   131         YYPARSE_ERROR( "Expected %s but got %s", TOKEN_NAMES[x], TOKEN_NAMES[_tok] ); \

   132     } \

   133 }

   135 static int iolex( int expectToken );

   136 static int iolex_open( const char *filename );

   137 static void iolex_close();

   139 static inline char *yystrdup()

   140 {

   141     char *str = g_malloc0(yytok.yylength+1);

   142     memcpy( str, yytok.yytext, yytok.yylength);

   143     return str;

   144 }

   146 static inline int yystrcasecmp(const char *cmp)

   147 {

   148     int len = strlen(cmp);

   149     if( len != yytok.yylength ) {

   150         return yytok.yylength - len;

   151     }

   152     return strncasecmp(yytok.yytext, cmp, yytok.yylength);

   153 }

   155 static int yymatch( const char *arr[], unsigned numOptions )

   156 {

   157     for( unsigned i=0; i<numOptions; i++ ) {

   158         if( yystrcasecmp( arr[i] ) == 0 ) {

   159             return i;

   160         }

   161     }

   162     return -1;

   163 }

   165 static gint register_block_sort_cb( gconstpointer a, gconstpointer b )

   166 {

   167     struct regblock *blocka = (struct regblock *)a;

   168     struct regblock *blockb = (struct regblock *)b;

   169     return blocka->address - blockb->address;

   170 }

   172 static int register_ptr_sort_cb( const void *a, const void *b )

   173 {

   174     regdef_t *ptra = (regdef_t *)a;

   175     regdef_t *ptrb = (regdef_t *)b;

   176     if( (*ptra)->offset != (*ptrb)->offset )

   177         return (*ptra)->offset - (*ptrb)->offset;

   178     return (*ptra)->type - (*ptrb)->type;

   179 }

   181 static void ioparse_apval(int tok, union apval *apv, unsigned *numBytes)

   182 {

   183     if( tok == TOK_INTEGER ) {

   184         apv->i = yytok.v.i;

   185         if( *numBytes == 0 ) {

   186             YYPARSE_ERROR( "Expected string initializer but was an integer (0x%x)", yytok.v.i );

   187         }

   188     } else if( tok = TOK_STRING ) {

   189         if( *numBytes == 0 ) {

   190             *numBytes = yytok.slen;

   191             apv->s = yytok.v.s;

   192         } else {

   193             if( *numBytes != yytok.slen ) {

   194                 YYPARSE_ERROR( "Expected %d byte initializer but was %d", *numBytes, yytok.slen );

   195             }

   196             assert( *numBytes < sizeof(uint64_t) );

   197             apv->i = 0;

   198             /* FIXME: handle endian mismatches */

   199             memcpy( &apv->i, yytok.v.s, yytok.slen );

   200         }

   201     } else {

   202         YYPARSE_ERROR( "Expected literal (integer or string), but got %s", TOKEN_NAMES[tok] );

   203     }

   204 }

   206 static void ioparse_regflags( regflags_t flags, unsigned *numBytes )

   207 {

   208     do {

   209         int tok = iolex(TOK_RPAREN);

   210         switch(tok) {

   211         case TOK_FILL:

   212             READ(TOK_EQUALS);

   213             tok = iolex(TOK_INTEGER);

   214             flags->fillSizeBytes = 4;

   215             ioparse_apval( tok, &flags->fillValue, &flags->fillSizeBytes );

   216             break;

   217         case TOK_ACCESS:

   218             READ(TOK_EQUALS);

   219             READ(TOK_IDENTIFIER);

   220             flags->access = yymatch(ACCESS_NAMES,elementsof(ACCESS_NAMES));

   221             if( flags->access == -1 ) {

   222                 YYPARSE_ERROR("Unknown access mode '%s'", yystrdup());

   223             }

   224             break;

   225         case TOK_MASK:

   226             if( numBytes ) {

   227                 READ(TOK_EQUALS);

   228                 tok = iolex(TOK_INTEGER);

   229                 ioparse_apval( tok, &flags->maskValue, numBytes );

   230             } else {

   231                 YYPARSE_ERROR("mask is not valid on a register block",0);

   232             }

   233             break;

   234         case TOK_ENDIAN:

   235             READ(TOK_EQUALS);

   236             READ(TOK_IDENTIFIER);

   237             flags->endian = yymatch(ENDIAN_NAMES,elementsof(ENDIAN_NAMES));

   238             if( flags->endian == -1 ) {

   239                 YYPARSE_ERROR("Unknown endianness '%s'", yystrdup());

   240             }

   241             break;

   242         case TOK_TRACE:

   243             READ(TOK_EQUALS);

   244             READ(TOK_IDENTIFIER);

   245             flags->traceFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES));

   246             if( flags->traceFlag == -1 ) {

   247                 YYPARSE_ERROR("Unknown trace flag '%s'", yystrdup());

   248             }

   249             break;

   250         case TOK_TEST:

   251             READ(TOK_EQUALS);

   252             READ(TOK_IDENTIFIER);

   253             flags->testFlag = yymatch(TRACE_NAMES,elementsof(TRACE_NAMES));

   254             if( flags->testFlag == -1 ) {

   255                 YYPARSE_ERROR("Unknown test flag '%s'", yystrdup());

   256             }

   257             break;

   258         case TOK_COMMA:

   259             break;

   260         case TOK_RPAREN:

   261             return;

   262         default:

   263             YYPARSE_ERROR("Expected flag or ')' but was %s", TOKEN_NAMES[tok]);

   264         }

   265     } while(1);

   266 }

   268 static void ioparse_regdef( struct regdef *reg )

   269 {

   270     reg->offset = yytok.v.i;

   271     reg->numElements = 1;

   272     reg->numBytes = 0;

   273     reg->initValue.i = 0;

   274     reg->flags.maskValue.i = -1;

   275     unsigned rangeOffset = reg->offset;

   277     int tok = iolex(TOK_COLON);

   278     if( tok == TOK_RANGE ) {

   279         READ(TOK_INTEGER);

   280         rangeOffset = yytok.v.i;

   281         if( rangeOffset < reg->offset ) {

   282             YYPARSE_ERROR( "Range end (0x%x) must be greater than the range start (0x%x)", rangeOffset, reg->offset );

   283         }

   284         READ(TOK_COLON);

   285     } else if( tok != TOK_COLON ) {

   286         YYPARSE_ERROR( "Expected ':' but was %s\n", TOKEN_NAMES[tok] );

   287     }

   288     READ(TOK_IDENTIFIER);

   289     reg->mode = yymatch(MODE_NAMES, elementsof(MODE_NAMES));

   290     if( reg->mode == -1 ) {

   291         YYPARSE_ERROR( "Unknown register mode '%s'", yystrdup() );

   292     }

   293     if( reg->mode == REG_MIRROR ) {

   294         /* Mirror regions have a target range only */

   295         READ(TOK_INTEGER);

   296         reg->initValue.i = yytok.v.i;

   297         reg->type = REG_I8;

   298         tok = iolex(TOK_RANGE);

   299         if( tok == TOK_RANGE ) {

   300             READ(TOK_INTEGER);

   301             if( yytok.v.i < reg->initValue.i ) {

   302                 YYPARSE_ERROR( "Invalid mirror target range 0x%x..0x%x", reg->initValue.i, yytok.v.i );

   303             }

   304             reg->numBytes = yytok.v.i - reg->initValue.i + 1;

   305             tok = iolex(TOK_STRIDE);

   306         }

   307         if( tok == TOK_STRIDE ) {

   308             READ(TOK_INTEGER);

   309             reg->stride = yytok.v.i;

   310             tok = iolex(TOK_ACTION);

   311         } else {

   312             reg->stride = reg->numBytes;

   313         }

   315         if( tok != TOK_SEMI ) {

   316             YYPARSE_ERROR( "Expected ; but gut %s", TOKEN_NAMES[tok] );

   317         }

   319         if( reg->stride < reg->numBytes ) {

   320             YYPARSE_ERROR( "Invalid mirror stride: %x is less than block size %x\n", reg->stride, reg->numBytes );

   321         }

   323         if( rangeOffset != reg->offset ) {

   324             reg->numElements *= ((rangeOffset - reg->offset) + reg->stride - 1) / reg->stride;

   325         }

   327     } else {

   328         READ(TOK_IDENTIFIER);

   329         reg->type = yymatch(TYPE_NAMES, elementsof(TYPE_NAMES));

   330         if( reg->type == -1 ) {

   331             YYPARSE_ERROR( "Unknown register type '%s'", yystrdup() );

   332         }

   333         reg->numBytes = TYPE_SIZES[reg->type];

   334         tok = iolex(TOK_IDENTIFIER);

   335         if( tok == TOK_IDENTIFIER ) {

   336             reg->name = yystrdup();

   337             tok = iolex(TOK_ACTION);

   338         }

   339         if( tok == TOK_STRING ) {

   340             reg->description = yytok.v.s;

   341             tok = iolex(TOK_ACTION);

   342         }

   343         if( tok == TOK_LPAREN ) {

   344             ioparse_regflags(&reg->flags, &reg->numBytes);

   345             tok = iolex(TOK_ACTION);

   346         }

   347         if( tok == TOK_EQUALS ) {

   348             tok = iolex(TOK_INTEGER);

   349             if( tok == TOK_UNDEFINED ) {

   350                 reg->initValue.i = 0xDEADBEEFDEADBEEF;

   351                 reg->initUndefined = TRUE;

   352             } else {

   353                 ioparse_apval(tok, &reg->initValue, &reg->numBytes);

   354             }

   355             tok = iolex(TOK_ACTION);

   356         } else if( reg->type == REG_STRING ) {

   357             YYPARSE_ERROR( "String declarations must have an initializer (ie = 'abcd')",0 );

   358         }

   359         if( tok == TOK_ACTION ) {

   360             // reg->action = yystrdup();

   361         } else if( tok != TOK_SEMI ) {

   362             YYPARSE_ERROR( "Expected ; or {, but got %s", TOKEN_NAMES[tok] );

   363         }

   365         if( rangeOffset != reg->offset ) {

   366             reg->numElements *= ((rangeOffset - reg->offset) + reg->numBytes - 1) / reg->numBytes;

   367         }

   368     }

   370 }

   372 static struct regblock *ioparse_regblock( )

   373 {

   374     unsigned regsAllocSize = 128;

   375     struct regblock *block = g_malloc0(sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);

   376     block->numRegs = 0;

   378     READ(TOK_IDENTIFIER);

   379     block->name = yystrdup();

   380     int tok = iolex(TOK_AT);

   381     if( tok == TOK_STRING ) {

   382         block->description = yytok.v.s;

   383         tok = iolex(TOK_AT);

   384     }

   385     if( tok != TOK_AT ) {

   386         YYPARSE_ERROR("Expected AT but got %s\n", TOKEN_NAMES[tok] );

   387     }

   388     READ(TOK_INTEGER);

   389     block->address = yytok.v.i;

   391     tok = iolex(TOK_LBRACE);

   392     if( tok == TOK_LPAREN) {

   393         ioparse_regflags(&block->flags, NULL);

   394         READ(TOK_LBRACE);

   395     } else if( tok != TOK_LBRACE ) {

   396         YYPARSE_ERROR("Expected { but got %s\n", TOKEN_NAMES[tok] );

   397     }

   399     tok = iolex(TOK_INTEGER);

   400     while( tok != TOK_RBRACE ) {

   401         if( tok != TOK_INTEGER ) {

   402             YYPARSE_ERROR("Expected INTEGER or } but got %s\n", TOKEN_NAMES[tok]);

   403         }

   404         struct regdef *regdef = g_malloc0(sizeof(struct regdef));

   405         if( block->numRegs >= regsAllocSize ) {

   406             regsAllocSize *= 2;

   407             block = g_realloc(block, sizeof(struct regblock) + sizeof(regdef_t)*regsAllocSize);

   408         }

   409         block->regs[block->numRegs++] = regdef;

   410         ioparse_regdef(regdef);

   412         tok = iolex(TOK_INTEGER);

   413     }

   415     qsort( &block->regs[0], block->numRegs, sizeof(block->regs[0]), register_ptr_sort_cb );

   417     return block;

   418 }

   420 GList *ioparse( const char *filename, GList *list )

   421 {

   422     GList *blocks = list;

   423     int count = 0;

   425     if( iolex_open(filename) != 0  )

   426         return blocks;

   428     int tok;

   429     do {

   430         tok = iolex(TOK_REGISTERS);

   431         if( tok == TOK_EOF ) {

   432             return blocks;

   433         } else if( tok == TOK_INCLUDE) {

   434             READ(TOK_STRING);

   436         } else if( tok == TOK_SPACE ) {

   437         } else if( tok == TOK_REGISTERS ) {

   438             struct regblock *block = ioparse_regblock(block);

   439             count++;

   440             blocks = g_list_insert_sorted(blocks, block, register_block_sort_cb);

   441         } else {

   442             YYPARSE_ERROR("Expected REGISTERS but got %s\n", TOKEN_NAMES[tok] );

   443         }

   444     } while( tok != TOK_EOF );

   446     iolex_close();

   447     if( count == 0 ) {

   448         fprintf( stderr, "Warning: input file '%s' did not contain any register definitions\n" );

   449     }

   451     return blocks;

   452 }

   454 /**************************** Lexical analyser ***************************/

   456 static int iolex_open( const char *filename )

   457 {

   458     struct stat st;

   459     int fd = open( filename, O_RDONLY );

   460     if( fd == -1 ) {

   461         fprintf( stderr, "Error opening file '%s': %s\n", filename, strerror(errno) );

   462         return -1;

   463     }

   465     if( fstat( fd, &st ) != 0 ) {

   466         fprintf( stderr, "Error statting file '%s': %s\n", filename, strerror(errno) );

   467         close(fd);

   468         return -1;

   469     }

   471     char *data = g_malloc( st.st_size + 1 );

   472     if( read(fd, data, st.st_size) != st.st_size ) {

   473         fprintf( stderr, "Error reading file '%s': %s\n", filename, strerror(errno) );

   474         close(fd);

   475         return -1;

   476     }

   477     close(fd);

   478     data[st.st_size] = 0;

   480     yybuffer = yyposn = data;

   481     yyend = data + st.st_size;

   482     yyline = 1;

   483     yylineposn = yyposn;

   484     return 0;

   485 }

   487 static void iolex_close()

   488 {

   489     g_free(yybuffer);

   490     yybuffer = yyend = NULL;

   491 }

   493 #define YYRETURN(x) do{ \

   494     yytok.yylength = yyposn - yystart; \

   495     return (x); \

   496 } while(0)

   498 static int iolex_readhex(char *p, int digits)

   499 {

   500     int result = 0;

   501     for( int i=0; i < digits; i++ ) {

   502         result <<= 4;

   503         if( isdigit(p[i]) ) {

   504             result += p[i]-'0';

   505         } else if( p[i] >= 'a' && p[i] <= 'f' ) {

   506             result += p[i]-'a'+10;

   507         } else if( p[i] >= 'A' && p[i] <= 'F' ) {

   508             result += p[i]-'A'+10;

   509         } else {

   510             return (result >> 4);

   511         }

   513     }

   514     return result;

   515 }

   517 static int iolex_readoctal(char *p, int digits)

   518 {

   519     int result = 0;

   520     for( int i=0; i < digits; i++ ) {

   521         result <<= 3;

   522         if( p[i] >= '0' && p[i] <= '7' ) {

   523             result += p[i]-'0';

   524         } else {

   525             return (result >> 4);

   526         }

   527     }

   528     return result;

   529 }

   531 /**

   532  * Copy and interpret the string segment as a C string, replacing escape

   533  * expressions with the corresponding value.

   534  */

   535 static char *iolex_getcstring( char *start, char *end, int *len )

   536 {

   537    char *result = g_malloc0(end-start+1); /* Allocate enough memory for the string as-is */

   538    char *q = result;

   540    for( char *p = start; p != end; p++ ) {

   541        if( *p == '\\' ) {

   542            if( ++p == end ) {

   543                *q++ = '\\';

   544                break;

   545            }

   546            if( p[0] >= '0' && p[0] <= '3' && p+3 <= end &&

   547                p[1] >= '0' && p[1] <= '7' &&

   548                p[2] >= '0' && p[2] <= '7' ) {

   549                *q++ = (char)iolex_readoctal(p,3);

   550                p+=2;

   551            } else {

   552                switch( *p ) {

   553                case 'n':

   554                    *q++ = '\n';

   555                    break;

   556                case 'r':

   557                    *q++ = '\r';

   558                    break;

   559                case 't':

   560                    *q++ = '\t';

   561                    break;

   562                case 'x': /* hex */

   563                    if( p + 3 > end || !isxdigit(p[1]) || !isxdigit(p[2]) ) {

   564                        *q++ = '\\';

   565                        *q++ = *p;

   566                    } else {

   567                        *q++ = (char)iolex_readhex(p+1, 2);

   568                        p+=2;

   569                    }

   570                    break;

   571                default:

   572                    *q++ = '\\';

   573                    *q++ = *p;

   574                }

   575            }

   576        } else {

   577            *q++ = *p;

   578        }

   579    }

   580    *len = q - result;

   581    return result;

   582 }

   584 int iolex( int expectToken )

   585 {

   586     int count = 0;

   587     while( yyposn < yyend ) {

   588         char *yystart = yytok.yytext = yyposn;

   589         yytok.yylength = 1;

   590         yytok.yyline = yyline;

   591         yytok.yycol = yyposn - yylineposn+1;

   592         int ch = *yyposn++;

   593         if( isdigit(ch) ) {

   594             /* INTEGER */

   595             if( ch == '0' ) {

   596                 if( *yyposn == 'x' ) {

   597                     while( yyposn < yyend && isxdigit(*++yyposn) ) ;

   598                 } else {

   599                     while( yyposn < yyend && *yyposn >= '0' && *yyposn <= '7' )

   600                         yyposn++;

   601                 }

   602             } else {

   603                 while( yyposn < yyend && isdigit(*yyposn) )

   604                     yyposn++;

   605             }

   606             yytok.v.i = strtol( yystart, NULL, 0 );

   607             YYRETURN(TOK_INTEGER);

   608         } else if( isalpha(ch) || ch == '_' ) {

   609             /* IDENTIFIER */

   610             while( yyposn < yyend && (isalnum(*yyposn) || *yyposn == '_') )

   611                 yyposn++;

   612             yytok.yylength = yyposn - yystart;

   613             if( expectToken == TOK_IDENTIFIER ) {

   614                 YYRETURN(TOK_IDENTIFIER);

   615             }

   616             /* Otherwise check for keywords */

   617             for( int i=FIRST_KEYWORD; i <= LAST_KEYWORD; i++ ) {

   618                 if( strlen(TOKEN_NAMES[i]) == yytok.yylength &&

   619                     strncasecmp(TOKEN_NAMES[i], yystart, yytok.yylength ) == 0 ) {

   620                     YYRETURN(i);

   621                 }

   622             }

   623             YYRETURN(TOK_IDENTIFIER);

   624         } else if( isspace(ch) ) {

   625             if( ch == '\n' ) {

   626                 yyline++;

   627                 yylineposn = yyposn;

   628             }

   629             while( isspace(*yyposn) ) {

   630                 if( *yyposn == '\n' ) {

   631                     yyline++;

   632                     yylineposn = yyposn+1;

   633                 }

   634                 yyposn++;

   635             }

   636         } else {

   637             switch( ch ) {

   638             case '(': YYRETURN(TOK_LPAREN);

   639             case ')': YYRETURN(TOK_RPAREN);

   640             case '[': YYRETURN(TOK_LSQUARE);

   641             case ']': YYRETURN(TOK_RSQUARE);

   642             case ',': YYRETURN(TOK_COMMA);

   643             case ':': YYRETURN(TOK_COLON);

   644             case ';': YYRETURN(TOK_SEMI);

   645             case '=': YYRETURN(TOK_EQUALS);

   646             case '/':

   647                 if( *yyposn == '/' ) { /* Line comment */

   648                     while( yyposn < yyend && *++yyposn != '\n' ) ;

   649                 } else if( *yyposn == '*' ) { /* C comment */

   650                     while( yyposn < yyend && (*++yyposn != '*' || *++yyposn != '/' ) ) {

   651                         if( *yyposn == '\n' ) {

   652                             yyline++;

   653                             yylineposn = yyposn+1;

   654                         }

   655                     }

   656                 }

   657                 break;

   658             case '\'': /* STRING */

   659                 while( *yyposn != '\'' ) {

   660                     if( *yyposn == '\n' ) {

   661                         fprintf( stderr, "Unexpected newline in string constant!\n" );

   662                         YYRETURN(TOK_ERROR);

   663                     } else if( yyposn >= yyend ) {

   664                         fprintf( stderr, "Unexpected EOF in string constant!\n" );

   665                         YYRETURN(TOK_ERROR);

   666                     } else if( *yyposn == '\\' && yyposn[1] == '\'' ) {

   667                         yyposn++;

   668                     }

   669                     yyposn++;

   670                 }

   671                 yyposn++;

   672                 yytok.v.s = iolex_getcstring(yystart+1, yyposn-1, &yytok.slen);

   673                 YYRETURN(TOK_STRING);

   674             case '\"': /* STRING */

   675                 while( *yyposn != '\"' ) {

   676                     if( *yyposn == '\n' ) {

   677                         fprintf( stderr, "Unexpected newline in string constant!\n" );

   678                         YYRETURN(TOK_ERROR);

   679                     } else if( yyposn >= yyend ) {

   680                         fprintf( stderr, "Unexpected EOF in string constant!\n" );

   681                         YYRETURN(TOK_ERROR);

   682                     } else if( *yyposn == '\\' && yyposn[1] == '\"' ) {

   683                         yyposn++;

   684                     }

   685                     yyposn++;

   686                 }

   687                 yyposn++;

   688                 yytok.v.s = iolex_getcstring(yystart+1, yyposn-1, &yytok.slen);

   689                 YYRETURN(TOK_STRING);

   690             case '}':

   691                 YYRETURN(TOK_RBRACE);

   692             case '{': /* ACTION or LBRACE */

   693                 if( expectToken == TOK_LBRACE ) {

   694                     YYRETURN(TOK_LBRACE);

   695                 } else {

   696                     count++;

   697                     while( count > 0 && yyposn < yyend ) {

   698                         if( *yyposn == '{' )

   699                             count++;

   700                         if( *yyposn == '}' )

   701                             count--;

   702                         yyposn++;

   703                     }

   704                     YYRETURN(TOK_ACTION);

   705                 }

   706             case '.':

   707                 if( *yyposn == '.' ) {

   708                     yyposn++;

   709                     YYRETURN(TOK_RANGE);

   710                 } else {

   711                     YYRETURN(TOK_PERIOD);

   712                 }

   713             default:

   714                 fprintf( stderr, "Illegal character: '%c'\n", ch );

   715                 YYRETURN(TOK_ERROR);

   716             }

   717         }

   719     }

   720     return TOK_EOF;

   721 }