cde/osf/uil/UilLexAna.c

   1 /*
   2  *  @OSF_COPYRIGHT@
   3  *  COPYRIGHT NOTICE
   4  *  Copyright (c) 1990, 1991, 1992, 1993 Open Software Foundation, Inc.
   5  *  ALL RIGHTS RESERVED (MOTIF). See the file named COPYRIGHT.MOTIF for
   6  *  the full copyright text.
   7 */
   8 /*
   9  * HISTORY
  10 */
  11 #ifdef REV_INFO
  12 #ifndef lint
  13 static char rcsid[] = "$TOG: UilLexAna.c /main/14 1997/03/12 15:10:52 dbl $"
  14 #endif
  15 #endif
  16
  17 /*
  18 *  (c) Copyright 1989, 1990, DIGITAL EQUIPMENT CORPORATION, MAYNARD, MASS. */
  19
  20 /*
  21 **++
  22 **  FACILITY:
  23 **
  24 **      User Interface Language Compiler (UIL)
  25 **
  26 **  ABSTRACT:
  27 **
  28 **      This module hold the routines that build tokens for the UIL
  29 **      compiler.
  30 **
  31 **--
  32 **/
  33
  34
  35 /*
  36 **
  37 **  INCLUDE FILES
  38 **
  39 **/
  40
  41 #include <Xm/Xm.h>
  42 /* I think this one should be public too, it's not the case right now,
  43    and I don't want to include XmP.h here - dd */
  44 extern char *_XmStringGetCurrentCharset ();
  45 #include <Xm/XmosP.h>   /* Need this for MB_CUR_MAX */
  46
  47 #include <Mrm/MrmosI.h> /* Need this for _MrmOSSetLocale. */
  48
  49 #include "UilDefI.h"
  50 #include <ctype.h>
  51
  52 double atof();
  53
  54 #include <errno.h>      /* needed to support conversion functions */
  55 #ifdef X_NOT_STDC_ENV
  56 extern int errno;
  57 #endif
  58
  59 /*
  60 **
  61 **  TABLE OF CONTENTS
  62 **
  63 */
  64
  65 /*
  66 ** FORWARD DECLARATIONS
  67 */
  68
  69 typedef struct  _lex_buffer_type
  70 {
  71     struct _lex_buffer_type     *az_next_buffer;
  72     unsigned char               c_text[ 1 ];
  73 } lex_buffer_type;
  74
  75
  76 static lex_buffer_type *get_lex_buffer  _ARGUMENTS(( lex_buffer_type *az_current_lex_buffer ));
  77 #if debug_version
  78 static void dump_token  _ARGUMENTS(( lex_buffer_type *az_current_lex_buffer , int l_lex_pos ));
  79 #endif
  80
  81
  82 /*
  83 **
  84 **  EXTERNAL DEFINITIONS
  85 **
  86 */
  87
  88
  89 /*    These values are set by Lex initiailize routine or if the charset     */
  90 /*    option is specified on the module declaration                         */
  91 externaldef(uil_comp_glbl) int                  Uil_lex_l_user_default_charset;
  92 externaldef(uil_comp_glbl) sym_value_entry_type *Uil_lex_az_charset_entry;
  93 externaldef(uil_comp_glbl) int                  Uil_lex_l_localized;
  94
  95
  96 /*  This is the most recent character set recognized by the grammar.  It    */
  97 /*  allows the use of the CHARACTER_SET function to prefix general string   */
  98 /*  literals.                                                               */
  99 externaldef(uil_comp_glbl) int                   Uil_lex_l_charset_specified;
 100 externaldef(uil_comp_glbl) int                   Uil_lex_l_literal_charset;
 101 externaldef(uil_comp_glbl) sym_value_entry_type *Uil_lex_az_literal_charset;
 102 /* %COMPLETE */
 103 externaldef(uil_comp_glbl) int Uil_characters_read;
 104
 105 /*    For portability, declare all yy* variables as extern.    */
 106
 107 extern yystype                  yylval;
 108 extern yystype                  prev_yylval;
 109
 110 /*    This stack entry is used as the result of epsilon productions.   */
 111
 112 extern yystype                  gz_yynullval;
 113
 114
 115 /*
 116  * Retain comments as scanned
 117  */
 118 #define INITIAL_COMMENT_SIZE 8000
 119 #define INCR_COMMENT_SIZE 2000
 120
 121
 122 externaldef(uil_comp_glbl) char *comment_text;
 123 externaldef(uil_comp_glbl) int comment_size;
 124
 125 static int last_token_seen = 0;
 126
 127
 128 \f
 129 /*
 130 **
 131 **  MACRO DEFINITIONS
 132 **
 133 **/
 134
 135 /*
 136 **  Each of the 256 possible input characters has an associated class.
 137 **  These defines gives names to the classes.
 138 */
 139
 140 #define     class_blank     0           /* white space */
 141 #define     class_punc      1           /* punctuation characters */
 142 #define     class_name      2           /* alphabetics - {eEntvbrf} + {$ _} */
 143 #define     class_e         3           /* e */
 144 #define     class_escape    4           /* n t v b r f */
 145 #define     class_ff        5           /* form feed */
 146 #define     class_digit     6           /* 0..9 */
 147 #define     class_dot       7           /* . */
 148 #define     class_sign      8           /* - + */
 149 #define     class_quote     9           /* ' */
 150 #define     class_bslash    10          /* \ */
 151 #define     class_slash     11          /* / */
 152 #define     class_star      12          /* * */
 153 #define     class_exclam    13          /* ! */
 154 #define     class_eol       14          /* end of line - ascii nul */
 155 #define     class_rest      15          /* remaining printable characters */
 156 #define     class_illegal   16          /* remaining non printable characters */
 157 #define     class_pound     17          /* # */
 158 #define     class_dquote    18          /* " */
 159 #define     class_langle    19          /* < */
 160 #define     class_rangle    20          /* > */
 161 #define     class_highbit   21          /* high order bit set */
 162 #define     max_class       21
 163
 164 /*
 165 **   states within the token table
 166 */
 167
 168 #define     state_initial   0           /* initial state of automaton */
 169 #define     state_name      1           /* looking for a name */
 170 #define     state_integer   2           /* looking for a integer */
 171 #define     state_real_1    3           /* looking for a real */
 172 #define     state_real_2    4           /* looking for a real */
 173 #define     state_exp_1     5           /* looking for a real with exponent */
 174 #define     state_exp_2     6           /* looking for a real with exponent */
 175 #define     state_exp_3     7           /* looking for a real with exponent */
 176 #define     state_str_1     8           /* looking for a string */
 177 #define     state_str_2     9           /* looking for a string */
 178 #define     state_str_3     10          /* looking for a string */
 179 #define     state_comment_1 11          /* looking for a comment */
 180 #define     state_comment_2 12          /* looking for a comment */
 181 #define     state_comment_3 13          /* looking for a comment */
 182 #define     state_comment_4 14          /* looking for a comment */
 183 #define     state_eat_rest  15          /* eat up rest characters */
 184 #define     state_gstr_1    16          /* looking for a general string */
 185 #define     state_gstr_2    17          /* looking for a general string */
 186 #define     state_gstr_3    18          /* looking for a general string */
 187 #define     state_gstr_4    19          /* looking for a general string */
 188 #define     state_gstr_5    20          /* looking for a general string */
 189 #define     state_langle    21          /* looking for << */
 190 #define     state_rangle    22          /* looking for >> */
 191 #define     max_state       22
 192
 193 /*
 194 **   actions in token table
 195 **       all actions requiring a move of the current character have
 196 **       a negative value
 197 */
 198
 199 #define     NEGATIVE 0x40
 200
 201 #define     min_action      1
 202 #define     move_advance    (NEGATIVE | 1)      /* save character - goto next state */
 203 #define     advance         1           /* goto next state */
 204 #define     move_final      (NEGATIVE | 2)      /* save character - found token */
 205 #define     final           2           /* found token */
 206 #define     move_error      (NEGATIVE | 3)      /* save character - found error */
 207 #define     error           3           /* found error */
 208 #define     move_special    (NEGATIVE | 4)              /* special action */
 209 #define     special         4           /* save character - special action */
 210 #define     reset           5           /* reset the analyzer */
 211 #define     final_comment   (NEGATIVE | 6)       /* save whole comment  RAP */
 212
 213 /*
 214 **  errors encoded in the token table
 215 */
 216
 217 #define     min_error       0
 218 #define     bad_prefix      0           /* junk starts a token */
 219 #define     error_max       0
 220
 221 /*
 222 **  final states in the token table
 223 */
 224
 225 #define     token_min           1
 226 #define     token_punc          1       /* punctuation character */
 227 #define     token_eol           2       /* end of line */
 228 #define     token_real          3       /* real number */
 229 #define     token_integer       4       /* integer */
 230 #define     token_name          5       /* identifier */
 231 #define     token_ustring       6       /* unterminated string */
 232 #define     token_string        7       /* string */
 233 #define     token_ff            8       /* form feed */
 234 #define     token_ugstr         9       /* unterminated general string */
 235 #define     token_gstr          10      /* general string */
 236 #define     token_punc2         11      /* 2 character punctuation */
 237 #define     token_comment       12      /* comment block RAP */
 238 #define     token_lstr          13      /* localized general string */
 239 #define     token_max           13
 240
 241
 242 /*
 243 **  special actions that take place
 244 */
 245
 246 #define     control_char    1           /* unprintable character in construct */
 247 #define     start_bslash    2           /* start of \ construct in string */
 248 #define     ignore_bslash   3           /* \ not followed by correct sequence */
 249 #define     found_digit     4           /* digit in \ sequence */
 250 #define     end_digits      5           /* end of \digit...\ sequence */
 251 #define     insert_char     6           /* end of \x sequence */
 252 #define     missing_bslash  7           /* \ at end of \digit...\ missing */
 253 #define     string_wrap     8           /* string wraps to next line */
 254 #define     comment_wrap    9           /* comment wraps to next line */
 255 #define     charset_gstr    10          /* found char set for general string */
 256 #define     nocharset_gstr  11          /* default char set for general string */
 257 #define     highbit_char    12          /* default char set for general string */
 258
 259 /*
 260 **  define backup field values of a token table cell
 261 */
 262
 263 #define     not_used        0           /* value not defined for this action */
 264 #define     backup_0        0           /* no backup required */
 265 #define     backup_1        1           /* 1 character backup needed */
 266 #define     backup_2        2           /* 2 character backup needed */
 267 #define     backup_3        3           /* 3 character backup needed */
 268
 269 /*
 270 **  form of each cell in the token table
 271 */
 272
 273 \f
 274 typedef struct
 275 {
 276     char        action;                     /* action to be taken */
 277     char        next_state;                 /* next state */
 278                                             /* terminal found */
 279                                             /* error found */
 280     char        backup;                     /* number of character to backup */
 281                                             /* action for special cases */
 282     char        unused;
 283 } cell;
 284
 285 /*
 286 **
 287 **  OWN Storage for TOKEN BUILDER
 288 **
 289 */
 290
 291 #define l_max_lex_buffer_pos  127
 292
 293 static  lex_buffer_type  *az_first_lex_buffer;
 294
 295 /*
 296 **  the actual token table
 297 */
 298
 299 static cell XmConst token_table[ max_state+1][ max_class+1] =
 300   {
 301     { /* state_initial */
 302         /* class_blank   */ { reset,            state_initial,      not_used },
 303         /* class_punc    */ { move_final,       token_punc,         backup_0 },
 304         /* class_name    */ { move_advance,     state_name,         not_used },
 305         /* class_e       */ { move_advance,     state_name,         not_used },
 306         /* class_escape  */ { move_advance,     state_name,         not_used },
 307         /* class_ff      */ { final,            token_ff,           backup_0 },
 308         /* class_digit   */ { move_advance,     state_integer,      not_used },
 309         /* class_dot     */ { move_advance,     state_real_1,       not_used },
 310         /* class_sign    */ { move_final,       token_punc,         backup_0 },
 311         /* class_quote   */ { advance,          state_str_1,        not_used },
 312         /* class_bslash  */ { move_advance,     state_eat_rest,     not_used },
 313         /* class_slash   */ { move_advance,     state_comment_1,    not_used },
 314         /* class_star    */ { move_final,       token_punc,         backup_0 },
 315         /* class_exclam  */ { move_advance,     state_comment_4,    not_used },
 316         /* class_eol     */ { final,            token_eol,          backup_0 },
 317         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 318         /* class_illegal */ { special,          state_initial,  control_char },
 319         /* class_pound   */ { move_final,       token_punc,         backup_0 },
 320         /* class_dquote  */ { special,          state_gstr_3, nocharset_gstr },
 321         /* class_langle  */ { move_advance,     state_langle,       not_used },
 322         /* class_rangle  */ { move_advance,     state_rangle,       not_used },
 323         /* class_highbit */ { move_advance,     state_eat_rest,     not_used },
 324     },
 325     { /* state_name */
 326         /* class_blank   */ { final,            token_name,         backup_0 },
 327         /* class_punc    */ { move_final,       token_name,         backup_1 },
 328         /* class_name    */ { move_advance,     state_name,         not_used },
 329         /* class_e       */ { move_advance,     state_name,         not_used },
 330         /* class_escape  */ { move_advance,     state_name,         not_used },
 331         /* class_ff      */ { move_final,       token_name,         backup_1 },
 332         /* class_digit   */ { move_advance,     state_name,         not_used },
 333         /* class_dot     */ { move_final,       token_name,         backup_1 },
 334         /* class_sign    */ { move_final,       token_name,         backup_1 },
 335         /* class_quote   */ { move_final,       token_name,         backup_1 },
 336         /* class_bslash  */ { move_final,       token_name,         backup_1 },
 337         /* class_slash   */ { move_final,       token_name,         backup_1 },
 338         /* class_star    */ { move_final,       token_name,         backup_1 },
 339         /* class_exclam  */ { move_final,       token_name,         backup_1 },
 340         /* class_eol     */ { move_final,       token_name,         backup_1 },
 341         /* class_rest    */ { move_final,       token_name,         backup_1 },
 342         /* class_illegal */ { move_final,       token_name,         backup_1 },
 343         /* class_pound   */ { move_final,       token_name,         backup_1 },
 344         /* class_dquote  */ { move_final,       token_name,         backup_1 },
 345         /* class_langle  */ { move_final,       token_name,         backup_1 },
 346         /* class_rangle  */ { move_final,       token_name,         backup_1 },
 347         /* class_highbit */ { move_final,       token_name,         backup_1 },
 348     },
 349     { /* state_integer */
 350         /* class_blank   */ { final,            token_integer,      backup_0 },
 351         /* class_punc    */ { move_final,       token_integer,      backup_1 },
 352         /* class_name    */ { move_final,       token_integer,      backup_1 },
 353         /* class_e       */ { move_final,       token_integer,      backup_1 },
 354         /* class_escape  */ { move_final,       token_integer,      backup_1 },
 355         /* class_ff      */ { move_final,       token_integer,      backup_1 },
 356         /* class_digit   */ { move_advance,     state_integer,      not_used },
 357         /* class_dot     */ { move_advance,     state_real_2,       not_used },
 358         /* class_sign    */ { move_final,       token_integer,      backup_1 },
 359         /* class_quote   */ { move_final,       token_integer,      backup_1 },
 360         /* class_bslash  */ { move_final,       token_integer,      backup_1 },
 361         /* class_slash   */ { move_final,       token_integer,      backup_1 },
 362         /* class_star    */ { move_final,       token_integer,      backup_1 },
 363         /* class_exclam  */ { move_final,       token_integer,      backup_1 },
 364         /* class_eol     */ { move_final,       token_integer,      backup_1 },
 365         /* class_rest    */ { move_final,       token_integer,      backup_1 },
 366         /* class_illegal */ { move_final,       token_integer,      backup_1 },
 367         /* class_pound   */ { move_final,       token_integer,      backup_1 },
 368         /* class_dquote  */ { move_final,       token_integer,      backup_1 },
 369         /* class_langle  */ { move_final,       token_integer,      backup_1 },
 370         /* class_rangle  */ { move_final,       token_integer,      backup_1 },
 371         /* class_highbit */ { move_final,       token_integer,      backup_1 },
 372     },
 373     { /* state_real_1 */
 374         /* class_blank   */ { move_advance,     state_eat_rest,     not_used },
 375         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 376         /* class_name    */ { move_error,       bad_prefix,         backup_1 },
 377         /* class_e       */ { move_error,       bad_prefix,         backup_1 },
 378         /* class_escape  */ { move_error,       bad_prefix,         backup_1 },
 379         /* class_ff      */ { special,          state_real_1,   control_char },
 380         /* class_digit   */ { move_advance,     state_real_2,       not_used },
 381         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 382         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 383         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 384         /* class_bslash  */ { move_error,       bad_prefix,         backup_1 },
 385         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 386         /* class_star    */ { move_error,       bad_prefix,         backup_1 },
 387         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 388         /* class_eol     */ { move_error,       bad_prefix,         backup_1 },
 389         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 390         /* class_illegal */ { special,          state_real_1,   control_char },
 391         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 392         /* class_dquote  */ { move_error,       bad_prefix,         backup_1 },
 393         /* class_langle  */ { move_error,       bad_prefix,         backup_1 },
 394         /* class_rangle  */ { move_error,       bad_prefix,         backup_1 },
 395         /* class_highbit */ { move_advance,     state_eat_rest,     not_used },
 396     },
 397     { /* state_real_2 */
 398         /* class_blank   */ { final,            token_real,         backup_0 },
 399         /* class_punc    */ { move_final,       token_real,         backup_1 },
 400         /* class_name    */ { move_final,       token_real,         backup_1 },
 401         /* class_e       */ { move_advance,     state_exp_1,        not_used },
 402         /* class_escape  */ { move_final,       token_real,         backup_1 },
 403         /* class_ff      */ { move_final,       token_real,         backup_1 },
 404         /* class_digit   */ { move_advance,     state_real_2,       not_used },
 405         /* class_dot     */ { move_final,       token_real,         backup_1 },
 406         /* class_sign    */ { move_final,       token_real,         backup_1 },
 407         /* class_quote   */ { move_final,       token_real,         backup_1 },
 408         /* class_bslash  */ { move_final,       token_real,         backup_1 },
 409         /* class_slash   */ { move_final,       token_real,         backup_1 },
 410         /* class_star    */ { move_final,       token_real,         backup_1 },
 411         /* class_exclam  */ { move_final,       token_real,         backup_1 },
 412         /* class_eol     */ { move_final,       token_real,         backup_1 },
 413         /* class_rest    */ { move_final,       token_real,         backup_1 },
 414         /* class_illegal */ { move_final,       token_real,         backup_1 },
 415         /* class_pound   */ { move_final,       token_real,         backup_1 },
 416         /* class_dquote  */ { move_final,       token_real,         backup_1 },
 417         /* class_langle  */ { move_final,       token_real,         backup_1 },
 418         /* class_rangle  */ { move_final,       token_real,         backup_1 },
 419         /* class_highbit */ { move_final,       token_real,         backup_1 },
 420     },
 421     { /* state_exp_1 */
 422         /* class_blank   */ { move_final,       token_real,         backup_2 },
 423         /* class_punc    */ { move_final,       token_real,         backup_2 },
 424         /* class_name    */ { move_final,       token_real,         backup_2 },
 425         /* class_e       */ { move_final,       token_real,         backup_2 },
 426         /* class_escape  */ { move_final,       token_real,         backup_2 },
 427         /* class_ff      */ { move_final,       token_real,         backup_2 },
 428         /* class_digit   */ { move_advance,     state_exp_3,        not_used },
 429         /* class_dot     */ { move_final,       token_real,         backup_2 },
 430         /* class_sign    */ { move_advance,     state_exp_2,        not_used },
 431         /* class_quote   */ { move_final,       token_real,         backup_2 },
 432         /* class_bslash  */ { move_final,       token_real,         backup_2 },
 433         /* class_slash   */ { move_final,       token_real,         backup_2 },
 434         /* class_star    */ { move_final,       token_real,         backup_2 },
 435         /* class_exclam  */ { move_final,       token_real,         backup_2 },
 436         /* class_eol     */ { move_final,       token_real,         backup_2 },
 437         /* class_rest    */ { move_final,       token_real,         backup_2 },
 438         /* class_illegal */ { move_final,       token_real,         backup_2 },
 439         /* class_pound   */ { move_final,       token_real,         backup_2 },
 440         /* class_dquote  */ { move_final,       token_real,         backup_2 },
 441         /* class_langle  */ { move_final,       token_real,         backup_2 },
 442         /* class_rangle  */ { move_final,       token_real,         backup_2 },
 443         /* class_highbit */ { move_final,       token_real,         backup_2 },
 444     },
 445     { /* state_exp_2 */
 446         /* class_blank   */ { move_final,       token_real,         backup_3 },
 447         /* class_punc    */ { move_final,       token_real,         backup_3 },
 448         /* class_name    */ { move_final,       token_real,         backup_3 },
 449         /* class_e       */ { move_final,       token_real,         backup_3 },
 450         /* class_escape  */ { move_final,       token_real,         backup_3 },
 451         /* class_ff      */ { move_final,       token_real,         backup_3 },
 452         /* class_digit   */ { move_advance,     state_exp_3,        not_used },
 453         /* class_dot     */ { move_final,       token_real,         backup_3 },
 454         /* class_sign    */ { move_final,       token_real,         backup_3 },
 455         /* class_quote   */ { move_final,       token_real,         backup_3 },
 456         /* class_bslash  */ { move_final,       token_real,         backup_3 },
 457         /* class_slash   */ { move_final,       token_real,         backup_3 },
 458         /* class_star    */ { move_final,       token_real,         backup_3 },
 459         /* class_exclam  */ { move_final,       token_real,         backup_3 },
 460         /* class_eol     */ { move_final,       token_real,         backup_3 },
 461         /* class_rest    */ { move_final,       token_real,         backup_3 },
 462         /* class_illegal */ { move_final,       token_real,         backup_3 },
 463         /* class_pound   */ { move_final,       token_real,         backup_3 },
 464         /* class_dquote  */ { move_final,       token_real,         backup_3 },
 465         /* class_langle  */ { move_final,       token_real,         backup_3 },
 466         /* class_rangle  */ { move_final,       token_real,         backup_3 },
 467         /* class_highbit */ { move_final,       token_real,         backup_3 },
 468     },
 469     { /* state_exp_3 */
 470         /* class_blank   */ { final,            token_real,         backup_0 },
 471         /* class_punc    */ { move_final,       token_real,         backup_1 },
 472         /* class_name    */ { move_final,       token_real,         backup_1 },
 473         /* class_e       */ { move_final,       token_real,         backup_1 },
 474         /* class_escape  */ { move_final,       token_real,         backup_1 },
 475         /* class_ff      */ { move_final,       token_real,         backup_1 },
 476         /* class_digit   */ { move_advance,     state_exp_3,        not_used },
 477         /* class_dot     */ { move_final,       token_real,         backup_1 },
 478         /* class_sign    */ { move_final,       token_real,         backup_1 },
 479         /* class_quote   */ { move_final,       token_real,         backup_1 },
 480         /* class_bslash  */ { move_final,       token_real,         backup_1 },
 481         /* class_slash   */ { move_final,       token_real,         backup_1 },
 482         /* class_star    */ { move_final,       token_real,         backup_1 },
 483         /* class_exclam  */ { move_final,       token_real,         backup_1 },
 484         /* class_eol     */ { move_final,       token_real,         backup_1 },
 485         /* class_rest    */ { move_final,       token_real,         backup_1 },
 486         /* class_illegal */ { move_final,       token_real,         backup_1 },
 487         /* class_pound   */ { move_final,       token_real,         backup_1 },
 488         /* class_dquote  */ { move_final,       token_real,         backup_1 },
 489         /* class_langle  */ { move_final,       token_real,         backup_1 },
 490         /* class_rangle  */ { move_final,       token_real,         backup_1 },
 491         /* class_highbit */ { move_final,       token_real,         backup_1 },
 492     },
 493     { /* state_str_1 */
 494         /* class_blank   */ { move_advance,     state_str_1,        not_used },
 495         /* class_punc    */ { move_advance,     state_str_1,        not_used },
 496         /* class_name    */ { move_advance,     state_str_1,        not_used },
 497         /* class_e       */ { move_advance,     state_str_1,        not_used },
 498         /* class_escape  */ { move_advance,     state_str_1,        not_used },
 499         /* class_ff      */ { special,          state_str_1,    control_char },
 500         /* class_digit   */ { move_advance,     state_str_1,        not_used },
 501         /* class_dot     */ { move_advance,     state_str_1,        not_used },
 502         /* class_sign    */ { move_advance,     state_str_1,        not_used },
 503         /* class_quote   */ { final,            token_string,       backup_0 },
 504         /* class_bslash  */ { special,          state_str_2,    start_bslash },
 505         /* class_slash   */ { move_advance,     state_str_1,        not_used },
 506         /* class_star    */ { move_advance,     state_str_1,        not_used },
 507         /* class_exclam  */ { move_advance,     state_str_1,        not_used },
 508         /* class_eol     */ { move_final,       token_ustring,      backup_1 },
 509         /* class_rest    */ { move_advance,     state_str_1,        not_used },
 510         /* class_illegal */ { special,          state_str_1,    control_char },
 511         /* class_pound   */ { move_advance,     state_str_1,        not_used },
 512         /* class_dquote  */ { move_advance,     state_str_1,        not_used },
 513         /* class_langle  */ { move_advance,     state_str_1,        not_used },
 514         /* class_rangle  */ { move_advance,     state_str_1,        not_used },
 515         /* class_highbit */ { move_advance,     state_str_1,        not_used },
 516     },
 517     { /* state_str_2 */
 518         /* class_blank   */ { move_special,     state_str_1,   ignore_bslash },
 519         /* class_punc    */ { move_special,     state_str_1,   ignore_bslash },
 520         /* class_name    */ { move_special,     state_str_1,   ignore_bslash },
 521         /* class_e       */ { move_special,     state_str_1,   ignore_bslash },
 522         /* class_escape  */ { special,          state_str_1,     insert_char },
 523         /* class_ff      */ { special,          state_str_1,    control_char },
 524         /* class_digit   */ { special,          state_str_3,     found_digit },
 525         /* class_dot     */ { move_special,     state_str_1,   ignore_bslash },
 526         /* class_sign    */ { move_special,     state_str_1,   ignore_bslash },
 527         /* class_quote   */ { special,          state_str_1,     insert_char },
 528         /* class_bslash  */ { special,          state_str_1,     insert_char },
 529         /* class_slash   */ { move_special,     state_str_1,   ignore_bslash },
 530         /* class_star    */ { move_special,     state_str_1,   ignore_bslash },
 531         /* class_exclam  */ { move_special,     state_str_1,   ignore_bslash },
 532         /* class_eol     */ { special,          state_str_1,     string_wrap },
 533         /* class_rest    */ { move_special,     state_str_1,   ignore_bslash },
 534         /* class_illegal */ { special,          state_str_1,    control_char },
 535         /* class_pound   */ { move_special,     state_str_1,   ignore_bslash },
 536         /* class_dquote  */ { special,          state_str_1,     insert_char },
 537         /* class_langle  */ { move_special,     state_str_1,   ignore_bslash },
 538         /* class_rangle  */ { move_special,     state_str_1,   ignore_bslash },
 539         /* class_highbit */ { move_special,     state_str_1,   ignore_bslash },
 540     },
 541     { /* state_str_3 */
 542         /* class_blank   */ { move_special,     state_str_1,  missing_bslash },
 543         /* class_punc    */ { move_special,     state_str_1,  missing_bslash },
 544         /* class_name    */ { move_special,     state_str_1,  missing_bslash },
 545         /* class_e       */ { move_special,     state_str_1,  missing_bslash },
 546         /* class_escape  */ { move_special,     state_str_1,  missing_bslash },
 547         /* class_ff      */ { special,          state_str_1,    control_char },
 548         /* class_digit   */ { special,          state_str_3,     found_digit },
 549         /* class_dot     */ { move_special,     state_str_1,  missing_bslash },
 550         /* class_sign    */ { move_special,     state_str_1,  missing_bslash },
 551         /* class_quote   */ { move_special,     state_str_1,  missing_bslash },
 552         /* class_bslash  */ { special,          state_str_1,      end_digits },
 553         /* class_slash   */ { move_special,     state_str_1,  missing_bslash },
 554         /* class_star    */ { move_special,     state_str_1,  missing_bslash },
 555         /* class_exclam  */ { move_special,     state_str_1,  missing_bslash },
 556         /* class_eol     */ { move_final,       token_ustring,      backup_1 },
 557         /* class_rest    */ { move_special,     state_str_1,  missing_bslash },
 558         /* class_illegal */ { special,          state_str_3,    control_char },
 559         /* class_pound   */ { move_special,     state_str_1,  missing_bslash },
 560         /* class_dquote  */ { move_special,     state_str_1,  missing_bslash },
 561         /* class_langle  */ { move_special,     state_str_1,  missing_bslash },
 562         /* class_rangle  */ { move_special,     state_str_1,  missing_bslash },
 563         /* class_highbit */ { move_special,     state_str_1,  missing_bslash },
 564     },
 565     { /* state_comment_1 */
 566         /* class_blank   */ { move_final,       token_punc,         backup_0 },
 567         /* class_punc    */ { move_final,       token_punc,         backup_1 },
 568         /* class_name    */ { move_final,       token_punc,         backup_1 },
 569         /* class_e       */ { move_final,       token_punc,         backup_1 },
 570         /* class_escape  */ { move_final,       token_punc,         backup_1 },
 571         /* class_ff      */ { special,          state_comment_1,control_char },
 572         /* class_digit   */ { move_final,       token_punc,         backup_1 },
 573         /* class_dot     */ { move_final,       token_punc,         backup_1 },
 574         /* class_sign    */ { move_final,       token_punc,         backup_1 },
 575         /* class_quote   */ { move_final,       token_punc,         backup_1 },
 576         /* class_bslash  */ { move_final,       token_punc,         backup_1 },
 577         /* class_slash   */ { move_final,       token_punc,         backup_1 },
 578         /* class_star    */ { move_advance,     state_comment_2,    not_used },
 579         /* class_exclam  */ { move_final,       token_punc,         backup_1 },
 580         /* class_eol     */ { move_final,       token_punc,         backup_1 },
 581         /* class_rest    */ { move_final,       token_punc,         backup_1 },
 582         /* class_illegal */ { special,          state_comment_1,control_char },
 583         /* class_pound   */ { move_final,       token_punc,         backup_1 },
 584         /* class_dquote  */ { move_final,       token_punc,         backup_1 },
 585         /* class_langle  */ { move_final,       token_punc,         backup_1 },
 586         /* class_rangle  */ { move_final,       token_punc,         backup_1 },
 587         /* class_highbit */ { move_final,       token_punc,         backup_1 },
 588     },
 589     { /* state_comment_2 */
 590         /* class_blank   */ { move_advance,     state_comment_2,    not_used },
 591         /* class_punc    */ { move_advance,     state_comment_2,    not_used },
 592         /* class_name    */ { move_advance,     state_comment_2,    not_used },
 593         /* class_e       */ { move_advance,     state_comment_2,    not_used },
 594         /* class_escape  */ { move_advance,     state_comment_2,    not_used },
 595         /* class_ff      */ { move_special,     state_comment_2,control_char },
 596         /* class_digit   */ { move_advance,     state_comment_2,    not_used },
 597         /* class_dot     */ { move_advance,     state_comment_2,    not_used },
 598         /* class_sign    */ { move_advance,     state_comment_2,    not_used },
 599         /* class_quote   */ { move_advance,     state_comment_2,    not_used },
 600         /* class_bslash  */ { move_advance,     state_comment_2,    not_used },
 601         /* class_slash   */ { move_advance,     state_comment_2,    not_used },
 602         /* class_star    */ { move_advance,     state_comment_3,    not_used },
 603         /* class_exclam  */ { move_advance,     state_comment_2,    not_used },
 604         /* class_eol     */ { move_special,     state_comment_2,comment_wrap },
 605         /* class_rest    */ { move_advance,     state_comment_2,    not_used },
 606         /* class_illegal */ { move_special,     state_comment_2,control_char },
 607         /* class_pound   */ { move_advance,     state_comment_2,    not_used },
 608         /* class_dquote  */ { move_advance,     state_comment_2,    not_used },
 609         /* class_langle  */ { move_advance,     state_comment_2,    not_used },
 610         /* class_rangle  */ { move_advance,     state_comment_2,    not_used },
 611         /* class_highbit */ { move_advance,     state_comment_2,    not_used },
 612     },
 613     { /* state_comment_3 */
 614         /* class_blank   */ { move_advance,     state_comment_2,    not_used },
 615         /* class_punc    */ { move_advance,     state_comment_2,    not_used },
 616         /* class_name    */ { move_advance,     state_comment_2,    not_used },
 617         /* class_e       */ { move_advance,     state_comment_2,    not_used },
 618         /* class_escape  */ { move_advance,     state_comment_2,    not_used },
 619         /* class_ff      */ { move_special,     state_comment_2,control_char },
 620         /* class_digit   */ { move_advance,     state_comment_2,    not_used },
 621         /* class_dot     */ { move_advance,     state_comment_2,    not_used },
 622         /* class_sign    */ { move_advance,     state_comment_2,    not_used },
 623         /* class_quote   */ { move_advance,     state_comment_2,    not_used },
 624         /* class_bslash  */ { move_advance,     state_comment_2,    not_used },
 625         /* class_slash   */ { final_comment,    token_comment,      not_used },
 626         /* class_star    */ { move_advance,     state_comment_3,    not_used },
 627         /* class_exclam  */ { move_advance,     state_comment_2,    not_used },
 628         /* class_eol     */ { move_special,     state_comment_2,comment_wrap },
 629         /* class_rest    */ { move_advance,     state_comment_2,    not_used },
 630         /* class_illegal */ { move_special,     state_comment_2,control_char },
 631         /* class_pound   */ { move_advance,     state_comment_2,    not_used },
 632         /* class_dquote  */ { move_advance,     state_comment_2,    not_used },
 633         /* class_langle  */ { move_advance,     state_comment_2,    not_used },
 634         /* class_rangle  */ { move_advance,     state_comment_2,    not_used },
 635         /* class_highbit */ { move_advance,     state_comment_2,    not_used },
 636     },
 637     { /* state_comment_4 */
 638         /* class_blank   */ { move_advance,     state_comment_4,    not_used },
 639         /* class_punc    */ { move_advance,     state_comment_4,    not_used },
 640         /* class_name    */ { move_advance,     state_comment_4,    not_used },
 641         /* class_e       */ { move_advance,     state_comment_4,    not_used },
 642         /* class_escape  */ { move_advance,     state_comment_4,    not_used },
 643         /* class_ff      */ { move_special,     state_comment_4,control_char },
 644         /* class_digit   */ { move_advance,     state_comment_4,    not_used },
 645         /* class_dot     */ { move_advance,     state_comment_4,    not_used },
 646         /* class_sign    */ { move_advance,     state_comment_4,    not_used },
 647         /* class_quote   */ { move_advance,     state_comment_4,    not_used },
 648         /* class_bslash  */ { move_advance,     state_comment_4,    not_used },
 649         /* class_slash   */ { move_advance,     state_comment_4,    not_used },
 650         /* class_star    */ { move_advance,     state_comment_4,    not_used },
 651         /* class_exclam  */ { move_advance,     state_comment_4,    not_used },
 652         /* class_eol     */ { final_comment,    token_comment,      backup_1 },
 653         /* class_rest    */ { move_advance,     state_comment_4,    not_used },
 654         /* class_illegal */ { move_special,     state_comment_4,control_char },
 655         /* class_pound   */ { move_advance,     state_comment_4,    not_used },
 656         /* class_dquote  */ { move_advance,     state_comment_4,    not_used },
 657         /* class_langle  */ { move_advance,     state_comment_4,    not_used },
 658         /* class_rangle  */ { move_advance,     state_comment_4,    not_used },
 659         /* class_highbit */ { move_advance,     state_comment_4,    not_used },
 660     },
 661     { /* state_eat_rest */
 662         /* class_blank   */ { move_advance,     state_eat_rest,     not_used },
 663         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 664         /* class_name    */ { move_error,       bad_prefix,         backup_1 },
 665         /* class_e       */ { move_error,       bad_prefix,         backup_1 },
 666         /* class_escape  */ { move_error,       bad_prefix,         backup_1 },
 667         /* class_ff      */ { special,          state_eat_rest, control_char },
 668         /* class_digit   */ { move_error,       bad_prefix,         backup_1 },
 669         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 670         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 671         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 672         /* class_bslash  */ { move_advance,     state_eat_rest,     not_used },
 673         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 674         /* class_star    */ { move_advance,     state_eat_rest,     not_used },
 675         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 676         /* class_eol     */ { move_error,       bad_prefix,         backup_1 },
 677         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 678         /* class_illegal */ { special,          state_eat_rest, control_char },
 679         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 680         /* class_dquote  */ { move_error,       bad_prefix,         backup_1 },
 681         /* class_langle  */ { move_error,       bad_prefix,         backup_1 },
 682         /* class_rangle  */ { move_error,       bad_prefix,         backup_1 },
 683         /* class_highbit */ { move_advance,     state_eat_rest,     not_used },
 684     },
 685     { /* state_gstr_1 */
 686         /* class_blank   */ { move_error,       bad_prefix,         backup_1 },
 687         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 688         /* class_name    */ { move_advance,     state_gstr_2,       not_used },
 689         /* class_e       */ { move_advance,     state_gstr_2,       not_used },
 690         /* class_escape  */ { move_advance,     state_gstr_2,       not_used },
 691         /* class_ff      */ { special,          state_gstr_1,   control_char },
 692         /* class_digit   */ { move_error,       bad_prefix,         backup_1 },
 693         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 694         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 695         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 696         /* class_bslash  */ { move_advance,     state_eat_rest,     not_used },
 697         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 698         /* class_star    */ { move_advance,     state_eat_rest,     not_used },
 699         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 700         /* class_eol     */ { move_error,       bad_prefix,         backup_1 },
 701         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 702         /* class_illegal */ { special,          state_eat_rest, control_char },
 703         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 704         /* class_dquote  */ { move_error,       bad_prefix,         backup_1 },
 705         /* class_langle  */ { move_error,       bad_prefix,         backup_1 },
 706         /* class_rangle  */ { move_error,       bad_prefix,         backup_1 },
 707         /* class_highbit */ { move_advance,     state_eat_rest,     not_used },
 708     },
 709     { /* state_gstr_2 */
 710         /* class_blank   */ { move_error,       bad_prefix,         backup_1 },
 711         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 712         /* class_name    */ { move_advance,     state_gstr_2,       not_used },
 713         /* class_e       */ { move_advance,     state_gstr_2,       not_used },
 714         /* class_escape  */ { move_advance,     state_gstr_2,       not_used },
 715         /* class_ff      */ { special,          state_gstr_2,   control_char },
 716         /* class_digit   */ { move_advance,     state_gstr_2,       not_used },
 717         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 718         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 719         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 720         /* class_bslash  */ { move_advance,     state_eat_rest,     not_used },
 721         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 722         /* class_star    */ { move_advance,     state_eat_rest,     not_used },
 723         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 724         /* class_eol     */ { move_error,       bad_prefix,         backup_1 },
 725         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 726         /* class_illegal */ { special,          state_eat_rest, control_char },
 727         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 728         /* class_dquote  */ { special,          state_gstr_3,   charset_gstr },
 729         /* class_langle  */ { move_error,       bad_prefix,         backup_1 },
 730         /* class_rangle  */ { move_error,       bad_prefix,         backup_1 },
 731         /* class_highbit */ { move_advance,     state_eat_rest,     not_used },
 732     },
 733     { /* state_gstr_3 */
 734         /* class_blank   */ { move_advance,     state_gstr_3,       not_used },
 735         /* class_punc    */ { move_advance,     state_gstr_3,       not_used },
 736         /* class_name    */ { move_advance,     state_gstr_3,       not_used },
 737         /* class_e       */ { move_advance,     state_gstr_3,       not_used },
 738         /* class_escape  */ { move_advance,     state_gstr_3,       not_used },
 739         /* class_ff      */ { special,          state_gstr_3,   control_char },
 740         /* class_digit   */ { move_advance,     state_gstr_3,       not_used },
 741         /* class_dot     */ { move_advance,     state_gstr_3,       not_used },
 742         /* class_sign    */ { move_advance,     state_gstr_3,       not_used },
 743         /* class_quote   */ { move_advance,     state_gstr_3,       not_used },
 744         /* class_bslash  */ { special,          state_gstr_4,   start_bslash },
 745         /* class_slash   */ { move_advance,     state_gstr_3,       not_used },
 746         /* class_star    */ { move_advance,     state_gstr_3,       not_used },
 747         /* class_exclam  */ { move_advance,     state_gstr_3,       not_used },
 748         /* class_eol     */ { move_final,       token_ugstr,        backup_1 },
 749         /* class_rest    */ { move_advance,     state_gstr_3,       not_used },
 750         /* class_illegal */ { special,          state_gstr_3,   control_char },
 751         /* class_pound   */ { move_advance,     state_gstr_3,       not_used },
 752         /* class_dquote  */ { final,            token_gstr,         backup_0 },
 753         /* class_langle  */ { move_advance,     state_gstr_3,       not_used },
 754         /* class_rangle  */ { move_advance,     state_gstr_3,       not_used },
 755         /* class_highbit */ { move_special,     state_gstr_3,   highbit_char },
 756     },
 757     { /* state_gstr_4 */
 758         /* class_blank   */ { move_special,     state_gstr_3,  ignore_bslash },
 759         /* class_punc    */ { move_special,     state_gstr_3,  ignore_bslash },
 760         /* class_name    */ { move_special,     state_gstr_3,  ignore_bslash },
 761         /* class_e       */ { move_special,     state_gstr_3,  ignore_bslash },
 762         /* class_escape  */ { special,          state_gstr_3,    insert_char },
 763         /* class_ff      */ { special,          state_gstr_3,   control_char },
 764         /* class_digit   */ { special,          state_gstr_5,    found_digit },
 765         /* class_dot     */ { move_special,     state_gstr_3,  ignore_bslash },
 766         /* class_sign    */ { move_special,     state_gstr_3,  ignore_bslash },
 767         /* class_quote   */ { special,          state_gstr_3,    insert_char },
 768         /* class_bslash  */ { special,          state_gstr_3,    insert_char },
 769         /* class_slash   */ { move_special,     state_gstr_3,  ignore_bslash },
 770         /* class_star    */ { move_special,     state_gstr_3,  ignore_bslash },
 771         /* class_exclam  */ { move_special,     state_gstr_3,  ignore_bslash },
 772         /* class_eol     */ { move_final,       token_ugstr,        backup_1 },
 773         /* class_rest    */ { move_special,     state_gstr_3,  ignore_bslash },
 774         /* class_illegal */ { special,          state_gstr_3,   control_char },
 775         /* class_pound   */ { move_special,     state_gstr_3,  ignore_bslash },
 776         /* class_dquote  */ { special,          state_gstr_3,    insert_char },
 777         /* class_langle  */ { move_special,     state_gstr_3,  ignore_bslash },
 778         /* class_rangle  */ { move_special,     state_gstr_3,  ignore_bslash },
 779         /* class_highbit */ { move_special,     state_gstr_3,  ignore_bslash },
 780     },
 781     { /* state_gstr_5 */
 782         /* class_blank   */ { move_special,     state_gstr_3, missing_bslash },
 783         /* class_punc    */ { move_special,     state_gstr_3, missing_bslash },
 784         /* class_name    */ { move_special,     state_gstr_3, missing_bslash },
 785         /* class_e       */ { move_special,     state_gstr_3, missing_bslash },
 786         /* class_escape  */ { move_special,     state_gstr_3, missing_bslash },
 787         /* class_ff      */ { special,          state_gstr_3,   control_char },
 788         /* class_digit   */ { special,          state_gstr_5,    found_digit },
 789         /* class_dot     */ { move_special,     state_gstr_3, missing_bslash },
 790         /* class_sign    */ { move_special,     state_gstr_3, missing_bslash },
 791         /* class_quote   */ { move_special,     state_gstr_3, missing_bslash },
 792         /* class_bslash  */ { special,          state_gstr_3,     end_digits },
 793         /* class_slash   */ { move_special,     state_gstr_3, missing_bslash },
 794         /* class_star    */ { move_special,     state_gstr_3, missing_bslash },
 795         /* class_exclam  */ { move_special,     state_gstr_3, missing_bslash },
 796         /* class_eol     */ { move_final,       token_ugstr,        backup_1 },
 797         /* class_rest    */ { move_special,     state_gstr_3, missing_bslash },
 798         /* class_illegal */ { special,          state_str_3,    control_char },
 799         /* class_pound   */ { move_special,     state_gstr_3, missing_bslash },
 800         /* class_dquote  */ { move_special,     state_gstr_3, missing_bslash },
 801         /* class_langle  */ { move_special,     state_gstr_3, missing_bslash },
 802         /* class_rangle  */ { move_special,     state_gstr_3, missing_bslash },
 803         /* class_highbit */ { move_special,     state_gstr_3, missing_bslash },
 804     },
 805     { /* state_langle */
 806         /* class_blank   */ { move_error,       bad_prefix,         backup_1 },
 807         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 808         /* class_name    */ { move_error,       bad_prefix,         backup_1 },
 809         /* class_e       */ { move_error,       bad_prefix,         backup_1 },
 810         /* class_escape  */ { move_error,       bad_prefix,         backup_1 },
 811         /* class_ff      */ { special,          state_langle,   control_char },
 812         /* class_digit   */ { move_error,       bad_prefix,         backup_1 },
 813         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 814         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 815         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 816         /* class_bslash  */ { move_error,       bad_prefix,         backup_1 },
 817         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 818         /* class_star    */ { move_error,       bad_prefix,         backup_1 },
 819         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 820         /* class_eol     */ { move_final,       token_ugstr,        backup_1 },
 821         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 822         /* class_illegal */ { special,          state_langle,   control_char },
 823         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 824         /* class_dquote  */ { move_error,       bad_prefix,         backup_1 },
 825         /* class_langle  */ { move_final,       token_punc2,        backup_0 },
 826         /* class_rangle  */ { move_error,       bad_prefix,         backup_1 },
 827         /* class_highbit */ { move_error,       bad_prefix,         backup_1 },
 828     },
 829     { /* state_rangle */
 830         /* class_blank   */ { move_error,       bad_prefix,         backup_1 },
 831         /* class_punc    */ { move_error,       bad_prefix,         backup_1 },
 832         /* class_name    */ { move_error,       bad_prefix,         backup_1 },
 833         /* class_e       */ { move_error,       bad_prefix,         backup_1 },
 834         /* class_escape  */ { move_error,       bad_prefix,         backup_1 },
 835         /* class_ff      */ { special,          state_langle,   control_char },
 836         /* class_digit   */ { move_error,       bad_prefix,         backup_1 },
 837         /* class_dot     */ { move_error,       bad_prefix,         backup_1 },
 838         /* class_sign    */ { move_error,       bad_prefix,         backup_1 },
 839         /* class_quote   */ { move_error,       bad_prefix,         backup_1 },
 840         /* class_bslash  */ { move_error,       bad_prefix,         backup_1 },
 841         /* class_slash   */ { move_error,       bad_prefix,         backup_1 },
 842         /* class_star    */ { move_error,       bad_prefix,         backup_1 },
 843         /* class_exclam  */ { move_error,       bad_prefix,         backup_1 },
 844         /* class_eol     */ { move_final,       token_ugstr,        backup_1 },
 845         /* class_rest    */ { move_advance,     state_eat_rest,     not_used },
 846         /* class_illegal */ { special,          state_langle,   control_char },
 847         /* class_pound   */ { move_error,       bad_prefix,         backup_1 },
 848         /* class_dquote  */ { move_error,       bad_prefix,         backup_1 },
 849         /* class_langle  */ { move_error,       bad_prefix,         backup_1 },
 850         /* class_rangle  */ { move_final,       token_punc2,        backup_0 },
 851         /* class_highbit */ { move_error,       bad_prefix,         backup_1 },
 852     },
 853   };
 854
 855 \f
 856 static char class_table[ 256 ] =
 857   {
 858 /* 00 */    class_eol,      class_illegal,  class_illegal,  class_illegal,
 859 /* 04 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 860 /* 08 */    class_illegal,  class_blank,    class_illegal,  class_illegal,
 861 /* 0C */    class_ff,       class_illegal,  class_illegal,  class_illegal,
 862 /* 10 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 863 /* 14 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 864 /* 18 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 865 /* 1C */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 866 /* 20 */    class_blank,    class_exclam,   class_dquote,   class_pound,
 867 /* 24 */    class_name,     class_rest,     class_punc,     class_quote,
 868 /* 28 */    class_punc,     class_punc,     class_star,     class_sign,
 869 /* 2C */    class_punc,     class_sign,     class_dot,      class_slash,
 870 /* 30 */    class_digit,    class_digit,    class_digit,    class_digit,
 871 /* 34 */    class_digit,    class_digit,    class_digit,    class_digit,
 872 /* 38 */    class_digit,    class_digit,    class_punc,     class_punc,
 873 /* 3C */    class_langle,   class_punc,     class_rangle,   class_rest,
 874 /* 40 */    class_rest,     class_name,     class_name,     class_name,
 875 /* 44 */    class_name,     class_e,        class_name,     class_name,
 876 /* 48 */    class_name,     class_name,     class_name,     class_name,
 877 /* 4C */    class_name,     class_name,     class_name,     class_name,
 878 /* 50 */    class_name,     class_name,     class_name,     class_name,
 879 /* 54 */    class_name,     class_name,     class_name,     class_name,
 880 /* 58 */    class_name,     class_name,     class_name,     class_rest,
 881 /* 5C */    class_bslash,   class_rest,     class_punc,     class_name,
 882 /* 60 */    class_rest,     class_name,     class_escape,   class_name,
 883 /* 64 */    class_name,     class_e,        class_escape,   class_name,
 884 /* 68 */    class_name,     class_name,     class_name,     class_name,
 885 /* 6C */    class_name,     class_name,     class_escape,   class_name,
 886 /* 70 */    class_name,     class_name,     class_escape,   class_name,
 887 /* 74 */    class_escape,   class_name,     class_escape,   class_name,
 888 /* 78 */    class_name,     class_name,     class_name,     class_punc,
 889 /* 7C */    class_punc,     class_punc,     class_punc,     class_illegal,
 890 /* 80 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 891 /* 84 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 892 /* 88 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 893 /* 8C */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 894 /* 90 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 895 /* 94 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 896 /* 98 */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 897 /* 9C */    class_illegal,  class_illegal,  class_illegal,  class_illegal,
 898 /* A0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 899 /* A4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 900 /* A8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 901 /* AC */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 902 /* B0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 903 /* B4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 904 /* B8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 905 /* BC */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 906 /* C0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 907 /* C4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 908 /* C8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 909 /* CC */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 910 /* D0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 911 /* D4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 912 /* D8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 913 /* DC */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 914 /* E0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 915 /* E4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 916 /* E8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 917 /* EC */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 918 /* F0 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 919 /* F4 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 920 /* F8 */    class_highbit,  class_highbit,  class_highbit,  class_highbit,
 921 /* FC */    class_highbit,  class_highbit,  class_highbit,  class_highbit
 922
 923   };
 924
 925
 926 /*    Tables to correlate token numbers and ASCII values for
 927       punctuation characters.  Used by yylex and lex_issue_error.     */
 928
 929 #define tok_punc_token_num      17
 930 static unsigned char    punc_char[tok_punc_token_num] = {
 931                         '{', '}', '=', ';', '(', ')', ':', '+', '-',
 932                         ',', '&', '~', '*', '/', '^', '|', '#'};
 933 static int      punc_token[tok_punc_token_num] =
 934                     {   LEFT_BRACE,
 935                         RIGHT_BRACE,
 936                         EQUAL_SIGN,
 937                         SEMICOLON,
 938                         LEFT_PAREN,
 939                         RIGHT_PAREN,
 940                         COLON,
 941                         PLUS,
 942                         MINUS,
 943                         COMMA,
 944                         AND,
 945                         NOT,
 946                         MULTIPLY,
 947                         DIVIDE,
 948                         XOR,
 949                         OR,
 950                         POUND };
 951
 952 /*    Tables to correlate token numbers and ASCII value pairs for
 953       punctuation characters.  Used by yylex and lex_issue_error.     */
 954
 955 static unsigned char    punc2_char[2] = {'<', '>'};
 956 static int      punc2_token[2] =
 957                     {   LEFT_SHIFT,
 958                         RIGHT_SHIFT };
 959
 960 \f
 961 /*
 962 **++
 963 **  FUNCTIONAL DESCRIPTION:
 964 **
 965 **      This function returns the next token to be built in the UIL source
 966 **      program being read by the compiler.
 967 **
 968 **  FORMAL PARAMETERS:
 969 **
 970 **      none
 971 **
 972 **  IMPLICIT INPUTS:
 973 **
 974 **
 975 **
 976 **  IMPLICIT OUTPUTS:
 977 **
 978 **
 979 **
 980 **  FUNCTION VALUE:
 981 **
 982 **      number of the token generated
 983 **
 984 **  SIDE EFFECTS:
 985 **
 986 **
 987 **
 988 **
 989 **--
 990 **/
 991 int     yylex()
 992 {
 993     unsigned char c_char;           /* current character */
 994     int         l_class;            /* current character's class */
 995     int         l_state;            /* current token_table state */
 996     int         l_lex_pos;          /* next available position in c_lex_buffer*/
 997     cell        z_cell;             /* local copy of current token_table state*/
 998     int         l_bslash_value;     /* current value of \digit...\ construct */
 999     int         l_start_src_pos;    /* starting source position of a token */
1000     int         l_charset;          /* character set for strings */
1001     int         l_write_direction;          /* writing direction */
1002     int         l_parse_direction;          /* parsing direction */
1003     int         l_charset_sixteen_bit = FALSE; /* true if charset is 16-bit */
1004     sym_value_entry_type
1005                 *az_charset_entry;  /* value entry for the current charset */
1006     int         l_16bit_char_count = 0; /* for strings, count 16-bit chars */
1007     int         l_16bit_chars_only; /* True if the gstr we are */
1008                                     /* processing only has 16-bit chars */
1009     src_source_record_type
1010                 *az_start_src_record;/* starting source record of a token */
1011     lex_buffer_type
1012                 *az_current_lex_buffer; /* current lexical buffer */
1013
1014         az_charset_entry = (sym_value_entry_type *) 0;
1015
1016     /*
1017     **  Call the Status callback routine to report our progress.
1018     */
1019     /* %COMPLETE  (between 0-50) */
1020     Uil_percent_complete =
1021       CEIL((int)( .5 * ((float)Uil_characters_read/(float)Uil_file_size))*100, 50);
1022     if (Uil_cmd_z_command.status_cb != (Uil_continue_type(*)())NULL)
1023         diag_report_status();
1024
1025
1026 initialize_token_builder:
1027
1028     /* initialize the lexical analyzer by
1029      *      saving starting source position of the token
1030      *      resetting the lexical buffer
1031      *      putting the analyser in its initial state */
1032
1033     az_start_src_record = src_az_current_source_record;
1034     l_start_src_pos = src_az_current_source_buffer->w_current_position;
1035
1036     az_current_lex_buffer = az_first_lex_buffer;
1037     l_lex_pos = 0;
1038     l_charset = lex_k_default_charset;
1039     l_16bit_chars_only = FALSE;
1040     l_state = state_initial;
1041
1042     /* start looking for the token */
1043
1044 continue_in_next_state:
1045
1046     for (;;)
1047     {
1048                                                     /* get next input char */
1049                                                     /* advance source too */
1050         c_char = src_az_current_source_buffer->c_text
1051                     [ src_az_current_source_buffer->w_current_position++ ];
1052
1053         /* %COMPLETE */
1054         Uil_characters_read++;
1055
1056         l_class = class_table[ c_char ];            /* determine its class */
1057         z_cell = token_table[ l_state][l_class ];   /* load state cell */
1058
1059         /* pick up the next state, or terminal, or error */
1060
1061         l_state = z_cell.next_state;
1062
1063         /* l_state is negative for action states requiring the current
1064          * character be saved in the current lexical buffer */
1065
1066         if (z_cell.action & NEGATIVE)
1067         {
1068             if (l_lex_pos > l_max_lex_buffer_pos )
1069             {
1070                 az_current_lex_buffer = get_lex_buffer( az_current_lex_buffer );
1071                 l_lex_pos = 0;
1072             }
1073             az_current_lex_buffer->c_text[ l_lex_pos ] = c_char;
1074             l_lex_pos++;
1075         }
1076
1077         /* next step is based on action */
1078
1079         switch (z_cell.action)
1080         {
1081         case move_advance:
1082         case advance:
1083             if (l_16bit_chars_only) goto found_16bit_char;
1084             continue;
1085
1086         case reset:
1087
1088             goto initialize_token_builder;
1089
1090         case move_final:
1091         case final:
1092         case final_comment:  /* RAP retain comments */
1093
1094             goto found_token;
1095
1096         case move_error:
1097         case error:
1098
1099             goto found_error;
1100
1101         case move_special:
1102         case special:
1103
1104             goto special_processing;
1105
1106         default:
1107
1108             _assert( FALSE, "unknown token_table action" );
1109
1110         }
1111     }
1112
1113     /* process special actions */
1114
1115 special_processing:
1116
1117     switch (z_cell.backup)      /* backup holds special processing code */
1118     {
1119     case control_char:          /* encountered a control char in a string or
1120                                  * comment - issue a diagnotic and continue */
1121         issue_control_char_diagnostic( c_char );
1122         break;
1123
1124     case start_bslash:          /* start of a \ construct in a string */
1125
1126         l_bslash_value = 0;         /* initialize collection cell */
1127         break;
1128
1129     case found_digit:           /* next digit in a \digit...\ sequence */
1130
1131         if (l_bslash_value < 256 )  /* if still in range add in next digit */
1132             l_bslash_value = l_bslash_value * 10 + ( c_char - 48 );
1133         break;
1134
1135     case end_digits:            /* end of \digit...\ sequence */
1136
1137         if (l_bslash_value >= 256 )  /* issue a diagnostic */
1138         {   diag_issue_diagnostic
1139                 ( d_out_range,
1140                   src_az_current_source_record,
1141                   src_az_current_source_buffer->w_current_position - 1,
1142                   "\\digit...\\ sequence",
1143                   "0-255" );
1144             l_bslash_value = lex_k_unprint_sub;
1145         }
1146
1147         if (l_bslash_value == 0 )  /* issue a diagnostic */
1148         {   diag_issue_diagnostic
1149                 ( d_null,
1150                   src_az_current_source_record,
1151                   src_az_current_source_buffer->w_current_position - 1 );
1152         }
1153
1154         if (l_lex_pos > l_max_lex_buffer_pos )
1155         {
1156             az_current_lex_buffer = get_lex_buffer( az_current_lex_buffer );
1157             l_lex_pos = 0;
1158         }
1159         az_current_lex_buffer->c_text[ l_lex_pos++ ] = l_bslash_value;
1160         break;
1161
1162     case insert_char:           /* place special character in lex buffer */
1163     {
1164
1165         static unsigned char c_bslash_char[10]  = {
1166                 '\\', '\'', 'n', 't', 'v', 'b', 'r', 'f', '"'};
1167         static unsigned char ab_bslash_value[9] =
1168                  { 0x5C, 0x27, 0x0A, 0x09, 0x0B, 0x08, 0x0D, 0x0C, 0x22 };
1169
1170         if (l_lex_pos > l_max_lex_buffer_pos )
1171         {
1172             az_current_lex_buffer = get_lex_buffer( az_current_lex_buffer );
1173             l_lex_pos = 0;
1174         }
1175         az_current_lex_buffer->c_text[ l_lex_pos++ ] =
1176             ab_bslash_value
1177                 [ _index( c_char, c_bslash_char, sizeof( c_bslash_char )-1 )];
1178         break;
1179     }
1180
1181     case missing_bslash:        /* \digit...\ sequence not terminated */
1182
1183         diag_issue_diagnostic
1184                 ( d_unterm_seq,
1185                   src_az_current_source_record,
1186                   src_az_current_source_buffer->w_current_position - 1,
1187                   "\\digit...\\ sequence",
1188                   "with \\" );
1189         break;
1190
1191     case ignore_bslash:         /* \ not followed by valid character */
1192
1193         diag_issue_diagnostic
1194                 ( d_backslash_ignored,
1195                   src_az_current_source_record,
1196                   src_az_current_source_buffer->w_current_position - 1,
1197                   c_char );
1198         break;
1199
1200     case string_wrap:
1201
1202         if (src_get_source_line( ) == src_k_end_source)
1203         {
1204             diag_issue_diagnostic
1205                     ( d_unterm_seq,
1206                       src_az_current_source_record,
1207                       src_az_current_source_buffer->w_current_position - 1,
1208                       "character string",
1209                       "before end of source" );
1210
1211             src_az_current_source_buffer->w_current_position--;
1212         }
1213
1214         break;
1215
1216     case comment_wrap:
1217
1218         if (src_get_source_line( ) == src_k_end_source)
1219         {
1220             diag_issue_diagnostic
1221                     ( d_unterm_seq,
1222                       src_az_current_source_record,
1223                       src_az_current_source_buffer->w_current_position - 1,
1224                       "comment",
1225                       "before end of source" );
1226
1227             src_az_current_source_buffer->w_current_position--;
1228             return UILEOF;
1229         }
1230         az_current_lex_buffer->
1231             c_text[strlen((char *)az_current_lex_buffer->c_text)] = '\n';
1232         break;
1233
1234     case highbit_char:          /* check if must accept extra chars */
1235 found_16bit_char:
1236     {
1237         unsigned char    next_char;
1238
1239
1240         /*
1241         **  If the current character set allows 16-bit characters, then
1242         **  process them specially.
1243         */
1244         if ( l_charset_sixteen_bit )
1245         {
1246             /* need to:
1247             **  1) get next input char and advance the source
1248             **  2) check that the next is not a control character
1249             **  3) place the next character in the lex buffer
1250             */
1251
1252             next_char = src_az_current_source_buffer->c_text
1253                     [ src_az_current_source_buffer->w_current_position++ ];
1254
1255             switch (class_table[ next_char ])
1256             {
1257
1258             case class_eol:
1259                 src_az_current_source_buffer->w_current_position--;
1260             case class_illegal:
1261                 issue_control_char_diagnostic( next_char );
1262                 break;
1263             default:
1264                 break;
1265             }
1266
1267             if (l_lex_pos > l_max_lex_buffer_pos )
1268             {
1269                 az_current_lex_buffer = get_lex_buffer( az_current_lex_buffer );
1270                 l_lex_pos = 0;
1271             }
1272
1273             az_current_lex_buffer->c_text[ l_lex_pos++ ] = next_char;
1274             l_16bit_char_count ++;
1275         }
1276
1277         break;
1278     }
1279
1280     case charset_gstr:          /* set l_charset with the char set */
1281     {
1282         _assert( FALSE, "Should never get to charset_gstr" );
1283         break;
1284     }
1285
1286     case nocharset_gstr:
1287         if (Uil_lex_l_charset_specified) {
1288             /* use the specified charset*/
1289             l_charset = Uil_lex_l_literal_charset;
1290             az_charset_entry = Uil_lex_az_literal_charset;
1291             }
1292         else if (Uil_lex_l_localized) goto found_localized_string;
1293         else {
1294             /* No charset specified, use the default */
1295             l_charset = Uil_lex_l_user_default_charset;
1296             az_charset_entry = Uil_lex_az_charset_entry;
1297             }
1298
1299
1300
1301         /* Get the charset information */
1302         sem_charset_info
1303             (l_charset,
1304              az_charset_entry,
1305              &l_write_direction,
1306              &l_parse_direction,
1307              &l_charset_sixteen_bit);
1308
1309         /* reset 16 bit character count to 0 */
1310
1311         l_16bit_char_count = 0;
1312
1313         /*
1314         **  if this is a user-defined, 16-bit charset then treat all
1315         **  as 16-bit.
1316         */
1317         if ((l_charset_sixteen_bit) && (l_charset == lex_k_userdefined_charset))
1318             l_16bit_chars_only = TRUE;
1319
1320         break;
1321
1322     default:
1323         _assert( FALSE, "unknown token_table special action" );
1324
1325     }
1326
1327     /* Next state of the token builder is should already be in l_state.
1328      * Continue at this point */
1329
1330     goto continue_in_next_state;
1331
1332
1333 found_localized_string:
1334     {
1335       /* Local variables */
1336       int               mb_len, i;
1337       unsigned char     mb_byte;
1338
1339       /* Should be looking at the first byte of the string. */
1340       /* Localize... */
1341       _MrmOSSetLocale("");
1342
1343       /* Parse the string. */
1344       while (TRUE)
1345         {
1346           mb_len = mblen((char *)&src_az_current_source_buffer->c_text
1347                          [src_az_current_source_buffer->w_current_position],
1348                          MB_CUR_MAX);
1349
1350           mb_byte = src_az_current_source_buffer->c_text
1351             [src_az_current_source_buffer->w_current_position];
1352
1353           if (mb_len == 1)
1354             switch (class_table[mb_byte])
1355               {
1356               case class_eol:
1357                 z_cell.backup = backup_0;
1358                 l_state = token_ugstr;
1359
1360                 if (l_lex_pos > l_max_lex_buffer_pos )
1361                   {
1362                     az_current_lex_buffer =
1363                       get_lex_buffer( az_current_lex_buffer );
1364                     l_lex_pos = 0;
1365                   }
1366                 az_current_lex_buffer->c_text[ l_lex_pos++ ] = mb_byte;
1367                 _MrmOSSetLocale("C");
1368                 goto found_token;
1369
1370               case class_dquote:
1371                 z_cell.backup = backup_0;
1372                 l_state = token_lstr;
1373                 src_az_current_source_buffer->w_current_position++;
1374                 _MrmOSSetLocale("C");
1375                 goto found_token;
1376
1377               default:
1378                 break;
1379               }
1380
1381           if (l_lex_pos > l_max_lex_buffer_pos )
1382             {
1383               az_current_lex_buffer = get_lex_buffer( az_current_lex_buffer );
1384               l_lex_pos = 0;
1385             }
1386
1387           for (i = 0; i < mb_len; i++)
1388             {
1389               if (l_lex_pos > l_max_lex_buffer_pos )
1390                 {
1391                   az_current_lex_buffer = get_lex_buffer(az_current_lex_buffer);
1392                   l_lex_pos = 0;
1393                 }
1394               az_current_lex_buffer->c_text[l_lex_pos++] =
1395                 src_az_current_source_buffer->c_text
1396                   [src_az_current_source_buffer->w_current_position++];
1397             }
1398         }
1399     }
1400
1401 found_token:
1402
1403     /* do any backup of the source buffer position and lex buffer */
1404
1405     src_az_current_source_buffer->w_current_position -= z_cell.backup;
1406     l_lex_pos -= z_cell.backup;
1407
1408     /* put a null at the end of the current lex buffer */
1409
1410     az_current_lex_buffer->c_text[ l_lex_pos ] = 0;
1411
1412     /* case on the token found */
1413
1414     switch (l_state)            /* l_state holds the token found */
1415     {
1416     case token_name:
1417     {
1418
1419         key_keytable_entry_type *az_keyword;
1420
1421         /* check the case sensitivity flag and change case if necessary */
1422
1423         if (! uil_v_case_sensitive)
1424         {
1425             char        * ptr;
1426             for ( ptr = (char *)(az_current_lex_buffer->c_text);
1427                   (* ptr) != 0;
1428                   (* ptr) = _upper (* ptr), ptr++)
1429             {}
1430         }
1431
1432         /* check if the name is a keyword */
1433
1434         az_keyword =
1435             key_find_keyword( l_lex_pos, (char *)az_current_lex_buffer->c_text );
1436         if( az_keyword != NULL)
1437             {
1438               /* check that the length of the name is in range */
1439
1440               if (l_lex_pos > key_k_keyword_max_length)
1441                 {
1442                   l_lex_pos = key_k_keyword_max_length;
1443                   az_current_lex_buffer->c_text[ l_lex_pos ] = 0;
1444                   diag_issue_diagnostic
1445                     ( d_name_too_long,
1446                      az_start_src_record,
1447                      l_start_src_pos,
1448                      az_current_lex_buffer->c_text );
1449                 }
1450
1451               yylval.value.az_keyword_entry = az_keyword;
1452               yylval.b_type = az_keyword->b_token;
1453               break;
1454             }
1455
1456         /* process the name as an identifier */
1457
1458         /* check that the length of the identifier is in range */
1459         /* Added for fix to CR 5566 */
1460
1461         if (l_lex_pos > lex_k_identifier_max_length)
1462             {
1463             l_lex_pos = lex_k_identifier_max_length;
1464             az_current_lex_buffer->c_text[ l_lex_pos ] = 0;
1465             diag_issue_diagnostic
1466                 ( d_name_too_long,
1467                   az_start_src_record,
1468                   l_start_src_pos,
1469                   lex_k_identifier_max_length,
1470                   az_current_lex_buffer->c_text );
1471             }
1472
1473         yylval.value.az_symbol_entry =
1474             (sym_entry_type *) sym_insert_name( l_lex_pos, (char *)az_current_lex_buffer->c_text );
1475
1476         yylval.b_type = NAME;
1477         break;
1478     }
1479
1480     case token_punc:
1481     {
1482         int             l_token;
1483
1484         /* found a punctuation mark - look up its token number in a table */
1485
1486         l_token = punc_token
1487                     [ _index( az_current_lex_buffer->c_text[ 0 ],
1488                               punc_char, sizeof( punc_char )) ];
1489         yylval.b_type = l_token;
1490
1491         break;
1492     }
1493
1494     case token_punc2:
1495     {
1496         int             l_token;
1497
1498         /* found a punctuation mark - look up its token number in a table */
1499
1500         l_token = punc2_token
1501                     [ _index( az_current_lex_buffer->c_text[ 0 ],
1502                               punc2_char, sizeof( punc2_char )) ];
1503         yylval.b_type = l_token;
1504
1505         break;
1506     }
1507
1508     case token_eol:
1509
1510         /* if there is no more source
1511          *   then return an end of file
1512          *   otherwise go look for the next token */
1513
1514         if (src_get_source_line( ) == src_k_end_source)
1515             return UILEOF;
1516
1517         goto initialize_token_builder;
1518
1519     case token_integer:
1520     {
1521         long                    l_integer;
1522
1523         yylval.b_type = UNS_INT_LITERAL;
1524
1525         /* convert the text to binary
1526          *    the sign of the number is applied as part of semantic
1527          *    analysis; thus we only handle integers in range 0..2**31-1
1528          */
1529
1530         errno = 0;
1531         l_integer = cvt_ascii_to_long(az_current_lex_buffer->c_text);
1532
1533         if (errno != 0)
1534             diag_issue_diagnostic
1535                 ( d_out_range,
1536                   az_start_src_record,
1537                   l_start_src_pos,
1538                   "integer",
1539                   " " );
1540
1541         yylval.value.az_symbol_entry =
1542                 (sym_entry_type *)sem_create_value_entry
1543                     ( (char *)&l_integer, sizeof( long ), sym_k_integer_value );
1544         break;
1545     }
1546
1547     case token_ustring:
1548
1549         diag_issue_diagnostic
1550                 ( d_unterm_seq,
1551                   src_az_current_source_record,
1552                   src_az_current_source_buffer->w_current_position - 1,
1553                   "character string",
1554                   "before end of line" );
1555
1556   case token_comment:       /* RAP preserve comments */
1557     {
1558       int size;
1559
1560       if (last_token_seen != token_comment)
1561         comment_text[0]=0;
1562
1563       size = (int)strlen((char *)az_current_lex_buffer->c_text)+1;
1564       if ((size  + (int)strlen (comment_text)) >= comment_size)
1565         {
1566           comment_text = XtRealloc(comment_text, INCR_COMMENT_SIZE + strlen(comment_text));
1567           comment_size = INCR_COMMENT_SIZE + strlen (comment_text);
1568         }
1569
1570       strcat (comment_text, (char *)az_current_lex_buffer->c_text);
1571       strcat (comment_text, "\n");
1572       last_token_seen = token_comment;
1573             goto initialize_token_builder;
1574     }
1575
1576
1577     case token_string:
1578 found_primitive_string:
1579     {
1580         int                     l_length;
1581         lex_buffer_type         *az_lex_buffer;
1582         sym_value_entry_type    *az_value;
1583         unsigned char           *c_char;
1584
1585         l_length = l_lex_pos;
1586
1587         for (az_lex_buffer = az_first_lex_buffer;
1588              az_lex_buffer != az_current_lex_buffer;
1589              az_lex_buffer = az_lex_buffer->az_next_buffer)
1590             l_length = l_length + l_max_lex_buffer_pos + 1;
1591
1592         az_value = create_str_entry( l_length, l_charset, az_charset_entry );
1593
1594         c_char = (unsigned char *)az_value->value.c_value;
1595
1596         for (az_lex_buffer = az_first_lex_buffer;
1597              az_lex_buffer != az_current_lex_buffer;
1598              az_lex_buffer = az_lex_buffer->az_next_buffer)
1599         {
1600             _move( c_char, az_lex_buffer->c_text, l_max_lex_buffer_pos + 1);
1601             c_char = c_char + l_max_lex_buffer_pos + 1;
1602         }
1603
1604         _move( c_char, az_lex_buffer->c_text, l_lex_pos );
1605
1606         yylval.value.az_symbol_entry = (sym_entry_type *) az_value;
1607         yylval.b_type = CHAR_8_LITERAL;
1608
1609         break;
1610     }
1611
1612     case token_real:
1613     {
1614         double  d_real;
1615
1616         yylval.b_type = UNS_FLOAT_LITERAL;
1617
1618         errno = 0;
1619         d_real = atof((char *)az_current_lex_buffer->c_text);
1620
1621         if (errno != 0)
1622             diag_issue_diagnostic
1623                 ( d_out_range,
1624                   az_start_src_record,
1625                   l_start_src_pos,
1626                   "real",
1627                   " " );
1628
1629         yylval.value.az_symbol_entry =
1630                 (sym_entry_type *)sem_create_value_entry
1631                     ( (char *)&d_real, sizeof( double ), sym_k_float_value );
1632         break;
1633     }
1634
1635     case token_ff:
1636
1637         if (l_start_src_pos != 0)
1638             issue_control_char_diagnostic( c_char );
1639
1640         src_az_current_source_record->b_flags |= src_m_form_feed;
1641
1642         goto initialize_token_builder;
1643
1644     case token_ugstr:
1645
1646         diag_issue_diagnostic
1647                 ( d_unterm_seq,
1648                   src_az_current_source_record,
1649                   src_az_current_source_buffer->w_current_position - 1,
1650                   "character string",
1651                   "before end of line" );
1652
1653     case token_gstr:
1654
1655         /*
1656         ** Some general strings require special processing.  Those
1657         ** that do not can go thru the normal string code.
1658         */
1659         if ( l_parse_direction == XmSTRING_DIRECTION_R_TO_L )
1660         {
1661             int             i,j;
1662             unsigned char   tmp1;
1663             unsigned char   tmp2;
1664
1665             /* assuming the string is confined to a single lex buffer.      */
1666             /* just flip the characters around.  16-bit characters need to  */
1667             /* be done in groups of two bytes                               */
1668             if (l_charset_sixteen_bit != TRUE)
1669                 /*
1670                 **  Just reverse the bytes from the first to last
1671                 */
1672                 for (i=0, j=l_lex_pos-1;  i < (l_lex_pos>>1);  i++,j--)
1673                 {
1674                     tmp1 = az_current_lex_buffer->c_text[ i ];
1675                     az_current_lex_buffer->c_text[ i ] =
1676                         az_current_lex_buffer->c_text[ j ];
1677                     az_current_lex_buffer->c_text[ j ] = tmp1;
1678                 }
1679
1680             /*
1681             **  Don't reverse the string if have less than 2 characters (4 bytes)
1682             */
1683             else if ((l_lex_pos>>1) > 1)
1684                 /*
1685                 **  This reversing doesn't work for mixed 8/16-bit character
1686                 **  sets, but only built-in character sets allow mixing and
1687                 **  they are not right-to-left.  We do the same copying as in
1688                 **  the 8-bit case above, but we move two bytes at a time and
1689                 **  reverse the order as we copy so they end up correct.
1690                 */
1691                 for (i=0, j=l_lex_pos-1;  i < (l_lex_pos>>1);  i+=2,j-=2)
1692                 {
1693                     tmp1 = az_current_lex_buffer->c_text[ i ];
1694                     tmp2 = az_current_lex_buffer->c_text[ i + 1 ];
1695                     az_current_lex_buffer->c_text[ i ] =
1696                         az_current_lex_buffer->c_text[ j - 1 ];
1697                     az_current_lex_buffer->c_text[ i + 1 ] =
1698                         az_current_lex_buffer->c_text[ j ];
1699                     az_current_lex_buffer->c_text[ j ] = tmp2;
1700                     az_current_lex_buffer->c_text[ j - 1 ] = tmp1;
1701                 }
1702         }
1703
1704
1705         /*
1706         **  If the string isn't 16-bit or it is userdefined and thus
1707         **  cannot be mixed 8/16-bit then we can just make a primitive
1708         **  string.
1709         */
1710         if ((l_charset_sixteen_bit != TRUE) ||
1711             (l_charset == lex_k_userdefined_charset))
1712             goto found_primitive_string;
1713         else
1714         {
1715
1716             sym_value_entry_type    *cstr_entry;
1717             sym_value_entry_type    *str_entry;
1718             int                     a_off, off;
1719
1720             /*
1721             **  if string consists solely of 8-bit ascii characters,
1722             **  l_16bit_char_count will be zero.
1723             **  if string consists solely of 16 bit characters,
1724             **  l_16bit_char_count*2 will equal l_lex_pos.
1725             **  In either of these cases, the result is still a
1726             **  primitive string.
1727             */
1728
1729             /*
1730             **  For KANJI and HANZI treat 8-bit characters as ISO_LATIN1.
1731             */
1732             if (l_16bit_char_count == 0)
1733             {
1734                 l_charset = uil_sym_isolatin1_charset;
1735                 goto found_primitive_string;
1736             }
1737
1738             /*
1739             **  If the string only contains 16-bit characters,
1740             **  it still can be stored as a primitive string.
1741             */
1742             if ((l_16bit_char_count<<1) == l_lex_pos)
1743                 goto found_primitive_string;
1744
1745
1746             /*
1747             **  lex buffer is a mix of 8 and 16 bit characters.
1748             **  need to build a compound string.
1749             */
1750
1751             cstr_entry = sem_create_cstr();
1752
1753             for ( a_off = 0,
1754                   off = 0;
1755                   off < l_lex_pos;
1756                 )
1757
1758             {
1759
1760                 for (off = a_off;  off < l_lex_pos;  off++)
1761                     if (az_current_lex_buffer->c_text[ off ] > 0x97)
1762                         break;
1763
1764 /*
1765  * Create the 8 bit string with iso_latin1
1766  */
1767
1768                 if (off > a_off)
1769                 {
1770                     str_entry = create_str_entry
1771                         ( off - a_off,
1772                          uil_sym_isolatin1_charset,
1773                          az_charset_entry );
1774
1775                     _move( str_entry->value.c_value,
1776                            &az_current_lex_buffer->c_text[ a_off ],
1777                            off-a_off );
1778
1779                     sem_append_str_to_cstr( cstr_entry, str_entry, TRUE );
1780                 }
1781
1782                 for (a_off = off;  a_off < l_lex_pos;  a_off += 2)
1783                     if (az_current_lex_buffer->c_text[ a_off ] <= 0x97)
1784                         break;
1785
1786 /*
1787  * Create the 16 bit string with its charset
1788  */
1789
1790                 if (a_off > off)
1791                 {
1792                     str_entry =
1793                         create_str_entry( a_off - off, l_charset, az_charset_entry );
1794
1795                     _move( str_entry->value.c_value,
1796                            &az_current_lex_buffer->c_text[ off ],
1797                            a_off-off );
1798
1799                     sem_append_str_to_cstr( cstr_entry, str_entry, TRUE );
1800                 }
1801
1802             }
1803
1804             yylval.value.az_symbol_entry = (sym_entry_type *)cstr_entry;
1805             yylval.b_type = COMP_STRING;
1806
1807         }
1808         break;
1809
1810   case token_lstr:
1811     {
1812       int                       l_length = 0;
1813       lex_buffer_type           *az_lex_buffer;
1814       sym_value_entry_type      *str_entry;
1815       unsigned char             *c_char;
1816
1817       l_length = l_lex_pos;
1818
1819       for (az_lex_buffer = az_first_lex_buffer;
1820            az_lex_buffer != az_current_lex_buffer;
1821            az_lex_buffer = az_lex_buffer->az_next_buffer)
1822         l_length = l_length + l_max_lex_buffer_pos + 1;
1823
1824       str_entry = create_str_entry(l_length, lex_k_fontlist_default_tag,
1825                                   az_charset_entry );
1826
1827       c_char = (unsigned char *)str_entry->value.c_value;
1828
1829       for (az_lex_buffer = az_first_lex_buffer;
1830            az_lex_buffer != az_current_lex_buffer;
1831            az_lex_buffer = az_lex_buffer->az_next_buffer)
1832         {
1833           _move( c_char, az_lex_buffer->c_text, l_max_lex_buffer_pos + 1);
1834           c_char = c_char + l_max_lex_buffer_pos + 1;
1835         }
1836
1837       _move( c_char, az_lex_buffer->c_text, l_lex_pos );
1838
1839       yylval.value.az_symbol_entry = (sym_entry_type *)str_entry;
1840       yylval.b_type = LOC_STRING;
1841
1842       break;
1843     }
1844
1845     default:
1846         _assert( FALSE, "unknown token table final state" );
1847   }
1848
1849 /* RAP we want to keep track of whether we are appending sequential comments */
1850     last_token_seen = l_state;
1851
1852     /* set position information in token value */
1853
1854     yylval.az_source_record = az_start_src_record;
1855     yylval.b_source_pos = l_start_src_pos;
1856     yylval.b_source_end = src_az_current_source_buffer->w_current_position;  /*  was "l_start_src_pos + l_lex_pos;" */
1857     yylval.b_tag = sar_k_token_frame;
1858
1859     /* dump the token if requested */
1860
1861 #if debug_version
1862     if (uil_v_dump_tokens)
1863         dump_token( az_current_lex_buffer, l_lex_pos );
1864 #endif
1865
1866     /*
1867     ** save this token
1868     */
1869
1870     prev_yylval = yylval;
1871
1872     /* return the token generated */
1873
1874     return yylval.b_type;
1875
1876 found_error:
1877
1878     /* do any backup of the source buffer position and lex buffer */
1879
1880     src_az_current_source_buffer->w_current_position -= z_cell.backup;
1881     l_lex_pos -= z_cell.backup;
1882
1883     /* put a null at the end of the current lex buffer */
1884
1885     az_current_lex_buffer->c_text[ l_lex_pos ] = 0;
1886
1887     /* case on the type of error */
1888
1889     switch (l_state)            /* contains the type of error */
1890     {
1891     case bad_prefix:
1892
1893         /* printable characters that are not part of a token were found */
1894
1895         diag_issue_diagnostic
1896                 ( d_unknown_seq,
1897                   az_start_src_record,
1898                   l_start_src_pos,
1899                   az_current_lex_buffer->c_text );
1900
1901         break;
1902
1903     default:
1904         _assert( FALSE, "unknown token table error state" );
1905         break;
1906       }
1907
1908     goto initialize_token_builder;
1909
1910   }
1911
1912 \f
1913 /*
1914 **++
1915 **  FUNCTIONAL DESCRIPTION:
1916 **
1917 **      This function initializes the lexical analyzer.
1918 **
1919 **  FORMAL PARAMETERS:
1920 **
1921 **      none
1922 **
1923 **  IMPLICIT INPUTS:
1924 **
1925 **      none
1926 **
1927 **  IMPLICIT OUTPUTS:
1928 **
1929 **      az_first_lex_buffer
1930 **
1931 **  FUNCTION VALUE:
1932 **
1933 **      void
1934 **
1935 **  SIDE EFFECTS:
1936 **
1937 **      lexical buffer is allocated
1938 **
1939 **--
1940 **/
1941
1942 #define UNSCHAR_MINUS_ONE (unsigned char) 255;
1943
1944 void  lex_initialize_analyzer( )
1945
1946 {
1947 String language;
1948
1949 /* RAP preserve comments */
1950 comment_text = (char *) _get_memory(INITIAL_COMMENT_SIZE);
1951 comment_size = INITIAL_COMMENT_SIZE;
1952
1953 comment_text[0] = '\0';
1954
1955 /* BEGIN OSF Fix CR 4749 */
1956 /* The lex algorithm has the potential to write
1957  * into index l_max_lex_buffer_pos + 1, so allocate l_max_lex_buffer_pos
1958  * plus 2 positions in buffer.
1959  */
1960 az_first_lex_buffer =
1961     (lex_buffer_type *) _get_memory (l_max_lex_buffer_pos + 2 +
1962                                      sizeof(lex_buffer_type *));
1963 /* END OSF Fix CR 4749 */
1964 az_first_lex_buffer->az_next_buffer = NULL;
1965
1966 /*   Initialize the stack frame entry for epsilon productions.   */
1967
1968 gz_yynullval.b_tag = sar_k_null_frame;
1969
1970 /*   Initialize the default character set  */
1971
1972 language = (char *) _XmStringGetCurrentCharset();
1973 if ( language == NULL )
1974     Uil_lex_l_user_default_charset = lex_k_default_charset;
1975 else
1976     {
1977     Uil_lex_l_user_default_charset = sem_charset_lang_name (language);
1978     if (Uil_lex_l_user_default_charset == sym_k_error_charset)
1979         {
1980         diag_issue_diagnostic
1981             ( d_bad_lang_value,
1982              diag_k_no_source,
1983              diag_k_no_column);
1984         Uil_lex_l_user_default_charset = lex_k_default_charset;
1985         }
1986     }
1987 Uil_lex_az_charset_entry = NULL;
1988
1989 /* Determine if localized strings are possible */
1990 if (Uil_cmd_z_command.v_use_setlocale == FALSE)
1991   Uil_lex_l_localized = FALSE;
1992 else
1993   {
1994     Uil_lex_l_localized = TRUE;
1995     _MrmOSSetLocale("C");
1996   }
1997
1998 /*   Initialize the current character set */
1999 Uil_lex_l_charset_specified = FALSE;
2000
2001 /*    Initialize the source position and record */
2002
2003 prev_yylval.b_source_end = UNSCHAR_MINUS_ONE;
2004 prev_yylval.az_source_record = src_az_current_source_record;
2005
2006 }
2007
2008 \f
2009 /*
2010 **++
2011 **  FUNCTIONAL DESCRIPTION:
2012 **
2013 **      This function performs the cleanup processing of the lexical analyzer.
2014 **
2015 **  FORMAL PARAMETERS:
2016 **
2017 **      none
2018 **
2019 **  IMPLICIT INPUTS:
2020 **
2021 **      az_first_lex_buffer
2022 **
2023 **  IMPLICIT OUTPUTS:
2024 **
2025 **      az_first_lex_buffer
2026 **
2027 **  FUNCTION VALUE:
2028 **
2029 **      void
2030 **
2031 **  SIDE EFFECTS:
2032 **
2033 **      lexical buffer is freed
2034 **
2035 **--
2036 **/
2037
2038 void  Uil_lex_cleanup_analyzer( )
2039
2040 {
2041     /*  pointer to next buffer to free  */
2042     lex_buffer_type  *az_buffer_to_free;
2043
2044     /* Loop through the list of buffers freeing them all */
2045     while (az_first_lex_buffer != NULL) {
2046         az_buffer_to_free = az_first_lex_buffer;
2047         az_first_lex_buffer = az_first_lex_buffer->az_next_buffer;
2048         _free_memory((char*)az_buffer_to_free);
2049         }
2050 }
2051
2052 \f
2053 /*
2054 **++
2055 **  FUNCTIONAL DESCRIPTION:
2056 **
2057 **      This function issues a syntax error.  It is called from the
2058 **      error handling mechanism in the parser.
2059 **
2060 **  FORMAL PARAMETERS:
2061 **
2062 **      restart_token           the token number for the punctuation
2063 **                              character where parsing will resume after
2064 **                              this error is issued.
2065 **
2066 **  IMPLICIT INPUTS:
2067 **
2068 **      current lex buffer
2069 **      punc_token and punc_char tables
2070 **
2071 **  IMPLICIT OUTPUTS:
2072 **
2073 **      none
2074 **
2075 **  FUNCTION VALUE:
2076 **
2077 **      void
2078 **
2079 **  SIDE EFFECTS:
2080 **
2081 **      issue a diagnostic
2082 **
2083 **--
2084 **/
2085
2086 void  lex_issue_error( restart_token )
2087
2088 int             restart_token;
2089
2090 {
2091
2092     int             i, token_num;
2093     unsigned char   c_char = '.';
2094     char            * tok_name;
2095
2096 /*    Find the token number for the restart character in the table.
2097       It should be there.  Get the corresponding character for this
2098       token.    */
2099
2100     for ( i = 0 ; i<tok_punc_token_num ; i++ )
2101         {
2102         if (restart_token == punc_token [i])
2103             {
2104             c_char = punc_char [i];
2105             break;
2106             }
2107         }
2108
2109 /*    Get the text of the token name which caused the error.    */
2110
2111     token_num = yylval.b_type;
2112     if ( (token_num < 0) || (token_num > tok_num_tokens) )
2113         tok_name = "UNKNOWN_TOKEN";
2114     else
2115         tok_name = tok_token_name_table[token_num];
2116
2117 /*    Issue the error.    */
2118
2119     diag_issue_diagnostic
2120         (d_syntax,
2121          yylval.az_source_record,
2122          yylval.b_source_pos,
2123          tok_name,
2124          c_char);
2125
2126 }
2127
2128 \f
2129
2130 /*
2131 **
2132 **  LOCAL FUNCTIONS
2133 **
2134 */
2135
2136 \f
2137 /*
2138 **++
2139 **  FUNCTIONAL DESCRIPTION:
2140 **
2141 **      Issue an error for an illegal control character.
2142 **
2143 **  FORMAL PARAMETERS:
2144 **
2145 **      c_char
2146 **
2147 **  IMPLICIT INPUTS:
2148 **
2149 **      current source position
2150 **
2151 **  IMPLICIT OUTPUTS:
2152 **
2153 **      none
2154 **
2155 **  FUNCTION VALUE:
2156 **
2157 **      void
2158 **
2159 **  SIDE EFFECTS:
2160 **
2161 **      issue a diagnostic
2162 **
2163 **--
2164 **/
2165
2166 void issue_control_char_diagnostic
2167
2168         (unsigned char c_char )
2169
2170 {
2171
2172     diag_issue_diagnostic
2173         ( d_control_char,
2174           src_az_current_source_record,
2175           src_az_current_source_buffer->w_current_position - 1,
2176           c_char );
2177
2178     src_az_current_source_record->b_flags |= src_m_unprintable_chars;
2179
2180     return;
2181
2182 }
2183
2184 \f
2185 /*
2186 **++
2187 **  FUNCTIONAL DESCRIPTION:
2188 **
2189 **      This function obtains another lexical buffer.
2190 **
2191 **  FORMAL PARAMETERS:
2192 **
2193 **      az_current_lex_buffer
2194 **
2195 **  IMPLICIT INPUTS:
2196 **
2197 **      none
2198 **
2199 **  IMPLICIT OUTPUTS:
2200 **
2201 **      none
2202 **
2203 **  FUNCTION VALUE:
2204 **
2205 **      address of a new lexical buffer
2206 **
2207 **  SIDE EFFECTS:
2208 **
2209 **      another lexical buffer may be allocated
2210 **
2211 **--
2212 **/
2213
2214 static lex_buffer_type *get_lex_buffer( az_current_lex_buffer )
2215
2216 lex_buffer_type *az_current_lex_buffer;
2217
2218 {
2219     lex_buffer_type *az_lex_buffer;
2220
2221     /* check to see if another buffer is available - if not allocate one */
2222
2223     az_lex_buffer = az_current_lex_buffer->az_next_buffer;
2224
2225     if (az_lex_buffer == NULL)
2226     {
2227 /* BEGIN OSF Fix CR 4749 */
2228       /* The lex algorithm has the potential to write
2229        * into index l_max_lex_buffer_pos + 1, so allocate l_max_lex_buffer_pos
2230        * plus 2 positions in buffer.
2231        */
2232         az_lex_buffer =
2233             (lex_buffer_type *)_get_memory( l_max_lex_buffer_pos + 2 +
2234                                            sizeof(lex_buffer_type *));
2235 /* END OSF Fix CR 4749 */
2236         az_current_lex_buffer->az_next_buffer = az_lex_buffer;
2237         az_lex_buffer->az_next_buffer = NULL;
2238     }
2239
2240     return az_lex_buffer;
2241
2242 }
2243
2244 \f
2245 /*
2246 **++
2247 **  FUNCTIONAL DESCRIPTION:
2248 **
2249 **      This procedure will change all the unprintable characters in
2250 **      a buffer to lex_k_unprint_sub.
2251 **
2252 **  FORMAL PARAMETERS:
2253 **
2254 **      buffer          buffer to be checked
2255 **      length          length of the buffer
2256 **      flags           lex_m_filter_xxx flags to indicate if additional
2257 **                      characters should be filtered.
2258 **
2259 **  IMPLICIT INPUTS:
2260 **
2261 **      class_table     gives the unprintable characters
2262 **
2263 **  IMPLICIT OUTPUTS:
2264 **
2265 **      none
2266 **
2267 **  FUNCTION VALUE:
2268 **
2269 **      void
2270 **
2271 **  SIDE EFFECTS:
2272 **
2273 **      none
2274 **
2275 **--
2276 **/
2277
2278 void lex_filter_unprintable_chars
2279 (unsigned char  *buffer,
2280  int            length,
2281  unsigned long  flags )
2282 {
2283     int             i;
2284
2285     for (i=0;  i<length;  i++)
2286     {
2287         if ((class_table[ buffer[ i ] ] == class_illegal) ||
2288             (buffer[ i ] == 12) ||  /* form feed */
2289             (buffer[ i ] == 0) ||    /* null */
2290             ( (flags & lex_m_filter_tab)
2291                 && buffer[ i ] == 9 )/* horizontal tab */
2292            )
2293             buffer[ i ] = lex_k_unprint_sub;
2294     }
2295
2296 }
2297
2298 \f
2299 #if debug_version
2300 /*
2301 **++
2302 **  FUNCTIONAL DESCRIPTION:
2303 **
2304 **      This procedure will dump a token.
2305 **
2306 **  FORMAL PARAMETERS:
2307 **
2308 **
2309 **  IMPLICIT INPUTS:
2310 **
2311 **
2312 **  IMPLICIT OUTPUTS:
2313 **
2314 **
2315 **  SIDE EFFECTS:
2316 **
2317 **
2318 **--
2319 **/
2320
2321 static void     dump_token( az_current_lex_buffer,
2322                     l_lex_pos)
2323
2324 lex_buffer_type *az_current_lex_buffer;
2325 int             l_lex_pos;
2326 {
2327     unsigned char    c_buffer[l_max_lex_buffer_pos +2];
2328     lex_buffer_type *az_lex_buffer;
2329     int             i;
2330     int             last;
2331     int             last_buffer;
2332     unsigned char   c_char;
2333
2334
2335     _debug_output
2336          ("token: %d start: %d, %d  end: %d, %d \n",
2337            yylval.b_type,
2338            yylval.az_source_record->w_line_number,
2339            yylval.b_source_pos,
2340            src_az_current_source_record->w_line_number,
2341            src_az_current_source_buffer->w_current_position );
2342
2343     for (az_lex_buffer = az_first_lex_buffer;
2344          ;
2345          az_lex_buffer = az_lex_buffer->az_next_buffer)
2346     {
2347         last_buffer = ( az_lex_buffer == az_current_lex_buffer );
2348         if (last_buffer)
2349             last = l_lex_pos;
2350         else
2351             last = l_max_lex_buffer_pos+1;
2352
2353         _move( c_buffer, az_lex_buffer->c_text, last );
2354
2355         lex_filter_unprintable_chars (c_buffer, last, 0);
2356
2357         c_buffer[ last ] = 0;
2358         _debug_output("%s \n", c_buffer);
2359
2360         if (last_buffer)
2361             return;
2362     }
2363 }
2364 #endif
2365 \f
2366 /*
2367 **++
2368 **  FUNCTIONAL DESCRIPTION:
2369 **
2370 **      This function converts a null terminated string to a
2371 **      longword integer in the range 0..2**31-1.  If the ascii value is
2372 **      outside that range, the external variable errno is set to ERANGE
2373 **      and the value returned is 2**31-1
2374 **
2375 **  FORMAL PARAMETERS:
2376 **
2377 **      c_text          null terminate string holding integer in ascii
2378 **
2379 **  IMPLICIT INPUTS:
2380 **
2381 **      none
2382 **
2383 **  IMPLICIT OUTPUTS:
2384 **
2385 **      errno           set if overflow occurs
2386 **
2387 **  FUNCTION VALUE:
2388 **
2389 **      long            integer value of c_text
2390 **
2391 **  SIDE EFFECTS:
2392 **
2393 **      none
2394 **
2395 **--
2396 **/
2397
2398 #define k_max_int 2147483647
2399 #define k_max_div_10 214748364
2400
2401 long            cvt_ascii_to_long(c_text)
2402 unsigned char XmConst   *c_text;
2403 {
2404     unsigned long       l_value;
2405     int                 pos;
2406
2407     l_value = 0;
2408     for (pos = 0;  c_text[ pos ] != 0;  pos++)
2409     {
2410         if (l_value >= k_max_div_10)
2411             goto potential_overflow;
2412         l_value = (l_value * 10) + c_text[ pos ] - '0';
2413     }
2414
2415     return l_value;
2416
2417 potential_overflow:
2418     if (l_value > k_max_div_10)
2419     {
2420         errno = ERANGE;
2421         return k_max_int;
2422     }
2423
2424     l_value = (l_value * 10) + c_text[ pos ] - '0';
2425
2426     if (l_value > k_max_int)
2427     {
2428         errno = ERANGE;
2429         return k_max_int;
2430     }
2431
2432     return l_value;
2433 }
2434
2435 \f
2436 /*
2437 **++
2438 **  FUNCTIONAL DESCRIPTION:
2439 **
2440 **      This function creates a symbol entry for a primitive string.
2441 **
2442 **  FORMAL PARAMETERS:
2443 **
2444 **      l_size      number of bytes to allocate
2445 **      l_charset   charset of the string (token value)
2446 **      az_charset_entry   charset of the string (symbol table value entry)
2447 **
2448 **  IMPLICIT INPUTS:
2449 **
2450 **      yylval
2451 **
2452 **  IMPLICIT OUTPUTS:
2453 **
2454 **      none
2455 **
2456 **  FUNCTION VALUE:
2457 **
2458 **      symbol node created
2459 **
2460 **  SIDE EFFECTS:
2461 **
2462 **      none
2463 **
2464 **--
2465 **/
2466
2467 sym_value_entry_type *create_str_entry (l_size, l_charset, az_charset_entry)
2468
2469     int                         l_size;
2470     int                         l_charset;
2471     sym_value_entry_type        *az_charset_entry;
2472
2473
2474 {
2475
2476     sym_value_entry_type        *node;
2477     int                         charset;        /* from sym_k_..._charset */
2478     unsigned char               direction;      /* writing direction */
2479
2480     /*
2481      *  Determine character set and writing direction
2482      */
2483     if (l_charset != lex_k_userdefined_charset)
2484         {
2485         charset = sem_map_subclass_to_charset( l_charset );
2486         direction = charset_writing_direction_table[charset];
2487         }
2488     else
2489         {
2490         charset = sym_k_userdefined_charset;
2491         direction = az_charset_entry->b_direction;
2492         }
2493
2494
2495     /* size of entry
2496      *  sym_k_value_entry for common part of a value entry
2497      *  l_size for the string
2498      *  1 for the null on string
2499      */
2500
2501     node = (sym_value_entry_type *)
2502             sem_allocate_node
2503                 ( sym_k_value_entry,
2504                   sym_k_value_entry_size );
2505
2506     node->value.c_value = XtCalloc(1, l_size + 1);
2507
2508     node->header.az_src_rec = yylval.az_source_record;
2509     node->header.b_src_pos = yylval.b_source_pos;
2510     node->header.b_end_pos = yylval.b_source_end;
2511
2512     node->b_type = sym_k_char_8_value;
2513     node->w_length = l_size;
2514     node->b_charset = charset;
2515     node->b_direction = direction;
2516     node->az_charset_value = az_charset_entry;
2517     node->obj_header.b_flags = sym_m_private | sym_m_builtin;
2518
2519     return node;
2520
2521     }
2522