Fixed problem with assertion checking for negative shifts in

[splint.git] / src / cscannerHelp.c

/*
** Splint - annotation-assisted static program checker
** Copyright (C) 1994-2003 University of Virginia,
**         Massachusetts Institute of Technology
**
** This program is free software; you can redistribute it and/or modify it
** under the terms of the GNU General Public License as published by the
** Free Software Foundation; either version 2 of the License, or (at your
** option) any later version.
** 
** This program is distributed in the hope that it will be useful, but
** WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
** General Public License for more details.
** 
** The GNU General Public License is available from http://www.gnu.org/ or
** the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
** MA 02111-1307, USA.
**
** For information on splint: info@splint.org
** To report a bug: splint-bug@splint.org
** For more information: http://www.splint.org
*/
/*
** cscannerHelp.c - procedures for scanning C
**
** Most of this code was in cscanner.l, but moved here to separate it
** from the flex-generated code.
*/

# include "splintMacros.nf"
# include "basic.h"
# include "cscannerHelp.h"
# include "cscanner.h"
# include "cgrammar_tokens.h"
# include "osd.h"

static int lminput (void);
static int s_tokLength = 0;

static /*@owned@*/ cstring s_lastidprocessed = cstring_undefined;
static bool s_inSpecPart = FALSE;
static int s_whichSpecPart;
static char s_savechar = '\0';
static bool s_expectingMetaStateName = FALSE;
static bool s_lastWasString = FALSE;
static bool s_expectingTypeName = TRUE;

struct skeyword
{
  /*@null@*/ /*@observer@*/ char *name;
  int token;
} ;

/*
** These tokens are followed by syntax that is parsed by the 
** grammar proper.
*/

static struct skeyword s_parsetable[] = {
  { "modifies", QMODIFIES } ,
  { "globals", QGLOBALS } ,
  { "alt", QALT } ,
  { "warn", QWARN } ,
  { "constant", QCONSTANT } ,
  { "function", QFUNCTION } ,
  { "iter", QITER } ,
  { "defines", QDEFINES } ,
  { "uses", QUSES } ,
  { "allocates", QALLOCATES } ,
  { "sets", QSETS } ,
  { "releases", QRELEASES } ,
  { "pre", QPRECLAUSE } ,
  { "post", QPOSTCLAUSE } ,
  { "setBufferSize", QSETBUFFERSIZE},
  { "setStringLength", QSETSTRINGLENGTH},
  { "testinRange", QTESTINRANGE},
  { "requires", QPRECLAUSE } ,
  { "ensures", QPOSTCLAUSE } ,
  { "invariant", QINVARIANT} ,
  { NULL, BADTOK } 
} ;

/*
** These tokens are either stand-alone tokens, or followed by 
** token-specific text.
*/

static struct skeyword s_keytable[] = {
  { "anytype", QANYTYPE } ,
  { "integraltype", QINTEGRALTYPE } ,
  { "unsignedintegraltype", QUNSIGNEDINTEGRALTYPE } ,
  { "signedintegraltype", QSIGNEDINTEGRALTYPE } ,
  { "out", QOUT } ,
  { "in", QIN } ,
  { "only", QONLY } , 
  { "owned", QOWNED } ,
  { "dependent", QDEPENDENT } ,
  { "partial", QPARTIAL } ,
  { "special", QSPECIAL } ,
  { "truenull", QTRUENULL } ,
  { "falsenull", QFALSENULL } ,
  { "nullwhentrue", QTRUENULL } ,
  { "falsewhennull", QFALSENULL } ,
  { "keep", QKEEP } ,
  { "kept", QKEPT } ,
  { "notnull", QNOTNULL } ,
  { "abstract", QABSTRACT } ,
  { "numabstract", QNUMABSTRACT } ,
  { "concrete", QCONCRETE } ,
  { "mutable", QMUTABLE } ,
  { "immutable", QIMMUTABLE } ,
  { "unused", QUNUSED } ,
  { "external", QEXTERNAL } ,
  { "sef", QSEF } ,
  { "unique", QUNIQUE } ,
  { "returned", QRETURNED } ,
  { "exposed", QEXPOSED } ,
  { "refcounted", QREFCOUNTED } ,
  { "refs", QREFS } ,
  { "newref", QNEWREF } ,
  { "tempref", QTEMPREF } ,
  { "killref", QKILLREF } ,
  { "null", QNULL } ,
  { "relnull", QRELNULL } ,
  { "nullterminated", QNULLTERMINATED }, 
  { "setBufferSize", QSETBUFFERSIZE },
  { "testInRange", QTESTINRANGE},
  { "isnull", QISNULL }, 
  { "MaxSet", QMAXSET},
  { "MaxRead", QMAXREAD},
  { "maxSet", QMAXSET},
  { "maxRead", QMAXREAD},
  { "reldef", QRELDEF } ,
  { "observer", QOBSERVER } ,
  { "exits", QEXITS } ,
  { "noreturn", QEXITS } ,
  { "mayexit", QMAYEXIT } ,
  { "maynotreturn", QMAYEXIT } ,
  { "trueexit", QTRUEEXIT } ,
  { "falseexit", QFALSEEXIT } ,
  { "noreturnwhentrue", QTRUEEXIT } ,
  { "noreturnwhenfalse", QFALSEEXIT } ,
  { "neverexit", QNEVEREXIT } ,
  { "alwaysreturns", QNEVEREXIT } ,
  { "temp", QTEMP } ,
  { "shared", QSHARED } ,
  { "ref", QREF } ,
  { "unchecked", QUNCHECKED } ,
  { "checked", QCHECKED } ,
  { "checkmod", QCHECKMOD } ,
  { "checkedstrict", QCHECKEDSTRICT } ,
  { "innercontinue", QINNERCONTINUE } ,
  { "innerbreak", QINNERBREAK } ,
  { "loopbreak", QLOOPBREAK } ,
  { "switchbreak", QSWITCHBREAK } ,
  { "safebreak", QSAFEBREAK } , 
  { "fallthrough", QFALLTHROUGH } ,
  { "l_fallthrou", QLINTFALLTHROUGH } , 
  { "l_fallth", QLINTFALLTHRU } ,
  { "notreached", QNOTREACHED } ,
  { "l_notreach", QLINTNOTREACHED } ,
  { "printflike", QPRINTFLIKE } ,
  { "l_printfli", QLINTPRINTFLIKE } ,
  { "scanflike", QSCANFLIKE } ,
  { "messagelike", QMESSAGELIKE } ,
  { "l_argsus", QARGSUSED } ,
  { NULL, BADTOK } 
} ;

/*
** would be better if these weren't hard coded...
*/

static bool isArtificial (cstring s)
{
  return (cstring_equalLit (s, "modifies") 
          || cstring_equalLit (s, "globals") 
          || cstring_equalLit (s, "warn")
          || cstring_equalLit (s, "alt"));
}

void cscannerHelp_swallowMacro (void)
{
  int i;
  bool skipnext = FALSE;

  while ((i = lminput ()) != EOF)
    {
      char c = (char) i;
      
      if (c == '\\')
        {
          skipnext = TRUE;
        }
      else if (c == '\n')
        {
          if (skipnext)
            {
              skipnext = FALSE;
            }
          else
            {
              reader_checkUngetc (i, yyin);
              return;
            }
        }
      else
        {
          ;
        }
    }

  if (i != EOF)
    {
      reader_checkUngetc (i, yyin);
    }
}

static int commentMarkerToken (cstring s)
{
  int i = 0;
  
  while (s_parsetable[i].name != NULL) 
    {
      DPRINTF (("Try :%s:%s:", s, s_parsetable[i].name));

      if (cstring_equalLit (s, s_parsetable[i].name))
        {
          return s_parsetable[i].token;
        }

      i++;
    }

  return BADTOK;
}

static int tokenMacroCode (cstring s)
{
  int i = 0;
  
  while (s_keytable[i].name != NULL) 
    {
      if (cstring_equalLit (s, s_keytable[i].name)) 
        {
          if (s_keytable[i].token == QLINTFALLTHROUGH) 
            {
              voptgenerror
                (FLG_WARNLINTCOMMENTS,
                 cstring_makeLiteral
                 ("Traditional lint comment /*FALLTHROUGH*/ used. "
                  "Splint interprets this in the same way as most Unix lints, but it is "
                  "preferable to replace it with the /*@fallthrough@*/ "
                  "semantic comment"),
                 g_currentloc);
              return QFALLTHROUGH;            
            }
          else if (s_keytable[i].token == QLINTFALLTHRU)
            {
              voptgenerror 
                (FLG_WARNLINTCOMMENTS,
                 cstring_makeLiteral
                 ("Traditional lint comment /*FALLTHRU*/ used. "
                  "Splint interprets this in the same way as most Unix lints, but it is "
                  "preferable to replace it with the /*@fallthrough@*/ "
                  "semantic comment"),
                 g_currentloc);
              return QFALLTHROUGH;
            }
          else if (s_keytable[i].token == QLINTNOTREACHED)
            {
              voptgenerror 
                (FLG_WARNLINTCOMMENTS,
                 cstring_makeLiteral
                 ("Traditional lint comment /*NOTREACHED*/ used. "
                  "Splint interprets this in the same way as most Unix lints, but it is "
                  "preferable to replace it with the /*@notreached@*/ "
                  "semantic comment."),
                 g_currentloc);
              
              return QNOTREACHED;
            }
          else if (s_keytable[i].token == QPRINTFLIKE)
            {
              setSpecialFunction (qual_createPrintfLike ());
              return SKIPTOK;
            }
          else if (s_keytable[i].token == QLINTPRINTFLIKE)
            {         
              voptgenerror 
                (FLG_WARNLINTCOMMENTS,
                 cstring_makeLiteral
                 ("Traditional lint comment /*PRINTFLIKE*/ used. "
                  "Splint interprets this in the same way as most Unix lints, but it is "
                  "preferable to replace it with either /*@printflike@*/, "
                  "/*@scanflike@*/ or /*@messagelike@*/."),
                 g_currentloc);
              
              setSpecialFunction (qual_createPrintfLike ());
              return SKIPTOK;
            }
          else if (s_keytable[i].token == QSCANFLIKE)
            {
              setSpecialFunction (qual_createScanfLike ());
              return SKIPTOK;
            }
          else if (s_keytable[i].token == QMESSAGELIKE)
            {
              setSpecialFunction (qual_createMessageLike ());
              return SKIPTOK;
            }
          else if (s_keytable[i].token == QARGSUSED)
            {
              voptgenerror
                (FLG_WARNLINTCOMMENTS,
                 cstring_makeLiteral
                 ("Traditional lint comment /*ARGSUSED*/ used. "
                  "Splint interprets this in the same way as most Unix lints, but it is "
                  "preferable to use /*@unused@*/ annotations on "
                  "the unused parameters."),
                 g_currentloc);
              
              setArgsUsed ();
              return SKIPTOK;
            }
          else
            {
              return s_keytable[i].token;
            }
        }

      i++;
    }
  
  return BADTOK;
}

static int lminput ()
{
  if (s_savechar == '\0')
    {
      incColumn ();
      return (cscanner_input ());
    }
  else
    {
      int save = (int) s_savechar;
      s_savechar = '\0';
      return save;
    }
}

static void lmsavechar (char c)
{
  if (s_savechar == '\0')
    {
      s_savechar = c;
    }
  else
    {
      llbuglit ("lmsavechar: override");
    }
}

int cscannerHelp_ninput ()  
{
  int c = lminput ();

  if (c != EOF && ((char)c == '\n'))
    {
      context_incLineno ();
    }

  return c;
}

static char macro_nextChar (void)
{
  static bool in_quote = FALSE, in_escape = FALSE, in_char = FALSE;
  int ic;
  char c;

  ic = lminput ();
  c = char_fromInt (ic);
  
  if (!in_quote && !in_char && (c == '\\' || c == BEFORE_COMMENT_MARKER[0]))
    {
      if (c == '\\')
        {
          while ((c = char_fromInt (lminput ())) != '\0' && c != '\n')
            {
              ; /* skip to newline */
            }
          
          context_incLineno ();
          
          if (c != '\0')
            {
              return macro_nextChar ();
            }
          else 
            {
              return c;
            }
        }
      else /* if (c == '@') */
        {
          llassert (FALSE); 

          if (cscannerHelp_handleLlSpecial () != BADTOK)
            {
              llerrorlit (FLG_SYNTAX, "Macro cannot use special syntax");
            }

          return macro_nextChar ();
        }
    }
  else if (!in_escape && c == '\"')
    {
      in_quote = !in_quote;
    }
  else if (!in_escape && c == '\'')
    {
      in_char = !in_char;
    }
  else if ((in_quote || in_char) && c == '\\')
    {
      in_escape = !in_escape;
    }
  else if ((in_quote || in_char) && in_escape)
    {
      in_escape = FALSE;
    }
  else if (!in_quote && c == '/')
    {
      char c2;
      
      if ((c2 = char_fromInt (lminput ())) == '*')
        {
          while (c2 != '\0')
            {
              while ((c2 = char_fromInt (lminput ())) != '\0'
                     && c2 != '\n' && c2 != '*')
                {
                  ;
                }
              
              if (c2 == '*')
                {
                  while ((c2 = char_fromInt (lminput ())) != '\0' 
                         && c2 == '*')
                    {
                      ;
                    }

                  if (c2 == '/')
                    {
                      goto outofcomment;
                    }
                }
              else 
                {
                  llfatalerror (cstring_makeLiteral ("Macro: bad comment!"));
                }
            }
        outofcomment:
          return macro_nextChar ();
        }
      else
        {
          /*** putchar does not work!  why?  puts to stdio...??! ***/
          lmsavechar (c2);
        }
    }
  else
    {
      ;
    }

  return c;
}

/*
** keeps semantic comments
*/

static char macro_nextCharC (void)
{
  static bool in_quote = FALSE, in_escape = FALSE, in_char = FALSE;
  char c;

  c = char_fromInt (lminput ());

  if (!in_quote && !in_char && c == '\\')
    {
      while ((c = char_fromInt (lminput ())) != '\0' && c != '\n')
        {
          ; /* skip to newline */
        }
      
      context_incLineno ();
      
      if (c != '\0')
        {
          return macro_nextCharC ();
        }
      else
        {
          return c;
        }
    }
  else if (!in_escape && c == '\"')
    {
      in_quote = !in_quote;
    }
  else if (!in_escape && c == '\'')
    {
      in_char = !in_char;
    }
  else if ((in_quote || in_char) && c == '\\')
    {
      in_escape = !in_escape;
    }
  else if ((in_quote || in_char) && in_escape)
    {
      in_escape = FALSE;
    }
  else if (!in_quote && c == '/')
    {
      char c2;
      
      if ((c2 = char_fromInt (lminput ())) == '*')
        {
          while (c2 != '\0')
            {
              while ((c2 = char_fromInt (lminput ())) != '\0' 
                     && c2 != '\n' && c2 != '*')
                {
                  ;
                }
              
              if (c2 == '*')
                {
                  while ((c2 = char_fromInt (lminput ())) != '\0'
                         && c2 == '*')
                    {
                      ;
                    }

                  if (c2 == '/') 
                    {
                      goto outofcomment;
                    }
                }
              else 
                {
                  llfatalerror (cstring_makeLiteral ("Macro: bad comment!"));
                }
            }
        outofcomment:
          return macro_nextCharC ();
        }
      else
        {
          lmsavechar (c2);
        }
    }
  else /* normal character */
    {
      ;
    }

  return c;
}

/*
** skips whitespace (handles line continuations)
** returns first non-whitespace character
*/

static char skip_whitespace (void)
{
  char c;

  while ((c = macro_nextChar ()) == ' ' || c == '\t')
    {
      ;
    }

  return c;
}

void cscannerHelp_handleMacro ()
{
  cstring mac = cstring_undefined;
  int macrocode;
  char c;

  while (currentColumn () > 2)
    {
      mac = cstring_appendChar (mac, ' ');
      cscannerHelp_setTokLength (-1);
    }

  c = macro_nextCharC ();

  if (c >= '0' && c <= '9')
    {
      int i;

      for (i = 0; i < (((int) (c - '0')) + 1); i++)
        {
          mac = cstring_appendChar (mac, ' ');
        }
    }
  else
    {
      BADBRANCH;
    }

  while (((c = macro_nextCharC ()) != '\0') && (c != '\n'))
    {
      mac = cstring_appendChar (mac, c);
    }

  
  macrocode = tokenMacroCode (mac);

  if (macrocode == BADTOK && !isArtificial (mac))
    {
      context_addMacroCache (mac);
    }
  else
    {
      cstring_free (mac);
    }

  if (c == '\n')
    {
      context_incLineno ();
    }
}

bool cscannerHelp_handleSpecial (char *yyt)
{
  char *l; /* !!  = mstring_create (MAX_NAME_LENGTH); */
  int lineno = 0;
  char c;
  char *ol;
  cstring olc;
  size_t len_yyt;

  len_yyt = strlen (yyt +1) ;

  l = mstring_copy (yyt + 1);
  
  while ((c = char_fromInt (lminput ())) != '\n' && c != '\0')
    {
      l = mstring_append(l, c);
    }

    /* Need to safe original l for deallocating. */
  ol = l;

  l += strlen (l);

  olc = cstring_fromChars (ol);
  
  if (cstring_equalPrefixLit (olc, "pragma"))
    {
      char *pname = mstring_create (size_fromInt (MAX_PRAGMA_LEN));
      char *opname = pname;
      char *ptr = ol + 6; /* pragma is six characters, plus space */
      int len = 0;
      
      
      /* skip whitespace */
      while (((c = *ptr) != '\0') && isspace (c))
        {
          ptr++;
        }

      
      while (((c = *ptr) != '\0') && !isspace (c))
        {
          len++;

          if (len > MAX_PRAGMA_LEN)
            {
              break;
            }

          ptr++;
          *pname++ = c;
        }

      *pname = '\0';
      
      if (len == PRAGMA_LEN_EXPAND 
          && mstring_equal (opname, PRAGMA_EXPAND))
        {
          cstring exname = cstring_undefined;
          uentry ue;
          
          ptr++; 
          while (((c = *ptr) != '\0') && !isspace (c))
            {
              exname = cstring_appendChar (exname, c);
              ptr++;
            }
             
          
          ue = usymtab_lookupExposeGlob (exname);
          
          if (uentry_isExpandedMacro (ue))
            {
              if (fileloc_isPreproc (uentry_whereDefined (ue)))
                {
                  fileloc_setColumn (g_currentloc, 1);
                  uentry_setDefined (ue, g_currentloc);
                }
            }

          cstring_free (exname);
        }

      (void) cscannerHelp_handleNewLine (); /* evans 2003-10-27: pragment increments line */
    }
  else if (cstring_equalPrefixLit (olc, "ident"))
    {
      /* Some pre-processors will leave these in the code.  Ignore rest of line */
      (void) cscannerHelp_handleNewLine (); /* evans 2003-10-27: ident increments line */
    }

  /*
  ** Yuk...Win32 filenames can have spaces in them...we need to read
  ** to the matching end quote.
  */
  else if ((sscanf (ol, "line %d \"", &lineno) == 1)
           || (sscanf (ol, " %d \"", &lineno) == 1))
    {
      char *tmp = ol;
      cstring fname;
      fileId fid;

      /*@access cstring@*/
      while (*tmp != '\"' && *tmp != '\0')
        {
          tmp++;
        }

      llassert (*tmp == '\"');
      tmp++;
      fname = tmp;
      
      while (*tmp != '\"' && *tmp != '\0')
        {
          tmp++;
        }

      llassert (*tmp == '\"');
      *tmp = '\0';

# if defined(OS2) || defined(MSDOS) || defined(WIN32)

      /*
      ** DOS-like path delimiters get delivered in pairs, something like 
      ** \"..\\\\file.h\", so we have to make it normal again. We do NOT
      ** remove the pre dirs yet as we usually specify tmp paths relative
      ** to the current directory, so tmp files would not get found in
      ** the hash table.  If this method fails we try it again later. 
      */

      {
        char *stmp = fname;
        
        /*
        ** Skip past the drive marker.
        */
        
        if (strchr (stmp, ':') != NULL)
          {
            stmp = strchr (stmp, ':') + 1;
          }
        
        while ((stmp = strchr (stmp, CONNECTCHAR)) != NULL )
          {
            if (*(stmp+1) == CONNECTCHAR)
              {
                memmove (stmp, stmp+1, strlen (stmp));
              }
            
            stmp++;
          }
        
        fid = fileTable_lookupBase (context_fileTable (), fname);
        if (!(fileId_isValid (fid)))
          {
            fname = removePreDirs (fname);
            fid = fileTable_lookupBase (context_fileTable (), fname);
          }
      }
# else  /* !defined(OS2) && !defined(MSDOS) */
      fname = removePreDirs (fname);
      fid = fileTable_lookupBase (context_fileTable (), fname);      
# endif /* !defined(OS2) && !defined(MSDOS) */
      
      if (!(fileId_isValid (fid)))
        {
          if (context_inXHFile ())
            {
              fid = fileTable_addXHFile (context_fileTable (), fname);
            }
          else if (isHeaderFile (fname))
            {
              fid = fileTable_addHeaderFile (context_fileTable (), fname);
            }
          else
            {
              fid = fileTable_addFile (context_fileTable (), fname);
            }
        }
      
      setFileLine (fid, lineno);
      /*@noaccess cstring@*/
    }
  else if ((sscanf (ol, "line %d", &lineno) == 1) 
           || (sscanf (ol, " %d", &lineno) == 1))
    {
      setLine (lineno); /* next line is <cr> */
    }
  else
    {
      if (mstring_equal (ol, "")) {
        DPRINTF (("Empty pp command!"));
        /*
        ** evs 2000-05-16: This is a horrible kludge, to get around a bug (well, difficulty) in the pre-processor.
        ** We handle a plain # in the input file, by echoing it, and ignoring it in the post-pp-file.
        */
        mstring_free (ol);
        (void) cscannerHelp_handleNewLine (); /* evans 2003-10-27: increments line */
        return FALSE;
      } else {
        voptgenerror
          (FLG_UNRECOGDIRECTIVE,
           message ("Unrecognized pre-processor directive: #%s", 
                    cstring_fromChars (ol)),
           g_currentloc);
        (void) cscannerHelp_handleNewLine (); /* evans 2003-10-27: increments line */
      }
      
      sfree (ol);
      return FALSE; /* evans 2001-12-30: was: TRUE; */
    }

  sfree (ol);
  return FALSE;
}
  
int cscannerHelp_handleLlSpecial (void)
{ 
  bool hasnl = FALSE;
  int ic; 
  char c;
  char *s = mstring_createEmpty ();
  char *os; 
  int tok;
  int charsread = 0;
  fileloc loc;

  loc = fileloc_copy (g_currentloc);
  DPRINTF (("Handle special: %s", fileloc_unparse (loc)));

  while (((ic = cscannerHelp_ninput ()) != 0) && isalpha (ic))
    {
      c = (char) ic;
      s = mstring_append (s, c);
      charsread++;
    }

  DPRINTF (("Read: %s / %s", s, fileloc_unparse (g_currentloc)));
  os = s;

  if (charsread == 0 && ic == (int) AFTER_COMMENT_MARKER[0])
    {
      ic = cscannerHelp_ninput ();

      llassert (ic == (int) AFTER_COMMENT_MARKER[1]);
            
      if (*s == '\0')
        {
          sfree (os);
          fileloc_free (loc);
          return QNOMODS; /* special token no modifications token */
        }
    }
  
  DPRINTF (("Coment marker: %s", os));
  tok = commentMarkerToken (cstring_fromChars (os));

  if (tok != BADTOK)
    {
      s_tokLength = charsread;
      sfree (os);
      s_inSpecPart = TRUE;
      s_whichSpecPart = tok;
      fileloc_free (loc);
      return tok;
    }

  DPRINTF (("Not a comment marker..."));
  /* Add rest of the comment */
  
  if (ic != 0 && ic != EOF)
    {
      c = (char) ic;
      
      s = mstring_append (s, c);
      charsread++;

      while (((ic = cscannerHelp_ninput ()) != 0) && (ic != EOF)
             && (ic != (int) AFTER_COMMENT_MARKER[0]))
        {
          c = (char) ic;

          /* evans 2001-09-01 added to prevent assertion failures for uncloses syntactic comments */

          if (c == '\n') {
            hasnl = TRUE; /* This prevents tokLength from being set later. */
            s_tokLength = 0; 

            voptgenerror
              (FLG_SYNTAX, 
               message ("Likely parse error: syntactic comment token spans multiple lines: %s",
                        cstring_fromChars (s)),
               loc);
          }

          s = mstring_append (s, c);
          charsread++;
        }
      /*@-branchstate@*/ 
    } /* spurious (?) warnings about s */
  /*@=branchstate@*/ 

  DPRINTF (("Read: %s / %s", s, fileloc_unparse (g_currentloc)));

  if (ic == (int) AFTER_COMMENT_MARKER[0]) 
    {
      int nc = cscannerHelp_ninput ();
      llassert ((char) nc ==  AFTER_COMMENT_MARKER[1]);
      charsread++;
    }

  os = s;

  while (*s == ' ' || *s == '\t' || *s == '\n') 
    {
      s++;
    }

  if (*s == '-' || *s == '+' || *s == '=') /* setting flags */
    {
      c = *s;

      while (c == '-' || c == '+' || c == '=')
        {
          ynm set = ynm_fromCodeChar (c);
          cstring thisflag;

          s++;
          
          thisflag = cstring_fromChars (s);
          
          while ((c = *s) != '\0' && (c != '-') && (c != '=')
                 && (c != '+') && (c != ' ') && (c != '\t') && (c != '\n'))
            {
              s++;
            }

          *s = '\0';

          if (!context_getFlag (FLG_NOCOMMENTS))
            {
              cstring flagname = thisflag;
              flagcode fflag = flags_identifyFlag (flagname);
              
              if (flagcode_isSkip (fflag))
                {
                  ;
                }
              else if (flagcode_isModeName (fflag))
                {
                  if (ynm_isMaybe (set))
                    {
                      llerror
                        (FLG_BADFLAG, 
                         message 
                         ("Semantic comment attempts to restore flag %s.  "
                          "A mode flag cannot be restored.",
                          flagname));
                    }
                  else
                    {
                      context_setMode (flagname);
                    }
                }
              else if (flagcode_isInvalid (fflag))
                {
                  voptgenerror
                    (FLG_UNRECOGFLAGCOMMENTS,
                     message ("Unrecognized option in semantic comment: %s", 
                              flagname),
                     loc);
                }
              else if (flagcode_isGlobalFlag (fflag))
                {
                  voptgenerror
                    (FLG_BADFLAG, 
                     message 
                     ("Semantic comment attempts to set global flag %s.  "
                      "A global flag cannot be set locally.",
                      flagname),
                     loc);
                }
              else
                {
                  context_fileSetFlag (fflag, set, loc);
                  
                  if (flagcode_hasArgument (fflag))
                    {
                      if (ynm_isMaybe (set))
                        {
                          voptgenerror
                            (FLG_BADFLAG, 
                             message 
                             ("Semantic comment attempts to restore flag %s.  "
                              "A flag for setting a value cannot be restored.",
                              flagname),
                             loc);
                        }
                      else
                        { /* cut-and-pastied from llmain...blecch */
                          cstring extra = cstring_undefined;
                          char *rest;
                          char *orest;
                          char rchar;
                          
                          *s = c;
                          rest = mstring_copy (s);
                          orest = rest;
                          *s = '\0';
                          
                          while ((rchar = *rest) != '\0'
                                 && (isspace (rchar)))
                            {
                              rest++;
                              s++;
                            }
                          
                          while ((rchar = *rest) != '\0'
                                 && !isspace (rchar))
                            {
                              extra = cstring_appendChar (extra, rchar);
                              rest++; 
                              s++;
                            }
                          s--; /* evans 2002-07-12: this was previously only in the else branch.
                                  Leads to an invalid read on the true branch.
                               */

                          sfree (orest);
                          
                          if (cstring_isUndefined (extra))
                            {
                              llerror 
                                (FLG_BADFLAG,
                                 message
                                 ("Flag %s (in semantic comment) must be followed by an argument",
                                  flagcode_unparse (fflag)));

                              cstring_free (extra);
                            }
                          else
                            {
                              if (flagcode_hasNumber (fflag))
                                {
                                  flags_setValueFlag (fflag, extra);
                                }
                              else if (flagcode_hasChar (fflag))
                                {
                                  flags_setValueFlag (fflag, extra);
                                }
                              else if (flagcode_hasString (fflag))
                                {
                                  flags_setStringFlag (fflag, extra);
                                }
                              else
                                {
                                  cstring_free (extra);
                                  BADEXIT;
                                }
                            }
                        }
                    }
                }
            }
          else
            {
              ;
            }

          *s = c;
          while ((c == ' ') || (c == '\t') || (c == '\n'))
            {
              c = *(++s);
            }
        } 

      if (context_inHeader () && !isArtificial (cstring_fromChars (os)))
        {
          DPRINTF (("Here adding comment: %s", os));
          context_addComment (cstring_fromCharsNew (os), loc);
        }
      else
        {
          ;
        }
    }
  else
    {
      char *t = s;
      int macrocode;
      char tchar = '\0';
      annotationInfo ainfo;

      while (*s != '\0' && *s != ' ' && *s != '\t' && *s != '\n') 
        {
          s++;
        }

      if (*s != '\0') 
        {
          tchar = *s;
          *s = '\0';
          s++;
        }
      
      t = cstring_toCharsSafe (cstring_downcase (cstring_fromChars (t)));
      macrocode = tokenMacroCode (cstring_fromChars (t));

      if (macrocode != BADTOK)
        {
          s_tokLength = hasnl ? 0 : size_toInt (mstring_length (t));
          
          sfree (t);
          sfree (os);
          fileloc_free (loc);

          if (macrocode == SKIPTOK)
            {
              return BADTOK;
            }

          return macrocode;
        }
      
      ainfo = context_lookupAnnotation (cstring_fromChars (os));
      
      if (annotationInfo_isDefined (ainfo)) {
        DPRINTF (("Found annotation: %s", annotationInfo_unparse (ainfo)));
        yylval.annotation = ainfo;
        s_tokLength = 0;
        sfree (os);
        sfree (t);
        fileloc_free (loc);
        return CANNOTATION;
      } 

      if (context_inHeader ())
        {
          if (tchar != '\0')
            {
              *(s-1) = tchar;
            }
          
          if ((context_inMacro () || context_inGlobalContext ())
              && macrocode != SKIPTOK
              && !isArtificial (cstring_fromChars (os))) 
            {
              if (context_processingMacros ())
                {
                  /* evans 2002-02-24: don't add comments when procssing macros */
                }
              else
                {
                  context_addComment (cstring_fromCharsNew (os), loc);
                }
            }
          else
            {
              ; 
            }
          
          if (tchar != '\0')
            {
              *(s-1) = '\0';
            }
        }

      if (mstring_equal (t, "ignore"))
        {
          if (!context_getFlag (FLG_NOCOMMENTS))
            {
              context_enterSuppressRegion (loc);
            }
        }
      else if ((*t == 'i' || *t == 't')
               && (*(t + 1) == '\0'))
        {
          if (!context_getFlag (FLG_NOCOMMENTS)
              && (*t == 'i' || context_getFlag (FLG_TMPCOMMENTS)))
            {
              context_enterSuppressLine (-1, loc); /* infinite suppression */
            }
        }
      else if (((*t == 'i') || (*t == 't'))
               && ((*(t + 1) >= '0' && *(t + 1) <= '9')))
        {
          bool tmpcomment = (*t == 't');
          int val = -1; 
          char *tt = t; /* don't mangle t, since it is free'd */
          char lc = *(++tt);

          if (lc >= '0' && lc <= '9')
            {
              val = (int)(lc - '0');
              
              lc = *(++tt);       
              while (lc >= '0' && lc <= '9')
                {
                  val *= 10;
                  val += (int) (lc - '0');
                  lc = *(++tt);
                }
            }
          
          if (!context_getFlag (FLG_NOCOMMENTS)
              && (!tmpcomment || context_getFlag (FLG_TMPCOMMENTS)))
            {
              DPRINTF (("Here: enter suppress: %s", fileloc_unparse (loc)));
              context_enterSuppressLine (val, loc);
            }
        }
      else if (mstring_equal (t, "end"))
        {
          if (!context_getFlag (FLG_NOCOMMENTS))
            {
              context_exitSuppressRegion (loc);
            }
        }
      else if (mstring_equal (t, "notfunction"))
        {
         ; /* handled by pcpp */
        }
      else if (mstring_equal (t, "access"))
        {
          cstring tname;
          
          while (TRUE)
            {
              while (((c = *s) != '\0') && (c == ' ' || c == '\t' || c == '\n'))
                {
                  s++;
                }
              
              if (c == '\0')
                {
                   break;
                }

              tname = cstring_fromChars (s);
              
              while ((c = *s) != '\0' && c != ' ' 
                     && c != '\t' && c != '\n' && c != ',') 
                {
                  s++;
                }

              *s = '\0';

              DPRINTF (("Access %s", tname));

              if (!context_getFlag (FLG_NOCOMMENTS) 
                  && !context_getFlag (FLG_NOACCESS))
                {
                  if (usymtab_existsType (tname))
                    {
                      typeId uid = usymtab_getTypeId (tname);
                      uentry ue = usymtab_getTypeEntry (uid);

                      if (uentry_isAbstractDatatype (ue))
                        {
                          context_addFileAccessType (uid);
                          DPRINTF (("Adding access to: %s / %d", tname, uid));
                        }
                      else
                        {
                          voptgenerror
                            (FLG_COMMENTERROR,
                             message
                             ("Non-abstract type %s used in access comment",
                              tname),
                             loc);
                        }
                    }
                  else
                    {
                      if (!(context_inSuppressRegion ()
                            || context_inSuppressZone (loc)))
                        {
                          voptgenerror
                            (FLG_COMMENTERROR,
                             message
                             ("Unrecognized type %s used in access comment",
                              tname),
                             loc);
                        }
                    }
                }
              
              if (c != '\0') 
                {
                  s++;
                }
              
              if (c != ',' && c != ' ')
                {
                  break;
                }
            }
        }
      else if (mstring_equal (t, "noaccess"))
        {
          cstring tname;
          char lc;
          
          while (TRUE)
            {
              while (((lc = *s) != '\0') && (lc == ' ' || lc == '\t' || lc == '\n')) 
                {
                  s++;
                }
              
              if (lc == '\0')
                {
                 break;
                }

              tname = cstring_fromChars (s);
              
              while ((lc = *s) != '\0' && lc != ' ' && lc != '\t' 
                     && lc != '\n' && lc != ',') 
                {
                  s++;
                }

              *s = '\0';

              if (!context_getFlag (FLG_NOCOMMENTS) 
                  && !context_getFlag (FLG_NOACCESS))
                {
                  if (usymtab_existsType (tname))
                    {
                      typeId tuid = usymtab_getTypeId (tname);
                      
                      if (context_couldHaveAccess (tuid))
                        {
                          DPRINTF (("Removing access: %s", tname));
                          context_removeFileAccessType (tuid);
                        }
                      else
                        {
                          if (!(context_inSuppressRegion () 
                                || context_inSuppressZone (loc)))
                            {
                              uentry ue = usymtab_getTypeEntry (tuid);
                              
                              if (uentry_isAbstractDatatype (ue))
                                {
                                  voptgenerror
                                    (FLG_COMMENTERROR,
                                     message
                                     ("Non-accessible abstract type %s used in noaccess comment",
                                      tname),
                                     loc);
                                }
                              else
                                {
                                  voptgenerror
                                    (FLG_COMMENTERROR,
                                     message
                                     ("Non-abstract type %s used in noaccess comment",
                                      tname),
                                     loc);
                                }
                            }
                        }
                    }
                  else
                    {
                      if (!(context_inSuppressRegion () 
                            || context_inSuppressZone (loc)))
                        {
                          voptgenerror
                            (FLG_COMMENTERROR,
                             message
                             ("Unrecognized type %s used in noaccess comment",
                              tname),
                             loc);
                        }
                    }
                }
              
              if (lc != '\0') 
                {
                  s++;
                }
              
              if (lc != ',' && lc != ' ')
                {
                  break;
                }
            }
        }
      else
        {
          voptgenerror (FLG_UNRECOGCOMMENTS, 
                        message ("Semantic comment unrecognized: %s", 
                                 cstring_fromChars (os)),
                        loc);
          /*@-branchstate@*/
        } /* spurious (?) warning about t */
      /*@=branchstate@*/
      
      sfree (t);
    }
  
  sfree (os);
  fileloc_free (loc);
  return BADTOK;
}

/*@only@*/ cstring cscannerHelp_makeIdentifier (char *s)
{
  char *c = mstring_create (strlen (s) + 1);
  cstring id = cstring_fromChars (c);

  while (isalnum (*s) || (*s == '_') || (*s == '$')) 
    {
      *c++ = *s++;
    }

  *c = '\0';
  return (id);
}

/*@observer@*/ /*@dependent@*/ uentry cscannerHelp_coerceId (cstring cn)
{
  if (!(usymtab_exists (cn)))
    {
      fileloc loc = fileloc_createExternal ();
      
      /*
      ** We need to put this in a global scope, otherwise the sRef will be deallocated.
      */
      
      uentry ce = uentry_makeUnrecognized (cn, loc);
      
      if (!context_inIterEnd ())
        {
          voptgenerror 
            (FLG_SYSTEMUNRECOG, 
             message ("Unrecognized (possibly system) identifier: %q", 
                      uentry_getName (ce)), 
             g_currentloc);
        }
      
      return ce;
    }
  
  return (usymtab_lookup (cn));
}

/*
** like, cscannerHelp_coerceId, but doesn't supercede for iters
*/

/*@observer@*/ uentry cscannerHelp_coerceIterId (cstring cn)
{
  if (!(usymtab_exists (cn)))
    {
      return uentry_undefined;
    }
  
  return (usymtab_lookup (cn));
}

/*
** Need to keep this in case there is a declaration that isn't processed until
** the scope exits.  Would be good to rearrange the symbol table so this doesn't
** happen, and save all the cstring copying.
*/

/*@observer@*/ cstring cscannerHelp_observeLastIdentifier ()
{
  cstring res = s_lastidprocessed;
  return res;
}

static void cscanner_setLastIdentifier (/*@keep@*/ cstring id) /*@modifies s_lastidprocessed@*/
{
  if (cstring_isDefined (s_lastidprocessed))
    {
      cstring_free (s_lastidprocessed); 
    }

  s_lastidprocessed = id;
}

int cscannerHelp_processIdentifier (cstring id)
{
  uentry le;

  if (context_getFlag (FLG_GRAMMAR))
    {
      lldiagmsg (message ("Process identifier: %s", id));
    }

  context_clearJustPopped ();
  cscanner_setLastIdentifier (id);

  DPRINTF (("Context: %s", context_unparse ()));

  if (context_inFunctionHeader ())
    {
      int tok = commentMarkerToken (id);
      DPRINTF (("in function decl: %s", id));

      if (tok != BADTOK)
        {
          return tok;
        }
      else 
        {
          tok = tokenMacroCode (id);
          
          if (tok != BADTOK)
            {
              return tok;
            }
          else 
            {
              annotationInfo ainfo;

              if (s_expectingMetaStateName) 
                {
                  metaStateInfo msinfo = context_lookupMetaStateInfo (id);

                  if (metaStateInfo_isDefined (msinfo))
                    {
                      yylval.msinfo = msinfo;
                      return METASTATE_NAME;
                    }
                  else
                    {
                      DPRINTF (("Not meta state name: %s", cstring_toCharsSafe (id)));
                    }
                }
              
              ainfo = context_lookupAnnotation (id);
              
              if (annotationInfo_isDefined (ainfo)) 
                {
                  DPRINTF (("Found annotation: %s", annotationInfo_unparse (ainfo)));
                  yylval.annotation = ainfo;
                  return CANNOTATION;
                }
              else
                {
                  DPRINTF (("Not annotation: %s", id));
                }
            }
        }
    }

  DPRINTF (("Here!"));

  /* Consider handling: Defined by C99 as static const char __func__[] */

  if (context_getFlag (FLG_GNUEXTENSIONS))
    {
      int tok = BADTOK;
      
      if (cstring_equalLit (id, "__stdcall")
          || cstring_equalLit (id, "__cdecl")
          || cstring_equalLit (id, "__extension__"))
        {
          return BADTOK;
        }
      else if (cstring_equalLit (id, "__volatile__"))
        {
          tok = QVOLATILE;
        }
      else if (cstring_equalLit (id, "__signed"))
        {
          tok = QSIGNED;
        }
      else if (cstring_equalLit (id, "__unsigned"))
        {
          tok = QUNSIGNED;
        }
      else if (cstring_equalLit (id, "__const__"))
        {
          tok = QCONST;
        }
      else if (cstring_equalLit (id, "__alignof__")) 
        {
          tok = CALIGNOF; /* alignof is parsed like sizeof */
        }
      else if (cstring_equalLit (id, "__typeof__")) 
        {
          tok = CTYPEOF;
        }
      else if (cstring_equalLit (id, "typeof")) 
        {
          tok = CTYPEOF;
        }
      else if (cstring_equalLit (id, "__FUNCTION__")
               || cstring_equalLit (id, "__PRETTY_FUNCTION__")) 
        {
          /* These tokens hold the name of the current function as strings */
          /* evans 2001-12-30: changed from exprNode_stringLiteral; bug reported by Jim Zelenka. */
          yylval.expr = exprNode_makeConstantString (id, fileloc_copy (g_currentloc));
          s_tokLength = 0;
          s_lastWasString = TRUE;
          tok = CCONSTANT;
          return tok;
        }
      else if (cstring_equalLit (id, "__attribute__")
               || cstring_equalLit (id, "__asm__")
               || cstring_equalLit (id, "_asm")
               || cstring_equalLit (id, "__asm")
               || cstring_equalLit (id, "__declspec"))
        {
          int depth = 0;
          bool useparens = FALSE;
          bool usebraces = FALSE;
          bool inquote = FALSE;
          bool inescape = FALSE;
          int ic;

          while ((ic = cscanner_input ()) != EOF)
            {
              char cc = (char) ic;

              if (inescape)
                {
                  inescape = FALSE;
                }
              else if (cc == '\\')
                {
                  inescape = TRUE;
                }
              else if (cc == '\"')
                {
                  inquote = !inquote;
                }
              else if (!inquote)
                {
                  if (cc == '(')
                    {
                      if (!useparens)
                        {
                          if (!usebraces)
                            {
                              useparens = TRUE;
                            }
                        }

                      if (useparens)
                        {
                          depth++;
                        }
                    }
                  else if (cc == '{')
                    {
                      if (!usebraces)
                        {
                          if (!useparens)
                            {
                              usebraces = TRUE;
                            }
                        }

                      if (usebraces)
                        {
                          depth++;
                        }
                    }
                  else if (cc == ')' && useparens)
                    {
                      depth--;
                      if (depth == 0) break;
                    }
                  else if (cc == '}' && usebraces)
                    {
                      depth--;
                      if (depth == 0) break;
                    }
                  else if (cc == '}' 
                           && !usebraces && !useparens
                           && cstring_equalLit (id, "__asm"))
                    {
                      /*
                      ** We need this because some MS VC++ include files
                      ** have __asm mov ... }
                      ** Its a kludge, but otherwise would need to parse
                      ** the asm code!
                      */ 
                      return TRBRACE;
                    }
                  else
                    {
                      ;
                    }
                }
              else
                {
                  ;
                }

              if (cc == '\n')
                {
                  context_incLineno ();

                  if (cstring_equalLit (id, "__asm")
                      && !useparens && !usebraces)
                    {
                      break;
                    }
                }
            }
          
          llassert ((useparens && ic == (int) ')')
                    || (usebraces && ic == (int) '}')
                    || (!useparens && !usebraces));

          return BADTOK;
        }
      else if (cstring_equalLit (id, "inline")
               || cstring_equalLit (id, "__inline")
               || cstring_equalLit (id, "_inline")
               || cstring_equalLit (id, "__inline__"))
        {
          tok = QINLINE;
        }
      else
        {
          ;
        }
      
      if (tok != BADTOK)
        {
          return (cscannerHelp_returnToken (tok));
        }
    }

  le = usymtab_lookupSafe (id);

  /*@-dependenttrans@*/
  
  if (uentry_isIter (le))
    {
      yylval.entry = le;
      return (ITER_NAME);
    }
  else if (uentry_isEndIter (le))
    {
      yylval.entry = le;
      return (ITER_ENDNAME);
    }
  else if (uentry_isUndefined (le))
    {
      yylval.cname = cstring_copy (id);

      /* avoid parse errors for certain system built ins */

      if (s_expectingTypeName && (cstring_firstChar (id) == '_')
          && (cstring_secondChar (id) == '_'))
        {
          return (TYPE_NAME_OR_ID);
        }

      return (NEW_IDENTIFIER);
    }
  else if (!uentry_isDeclared (le) && !uentry_isCodeDefined (le))
    {
      if (uentry_isDatatype (le))
        {
          yylval.cname = cstring_copy (id);
          return (NEW_IDENTIFIER);
        }
      else
        {
          yylval.entry = le;              
          return (IDENTIFIER); 
        }
    }
  else if (uentry_isDatatype (le))
    {
      if (!s_expectingTypeName)
        {
          yylval.cname = cstring_copy (id);

          return (NEW_IDENTIFIER);
        }
      else
        {
          yylval.ctyp = uentry_getAbstractType (le);
          
          uentry_setUsed (le, g_currentloc);
          return (TYPE_NAME);
        }
    }
  else
    {
      yylval.entry = le;            
      return (IDENTIFIER); 
    }

  /*@=dependenttrans@*/
}

bool cscannerHelp_processHashIdentifier (/*@only@*/ cstring id)
{
  if (context_inMacro () || context_inIterDef () ||
      context_inIterEnd ())
    {
      uentry le;
      
      context_clearJustPopped ();

      le = usymtab_lookupSafe (id);
      cscanner_setLastIdentifier (id);

      if (uentry_isParam (le) || uentry_isRefParam (le))
        {
          return TRUE;
        }
      else
        {
          return FALSE;
        }
    }
  else
    {
      /*
      ** Will be handled by handleLlSpecial
      */

      cstring_free (id);
      return FALSE;
    }
}

/*@only@*/ exprNode cscannerHelp_processString (void)
{
  exprNode res;
  fileloc loc;
  char *nl = strchr (yytext, '\n');
  cstring ns = cstring_fromCharsNew (yytext);

  if (nl == NULL)
    {
      loc = fileloc_copy (g_currentloc);
      addColumn (size_toInt (cstring_length (ns)));
    }
  else
    {
      char *lastnl = nl;

      loc = fileloc_copy (g_currentloc);

      context_incLineno ();
      
      while ((nl = strchr ((nl + 1), '\n')) != NULL)
        {
          context_incLineno ();
          lastnl = nl;
        }
    }

    
  res = exprNode_stringLiteral (ns, loc);
  return (res);
}

/*
** process a wide character string L"...."
*/

/*@only@*/ exprNode cscannerHelp_processWideString ()
{
  exprNode res;
  fileloc loc;
  char *nl = strchr (yytext, '\n');
  cstring ns;

  llassert (*yytext == 'L');
  yytext++;

  ns = cstring_fromCharsNew (yytext);
  
  if (nl == NULL)
    {
      loc = fileloc_copy (g_currentloc);
      addColumn (size_toInt (cstring_length (ns)));
    }
  else
    {
      char *lastnl = nl;

      loc = fileloc_copy (g_currentloc);

      context_incLineno ();
      
      while ((nl = strchr ((nl + 1), '\n')) != NULL)
        {
          context_incLineno ();
          lastnl = nl;
        }
    }

  res = exprNode_wideStringLiteral (ns, loc);
  return (res);
}

char cscannerHelp_processChar ()
{
  char fchar;
  char next;

  llassert (*yytext != '\0');
  fchar = *(yytext + 1);
  if (fchar != '\\') return fchar;
  
  next = *(yytext + 2);
  
  switch (next)
    {
    case 'n': return '\n';
    case 't': return '\t';
    case '\"': return '\"';
    case '\'': return '\'';
    case '\\': return '\\';
    default: return '\0';
    }
}

double cscannerHelp_processFloat ()
{
  double ret = atof (yytext);

    return (ret);
}

long cscannerHelp_processHex ()
{
  int index = 2;
  long val = 0;

  llassert (yytext[0] == '0'
            && (yytext[1] == 'X' || yytext[1] == 'x'));

  while (yytext[index] != '\0') {
    int tval;
    char c = yytext[index];

    if (c >= '0' && c <= '9') {
      tval = (int) c - (int) '0';
    } else if (c >= 'A' && c <= 'F') {
      tval = (int) c - (int) 'A' + 10;
    } else if (c >= 'a' && c <= 'f') {
      tval = (int) c - (int) 'a' + 10;
    } else if (c == 'U' || c == 'L' || c == 'u' || c == 'l') {
      index++;
      while (yytext[index] != '\0') {
        if (c == 'U' || c == 'L' || c == 'u' || c == 'l') {
          ;
        } else {
          voptgenerror
            (FLG_SYNTAX, 
             message ("Invalid character (%c) following specifier in hex constant: %s",
                      c, cstring_fromChars (yytext)),
             g_currentloc);
        }
        index++;
      }

      break;
    } else {
      voptgenerror
        (FLG_SYNTAX, 
         message ("Invalid character (%c) in hex constant: %s",
                  c, cstring_fromChars (yytext)),
         g_currentloc);
      break;
    }

    val = (val * 16) + tval;
    index++;
  }

  DPRINTF (("Hex constant: %s = %ld", yytext, val));
  return val;
}

long cscannerHelp_processOctal ()
{
  int index = 1;
  long val = 0;

  llassert (yytext[0] == '0' && yytext[1] != 'X' && yytext[1] != 'x');
    
  while (yytext[index] != '\0') {
    int tval;
    char c = yytext[index];
    
    if (c >= '0' && c <= '7') {
      tval = (int) c - (int) '0';
    } else if (c == 'U' || c == 'L' || c == 'u' || c == 'l') {
      index++;
      while (yytext[index] != '\0') {
        if (c == 'U' || c == 'L' || c == 'u' || c == 'l') {
          ;
        } else {
          voptgenerror
            (FLG_SYNTAX, 
             message ("Invalid character (%c) following specifier in octal constant: %s",
                      c, cstring_fromChars (yytext)),
             g_currentloc);
        }
        index++;
      }

      break;
    } else {
      voptgenerror
        (FLG_SYNTAX, 
         message ("Invalid character (%c) in octal constant: %s",
                  c, cstring_fromChars (yytext)),
         g_currentloc);
      break;
    }

    val = (val * 8) + tval;
    index++;
  }

  DPRINTF (("Octal constant: %s = %ld", yytext, val));
  return val;
}

long cscannerHelp_processDec ()
{
  return (atol (yytext));
}

int cscannerHelp_processSpec (int tok)
{
  size_t length = strlen (yytext);
  
  if (s_inSpecPart)
    {

      /*drl 12/11/2002
        patched to fix assert failures in constraint code.
        Added the else if test so that splint does not treat MaxSet and MaxRead
        as identifies*/
        
      if (s_whichSpecPart == QMODIFIES
          || s_whichSpecPart == QDEFINES
          || s_whichSpecPart == QUSES 
          || s_whichSpecPart == QALLOCATES
          || s_whichSpecPart == QSETS
          || s_whichSpecPart == QRELEASES)
        
        {
          DPRINTF((message("Treating specifaction keyword %s as an identifiers.  (This corresponds to"
                           " token %d and we're in the  specification denoted by %d see cgrammar_tokens.h"
                           " for an explanation of these numbers",
                           yytext, tok, s_whichSpecPart)
                   ));
                           
          ; /* Allow specificiation keywords to be used as identifiers in these contexts. */
        }
      else if ( (s_whichSpecPart == QPRECLAUSE
                 || s_whichSpecPart == QPOSTCLAUSE
                 || s_whichSpecPart == QINVARIANT )
                &&  (!cscannerHelp_isConstraintToken(tok) )
              )
        {
          DPRINTF((message("Treating specifaction keyword %s as an identifiers.  (This corresponds to"
                           " token %d and we're in the  specification denoted by %d see cgrammar_tokens.h"
                           " for an explanation of these numbers",
                           yytext, tok, s_whichSpecPart)
                   ));
          
          /* Allow specificiation keywords to be used as identifiers in these contexts. */
        }
      else
        {
          cscannerHelp_setTokLengthT (length);
          return cscannerHelp_returnToken (tok);
        }
    }
  
  context_saveLocation ();
  cscannerHelp_setTokLengthT (length);
  return (cscannerHelp_processIdentifier (cscannerHelp_makeIdentifier (yytext)));
}

void cscannerHelp_expectingMetaStateName ()
{
  llassert (!s_expectingMetaStateName);
  llassert (context_inFunctionHeader ());
  s_expectingMetaStateName = TRUE;
}

void cscannerHelp_clearExpectingMetaStateName ()
{
  llassert (s_expectingMetaStateName);
  s_expectingMetaStateName = FALSE;
}

bool cscannerHelp_isConstraintToken (int tok)
   /* drl added 12/11/2002
      Tell whether a token has special meaning
      within a function constraint
   */
{
  return (tok == QMAXSET || tok == QMAXREAD);
          /* || tok == QMINREAD || tok == QMINSET */
  /* uncomment the additional if statement tests when minSet and minRead are supported */
}

bool cscannerHelp_processMacro (void)
{
  uentry e2;
  ctype ct;
  int noparams = 0;
  cstring fname = cstring_undefined;
  bool res = TRUE;
  bool isspecfcn = FALSE;
  bool isiter = FALSE;
  bool skipparam = FALSE;
  bool isenditer = FALSE;
  bool unknownm = FALSE;
  bool hasParams = FALSE;
  bool emptyMacro = FALSE;
  char c = skip_whitespace ();
  fileloc loc = fileloc_noColumn (g_currentloc);

  /* are both of these necessary?  what do they mean? */
  uentryList specparams = uentryList_undefined;
  uentryList pn = uentryList_undefined;

  context_resetMacroMissingParams ();

  if (c == '\0' || c == '\n')
    {
      llcontbug (cstring_makeLiteral ("Bad macro"));
      fileloc_free (loc);
      return FALSE;
    }
  
  fname = cstring_appendChar (fname, c);  

  while ((c = macro_nextChar ()) != '(' && c != '\0'
         && c != ' ' && c != '\t' && c != '\n')
    {
      fname = cstring_appendChar (fname, c);
    }

  if (c == ' ' || c == '\t' || c == '\n')
    {
      char oldc = c;

      if (c != '\n')
        {
          while (c == ' ' || c == '\t')
            {
              c = macro_nextChar ();
            }
          cscanner_unput ((int) c);
        }

      if (c == '\n')
        {
          emptyMacro = TRUE;
          cscanner_unput ((int) c);
        }

      c = oldc;
    }

  hasParams = (c == '(');
  
  if (usymtab_exists (fname))
    {
      e2 = usymtab_lookupExpose (fname);
      ct = uentry_getType (e2);

      if (uentry_isCodeDefined (e2) 
          && fileloc_isUser (uentry_whereDefined (e2)))
        {
          if (optgenerror 
              (FLG_MACROREDEF,
               message ("Macro %s already defined", fname),
               loc))
            {
              uentry_showWhereDefined (e2);
              uentry_clearDefined (e2);
            }

          if (uentry_isFunction (e2))
            {
              uentry_setType (e2, ctype_unknown);
              ct = ctype_unknown;
              unknownm = TRUE;
              context_enterUnknownMacro (e2); 
            }
          else
            {
              context_enterConstantMacro (e2);
            }
        }
      else
        {
          if (uentry_isForward (e2) && uentry_isFunction (e2))
            {
              unknownm = TRUE;

              voptgenerror 
                (FLG_MACROFCNDECL,
                 message
                 ("Parameterized macro has no prototype or specification: %s ", 
                  fname),
                 loc);
              
              ct = ctype_unknown;
              uentry_setType (e2, ctype_unknown);
              uentry_setFunctionDefined (e2, loc); 
              uentry_setUsed (e2, fileloc_undefined);
              context_enterUnknownMacro (e2); 
            }
          else
            {
              if (uentry_isIter (e2))
                {
                  isiter = TRUE;
                  specparams = uentry_getParams (e2);
                  noparams = uentryList_size (specparams);
                  uentry_setDefined (e2, loc);
                  context_enterIterDef (e2); 
                }
              else if (uentry_isEndIter (e2))
                {
                  isenditer = TRUE;
                  uentry_setDefined (e2, loc);
                  context_enterIterEnd (e2); /* don't care about it now */
                  /* but should parse like an iter! */
                }
              else if (uentry_isConstant (e2))
                {
                  if (hasParams)
                    {
                      voptgenerror 
                        (FLG_INCONDEFS, 
                         message ("Constant %s implemented as parameterized macro",
                                  fname),
                         g_currentloc);
                      
                      uentry_showWhereSpecified (e2);
                      uentry_setType (e2, ctype_unknown);
                      uentry_makeConstantFunction (e2);
                      uentry_setDefined (e2, g_currentloc);
                      uentry_setFunctionDefined (e2, g_currentloc);
                      context_enterUnknownMacro (e2); 
                    }
                  else
                    {
                      if (!uentry_isSpecified (e2))
                        {
                          fileloc oloc = uentry_whereDeclared (e2);

                          if (fileloc_isLib (oloc))
                            {
                              ;
                            }
                          else if (fileloc_isUndefined (oloc)
                                   || fileloc_isPreproc (oloc))
                            {
                              if (!emptyMacro)
                                {
                                  voptgenerror
                                    (FLG_MACROCONSTDECL,
                                     message 
                                     ("Macro constant %q not declared",
                                      uentry_getName (e2)),
                                     loc);                       
                                }
                            }
                          else if (!fileloc_withinLines (oloc, loc, 2))
                            { /* bogus!  will give errors if there is too much whitespace */
                              voptgenerror
                                (FLG_SYNTAX,
                                 message 
                                 ("Macro constant name %s does not match name in "
                                  "previous constant declaration.  This constant "
                                  "is declared at %q", fname, 
                                  fileloc_unparse (oloc)),
                                 loc);
                            }
                          else
                            {
                              /* No warning */
                            }
                        }

                      context_enterConstantMacro (e2);        
                      cstring_free (fname);
                      fileloc_free (loc);
                      return res;
                    }

                }
              else if (ctype_isFunction (ct))
                {
                  isspecfcn = TRUE;
                  specparams = ctype_argsFunction (ct);
                  noparams = uentryList_size (specparams);
                  
                  uentry_setFunctionDefined (e2, loc); 
                  context_enterMacro (e2);
                }
              else if (uentry_isVar (e2))
                {
                  if (hasParams)
                    {
                      voptgenerror
                        (FLG_INCONDEFS,
                         message ("Variable %s implemented as parameterized macro", 
                                  fname),
                         loc);

                      uentry_showWhereSpecified (e2);
                      uentry_setType (e2, ctype_unknown);
                      uentry_makeVarFunction (e2);
                      uentry_setDefined (e2, g_currentloc);
                      uentry_setFunctionDefined (e2, g_currentloc);
                      context_enterUnknownMacro (e2); 
                    }
                  else
                    {
                      uentry ucons = uentry_makeConstant (fname,
                                                          ctype_unknown,
                                                          loc);
                      if (uentry_isExpandedMacro (e2))
                        {
                          ; /* okay */
                        }
                      else
                        {
                          if (optgenerror 
                              (FLG_INCONDEFS,
                               message ("Variable %s implemented by a macro",
                                        fname),
                               loc))
                            {
                              uentry_showWhereSpecified (e2);
                            }
                        }

                      uentry_setDefined (e2, loc);
                      uentry_setUsed (ucons, loc);

                      context_enterConstantMacro (ucons);
                      uentry_markOwned (ucons);
                      cstring_free (fname);
                      return res;
                    }
                }
              else
                {
                  if (uentry_isDatatype (e2))
                    {
                      vgenhinterror 
                        (FLG_SYNTAX,
                         message ("Type implemented as macro: %x", 
                                  uentry_getName (e2)),
                         message ("A type is implemented using a macro definition.  A "
                                  "typedef should be used instead."),
                         g_currentloc);

                      cscannerHelp_swallowMacro ();
                      /* Must exit scope (not sure why a new scope was entered?) */
                      usymtab_quietExitScope (g_currentloc);
                      uentry_setDefined (e2, g_currentloc);
                      res = FALSE;
                    }
                  else
                    {
                      llcontbug 
                        (message ("Unexpanded macro not function or constant: %q", 
                                  uentry_unparse (e2)));
                      uentry_setType (e2, ctype_unknown);
                      
                      if (hasParams)
                        {
                          uentry_makeVarFunction (e2);
                          uentry_setDefined (e2, g_currentloc);
                          uentry_setFunctionDefined (e2, g_currentloc);
                          context_enterUnknownMacro (e2); 
                        }
                    }
                }
            }
        }
    }
  else
    {
      uentry ce;

      /* evans 2001-09-09 - if it has params, assume a function */
      if (hasParams)
        {
          voptgenerror 
            (FLG_MACROMATCHNAME,
             message ("Unexpanded macro %s does not match name of a declared "
                      "function. The name used in the control "
                      "comment on the previous line should match.",
                      fname),
             loc);
          
          ce = uentry_makeFunction (fname, ctype_unknown, 
                                    typeId_invalid,
                                    globSet_undefined,
                                    sRefSet_undefined,
                                    warnClause_undefined,
                                    fileloc_undefined);      
          uentry_setUsed (ce, loc); /* perhaps bogus? */
          e2 = usymtab_supEntryReturn (ce);
          context_enterUnknownMacro (e2);             
        }
      else
        {
          voptgenerror 
            (FLG_MACROMATCHNAME,
             message ("Unexpanded macro %s does not match name of a constant "
                      "or iter declaration.  The name used in the control "
                      "comment on the previous line should match.  "
                      "(Assuming macro defines a constant.)", 
                      fname),
             loc);
          
          ce = uentry_makeConstant (fname, ctype_unknown, fileloc_undefined);      
          uentry_setUsed (ce, loc); /* perhaps bogus? */
          e2 = usymtab_supEntryReturn (ce);
          
          context_enterConstantMacro (e2);            
          cstring_free (fname);
          fileloc_free (loc);
          return res;
        }
    }
  
  /* in macros, ( must follow immediatetly after name */
  
  if (hasParams)
    {
      int paramno = 0;
      
      c = skip_whitespace ();

      while (c != ')' && c != '\0')
        {
          uentry  param;
          bool    suppress = context_inSuppressRegion ();
          cstring paramname = cstring_undefined;

          /*
          ** save the parameter location
          */

          decColumn ();
          context_saveLocation ();
          incColumn ();

          while (c != ' ' && c != '\t' && c != ',' && c != '\0' && c != ')')
            {
              paramname = cstring_appendChar (paramname, c);
              c = macro_nextChar ();
            }
          
          if (c == ' ' || c == '\t') c = skip_whitespace ();

          if (c == ',')
            {
              c = macro_nextChar ();
              if (c == ' ' || c == '\t') c = skip_whitespace ();
            }
          
          if (c == '\0')
            {
              llfatalerror (cstring_makeLiteral
                            ("Bad macro syntax: uentryList"));
            }
          
          if ((isspecfcn || isiter) && (paramno < noparams)
              && !uentry_isElipsisMarker (uentryList_getN 
                                          (specparams, paramno)))
            {
              fileloc sloc = context_getSaveLocation ();
              uentry decl = uentryList_getN (specparams, paramno);
              sRef sr;
              
              param = uentry_nameCopy (paramname, decl);
                              
              uentry_setParam (param);
              sr = sRef_makeParam (paramno, uentry_getType (param), 
                                   stateInfo_makeLoc (sloc, SA_DECLARED));

              if (sRef_getNullState (sr) == NS_ABSNULL)
                {
                  ctype pt = ctype_realType (uentry_getType (param));

                  if (ctype_isUser (pt))
                    {
                      uentry te = usymtab_getTypeEntrySafe (ctype_typeId (pt));
                      
                      if (uentry_isValid (te))
                        {
                          sRef_setStateFromUentry (sr, te);
                        }
                    }
                  else
                    {
                      sRef_setNullState (sr, NS_UNKNOWN, sloc);
                    }
                }

              uentry_setSref (param, sr);
              uentry_setDeclaredForceOnly (param, sloc);

              skipparam = isiter && uentry_isOut (uentryList_getN (specparams, paramno));
            }
          else
            {
              fileloc sloc = context_getSaveLocation ();

              param = uentry_makeVariableSrefParam 
                (paramname, ctype_unknown, fileloc_copy (sloc), 
                 sRef_makeParam (paramno, ctype_unknown,
                                 stateInfo_makeLoc (sloc, SA_DECLARED)));
              DPRINTF (("Unknown param: %s", uentry_unparseFull (param)));
              cstring_free (paramname);

              sRef_setPosNull  (uentry_getSref (param), sloc);
              uentry_setDeclaredForce (param, sloc);

              skipparam = FALSE;
              fileloc_free (sloc);
            }

          if (!skipparam)
            {
              llassert (!uentry_isElipsisMarker (param));

              if (!suppress)
                {
                  sRef_makeUnsafe (uentry_getSref (param));
                }
              
              pn = uentryList_add (pn, uentry_copy (param));
              usymtab_supEntry (param);
            }
          else
            {
              /* don't add param */
              uentry_free (param);
            }

          if (c == ',') 
            {
              (void) macro_nextChar ();
              c = skip_whitespace ();
            }

          paramno++;
        }
      
      if (c == ')')
        {
          if (isspecfcn || isiter)
            {
              if (paramno != noparams && noparams >= 0)
                {
                  cscannerHelp_advanceLine ();

                  voptgenerror 
                    (FLG_INCONDEFS,
                     message ("Macro %s specified with %d args, defined with %d", 
                              fname, noparams, paramno),
                     g_currentloc);

                  uentry_showWhereSpecified (e2);
                  uentry_resetParams (e2, pn);
                }
            }
          else
            {
              uentry_resetParams (e2, pn);
            }
        }
    }
  else
    {
      /*
      ** the form should be:
      **
      ** # define newname oldname
      ** where oldname refers to a function matching the specification
      ** of newname.
      */

      if (unknownm)
        {
          sRef_setGlobalScope ();
          usymtab_supGlobalEntry (uentry_makeVariableLoc (fname, ctype_unknown));
          sRef_clearGlobalScope ();
        }
      else
        {
          context_setMacroMissingParams ();
        }
    }
  
  
  /* context_setuentryList (pn); */
  usymtab_enterScope ();

  fileloc_free (loc);
  cstring_free (fname);

  return res;
}

void cscannerHelp_setTokLength (int len) 
{
  addColumn (len);
  s_tokLength = len;
  DPRINTF (("Set tok length: %d", len));
}

void cscannerHelp_setTokLengthT (size_t len)
{
  cscannerHelp_setTokLength (size_toInt (len));
}

void cscannerHelp_advanceLine (void)
{
  s_tokLength = 0;
  beginLine ();
}

int cscannerHelp_returnToken (int t)
{
  if (s_tokLength > fileloc_column (g_currentloc)) {
    yylval.tok = lltok_create (t, fileloc_copy (g_currentloc));
  } else {
    yylval.tok = lltok_create (t, fileloc_decColumn (g_currentloc, s_tokLength)); 
  }

  s_tokLength = 0; 
  s_lastWasString = FALSE; 
  return (t); 
}

int cscannerHelp_returnTokenLength (int t, int length)
{
  cscannerHelp_setTokLength (length);
  return cscannerHelp_returnToken (t);
}

int cscannerHelp_returnString (cstring s)
{
  yylval.expr = exprNode_stringLiteral (s, fileloc_decColumn (g_currentloc, s_tokLength)); 
  s_tokLength = 0;
  s_lastWasString = TRUE; 
  return (CCONSTANT); 
}

int cscannerHelp_returnInt (ctype ct, long val)
{
  ctype c = ct;

  if (ctype_equal (ct, ctype_int))
    {
      if (val == 0)
        {
          c = context_typeofZero ();
        }
      else if (val == 1)
        {
          c = context_typeofOne ();
        }
      else
        {
          ;
        }
    }
  
  yylval.expr = exprNode_numLiteral (c, cstring_fromChars (yytext), 
                                     fileloc_decColumn (g_currentloc, s_tokLength),
                                     val);   
  s_tokLength = 0; 
  s_lastWasString = FALSE;
  return (CCONSTANT);
}

int cscannerHelp_returnFloat (ctype ct, double f)
{
  yylval.expr = exprNode_floatLiteral (f, ct, cstring_fromChars (yytext), 
                                       fileloc_decColumn (g_currentloc, s_tokLength));
  s_tokLength = 0; 
  s_lastWasString = FALSE;
  return (CCONSTANT);
}

int cscannerHelp_returnChar (char c)
{
  yylval.expr = exprNode_charLiteral (c, cstring_fromChars (yytext), 
                                      fileloc_decColumn (g_currentloc, s_tokLength));
  s_tokLength = 0; 
  s_lastWasString = FALSE;
  return (CCONSTANT);
}

int cscannerHelp_returnType (int tok, ctype ct) 
{
  yylval.ctyp = ct; 
  s_tokLength = 0; 
  s_lastWasString = FALSE;
  return tok;
}

int cscannerHelp_returnExpr (exprNode e)
{
  yylval.expr = e; 
  s_tokLength = 0; 
  s_lastWasString = TRUE; 
  return (CCONSTANT); 
}

void cscannerHelp_setExpectingTypeName ()
{
  s_expectingTypeName = TRUE;
}

void cscannerHelp_clearExpectingTypeName ()
{
  s_expectingTypeName = FALSE;
}

bool cscannerHelp_isExpectingTypeName ()
{
  return s_expectingTypeName;
}

int cscannerHelp_processTextIdentifier (char *text)
{
  context_saveLocation (); 
  cscannerHelp_setTokLength (size_toInt (mstring_length (text))); 
  return cscannerHelp_processIdentifier (cscannerHelp_makeIdentifier (text)); 
}

static bool s_continueLine = FALSE;

int cscannerHelp_handleNewLine ()
{
  context_incLineno ();

  if (s_tokLength != 0) { 
    s_tokLength = 0; 
    /* No error to report 
       voptgenerror
       (FLG_SYNTAX, 
       message ("Likely parse error: token spans multiple lines."),
       g_currentloc);
    */
  }
  
  if (s_continueLine)
    {
      s_continueLine = FALSE;
    }
  else 
    {
      if (context_inMacro ())
        {
          /* Don't use return cscannerHelp_returnToken */
          /* !!! evans 2002-03-13 */
          yylval.tok = lltok_create (TENDMACRO, fileloc_copy (g_currentloc)); 
          s_lastWasString = FALSE;
          return TENDMACRO;
        }  
    }

  return BADTOK;
}

void cscannerHelp_setContinueLine ()
{
  s_continueLine = TRUE;
}

void cscannerHelp_exitSpecPart ()
{
  llassert (s_inSpecPart);
  s_inSpecPart = FALSE;
  s_whichSpecPart = BADTOK;
}