Tokenizer.hpp

Go to the documentation of this file.

#if defined(_MSC_VER) && (_MSC_VER >= 1200)
#pragma once
#endif
#ifndef COMMON_TOKENIZER_H_
#define COMMON_TOKENIZER_H_

#include "Error.hpp"

namespace common
{

class Stream;


class TokenizerError : public Error
{
private:
  size_t m_Char, m_Row, m_Col;
public:
  TokenizerError(size_t Char, size_t Row, size_t Col, const tstring &Msg = _T(""), const tstring &File = _T(""), int a_Line = 0);

  size_t GetCharNum() { return m_Char; }
  size_t GetRowNum() { return m_Row; }
  size_t GetColNum() { return m_Col; }
};

class Tokenizer_pimpl;

class Tokenizer
{
private:
  scoped_ptr<Tokenizer_pimpl> pimpl;

public:
  static const uint FLAG_TOKEN_EOL         = 0x01;
  static const uint FLAG_MULTILINE_STRINGS = 0x02;

  enum TOKEN
  {
    TOKEN_EOF,        
    TOKEN_EOL,        
    TOKEN_SYMBOL,     
    TOKEN_INTEGER,    
    TOKEN_FLOAT,      
    TOKEN_CHAR,       
    TOKEN_IDENTIFIER, 
    TOKEN_KEYWORD,    
    TOKEN_STRING,     
  };

  Tokenizer(const tchar *Input, size_t NumChars, uint Flags);
  Tokenizer(const tstring *Input, uint Flags);
  Tokenizer(Stream *Input, uint Flags);

  ~Tokenizer();

  void RegisterKeyword(uint Id, const tstring &Keyword);

  void RegisterKeywords(const tchar **Keywords, size_t KeywordCount);


  void Next();

  TOKEN GetToken();

  size_t GetCharNum();
  size_t GetRowNum();
  size_t GetColNum();

  const tstring & GetString();
  tchar GetChar();
  uint GetId();
  bool GetUint1(uint1 *Out);
  bool GetUint2(uint2 *Out);
  bool GetUint4(uint4 *Out);
  bool GetUint8(uint8 *Out);
  bool GetInt1(int1 *Out);
  bool GetInt2(int2 *Out);
  bool GetInt4(int4 *Out);
  bool GetInt8(int8 *Out);
  uint1 MustGetUint1();
  uint2 MustGetUint2();
  uint4 MustGetUint4();
  uint8 MustGetUint8();
  int1 MustGetInt1();
  int2 MustGetInt2();
  int4 MustGetInt4();
  int8 MustGetInt8();
  bool GetFloat(float *Out);
  bool GetDouble(double *Out);
  float MustGetFloat();
  double MustGetDouble();


  void CreateError();
  void CreateError(const tstring &Msg);

  bool QueryToken(TOKEN Token);
  bool QueryToken(TOKEN Token1, TOKEN Token2);
  bool QueryEOF();
  bool QueryEOL();
  bool QuerySymbol(tchar Symbol);
  bool QueryIdentifier(const tstring &Identifier);
  bool QueryKeyword(uint KeywordId);
  bool QueryKeyword(const tstring &Keyword);


  void AssertToken(TOKEN Token);
  void AssertToken(TOKEN Token1, TOKEN Token2);
  void AssertEOF();
  void AssertEOL();
  void AssertSymbol(tchar Symbol);
  void AssertIdentifier(const tstring &Identifier);
  void AssertKeyword(uint KeywordId);
  void AssertKeyword(const tstring &Keyword);

  static const tchar * GetTokenName(TOKEN Token);
};

class TokenWriter
{
public:
  static const uint ESCAPE_EOL   = 0x01; 
  static const uint ESCAPE_OTHER = 0x02; 


  static void Escape(tstring *Out, const tstring &In, uint EscapeFlags);

  TokenWriter(tstring *outStr);
  TokenWriter(common::Stream *outStream);

  //====== Configuration
  const tstring & GetEOL() { return m_EOL; }
  const tstring & GetIndent() { return m_Indent; }
  void SetEOL(const tstring &s) { m_EOL = s; }
  void SetIndent(const tstring &s) { m_Indent = s; }
  uint GetEscapeFlags() { return m_EscapeFlags; }
  void SetEscapeFlags(uint v) { m_EscapeFlags = v; }
  void RegisterSymbol(tchar ch, bool spaceBefore, bool spaceAfter, int levelDelta);
  void RegisterSymbol(const tstring &s, bool spaceBefore, bool spaceAfter, int levelDelta);
  void RegisterKeyword(const tstring &s, int levelDelta);
  
  uint GetLevel() { return m_Level; }

  void WriteSpace(tchar ch = _T(' '));
  void WriteSpace(const tstring &s);
  void WriteEOL();

  void WriteKeyword(const tstring &s);
  void WriteIdentifier(const tstring &s);
  void WriteSymbol(tchar ch);
  void WriteSymbol(const tstring &s);
  void WriteString(const tstring &s);
  void WriteUint1(uint1 v, bool hex = false);
  void WriteUint2(uint2 v, bool hex = false);
  void WriteUint4(uint4 v, bool hex = false);
  void WriteUint8(uint8 v, bool hex = false);
  void WriteInt1(int1 v);
  void WriteInt2(int2 v);
  void WriteInt4(int4 v);
  void WriteInt8(int8 v);
  void WriteFloat(float v, char Mode = 'g', int Precision = 6);
  void WriteDouble(double v, char Mode = 'g', int Precision = 6);
  void WriteNumberString(const tstring &v);
  void WriteComment(const tstring &s, bool alwaysMultiline = false);

private:
  tstring *m_OutString;
  common::Stream *m_OutStream;

  tstring m_EOL, m_Indent;
  uint m_EscapeFlags;
  tstring m_Symbols_Chars_SpaceBefore, m_Symbols_Chars_SpaceAfter;
  tstring m_Symbols_Chars_LevelInc, m_Symbols_Chars_LevelDec;
  STRING_VECTOR m_Symbols_Strings_SpaceBefore, m_Symbols_Strings_SpaceAfter;
  STRING_VECTOR m_Symbols_Strings_LevelInc, m_Symbols_Strings_LevelDec;
  STRING_VECTOR m_Keywords_LevelInc, m_Keywords_LevelDec;

  uint m_Level;
  bool m_WasEOL;
  bool m_SpaceRequired;

  void Ctor();
  void WriteIndent(uint level);
  void LevelInc() { m_Level++; }
  void LevelDec() { assert(m_Level > 0); m_Level--; }
  void WriteRaw(tchar ch);
  void WriteRaw(const tchar *s);
  void WriteRaw(const tstring &s);
  int GetKeywordLevelDelta(const tstring &s);
  int GetSymbolLevelDelta(tchar ch);
  int GetSymbolLevelDelta(const tstring &s);
  bool IsSymbolSpaceBefore(tchar ch);
  bool IsSymbolSpaceBefore(const tstring &s);
  bool IsSymbolSpaceAfter(tchar ch);
  bool IsSymbolSpaceAfter(const tstring &s);
  void WriteMultiLineComment(const tstring &s);
  void WriteSingleLineComment(const tstring &s);
};

// code_tokenizer

} // namespace common

#endif

---