tokenizer.h

Classes:

class
Detailed Description

  1
  2//-----------------------------------------------------------------------------
  3// Copyright (c) 2012 GarageGames, LLC
  4//
  5// Permission is hereby granted, free of charge, to any person obtaining a copy
  6// of this software and associated documentation files (the "Software"), to
  7// deal in the Software without restriction, including without limitation the
  8// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  9// sell copies of the Software, and to permit persons to whom the Software is
 10// furnished to do so, subject to the following conditions:
 11//
 12// The above copyright notice and this permission notice shall be included in
 13// all copies or substantial portions of the Software.
 14//
 15// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 18// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21// IN THE SOFTWARE.
 22//-----------------------------------------------------------------------------
 23
 24#ifndef _TOKENIZER_H_
 25#define _TOKENIZER_H_
 26
 27//Includes
 28#ifndef _PLATFORM_H_
 29#include "platform/platform.h"
 30#endif
 31
 32#ifndef _STREAM_H_
 33#include "core/stream/stream.h"
 34#endif
 35
 36#ifndef _TVECTOR_H_
 37#include "core/util/tVector.h"
 38#endif
 39
 40class SizedStream;
 41
 42class Tokenizer
 43{
 44  public:
 45   enum
 46   {
 47      MaxTokenSize = 1023
 48   };
 49
 50  private:
 51   char   mFileName[1024];
 52
 53   char* mpBuffer;
 54   U32   mBufferSize;
 55
 56   S32 mCurrPos;
 57   S32 mStartPos;
 58
 59   bool mTokenIsQuoted;
 60
 61   char   mCurrTokenBuffer[MaxTokenSize + 1];
 62   bool   mTokenIsCurrent;
 63
 64   char*  mSingleTokens;
 65
 66   Vector<U32> mLinePositions;
 67
 68  public:
 69   Tokenizer();
 70   ~Tokenizer();
 71
 72   bool openFile(const char* pFileName);
 73   bool openFile(Stream* pStream);
 74   void setBuffer(const char* buffer, U32 bufferSize);
 75
 76   void setSingleTokens(const char* singleTokens);
 77
 78   void buildLinePositions();
 79
 80   bool        advanceToken(const bool crossLine, const bool assertAvailable = false);
 81   bool        regressToken(const bool crossLine);
 82   bool        tokenAvailable();
 83
 84   const char* getToken() const;
 85   const char* getNextToken();
 86   bool tokenICmp(const char* pCmp) const;
 87   bool tokenIsQuoted() const { return mTokenIsQuoted; }
 88
 89   bool findToken(const char* pCmp);
 90   bool findToken(U32 start, const char* pCmp);
 91
 92   const char* getFileName() const     { return mFileName; }
 93
 94   U32   getLinePosition(const U32 pos, U32 lowIndex = 0, S32 highIndex = -1);
 95   U32   getCurrentLine();
 96   U32   getTokenLineOffset();
 97   U32   getCurrentPos() const { return mCurrPos; }
 98
 99   bool  setCurrentPos(U32 pos);
100
101   // Resets our active data but leaves the buffer intact
102   bool  reset();
103   // Clears the buffer and resets the active data
104   bool  clear();
105
106   bool  endOfFile();
107};
108
109
110#endif //_TOKENIZER_H_
111