Tokenizer

Engine/source/core/tokenizer.h

More...

Public Types

enum
_Anonymous_ {  MaxTokenSize = 1023
}

Private Attributes

Public Functions

bool
advanceToken(const bool crossLine, const bool assertAvailable)
bool
bool
bool
findToken(const char * pCmp)
bool
findToken(U32 start, const char * pCmp)
const char *
getLinePosition(const U32 pos, U32 lowIndex, S32 highIndex)
const char *
bool
openFile(const char * pFileName)
bool
openFile(Stream * pStream)
bool
regressToken(const bool crossLine)
bool
setBuffer(const char * buffer, U32 bufferSize)
bool
setSingleTokens(const char * singleTokens)
bool
tokenICmp(const char * pCmp)

Detailed Description

Public Types

@71

Enumerator

MaxTokenSize = 1023

Private Attributes

U32 mBufferSize 
S32 mCurrPos 
char mCurrTokenBuffer [MaxTokenSize+1]
char mFileName [1024]
Vector< U32 > mLinePositions 
char * mpBuffer 
char * mSingleTokens 
S32 mStartPos 
bool mTokenIsCurrent 
bool mTokenIsQuoted 

Public Functions

Tokenizer()

~Tokenizer()

advanceToken(const bool crossLine, const bool assertAvailable)

buildLinePositions()

clear()

endOfFile()

findToken(const char * pCmp)

findToken(U32 start, const char * pCmp)

getCurrentLine()

getCurrentPos()

getFileName()

getLinePosition(const U32 pos, U32 lowIndex, S32 highIndex)

getNextToken()

getToken()

getTokenLineOffset()

openFile(const char * pFileName)

openFile(Stream * pStream)

regressToken(const bool crossLine)

reset()

setBuffer(const char * buffer, U32 bufferSize)

setCurrentPos(U32 pos)

setSingleTokens(const char * singleTokens)

tokenAvailable()

tokenICmp(const char * pCmp)

tokenIsQuoted()