Tokenizer

_Anonymous_ {  MaxTokenSize = 1023
}

mBufferSize 

mCurrPos 

mCurrTokenBuffer [MaxTokenSize+1]

mFileName [1024]

mLinePositions 

mpBuffer 

mSingleTokens 

mStartPos 

mTokenIsCurrent 

mTokenIsQuoted 

Tokenizer()

~Tokenizer()

advanceToken(const bool crossLine, const bool assertAvailable)

buildLinePositions()

clear()

endOfFile()

findToken(const char * pCmp)

findToken(U32 start, const char * pCmp)

getCurrentLine()

getCurrentPos()

getFileName()

getLinePosition(const U32 pos, U32 lowIndex, S32 highIndex)

getNextToken()

getToken()

getTokenLineOffset()

openFile(const char * pFileName)

openFile(Stream * pStream)

regressToken(const bool crossLine)

reset()

setBuffer(const char * buffer, U32 bufferSize)

setCurrentPos(U32 pos)

setSingleTokens(const char * singleTokens)

tokenAvailable()

tokenICmp(const char * pCmp)

tokenIsQuoted()