antlr3tokenstream.c File Reference

---

Detailed Description

Default implementation of CommonTokenStream.

#include <antlr3tokenstream.h>

Include dependency graph for antlr3tokenstream.c:

Functions
static ANTLR3_UINT32	_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM	antlr3CommonTokenDebugStreamSourceNew (ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM	antlr3CommonTokenStreamNew (ANTLR3_UINT32 hint)
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM	antlr3CommonTokenStreamSourceNew (ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
static void	antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
static void	antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream)
ANTLR3_API pANTLR3_TOKEN_STREAM	antlr3TokenStreamNew ()
static void	consume (pANTLR3_INT_STREAM is)
static void	consumeInitialHiddenTokens (pANTLR3_INT_STREAM is)
	Debug only method to flag consumption of initial off-channel tokens in the input stream.
static void	dbgConsume (pANTLR3_INT_STREAM is)
static ANTLR3_UINT32	dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
static ANTLR3_MARKER	dbgMark (pANTLR3_INT_STREAM is)
static void	dbgRewindLast (pANTLR3_INT_STREAM is)
static void	dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
static void	dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
static pANTLR3_COMMON_TOKEN	dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
	As per the normal tokLT but sends information to the debugger.
static void	discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard)
static void	discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype)
static void	fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
static pANTLR3_COMMON_TOKEN	get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
static pANTLR3_STRING	getSourceName (pANTLR3_INT_STREAM is)
static pANTLR3_LIST	getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
static pANTLR3_VECTOR	getTokens (pANTLR3_COMMON_TOKEN_STREAM cts)
static pANTLR3_LIST	getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
static pANTLR3_TOKEN_SOURCE	getTokenSource (pANTLR3_TOKEN_STREAM ts)
static pANTLR3_LIST	getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
static pANTLR3_LIST	getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
static pANTLR3_COMMON_TOKEN	LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
static ANTLR3_MARKER	mark (pANTLR3_INT_STREAM is)
static void	release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
static void	rewindLast (pANTLR3_INT_STREAM is)
static void	rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
static void	seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
static void	setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
static void	setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource)
static void	setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
static ANTLR3_UINT32	size (pANTLR3_INT_STREAM is)
static ANTLR3_UINT32	skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
static ANTLR3_UINT32	skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
static ANTLR3_MARKER	tindex (pANTLR3_INT_STREAM is)
static pANTLR3_COMMON_TOKEN	tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
	Get the ith token from the current position 1.
static pANTLR3_STRING	toString (pANTLR3_TOKEN_STREAM ts)
static pANTLR3_STRING	toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
static pANTLR3_STRING	toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)

---

Function Documentation

static ANTLR3_UINT32 _LA	(	pANTLR3_INT_STREAM	is,
		ANTLR3_INT32	i
	)			[static]

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenDebugStreamSourceNew	(	ANTLR3_UINT32	hint,
		pANTLR3_TOKEN_SOURCE	source,
		pANTLR3_DEBUG_EVENT_LISTENER	debugger
	)

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_FALSE, antlr3CommonTokenStreamSourceNew(), ANTLR3_INT_STREAM_struct::consume, dbgConsume(), dbgLA(), dbgMark(), dbgRewindLast(), dbgRewindStream(), dbgSeek(), dbgTokLT(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_INT_STREAM_struct::seek, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.

Here is the call graph for this function:

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamNew

(

ANTLR3_UINT32

hint

)

References _LA(), ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_MALLOC, ANTLR3_TOKENSTREAM, antlr3IntStreamNew(), antlr3TokenStreamNew(), antlr3VectorNew(), consume(), ANTLR3_INT_STREAM_struct::consume, discardOffChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannelToks, discardTokenType(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardTokenType, ANTLR3_TOKEN_STREAM_struct::get, getSourceName(), ANTLR3_INT_STREAM_struct::getSourceName, getTokenRange(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokenRange, getTokens(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokens, getTokensList(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensList, getTokenSource(), ANTLR3_TOKEN_STREAM_struct::getTokenSource, getTokensSet(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, getTokensType(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensType, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, mark(), ANTLR3_INT_STREAM_struct::mark, ANTLR3_COMMON_TOKEN_STREAM_struct::p, release(), ANTLR3_INT_STREAM_struct::release, ANTLR3_INT_STREAM_struct::rewind, rewindLast(), ANTLR3_INT_STREAM_struct::rewindLast, rewindStream(), seek(), ANTLR3_INT_STREAM_struct::seek, setDebugListener(), ANTLR3_TOKEN_STREAM_struct::setDebugListener, setTokenSource(), ANTLR3_TOKEN_STREAM_struct::setTokenSource, setTokenTypeChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::setTokenTypeChannel, size(), ANTLR3_INT_STREAM_struct::size, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, tindex(), ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, tokLT(), toString(), ANTLR3_TOKEN_STREAM_struct::toString, toStringSS(), ANTLR3_TOKEN_STREAM_struct::toStringSS, toStringTT(), ANTLR3_TOKEN_STREAM_struct::toStringTT, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_INT_STREAM_struct::type.

Referenced by antlr3CommonTokenStreamSourceNew().

Here is the call graph for this function:

Here is the caller graph for this function:

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamSourceNew	(	ANTLR3_UINT32	hint,
		pANTLR3_TOKEN_SOURCE	source
	)

References ANTLR3_TOKEN_DEFAULT_CHANNEL, ANTLR3_TRUE, antlr3CommonTokenStreamNew(), antlr3CTSFree(), ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_COMMON_TOKEN_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::setTokenSource, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.

Referenced by antlr3CommonTokenDebugStreamSourceNew().

Here is the call graph for this function:

Here is the caller graph for this function:

static void antlr3CTSFree

(

pANTLR3_COMMON_TOKEN_STREAM

stream

)

[static]

References ANTLR3_FREE, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_LIST_struct::free, ANTLR3_TOKEN_STREAM_struct::free, ANTLR3_INT_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.

Referenced by antlr3CommonTokenStreamSourceNew().

Here is the caller graph for this function:

static void antlr3TokenStreamFree

(

pANTLR3_TOKEN_STREAM

stream

)

[static]

References ANTLR3_FREE.

Referenced by antlr3TokenStreamNew().

Here is the caller graph for this function:

ANTLR3_API pANTLR3_TOKEN_STREAM antlr3TokenStreamNew

(

)

References ANTLR3_MALLOC, antlr3TokenStreamFree(), and ANTLR3_TOKEN_STREAM_struct::free.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:

Here is the caller graph for this function:

static void consume

(

pANTLR3_INT_STREAM

is

)

[static]

static void consumeInitialHiddenTokens

(

pANTLR3_INT_STREAM

is

)

[static]

Debug only method to flag consumption of initial off-channel tokens in the input stream.

References ANTLR3_FALSE, ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.

Referenced by dbgTokLT().

Here is the caller graph for this function:

static void dbgConsume

(

pANTLR3_INT_STREAM

is

)

[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static ANTLR3_UINT32 dbgLA	(	pANTLR3_INT_STREAM	is,
		ANTLR3_INT32	i
	)			[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static ANTLR3_MARKER dbgMark

(

pANTLR3_INT_STREAM

is

)

[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static void dbgRewindLast

(

pANTLR3_INT_STREAM

is

)

[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static void dbgRewindStream	(	pANTLR3_INT_STREAM	is,
		ANTLR3_MARKER	marker
	)			[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static void dbgSeek	(	pANTLR3_INT_STREAM	is,
		ANTLR3_MARKER	index
	)			[static]

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the caller graph for this function:

static pANTLR3_COMMON_TOKEN dbgTokLT	(	pANTLR3_TOKEN_STREAM	ts,
		ANTLR3_INT32	k
	)			[static]

As per the normal tokLT but sends information to the debugger.

References ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, and tokLT().

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the call graph for this function:

Here is the caller graph for this function:

static void discardOffChannel	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_BOOLEAN	discard
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static void discardTokenType	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_INT32	ttype
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static void fillBuffer

(

pANTLR3_COMMON_TOKEN_STREAM

tokenStream

)

[static]

static pANTLR3_COMMON_TOKEN get	(	pANTLR3_TOKEN_STREAM	ts,
		ANTLR3_UINT32	i
	)			[static]

static pANTLR3_STRING getSourceName

(

pANTLR3_INT_STREAM

is

)

[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_LIST getTokenRange	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_UINT32	start,
		ANTLR3_UINT32	stop
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_VECTOR getTokens

(

pANTLR3_COMMON_TOKEN_STREAM

cts

)

[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_LIST getTokensList	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_UINT32	start,
		ANTLR3_UINT32	stop,
		pANTLR3_LIST	list
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_TOKEN_SOURCE getTokenSource

(

pANTLR3_TOKEN_STREAM

ts

)

[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_LIST getTokensSet	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_UINT32	start,
		ANTLR3_UINT32	stop,
		pANTLR3_BITSET	types
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_LIST getTokensType	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_UINT32	start,
		ANTLR3_UINT32	stop,
		ANTLR3_UINT32	type
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static pANTLR3_COMMON_TOKEN LB	(	pANTLR3_COMMON_TOKEN_STREAM	tokenStream,
		ANTLR3_INT32	i
	)			[static]

static ANTLR3_MARKER mark

(

pANTLR3_INT_STREAM

is

)

[static]

static void release	(	pANTLR3_INT_STREAM	is,
		ANTLR3_MARKER	mark
	)			[static]

static void rewindLast

(

pANTLR3_INT_STREAM

is

)

[static]

static void rewindStream	(	pANTLR3_INT_STREAM	is,
		ANTLR3_MARKER	marker
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static void seek	(	pANTLR3_INT_STREAM	is,
		ANTLR3_MARKER	index
	)			[static]

static void setDebugListener	(	pANTLR3_TOKEN_STREAM	ts,
		pANTLR3_DEBUG_EVENT_LISTENER	debugger
	)			[static]

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_FALSE, ANTLR3_INT_STREAM_struct::consume, dbgConsume(), dbgLA(), dbgMark(), dbgRewindLast(), dbgRewindStream(), dbgSeek(), dbgTokLT(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, and ANTLR3_INT_STREAM_struct::seek.

Here is the call graph for this function:

static void setTokenSource	(	pANTLR3_TOKEN_STREAM	ts,
		pANTLR3_TOKEN_SOURCE	tokenSource
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static void setTokenTypeChannel	(	pANTLR3_COMMON_TOKEN_STREAM	cts,
		ANTLR3_UINT32	ttype,
		ANTLR3_UINT32	channel
	)			[static]

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

static ANTLR3_UINT32 size

(

pANTLR3_INT_STREAM

is

)

[static]