github.com/antlr4-go/antlr/v4 v4.13.0: Tokenize text and inspect tokens with antlr.NewInputStream and antlr.NewCommonTokenStream

# antlr4-go Version Depth Notes

## Search Results
An initial search via `search_known_solution` for `pkg:golang/github.com/antlr4-go/antlr/v4@v4.13.0` yielded `NO_SAFE_MATCH`.

## Pinned Version
- Package: `github.com/antlr4-go/antlr/v4`
- Release: `v4.13.0` (in `go.mod` / `go.sum`)

## Observed Behavior
1. **Character Stream Operations (`antlr.NewInputStream`)**:
   - `antlr.NewInputStream(data)` instantiates a character stream conforming to `antlr.CharStream`.
   - `input.Size()` returns the character length of the input data.
   - `input.LA(1)` peeks the upcoming character returning its Unicode integer representation.
   - `input.GetText(start, stop)` and `input.GetTextFromInterval(antlr.Interval{Start, Stop})` extract substring slices.
   - `input.Seek(index)` repositioning updates `input.Index()`.

2. **Buffered Token Stream & Hidden Channels (`antlr.NewCommonTokenStream`)**:
   - `antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)` connects a lexer to an on-demand token buffer.
   - Calling `tokens.Fill()` buffers all tokens up to `TokenEOF`.
   - `tokens.Get(index)` returns individual indexed `antlr.Token` instances.
   - `tokens.GetHiddenTokensToRight(tokenIndex, channel)` queries tokens routed to off-default channels (e.g. whitespace on `TokenHiddenChannel`).
   - `tokens.LA(k)` and `tokens.LT(k)` perform channel-filtered token lookahead.

3. **In-place Lazy Stream Rewriting (`antlr.NewTokenStreamRewriter`)**:
   - `antlr.NewTokenStreamRewriter(tokens)` records non-destructive mutations over the token stream without modifying the underlying token stream buffer.
   - Methods like `rewriter.ReplaceTokenDefault(from, to, text)` and `rewriter.InsertAfterToken(program, token, text)` schedule edits executed when `rewriter.GetTextDefault()` is invoked.

package antlrsample_test

import (
	"testing"
	"unicode"

	"github.com/antlr4-go/antlr/v4"
)

const (
	TokenTypeIdentifier = 1
	TokenTypeNumber     = 2
	TokenTypeSymbol     = 3
	TokenTypeWhitespace = 4
)

// SimpleLexer is a hand-crafted Lexer for demonstration that embeds BaseLexer.
type SimpleLexer struct {
	*antlr.BaseLexer
}

func NewSimpleLexer(input antlr.CharStream) *SimpleLexer {
	l := &SimpleLexer{
		BaseLexer: antlr.NewBaseLexer(input),
	}
	l.Virt = l
	l.Interpreter = &antlr.LexerATNSimulator{
		Line:               1,
		CharPositionInLine: 0,
	}
	return l
}

func (l *SimpleLexer) consumeRune(input antlr.CharStream) rune {
	ch := input.LA(1)
	input.Consume()
	r := rune(ch)
	sim := l.Interpreter.(*antlr.LexerATNSimulator)
	if r == '\n' {
		sim.Line++
		sim.CharPositionInLine = 0
	} else {
		sim.CharPositionInLine++
	}
	return r
}

func (l *SimpleLexer) NextToken() antlr.Token {
	input := l.GetInputStream()
	sim := l.Interpreter.(*antlr.LexerATNSimulator)
	for {
		ch := input.LA(1)
		if ch == antlr.TokenEOF {
			return l.GetTokenFactory().Create(
				l.GetTokenSourceCharStreamPair(),
				antlr.TokenEOF,
				"",
				antlr.TokenDefaultChannel,
				input.Index(),
				input.Index(),
				sim.Line,
				sim.CharPositionInLine,
			)
		}

		start := input.Index()
		line := sim.Line
		col := sim.CharPositionInLine

		r := rune(ch)
		if unicode.IsSpace(r) {
			l.consumeRune(input)
			for unicode.IsSpace(rune(input.LA(1))) && input.LA(1) != antlr.TokenEOF {
				l.consumeRune(input)
			}
			stop := input.Index() - 1
			token := l.GetTokenFactory().Create(
				l.GetTokenSourceCharStreamPair(),
				TokenTypeWhitespace,
				input.GetText(start, stop),
				antlr.TokenHiddenChannel,
				start,
				stop,
				line,
				col,
			)
			return token
		}

		if unicode.IsLetter(r) || r == '_' {
			l.consumeRune(input)
			for (unicode.IsLetter(rune(input.LA(1))) || unicode.IsDigit(rune(input.LA(1))) || rune(input.LA(1)) == '_') && input.LA(1) != antlr.TokenEOF {
				l.consumeRune(input)
			}
			stop := input.Index() - 1
			token := l.GetTokenFactory().Create(
				l.GetTokenSourceCharStreamPair(),
				TokenTypeIdentifier,
				input.GetText(start, stop),
				antlr.TokenDefaultChannel,
				start,
				stop,
				line,
				col,
			)
			return token
		}

		if unicode.IsDigit(r) {
			l.consumeRune(input)
			for unicode.IsDigit(rune(input.LA(1))) && input.LA(1) != antlr.TokenEOF {
				l.consumeRune(input)
			}
			stop := input.Index() - 1
			token := l.GetTokenFactory().Create(
				l.GetTokenSourceCharStreamPair(),
				TokenTypeNumber,
				input.GetText(start, stop),
				antlr.TokenDefaultChannel,
				start,
				stop,
				line,
				col,
			)
			return token
		}

		// Single character symbol
		l.consumeRune(input)
		token := l.GetTokenFactory().Create(
			l.GetTokenSourceCharStreamPair(),
			TokenTypeSymbol,
			input.GetText(start, start),
			antlr.TokenDefaultChannel,
			start,
			start,
			line,
			col,
		)
		return token
	}
}

func TestInputStreamOperations(t *testing.T) {
	text := "SELECT user_id, count FROM orders WHERE status = 'active';"
	input := antlr.NewInputStream(text)

	if input.Size() != len(text) {
		t.Fatalf("expected size %d, got %d", len(text), input.Size())
	}
	if input.LA(1) != int('S') {
		t.Fatalf("expected 'S' (%d), got %d", int('S'), input.LA(1))
	}
	if input.GetText(0, 5) != "SELECT" {
		t.Fatalf("expected 'SELECT', got %q", input.GetText(0, 5))
	}

	interval := antlr.Interval{Start: 7, Stop: 13}
	if input.GetTextFromInterval(interval) != "user_id" {
		t.Fatalf("expected 'user_id', got %q", input.GetTextFromInterval(interval))
	}

	input.Seek(7)
	if input.Index() != 7 {
		t.Fatalf("expected index 7, got %d", input.Index())
	}
	if input.LA(1) != int('u') {
		t.Fatalf("expected 'u', got %c", input.LA(1))
	}
}

func TestCommonTokenStreamAndChannelFiltering(t *testing.T) {
	text := "let count = 42 + offset;"
	input := antlr.NewInputStream(text)
	lexer := NewSimpleLexer(input)
	tokens := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)

	tokens.Fill()
	allTokens := tokens.GetAllTokens()

	if len(allTokens) == 0 {
		t.Fatalf("expected non-empty tokens")
	}

	// First default token should be "let"
	tok0 := tokens.Get(0)
	if tok0.GetText() != "let" || tok0.GetTokenType() != TokenTypeIdentifier {
		t.Fatalf("unexpected token 0: %v", tok0)
	}

	// Channel filtering: get hidden whitespace tokens to the right of token 0
	hidden := tokens.GetHiddenTokensToRight(0, antlr.TokenHiddenChannel)
	if len(hidden) != 1 || hidden[0].GetText() != " " {
		t.Fatalf("expected single whitespace hidden token, got %v", hidden)
	}

	// Lookahead verification
	if tokens.LA(1) != TokenTypeIdentifier {
		t.Fatalf("expected LA(1) to be TokenTypeIdentifier, got %d", tokens.LA(1))
	}
	if tokens.LT(1).GetText() != "let" {
		t.Fatalf("expected LT(1) text 'let', got %q", tokens.LT(1).GetText())
	}
}

func TestTokenStreamRewriterMutations(t *testing.T) {
	text := "let count = 42 + offset;"
	input := antlr.NewInputStream(text)
	lexer := NewSimpleLexer(input)
	tokens := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
	tokens.Fill()

	rewriter := antlr.NewTokenStreamRewriter(tokens)

	// Replace "let" with "const"
	tok0 := tokens.Get(0)
	rewriter.ReplaceTokenDefault(tok0, tok0, "const")

	// Find the semicolon token and insert comment after it
	var semiTok antlr.Token
	for _, tok := range tokens.GetAllTokens() {
		if tok.GetText() == ";" {
			semiTok = tok
			break
		}
	}
	if semiTok == nil {
		t.Fatalf("semicolon token not found")
	}

	rewriter.InsertAfterToken(antlr.DefaultProgramName, semiTok, " // mutated")

	rewritten := rewriter.GetTextDefault()
	expected := "const count = 42 + offset; // mutated"
	if rewritten != expected {
		t.Fatalf("expected rewritten %q, got %q", expected, rewritten)
	}
}

{"case":{"caseId":"case:sha256:5a650ba77cbc0b2138b04c0dfb21500e07e1d200136a95140f2db835ba09f845","contract":["antlr.NewInputStream creates a CharStream from a string that tracks stream size, character lookahead with LA(1), substring slices via GetText, and position seeking.","antlr.NewCommonTokenStream consumes tokens from a Lexer, filters tokens to the default channel, and enables channel-specific lookback and lookahead with GetHiddenTokensToRight.","antlr.NewTokenStreamRewriter alters token streams by replacing tokens, inserting annotations relative to token positions, and producing transformed output without modifying the underlying token stream buffer."],"goal":"Tokenize text and inspect tokens with antlr.NewInputStream and antlr.NewCommonTokenStream","kind":"HOW","packages":["pkg:golang/github.com/antlr4-go/antlr/v4@v4.13.0"],"schemaVersion":1,"symbols":["antlr.NewCommonTokenStream","antlr.NewInputStream","antlr.NewTokenStreamRewriter","antlr.Interval"]},"contractCommand":["go","test","./..."],"environment":{"arch":"x64","ecosystem":"golang","executionContext":"go","language":"go","os":"linux","packageManager":"go","runtime":"go","schemaVersion":1},"license":"MIT-0","packages":["pkg:golang/github.com/antlr4-go/antlr/v4@v4.13.0"],"schemaVersion":1,"subject":"pkg:golang/github.com/antlr4-go/antlr/v4@v4.13.0","symbols":["antlr.NewCommonTokenStream","antlr.NewInputStream","antlr.NewTokenStreamRewriter","antlr.Interval"],"verifierAdapter":"golang@1"}

module example.com/antlrsample

go 1.26.6

require (
	github.com/antlr4-go/antlr/v4 v4.13.0
	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
)

github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=