Skip to main content
Tokens represent individual lexical elements in Lua source code, including their exact position and type information.

​
Token

A token consisting of its Position and a TokenType. 
pub struct Token {
    pub(crate) start_position: Position,
    pub(crate) end_position: Position,
    pub(crate) token_type: TokenType,
}

​
Methods

​
new

Creates a token with a zero position. 
pub fn new(token_type: TokenType) -> Token

​
start_position

The position a token begins at. 
pub fn start_position(&self) -> Position

​
end_position

The position a token ends at. 
pub fn end_position(&self) -> Position

​
token_type

The type of token as well as the data needed to represent it. If you don’t need any other information, use token_kind instead. 
pub fn token_type(&self) -> &TokenType

​
token_kind

The kind of token with no additional data. If you need any information such as identifier names, use token_type instead. 
pub fn token_kind(&self) -> TokenKind

​
TokenType

The type of tokens in parsed code. 
pub enum TokenType {
    /// End of file, should always be the very last token
    Eof,

    /// An identifier, such as `foo`
    Identifier {
        /// The identifier itself
        identifier: ShortString,
    },

    /// A multi line comment in the format of `--[[ comment ]]`
    MultiLineComment {
        /// Number of equals signs, if any, for the multi line comment
        /// For example, `--[=[` would have a `blocks` value of `1`
        blocks: usize,
        /// The comment itself, ignoring opening and closing tags
        comment: ShortString,
    },

    /// A literal number, such as `3.3`
    Number {
        /// The text representing the number, includes details such as `0x`
        text: ShortString,
    },

    /// A shebang line
    Shebang {
        /// The shebang line itself
        line: ShortString,
    },

    /// A single line comment, such as `-- comment`
    SingleLineComment {
        /// The comment, ignoring initial `--`
        comment: ShortString,
    },

    /// A literal string, such as "Hello, world"
    StringLiteral {
        /// The literal itself, ignoring quotation marks
        literal: ShortString,
        /// Number of equals signs used for a multi line string, if it is one
        /// For example, `[=[string]=]` would have a `multi_line_depth` value of 1
        /// `[[string]]` would have a `multi_line_depth` value of 0
        /// A string such as `"string"` would have also have a `multi_line_depth` value of 0
        multi_line_depth: usize,
        /// The type of quotation mark used to make the string
        quote_type: StringLiteralQuoteType,
    },

    /// A `Symbol`, such as `local` or `+`
    Symbol {
        /// The symbol itself
        symbol: Symbol,
    },

    /// Whitespace, such as tabs or new lines
    Whitespace {
        /// Characters consisting of the whitespace
        characters: ShortString,
    },

    /// Some form of interpolated string (Luau only)
    #[cfg(feature = "luau")]
    InterpolatedString {
        /// The literal itself, ignoring backticks
        literal: ShortString,
        /// The kind of interpolated string.
        /// If it is the beginning, middle, end, or a standalone string.
        kind: InterpolatedStringKind,
    },

    /// A single/multi line C-style comment, such as `/* comment */` (cfxlua only)
    #[cfg(feature = "cfxlua")]
    CStyleComment {
        /// The comment, ignoring initial `--`
        comment: ShortString,
    },
}

​
Methods

​
is_trivia

Returns whether a token can be practically ignored in most cases. Comments and whitespace will return true, everything else will return false. 
pub fn is_trivia(&self) -> bool

​
kind

Returns the kind of the token type. 
pub fn kind(&self) -> TokenKind
Example: 
use full_moon::{ShortString, tokenizer::{TokenKind, TokenType}};

assert_eq!(
    TokenType::Identifier {
        identifier: ShortString::new("hello")
    }.kind(),
    TokenKind::Identifier,
);

​
spaces

Returns a whitespace TokenType consisting of spaces. 
pub fn spaces(spaces: usize) -> Self

​
tabs

Returns a whitespace TokenType consisting of tabs. 
pub fn tabs(tabs: usize) -> Self

​
TokenKind

The kind of token. Contains no additional data. 
pub enum TokenKind {
    /// End of file, should always be the very last token
    Eof,
    /// An identifier, such as `foo`
    Identifier,
    /// A multi line comment in the format of `--[[ comment ]]`
    MultiLineComment,
    /// A literal number, such as `3.3`
    Number,
    /// The shebang line
    Shebang,
    /// A single line comment, such as `-- comment`
    SingleLineComment,
    /// A literal string, such as "Hello, world"
    StringLiteral,
    /// A `Symbol`, such as `local` or `+`
    Symbol,
    /// Whitespace, such as tabs or new lines
    Whitespace,

    #[cfg(feature = "luau")]
    /// Some form of interpolated string
    InterpolatedString,

    #[cfg(feature = "cfxlua")]
    /// A C-style comment, such as `/* comment */`
    CStyleComment,
}

​
TokenReference

A reference to a token used by ASTs. Dereferences to a Token. TokenReferences include “trivia” - leading and trailing whitespace and comments that are attached to the token but don’t affect its meaning. 
pub struct TokenReference {
    pub(crate) leading_trivia: Vec<Token>,
    pub(crate) token: Token,
    pub(crate) trailing_trivia: Vec<Token>,
}

​
Methods

​
new

Creates a TokenReference from leading/trailing trivia as well as the leading token. 
pub fn new(leading_trivia: Vec<Token>, token: Token, trailing_trivia: Vec<Token>) -> Self

​
symbol

Returns a symbol with the leading and trailing whitespace. Only whitespace is supported. 
pub fn symbol(text: &str) -> Result<Self, TokenizerErrorType>
Example: 
use full_moon::tokenizer::{Symbol, TokenReference, TokenType, TokenizerErrorType};

let symbol = TokenReference::symbol("\nreturn ")?;
assert_eq!(symbol.leading_trivia().next().unwrap().to_string(), "\n");
assert_eq!(symbol.token().token_type(), &TokenType::Symbol {
    symbol: Symbol::Return,
});
assert_eq!(symbol.trailing_trivia().next().unwrap().to_string(), " ");
assert!(TokenReference::symbol("isnt whitespace").is_err());
assert!(TokenReference::symbol(" notasymbol ").is_err());

​
symbol_specific_lua_version

Returns a symbol with the leading and trailing whitespace, much like TokenReference::symbol, but only if it’s valid for the given Lua version. 
pub fn symbol_specific_lua_version(
    text: &str,
    lua_version: LuaVersion,
) -> Result<Self, TokenizerErrorType>

​
token

Returns the inner token. 
pub fn token(&self) -> &Token

​
leading_trivia

Returns the leading trivia. 
pub fn leading_trivia(&self) -> impl Iterator<Item = &Token>

​
trailing_trivia

Returns the trailing trivia. 
pub fn trailing_trivia(&self) -> impl Iterator<Item = &Token>

​
with_token

Creates a clone of the current TokenReference with the new inner token, preserving trivia. 
pub fn with_token(&self, token: Token) -> Self

​
is_symbol

Checks if the token is the given symbol. 
pub fn is_symbol(&self, symbol: Symbol) -> bool

​
Symbol

A literal symbol, used for both words important to syntax (like while) and operators (like +). 
pub enum Symbol {
    And,
    Break,
    Do,
    Else,
    ElseIf,
    End,
    False,
    For,
    Function,
    If,
    In,
    Local,
    Nil,
    Not,
    Or,
    Repeat,
    Return,
    Then,
    True,
    Until,
    While,

    // Version-specific symbols
    Goto,           // lua52+, luajit
    PlusEqual,      // luau, cfxlua
    MinusEqual,     // luau, cfxlua
    StarEqual,      // luau, cfxlua
    SlashEqual,     // luau, cfxlua
    // ... and many more

    // Operators
    Caret,
    Colon,
    Comma,
    Dot,
    TwoDots,
    Ellipsis,
    Equal,
    TwoEqual,
    GreaterThan,
    GreaterThanEqual,
    Hash,
    LeftBrace,
    LeftBracket,
    LeftParen,
    LessThan,
    LessThanEqual,
    Minus,
    Percent,
    Plus,
    RightBrace,
    RightBracket,
    RightParen,
    Semicolon,
    Slash,
    Star,
    TildeEqual,
    // ... and more version-specific operators
}

​
Methods

​
from_str

Given just the symbol text (no whitespace) and the Lua version, returns the associated symbol, if it exists. 
pub fn from_str(symbol: &str, lua_version: LuaVersion) -> Option<Self>
Example: 
use full_moon::{LuaVersion, tokenizer::Symbol};

assert_eq!(Symbol::from_str("local", LuaVersion::lua51()), Some(Symbol::Local));

#[cfg(feature = "lua52")]
assert_eq!(Symbol::from_str("goto", LuaVersion::lua52()), Some(Symbol::Goto));
assert_eq!(Symbol::from_str("goto", LuaVersion::lua51()), None);

​
StringLiteralQuoteType

The types of quotes used in a Lua string. 
pub enum StringLiteralQuoteType {
    /// Strings formatted \[\[with brackets\]\]
    Brackets,
    /// Strings formatted "with double quotes"
    Double,
    /// Strings formatted 'with single quotes'
    Single,
    /// Strings in form of Jenkins Hash `with backticks` (cfxlua only)
    #[cfg(feature = "cfxlua")]
    Backtick,
}

Build docs developers (and LLMs) love

Get started for freeTalk to us