pastemyst | Tokenizer

Tokenizer

(untitled)

Perlcopy raw

use v6.d;

use Tok;

unit module Lex;

grammar Lexer is export {
    token TOP { <tok>* }

token tok {
        | <whitespace>
        | <dot>
        | <colon>
        | <hash>
        | <squote>
        | <dquote>
        | <dash>
        | <bar>
        | <comma>
        | <lbrace>
        | <rbrace>
        | <lsquare>
        | <rsquare>
        | <lparen>
        | <rparen>
        | <dothash>
        | <dotlbrace>
        | <dotrbrace>
        | <ident>
    }

token whitespace { <!ww> \s+ }
    token dot { '.' }
    token colon { ':' }
    token hash { '#' }
    token squote { '\'' }
    token dquote { '"' }
    token dash { '-' }
    token bar { '|' }
    token comma { ',' }
    token lbrace { '{' }
    token rbrace { '}' }
    token lsquare { '[' }
    token rsquare { ']' }
    token lparen { '(' }
    token rparen { ')' }
    token dothash { '.#' }
    token dotlbrace{ '.{' }
    token dotrbrace { '.}' }
    token ident { <ident_part>+ }

token ident_part { \w || '_' }
}

class Tokenizer is export {

method whitespace ($/) { make WS }
    method dot ($/) { make 1 }
    method colon ($/) { make Colon }
    method hash ($/) { make Hash }
    method squote ($/) { make SingleQuote }
    method dquote ($/) { make DoubleQuote }
    method dash ($/) { make Dash }
    method bar ($/) { make Bar }
    method comma ($/) { make Comma }
    method lbrace ($/) { make LeftBrace }
    method rbrace ($/) { make RightBrace }
    method lsquare ($/) { make LeftSquare }
    method rsquare ($/) { make RightSquare }
    method lparen ($/) { make LeftParen }
    method rparen ($/) { make RightParen }
    method dothash ($/) { make DotHash }
    method dotlbrace ($/) { make DotLBrace }
    method dotrbrace ($/) { make DotRBrace }
    method ident ($/) { make Ident[$<ident_part>.join].new }

method tok ($/) { say $/.made }

method TOP ($match) { $match.make: $match<tok>>>.made }
}

(untitled)

Perlcopy raw

created at:

stats: 113 lines, 500 words, 2.89kb