You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
275 lines
6.9 KiB
275 lines
6.9 KiB
import { CharacterStream } from './character_stream.mjs'; |
|
import { TemplateSyntaxError } from './errors/template_syntax_error.mjs'; |
|
|
|
export class TokenStream { |
|
static ESCAPE = '\\'; |
|
static OPEN_STATEMENT = '<{'; |
|
static CLOSE_STATEMENT = '}>'; |
|
static OPEN_INLINE = '<%'; |
|
static CLOSE_INLINE = '%>'; |
|
static OPEN_INLINE_UNESCAPED = '<!%'; |
|
static CLOSE_INLINE_UNESCAPED = '%>'; |
|
static OPEN_INLINE_STATEMENT = '<$%'; |
|
static CLOSE_INLINE_STATEMENT = '%>'; |
|
static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%'; |
|
static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>'; |
|
static STATEMENT = '$'; |
|
static COMMENT = '#'; |
|
static SOURCE = '.INCLUDE'; |
|
|
|
constructor() { |
|
this.reset(); |
|
} |
|
|
|
reset() { |
|
this._cs = new CharacterStream(); |
|
this._peekedToken = null; |
|
} |
|
|
|
loadFile(fileName) { |
|
this.reset(); |
|
return this._cs.loadFile(fileName); |
|
} |
|
|
|
peek() { |
|
if (!this._peekedToken) this._peekedToken = this._readNext(); |
|
return this._peekedToken; |
|
} |
|
|
|
next() { |
|
const token = this.peek(); |
|
this._peekedToken = null; |
|
return token; |
|
} |
|
|
|
eof() { |
|
return this.peek() === null; |
|
} |
|
|
|
_readNext() { |
|
let raw = ''; |
|
|
|
const flushRaw = () => { |
|
if (raw.length) { |
|
const value = raw; |
|
raw = ''; |
|
return {type: 'raw', value}; |
|
} |
|
} |
|
|
|
while (!this._cs.eof()) { |
|
if ( |
|
this._cs.line === 0 && |
|
this._cs.column === 0 && |
|
this._cs.peek(2) === '#!' |
|
) { |
|
// Skip shebang (can be used to execute templates directly with an |
|
// interpreter that first builds the template and then executes) |
|
this._skipShebang(); |
|
} else if (this._isNextEscapedTag([ |
|
// Allow escaping of open and full-line tags |
|
TokenStream.OPEN_STATEMENT, |
|
TokenStream.OPEN_INLINE, |
|
TokenStream.OPEN_INLINE_UNESCAPED, |
|
TokenStream.OPEN_INLINE_STATEMENT, |
|
TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, |
|
TokenStream.STATEMENT, |
|
TokenStream.COMMENT, |
|
TokenStream.SOURCE |
|
])) { |
|
// Skip escape |
|
this._cs.next(TokenStream.ESCAPE.length); |
|
// Treat everything as raw until next space |
|
raw += this._cs.nextWhile((c) => /\S/.test(c)); |
|
} else if (this._cs.isNext(TokenStream.OPEN_STATEMENT)) { |
|
return flushRaw() || this._readBlockStatement(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE)) { |
|
return flushRaw() || this._readInline(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) { |
|
return flushRaw() || this._readInlineUnescaped(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) { |
|
return flushRaw() || this._readInlineStatement(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) { |
|
return flushRaw() || this._readInlineStatementUnescaped(); |
|
} else if (this._matchFullLine(TokenStream.STATEMENT)) { |
|
return flushRaw() || this._readStatement(); |
|
} else if (this._matchFullLine(TokenStream.COMMENT)) { |
|
return flushRaw() || this._readComment(); |
|
} else if (this._matchFullLine(TokenStream.SOURCE)) { |
|
return flushRaw() || this._readSource(); |
|
} else { |
|
raw += this._cs.next(); |
|
} |
|
} |
|
|
|
return flushRaw() || null; |
|
} |
|
|
|
_skipShebang() { |
|
// Skip shebang |
|
this._cs.nextWhile((c) => c !== '\n'); |
|
|
|
// Skip new line |
|
this._cs.next(); |
|
} |
|
|
|
_missingOpenSpaceError(tag) { |
|
throw new TemplateSyntaxError( |
|
this._cs, |
|
`Tag: "${tag}" must be followed by a whitespace or new line` |
|
); |
|
} |
|
|
|
_missingCloseSpaceError(tag) { |
|
throw new TemplateSyntaxError( |
|
this._cs, |
|
`Tag: "${tag}" must be preceded by a whitespace or new line` |
|
); |
|
} |
|
|
|
_readBlockGeneric(tokenType, openTag, closeTag) { |
|
this._cs.next(openTag.length); |
|
|
|
// Check for mandatory space |
|
if (/\S/.test(this._cs.peek())) { |
|
this._missingOpenSpaceError(openTag); |
|
} |
|
this._cs.next(); |
|
|
|
let value = ''; |
|
while (!this._cs.eof() && !this._cs.isNext(closeTag)) { |
|
// Handle any escaped closing tag |
|
if (this._isNextEscapedTag([closeTag])) { |
|
// Skip escape |
|
this._cs.next(TokenStream.ESCAPE.length); |
|
|
|
// Treat everything as raw until next space or eof |
|
value += this._cs.nextWhile((c) => /\S/.test(c)); |
|
} else { |
|
const char = this._cs.next(); |
|
|
|
if (this._cs.isNext(closeTag)) { |
|
if (/\S/.test(char)) { |
|
this._missingCloseSpaceError(closeTag); |
|
} |
|
} else { |
|
value += char; |
|
} |
|
} |
|
} |
|
this._cs.next(closeTag.length); |
|
return {type: tokenType, value}; |
|
} |
|
|
|
_readBlockStatement() { |
|
return this._readBlockGeneric( |
|
'block_statement', |
|
TokenStream.OPEN_STATEMENT, |
|
TokenStream.CLOSE_STATEMENT |
|
); |
|
} |
|
|
|
_readInline() { |
|
return this._readBlockGeneric( |
|
'inline', |
|
TokenStream.OPEN_INLINE, |
|
TokenStream.CLOSE_INLINE |
|
); |
|
} |
|
|
|
_readInlineUnescaped() { |
|
return this._readBlockGeneric( |
|
'inline_unescaped', |
|
TokenStream.OPEN_INLINE_UNESCAPED, |
|
TokenStream.CLOSE_INLINE_UNESCAPED |
|
); |
|
} |
|
|
|
_readInlineStatement() { |
|
return this._readBlockGeneric( |
|
'inline_statement', |
|
TokenStream.OPEN_INLINE_STATEMENT, |
|
TokenStream.CLOSE_INLINE_STATEMENT |
|
); |
|
} |
|
|
|
_readInlineStatementUnescaped() { |
|
return this._readBlockGeneric( |
|
'inline_statement_unescaped', |
|
TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, |
|
TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED |
|
); |
|
} |
|
|
|
_missingFullLineSpaceError(tag) { |
|
throw new TemplateSyntaxError( |
|
this._cs, |
|
`Full-Line Tag: "${tag}" must be followed by a whitespace` |
|
); |
|
} |
|
|
|
_readFullLineGeneric(tokenType, tag) { |
|
// Skip whitespace |
|
this._cs.nextWhile(this._isSpace); |
|
|
|
// Skip tag |
|
this._cs.next(tag.length); |
|
|
|
// Check for mandatory space/tab/new line |
|
if (this._isSpace(this._cs.peek())) { |
|
// Consume space |
|
this._cs.next(); |
|
} else if (/[^\r\n]/.test(this._cs.peek())) { |
|
this._missingFullLineSpaceError(tag); |
|
} |
|
|
|
// Remainder of line is value |
|
const value = this._cs.nextWhile((c) => c !== '\n'); |
|
|
|
// Skip new line |
|
this._cs.next(); |
|
|
|
return {type: tokenType, value}; |
|
} |
|
|
|
_readStatement() { |
|
return this._readFullLineGeneric( |
|
'statement', |
|
TokenStream.STATEMENT |
|
); |
|
} |
|
|
|
_readComment() { |
|
return this._readFullLineGeneric( |
|
'comment', |
|
TokenStream.COMMENT |
|
); |
|
} |
|
|
|
_readSource() { |
|
return this._readFullLineGeneric( |
|
'source', |
|
TokenStream.SOURCE |
|
); |
|
} |
|
|
|
// Utility methods |
|
_isSpace(c) { |
|
return /[^\S\r\n]/.test(c); |
|
} |
|
|
|
_isNextEscapedTag(possibleTags=[]) { |
|
if (!this._cs.isNext(TokenStream.ESCAPE)) return false; |
|
|
|
return !!possibleTags.find(tag => |
|
this._cs.isNext(TokenStream.ESCAPE + tag) |
|
); |
|
} |
|
|
|
_matchFullLine(tag) { |
|
return ( |
|
this._cs.column === 0 && |
|
this._cs.peekAfter(this._isSpace, tag.length) === tag |
|
); |
|
} |
|
}
|
|
|