You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
212 lines
5.0 KiB
212 lines
5.0 KiB
import { CharacterStream } from './character_stream.mjs'; |
|
import { TemplateSyntaxError } from './errors/template_syntax_error.mjs'; |
|
|
|
export class TokenStream { |
|
static OPEN_INLINE = '<%'; |
|
static CLOSE_INLINE = '%>'; |
|
static OPEN_INLINE_UNESCAPED = '<!%'; |
|
static CLOSE_INLINE_UNESCAPED = '%>'; |
|
static OPEN_INLINE_STATEMENT = '<$%'; |
|
static CLOSE_INLINE_STATEMENT = '%>'; |
|
static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%'; |
|
static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>'; |
|
static STATEMENT = '$'; |
|
static COMMENT = '#'; |
|
|
|
constructor() { |
|
this.reset(); |
|
} |
|
|
|
reset() { |
|
this._cs = new CharacterStream(); |
|
this._peekedToken = null; |
|
} |
|
|
|
loadFile(fileName) { |
|
this.reset(); |
|
return this._cs.loadFile(fileName); |
|
} |
|
|
|
peek() { |
|
if (!this._peekedToken) this._peekedToken = this._readNext(); |
|
return this._peekedToken; |
|
} |
|
|
|
next() { |
|
const token = this.peek(); |
|
this._peekedToken = null; |
|
return token; |
|
} |
|
|
|
eof() { |
|
return this.peek() === null; |
|
} |
|
|
|
_readNext() { |
|
let raw = ''; |
|
|
|
const flushRaw = () => { |
|
if (raw.length) { |
|
const value = raw; |
|
raw = ''; |
|
return {type: 'raw', value}; |
|
} |
|
} |
|
|
|
while (!this._cs.eof()) { |
|
if ( |
|
this._cs.line === 0 && |
|
this._cs.column === 0 && |
|
this._cs.peek(2) === '#!' |
|
) { |
|
// Skip shebang (can be used to execute templates directly with an |
|
// interpreter that first builds the template and then executes) |
|
this._skipShebang(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE)) { |
|
return flushRaw() || this._readInline(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) { |
|
return flushRaw() || this._readInlineUnescaped(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) { |
|
return flushRaw() || this._readInlineStatement(); |
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) { |
|
return flushRaw() || this._readInlineStatementUnescaped(); |
|
} else if ( |
|
this._cs.column === 0 && |
|
this._cs.peekAfter(this._isWhitespace) === TokenStream.STATEMENT |
|
) { |
|
return flushRaw() || this._readStatement(); |
|
} else if ( |
|
this._cs.column === 0 && |
|
this._cs.peekAfter(this._isWhitespace) === TokenStream.COMMENT |
|
) { |
|
return flushRaw() || this._readComment(); |
|
} else { |
|
raw += this._cs.next(); |
|
} |
|
} |
|
|
|
return flushRaw() || null; |
|
} |
|
|
|
_skipShebang() { |
|
// Skip shebang |
|
this._cs.nextWhile((c) => c !== '\n'); |
|
|
|
// Skip new line |
|
this._cs.next(); |
|
} |
|
|
|
_missingInlineSpaceError(tag) { |
|
throw new TemplateSyntaxError( |
|
this._cs, |
|
`Inline tag: "${tag}" must be followed by a whitespace` |
|
); |
|
} |
|
|
|
_readInlineGeneric(tokenType, openTag, closeTag) { |
|
this._cs.next(openTag.length); |
|
|
|
// Check for mandatory space |
|
if (this._cs.peek() !== ' ') { |
|
this._missingInlineSpaceError(openTag); |
|
} |
|
this._cs.next(); |
|
|
|
let value = ''; |
|
while (!this._cs.eof() && !this._cs.isNext(closeTag)) { |
|
const char = this._cs.next(); |
|
|
|
if (this._cs.isNext(closeTag)) { |
|
if (char !== ' ') { |
|
this._missingInlineSpaceError(closeTag); |
|
} |
|
} else { |
|
value += char; |
|
} |
|
} |
|
this._cs.next(closeTag.length); |
|
return {type: tokenType, value}; |
|
} |
|
|
|
_readInline() { |
|
return this._readInlineGeneric( |
|
'inline', |
|
TokenStream.OPEN_INLINE, |
|
TokenStream.CLOSE_INLINE |
|
); |
|
} |
|
|
|
_readInlineUnescaped() { |
|
return this._readInlineGeneric( |
|
'inline_unescaped', |
|
TokenStream.OPEN_INLINE_UNESCAPED, |
|
TokenStream.CLOSE_INLINE_UNESCAPED |
|
); |
|
} |
|
|
|
_readInlineStatement() { |
|
return this._readInlineGeneric( |
|
'inline_statement', |
|
TokenStream.OPEN_INLINE_STATEMENT, |
|
TokenStream.CLOSE_INLINE_STATEMENT |
|
); |
|
} |
|
|
|
_readInlineStatementUnescaped() { |
|
return this._readInlineGeneric( |
|
'inline_statement_unescaped', |
|
TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, |
|
TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED |
|
); |
|
} |
|
|
|
_missingFullLineSpaceError(tag) { |
|
throw new TemplateSyntaxError( |
|
this._cs, |
|
`Full-Line Tag: "${tag}" must be followed by a whitespace` |
|
); |
|
} |
|
|
|
_readFullLineGeneric(tokenType, tag) { |
|
// Skip whitespace |
|
this._cs.nextWhile(this._isWhitespace); |
|
|
|
// Skip tag |
|
this._cs.next(tag.length); |
|
|
|
// Check for mandatory space |
|
if (this._cs.peek() !== ' ') { |
|
this._missingFullLineSpaceError(tag); |
|
} |
|
|
|
// Consume space |
|
this._cs.next(); |
|
|
|
// Remainder of line is value |
|
const value = this._cs.nextWhile((c) => c !== '\n'); |
|
|
|
// Skip new line |
|
this._cs.next(); |
|
|
|
return {type: tokenType, value}; |
|
} |
|
|
|
_readStatement() { |
|
return this._readFullLineGeneric( |
|
'statement', |
|
TokenStream.STATEMENT |
|
); |
|
} |
|
|
|
_readComment() { |
|
return this._readFullLineGeneric( |
|
'comment', |
|
TokenStream.COMMENT |
|
); |
|
} |
|
|
|
// Utility methods |
|
_isWhitespace(c) { |
|
return /\s/.test(c); |
|
} |
|
}
|
|
|