|
|
|
import { CharacterStream } from './character_stream.mjs';
|
|
|
|
import { TemplateSyntaxError } from './errors/template_syntax_error.mjs';
|
|
|
|
|
|
|
|
export class TokenStream {
|
|
|
|
static ESCAPE = '\\';
|
|
|
|
static OPEN_STATEMENT = '<{';
|
|
|
|
static CLOSE_STATEMENT = '}>';
|
|
|
|
static OPEN_INLINE = '<%';
|
|
|
|
static CLOSE_INLINE = '%>';
|
|
|
|
static OPEN_INLINE_UNESCAPED = '<!%';
|
|
|
|
static CLOSE_INLINE_UNESCAPED = '%>';
|
|
|
|
static OPEN_INLINE_STATEMENT = '<$%';
|
|
|
|
static CLOSE_INLINE_STATEMENT = '%>';
|
|
|
|
static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%';
|
|
|
|
static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>';
|
|
|
|
static STATEMENT = '$';
|
|
|
|
static COMMENT = '#';
|
|
|
|
|
|
|
|
constructor() {
|
|
|
|
this.reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
reset() {
|
|
|
|
this._cs = new CharacterStream();
|
|
|
|
this._peekedToken = null;
|
|
|
|
}
|
|
|
|
|
|
|
|
loadFile(fileName) {
|
|
|
|
this.reset();
|
|
|
|
return this._cs.loadFile(fileName);
|
|
|
|
}
|
|
|
|
|
|
|
|
peek() {
|
|
|
|
if (!this._peekedToken) this._peekedToken = this._readNext();
|
|
|
|
return this._peekedToken;
|
|
|
|
}
|
|
|
|
|
|
|
|
next() {
|
|
|
|
const token = this.peek();
|
|
|
|
this._peekedToken = null;
|
|
|
|
return token;
|
|
|
|
}
|
|
|
|
|
|
|
|
eof() {
|
|
|
|
return this.peek() === null;
|
|
|
|
}
|
|
|
|
|
|
|
|
_readNext() {
|
|
|
|
let raw = '';
|
|
|
|
|
|
|
|
const flushRaw = () => {
|
|
|
|
if (raw.length) {
|
|
|
|
const value = raw;
|
|
|
|
raw = '';
|
|
|
|
return {type: 'raw', value};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
while (!this._cs.eof()) {
|
|
|
|
if (
|
|
|
|
this._cs.line === 0 &&
|
|
|
|
this._cs.column === 0 &&
|
|
|
|
this._cs.peek(2) === '#!'
|
|
|
|
) {
|
|
|
|
// Skip shebang (can be used to execute templates directly with an
|
|
|
|
// interpreter that first builds the template and then executes)
|
|
|
|
this._skipShebang();
|
|
|
|
} else if (this._cs.isNext(TokenStream.ESCAPE)) {
|
|
|
|
// Skip escape
|
|
|
|
this._cs.next(TokenStream.ESCAPE.length);
|
|
|
|
// Treat everything as raw until next space
|
|
|
|
raw += this._cs.nextWhile((c) => /\S/.test(c));
|
|
|
|
} else if (this._cs.isNext(TokenStream.OPEN_STATEMENT)) {
|
|
|
|
return flushRaw() || this._readBlockStatement();
|
|
|
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE)) {
|
|
|
|
return flushRaw() || this._readInline();
|
|
|
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) {
|
|
|
|
return flushRaw() || this._readInlineUnescaped();
|
|
|
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) {
|
|
|
|
return flushRaw() || this._readInlineStatement();
|
|
|
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) {
|
|
|
|
return flushRaw() || this._readInlineStatementUnescaped();
|
|
|
|
} else if (
|
|
|
|
this._cs.column === 0 &&
|
|
|
|
this._cs.peekAfter(this._isSpace) === TokenStream.STATEMENT
|
|
|
|
) {
|
|
|
|
return flushRaw() || this._readStatement();
|
|
|
|
} else if (
|
|
|
|
this._cs.column === 0 &&
|
|
|
|
this._cs.peekAfter(this._isSpace) === TokenStream.COMMENT
|
|
|
|
) {
|
|
|
|
return flushRaw() || this._readComment();
|
|
|
|
} else {
|
|
|
|
raw += this._cs.next();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return flushRaw() || null;
|
|
|
|
}
|
|
|
|
|
|
|
|
_skipShebang() {
|
|
|
|
// Skip shebang
|
|
|
|
this._cs.nextWhile((c) => c !== '\n');
|
|
|
|
|
|
|
|
// Skip new line
|
|
|
|
this._cs.next();
|
|
|
|
}
|
|
|
|
|
|
|
|
_missingOpenSpaceError(tag) {
|
|
|
|
throw new TemplateSyntaxError(
|
|
|
|
this._cs,
|
|
|
|
`Tag: "${tag}" must be followed by a whitespace or new line`
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_missingCloseSpaceError(tag) {
|
|
|
|
throw new TemplateSyntaxError(
|
|
|
|
this._cs,
|
|
|
|
`Tag: "${tag}" must be preceded by a whitespace or new line`
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readBlockGeneric(tokenType, openTag, closeTag) {
|
|
|
|
this._cs.next(openTag.length);
|
|
|
|
|
|
|
|
// Check for mandatory space
|
|
|
|
if (/\S/.test(this._cs.peek())) {
|
|
|
|
this._missingOpenSpaceError(openTag);
|
|
|
|
}
|
|
|
|
this._cs.next();
|
|
|
|
|
|
|
|
let value = '';
|
|
|
|
while (!this._cs.eof() && !this._cs.isNext(closeTag)) {
|
|
|
|
// Handle any escaped closing tag
|
|
|
|
if (this._cs.isNext(TokenStream.ESCAPE)) {
|
|
|
|
// Skip escape
|
|
|
|
this._cs.next(TokenStream.ESCAPE.length);
|
|
|
|
|
|
|
|
// Treat everything as raw until next space or eof
|
|
|
|
value += this._cs.nextWhile((c) => /\S/.test(c));
|
|
|
|
} else {
|
|
|
|
const char = this._cs.next();
|
|
|
|
|
|
|
|
if (this._cs.isNext(closeTag)) {
|
|
|
|
if (/\S/.test(char)) {
|
|
|
|
this._missingCloseSpaceError(closeTag);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
value += char;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
this._cs.next(closeTag.length);
|
|
|
|
return {type: tokenType, value};
|
|
|
|
}
|
|
|
|
|
|
|
|
_readBlockStatement() {
|
|
|
|
return this._readBlockGeneric(
|
|
|
|
'block_statement',
|
|
|
|
TokenStream.OPEN_STATEMENT,
|
|
|
|
TokenStream.CLOSE_STATEMENT
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readInline() {
|
|
|
|
return this._readBlockGeneric(
|
|
|
|
'inline',
|
|
|
|
TokenStream.OPEN_INLINE,
|
|
|
|
TokenStream.CLOSE_INLINE
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readInlineUnescaped() {
|
|
|
|
return this._readBlockGeneric(
|
|
|
|
'inline_unescaped',
|
|
|
|
TokenStream.OPEN_INLINE_UNESCAPED,
|
|
|
|
TokenStream.CLOSE_INLINE_UNESCAPED
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readInlineStatement() {
|
|
|
|
return this._readBlockGeneric(
|
|
|
|
'inline_statement',
|
|
|
|
TokenStream.OPEN_INLINE_STATEMENT,
|
|
|
|
TokenStream.CLOSE_INLINE_STATEMENT
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readInlineStatementUnescaped() {
|
|
|
|
return this._readBlockGeneric(
|
|
|
|
'inline_statement_unescaped',
|
|
|
|
TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED,
|
|
|
|
TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_missingFullLineSpaceError(tag) {
|
|
|
|
throw new TemplateSyntaxError(
|
|
|
|
this._cs,
|
|
|
|
`Full-Line Tag: "${tag}" must be followed by a whitespace`
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readFullLineGeneric(tokenType, tag) {
|
|
|
|
// Skip whitespace
|
|
|
|
this._cs.nextWhile(this._isSpace);
|
|
|
|
|
|
|
|
// Skip tag
|
|
|
|
this._cs.next(tag.length);
|
|
|
|
|
|
|
|
// Check for mandatory space/tab/new line
|
|
|
|
if (this._isSpace(this._cs.peek())) {
|
|
|
|
// Consume space
|
|
|
|
this._cs.next();
|
|
|
|
} else if (/[^\r\n]/.test(this._cs.peek())) {
|
|
|
|
this._missingFullLineSpaceError(tag);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remainder of line is value
|
|
|
|
const value = this._cs.nextWhile((c) => c !== '\n');
|
|
|
|
|
|
|
|
// Skip new line
|
|
|
|
this._cs.next();
|
|
|
|
|
|
|
|
return {type: tokenType, value};
|
|
|
|
}
|
|
|
|
|
|
|
|
_readStatement() {
|
|
|
|
return this._readFullLineGeneric(
|
|
|
|
'statement',
|
|
|
|
TokenStream.STATEMENT
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
_readComment() {
|
|
|
|
return this._readFullLineGeneric(
|
|
|
|
'comment',
|
|
|
|
TokenStream.COMMENT
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Utility methods
|
|
|
|
_isSpace(c) {
|
|
|
|
return /[^\S\r\n]/.test(c);
|
|
|
|
}
|
|
|
|
}
|