You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
213 lines
5.0 KiB
213 lines
5.0 KiB
2 years ago
|
import { CharacterStream } from './character_stream.mjs';
|
||
|
import { TemplateSyntaxError } from './errors/template_syntax_error.mjs';
|
||
2 years ago
|
|
||
|
export class TokenStream {
|
||
|
static OPEN_INLINE = '<%';
|
||
|
static CLOSE_INLINE = '%>';
|
||
|
static OPEN_INLINE_UNESCAPED = '<!%';
|
||
|
static CLOSE_INLINE_UNESCAPED = '%>';
|
||
|
static OPEN_INLINE_STATEMENT = '<$%';
|
||
|
static CLOSE_INLINE_STATEMENT = '%>';
|
||
|
static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%';
|
||
|
static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>';
|
||
|
static STATEMENT = '$';
|
||
|
static COMMENT = '#';
|
||
|
|
||
|
constructor() {
|
||
|
this.reset();
|
||
|
}
|
||
|
|
||
|
reset() {
|
||
|
this._cs = new CharacterStream();
|
||
|
this._peekedToken = null;
|
||
|
}
|
||
|
|
||
|
loadFile(fileName) {
|
||
|
this.reset();
|
||
|
return this._cs.loadFile(fileName);
|
||
|
}
|
||
|
|
||
|
peek() {
|
||
|
if (!this._peekedToken) this._peekedToken = this._readNext();
|
||
|
return this._peekedToken;
|
||
|
}
|
||
|
|
||
|
next() {
|
||
|
const token = this.peek();
|
||
|
this._peekedToken = null;
|
||
|
return token;
|
||
|
}
|
||
|
|
||
|
eof() {
|
||
|
return this.peek() === null;
|
||
|
}
|
||
|
|
||
|
_readNext() {
|
||
|
let raw = '';
|
||
|
|
||
|
const flushRaw = () => {
|
||
|
if (raw.length) {
|
||
|
const value = raw;
|
||
|
raw = '';
|
||
|
return {type: 'raw', value};
|
||
|
}
|
||
|
}
|
||
|
|
||
|
while (!this._cs.eof()) {
|
||
2 years ago
|
if (
|
||
|
this._cs.line === 0 &&
|
||
|
this._cs.column === 0 &&
|
||
|
this._cs.peek(2) === '#!'
|
||
|
) {
|
||
|
// Skip shebang (can be used to execute templates directly with an
|
||
|
// interpreter that first builds the template and then executes)
|
||
|
this._skipShebang();
|
||
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE)) {
|
||
2 years ago
|
return flushRaw() || this._readInline();
|
||
2 years ago
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) {
|
||
2 years ago
|
return flushRaw() || this._readInlineUnescaped();
|
||
2 years ago
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) {
|
||
2 years ago
|
return flushRaw() || this._readInlineStatement();
|
||
2 years ago
|
} else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) {
|
||
2 years ago
|
return flushRaw() || this._readInlineStatementUnescaped();
|
||
|
} else if (
|
||
|
this._cs.column === 0 &&
|
||
|
this._cs.peekAfter(this._isWhitespace) === TokenStream.STATEMENT
|
||
|
) {
|
||
|
return flushRaw() || this._readStatement();
|
||
|
} else if (
|
||
|
this._cs.column === 0 &&
|
||
|
this._cs.peekAfter(this._isWhitespace) === TokenStream.COMMENT
|
||
|
) {
|
||
|
return flushRaw() || this._readComment();
|
||
|
} else {
|
||
|
raw += this._cs.next();
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return flushRaw() || null;
|
||
|
}
|
||
|
|
||
2 years ago
|
_skipShebang() {
|
||
|
// Skip shebang
|
||
|
this._cs.nextWhile((c) => c !== '\n');
|
||
|
|
||
|
// Skip new line
|
||
|
this._cs.next();
|
||
|
}
|
||
|
|
||
2 years ago
|
_missingInlineSpaceError(tag) {
|
||
|
throw new TemplateSyntaxError(
|
||
|
this._cs,
|
||
|
`Inline tag: "${tag}" must be followed by a whitespace`
|
||
|
);
|
||
|
}
|
||
|
|
||
|
_readInlineGeneric(tokenType, openTag, closeTag) {
|
||
|
this._cs.next(openTag.length);
|
||
|
|
||
|
// Check for mandatory space
|
||
|
if (this._cs.peek() !== ' ') {
|
||
|
this._missingInlineSpaceError(openTag);
|
||
|
}
|
||
|
this._cs.next();
|
||
|
|
||
|
let value = '';
|
||
|
while (!this._cs.eof() && !this._cs.isNext(closeTag)) {
|
||
|
const char = this._cs.next();
|
||
|
|
||
|
if (this._cs.isNext(closeTag)) {
|
||
|
if (char !== ' ') {
|
||
|
this._missingInlineSpaceError(closeTag);
|
||
|
}
|
||
|
} else {
|
||
|
value += char;
|
||
|
}
|
||
|
}
|
||
|
this._cs.next(closeTag.length);
|
||
|
return {type: tokenType, value};
|
||
|
}
|
||
|
|
||
|
_readInline() {
|
||
|
return this._readInlineGeneric(
|
||
|
'inline',
|
||
|
TokenStream.OPEN_INLINE,
|
||
|
TokenStream.CLOSE_INLINE
|
||
|
);
|
||
|
}
|
||
|
|
||
|
_readInlineUnescaped() {
|
||
|
return this._readInlineGeneric(
|
||
|
'inline_unescaped',
|
||
|
TokenStream.OPEN_INLINE_UNESCAPED,
|
||
|
TokenStream.CLOSE_INLINE_UNESCAPED
|
||
|
);
|
||
|
}
|
||
|
|
||
|
_readInlineStatement() {
|
||
|
return this._readInlineGeneric(
|
||
|
'inline_statement',
|
||
|
TokenStream.OPEN_INLINE_STATEMENT,
|
||
|
TokenStream.CLOSE_INLINE_STATEMENT
|
||
|
);
|
||
|
}
|
||
|
|
||
|
_readInlineStatementUnescaped() {
|
||
|
return this._readInlineGeneric(
|
||
|
'inline_statement_unescaped',
|
||
|
TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED,
|
||
|
TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED
|
||
|
);
|
||
|
}
|
||
|
|
||
2 years ago
|
_missingFullLineSpaceError(tag) {
|
||
|
throw new TemplateSyntaxError(
|
||
|
this._cs,
|
||
|
`Full-Line Tag: "${tag}" must be followed by a whitespace`
|
||
|
);
|
||
|
}
|
||
|
|
||
|
_readFullLineGeneric(tokenType, tag) {
|
||
2 years ago
|
// Skip whitespace
|
||
|
this._cs.nextWhile(this._isWhitespace);
|
||
|
|
||
2 years ago
|
// Skip tag
|
||
|
this._cs.next(tag.length);
|
||
2 years ago
|
|
||
|
// Check for mandatory space
|
||
|
if (this._cs.peek() !== ' ') {
|
||
2 years ago
|
this._missingFullLineSpaceError(tag);
|
||
2 years ago
|
}
|
||
|
|
||
|
// Consume space
|
||
|
this._cs.next();
|
||
|
|
||
2 years ago
|
// Remainder of line is value
|
||
2 years ago
|
const value = this._cs.nextWhile((c) => c !== '\n');
|
||
|
|
||
2 years ago
|
// Skip new line
|
||
2 years ago
|
this._cs.next();
|
||
|
|
||
2 years ago
|
return {type: tokenType, value};
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
_readStatement() {
|
||
|
return this._readFullLineGeneric(
|
||
|
'statement',
|
||
|
TokenStream.STATEMENT
|
||
|
);
|
||
|
}
|
||
2 years ago
|
|
||
2 years ago
|
_readComment() {
|
||
|
return this._readFullLineGeneric(
|
||
|
'comment',
|
||
|
TokenStream.COMMENT
|
||
|
);
|
||
2 years ago
|
}
|
||
|
|
||
|
// Utility methods
|
||
|
_isWhitespace(c) {
|
||
|
return /\s/.test(c);
|
||
|
}
|
||
|
}
|