import { CharacterStream } from './character_stream.mjs'; import { TemplateSyntaxError } from './errors/template_syntax_error.mjs'; export class TokenStream { static ESCAPE = '\\'; static OPEN_STATEMENT = '<{'; static CLOSE_STATEMENT = '}>'; static OPEN_INLINE = '<%'; static CLOSE_INLINE = '%>'; static OPEN_INLINE_UNESCAPED = ''; static OPEN_INLINE_STATEMENT = '<$%'; static CLOSE_INLINE_STATEMENT = '%>'; static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%'; static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>'; static STATEMENT = '$'; static COMMENT = '#'; static SOURCE = '.INCLUDE'; constructor() { this.reset(); } reset() { this._cs = new CharacterStream(); this._peekedToken = null; } loadFile(fileName) { this.reset(); return this._cs.loadFile(fileName); } peek() { if (!this._peekedToken) this._peekedToken = this._readNext(); return this._peekedToken; } next() { const token = this.peek(); this._peekedToken = null; return token; } eof() { return this.peek() === null; } _readNext() { let raw = ''; const flushRaw = () => { if (raw.length) { const value = raw; raw = ''; return {type: 'raw', value}; } } while (!this._cs.eof()) { if ( this._cs.line === 0 && this._cs.column === 0 && this._cs.peek(2) === '#!' ) { // Skip shebang (can be used to execute templates directly with an // interpreter that first builds the template and then executes) this._skipShebang(); } else if (this._isNextEscapedTag([ // Allow escaping of open and full-line tags TokenStream.OPEN_STATEMENT, TokenStream.OPEN_INLINE, TokenStream.OPEN_INLINE_UNESCAPED, TokenStream.OPEN_INLINE_STATEMENT, TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, TokenStream.STATEMENT, TokenStream.COMMENT, TokenStream.SOURCE ])) { // Skip escape this._cs.next(TokenStream.ESCAPE.length); // Treat everything as raw until next space raw += this._cs.nextWhile((c) => /\S/.test(c)); } else if (this._cs.isNext(TokenStream.OPEN_STATEMENT)) { return flushRaw() || this._readBlockStatement(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE)) { return flushRaw() || this._readInline(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) { return flushRaw() || this._readInlineUnescaped(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) { return flushRaw() || this._readInlineStatement(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) { return flushRaw() || this._readInlineStatementUnescaped(); } else if (this._matchFullLine(TokenStream.STATEMENT)) { return flushRaw() || this._readStatement(); } else if (this._matchFullLine(TokenStream.COMMENT)) { return flushRaw() || this._readComment(); } else if (this._matchFullLine(TokenStream.SOURCE)) { return flushRaw() || this._readSource(); } else { raw += this._cs.next(); } } return flushRaw() || null; } _skipShebang() { // Skip shebang this._cs.nextWhile((c) => c !== '\n'); // Skip new line this._cs.next(); } _missingOpenSpaceError(tag) { throw new TemplateSyntaxError( this._cs, `Tag: "${tag}" must be followed by a whitespace or new line` ); } _missingCloseSpaceError(tag) { throw new TemplateSyntaxError( this._cs, `Tag: "${tag}" must be preceded by a whitespace or new line` ); } _readBlockGeneric(tokenType, openTag, closeTag) { this._cs.next(openTag.length); // Check for mandatory space if (/\S/.test(this._cs.peek())) { this._missingOpenSpaceError(openTag); } this._cs.next(); let value = ''; while (!this._cs.eof() && !this._cs.isNext(closeTag)) { // Handle any escaped closing tag if (this._isNextEscapedTag([closeTag])) { // Skip escape this._cs.next(TokenStream.ESCAPE.length); // Treat everything as raw until next space or eof value += this._cs.nextWhile((c) => /\S/.test(c)); } else { const char = this._cs.next(); if (this._cs.isNext(closeTag)) { if (/\S/.test(char)) { this._missingCloseSpaceError(closeTag); } } else { value += char; } } } this._cs.next(closeTag.length); return {type: tokenType, value}; } _readBlockStatement() { return this._readBlockGeneric( 'block_statement', TokenStream.OPEN_STATEMENT, TokenStream.CLOSE_STATEMENT ); } _readInline() { return this._readBlockGeneric( 'inline', TokenStream.OPEN_INLINE, TokenStream.CLOSE_INLINE ); } _readInlineUnescaped() { return this._readBlockGeneric( 'inline_unescaped', TokenStream.OPEN_INLINE_UNESCAPED, TokenStream.CLOSE_INLINE_UNESCAPED ); } _readInlineStatement() { return this._readBlockGeneric( 'inline_statement', TokenStream.OPEN_INLINE_STATEMENT, TokenStream.CLOSE_INLINE_STATEMENT ); } _readInlineStatementUnescaped() { return this._readBlockGeneric( 'inline_statement_unescaped', TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED ); } _missingFullLineSpaceError(tag) { throw new TemplateSyntaxError( this._cs, `Full-Line Tag: "${tag}" must be followed by a whitespace` ); } _readFullLineGeneric(tokenType, tag) { // Skip whitespace this._cs.nextWhile(this._isSpace); // Skip tag this._cs.next(tag.length); // Check for mandatory space/tab/new line if (this._isSpace(this._cs.peek())) { // Consume space this._cs.next(); } else if (/[^\r\n]/.test(this._cs.peek())) { this._missingFullLineSpaceError(tag); } // Remainder of line is value const value = this._cs.nextWhile((c) => c !== '\n'); // Skip new line this._cs.next(); return {type: tokenType, value}; } _readStatement() { return this._readFullLineGeneric( 'statement', TokenStream.STATEMENT ); } _readComment() { return this._readFullLineGeneric( 'comment', TokenStream.COMMENT ); } _readSource() { return this._readFullLineGeneric( 'source', TokenStream.SOURCE ); } // Utility methods _isSpace(c) { return /[^\S\r\n]/.test(c); } _isNextEscapedTag(possibleTags=[]) { if (!this._cs.isNext(TokenStream.ESCAPE)) return false; return !!possibleTags.find(tag => this._cs.isNext(TokenStream.ESCAPE + tag) ); } _matchFullLine(tag) { return ( this._cs.column === 0 && this._cs.peekAfter(this._isSpace, tag.length) === tag ); } }