import { CharacterStream } from './character_stream.mjs'; import { TemplateSyntaxError } from './errors/template_syntax_error.mjs'; export class TokenStream { static OPEN_INLINE = '<%'; static CLOSE_INLINE = '%>'; static OPEN_INLINE_UNESCAPED = ''; static OPEN_INLINE_STATEMENT = '<$%'; static CLOSE_INLINE_STATEMENT = '%>'; static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%'; static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>'; static STATEMENT = '$'; static COMMENT = '#'; constructor() { this.reset(); } reset() { this._cs = new CharacterStream(); this._peekedToken = null; } loadFile(fileName) { this.reset(); return this._cs.loadFile(fileName); } peek() { if (!this._peekedToken) this._peekedToken = this._readNext(); return this._peekedToken; } next() { const token = this.peek(); this._peekedToken = null; return token; } eof() { return this.peek() === null; } _readNext() { let raw = ''; const flushRaw = () => { if (raw.length) { const value = raw; raw = ''; return {type: 'raw', value}; } } while (!this._cs.eof()) { if ( this._cs.line === 0 && this._cs.column === 0 && this._cs.peek(2) === '#!' ) { // Skip shebang (can be used to execute templates directly with an // interpreter that first builds the template and then executes) this._skipShebang(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE)) { return flushRaw() || this._readInline(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) { return flushRaw() || this._readInlineUnescaped(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) { return flushRaw() || this._readInlineStatement(); } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) { return flushRaw() || this._readInlineStatementUnescaped(); } else if ( this._cs.column === 0 && this._cs.peekAfter(this._isWhitespace) === TokenStream.STATEMENT ) { return flushRaw() || this._readStatement(); } else if ( this._cs.column === 0 && this._cs.peekAfter(this._isWhitespace) === TokenStream.COMMENT ) { return flushRaw() || this._readComment(); } else { raw += this._cs.next(); } } return flushRaw() || null; } _skipShebang() { // Skip shebang this._cs.nextWhile((c) => c !== '\n'); // Skip new line this._cs.next(); } _missingInlineSpaceError(tag) { throw new TemplateSyntaxError( this._cs, `Inline tag: "${tag}" must be followed by a whitespace` ); } _readInlineGeneric(tokenType, openTag, closeTag) { this._cs.next(openTag.length); // Check for mandatory space if (this._cs.peek() !== ' ') { this._missingInlineSpaceError(openTag); } this._cs.next(); let value = ''; while (!this._cs.eof() && !this._cs.isNext(closeTag)) { const char = this._cs.next(); if (this._cs.isNext(closeTag)) { if (char !== ' ') { this._missingInlineSpaceError(closeTag); } } else { value += char; } } this._cs.next(closeTag.length); return {type: tokenType, value}; } _readInline() { return this._readInlineGeneric( 'inline', TokenStream.OPEN_INLINE, TokenStream.CLOSE_INLINE ); } _readInlineUnescaped() { return this._readInlineGeneric( 'inline_unescaped', TokenStream.OPEN_INLINE_UNESCAPED, TokenStream.CLOSE_INLINE_UNESCAPED ); } _readInlineStatement() { return this._readInlineGeneric( 'inline_statement', TokenStream.OPEN_INLINE_STATEMENT, TokenStream.CLOSE_INLINE_STATEMENT ); } _readInlineStatementUnescaped() { return this._readInlineGeneric( 'inline_statement_unescaped', TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED, TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED ); } _missingFullLineSpaceError(tag) { throw new TemplateSyntaxError( this._cs, `Full-Line Tag: "${tag}" must be followed by a whitespace` ); } _readFullLineGeneric(tokenType, tag) { // Skip whitespace this._cs.nextWhile(this._isWhitespace); // Skip tag this._cs.next(tag.length); // Check for mandatory space if (this._cs.peek() !== ' ') { this._missingFullLineSpaceError(tag); } // Consume space this._cs.next(); // Remainder of line is value const value = this._cs.nextWhile((c) => c !== '\n'); // Skip new line this._cs.next(); return {type: tokenType, value}; } _readStatement() { return this._readFullLineGeneric( 'statement', TokenStream.STATEMENT ); } _readComment() { return this._readFullLineGeneric( 'comment', TokenStream.COMMENT ); } // Utility methods _isWhitespace(c) { return /\s/.test(c); } }