Template-Engine/src/token_stream.js

import { CharacterStream } from './character_stream.js';
import { TemplateSyntaxError } from './errors/template_syntax_error.js';

export class TokenStream {
  static OPEN_INLINE = '<%';
  static CLOSE_INLINE = '%>';
  static OPEN_INLINE_UNESCAPED = '<!%';
  static CLOSE_INLINE_UNESCAPED = '%>';
  static OPEN_INLINE_STATEMENT = '<$%';
  static CLOSE_INLINE_STATEMENT = '%>';
  static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%';
  static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>';
  static STATEMENT = '$';
  static COMMENT = '#';

  constructor() {
    this.reset();
  }

  reset() {
    this._cs = new CharacterStream();
    this._peekedToken = null;
  }

  loadFile(fileName) {
    this.reset();
    return this._cs.loadFile(fileName);
  }

  peek() {
    if (!this._peekedToken) this._peekedToken = this._readNext();
    return this._peekedToken;
  }

  next() {
    const token = this.peek();
    this._peekedToken = null;
    return token;
  }

  eof() {
    return this.peek() === null;
  }

  _readNext() {
    let raw = '';

    const flushRaw = () => {
      if (raw.length) {
        const value = raw;
        raw = '';
        return {type: 'raw', value};
      }
    }

    while (!this._cs.eof()) {
      if (
        this._cs.line === 0 &&
        this._cs.column === 0 &&
        this._cs.peek(2) === '#!'
      ) {
        // Skip shebang (can be used to execute templates directly with an
        // interpreter that first builds the template and then executes)
        this._skipShebang();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE)) {
        return flushRaw() || this._readInline();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) {
        return flushRaw() || this._readInlineUnescaped();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) {
        return flushRaw() || this._readInlineStatement();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) {
        return flushRaw() || this._readInlineStatementUnescaped();
      } else if (
        this._cs.column === 0 &&
        this._cs.peekAfter(this._isWhitespace) === TokenStream.STATEMENT
      ) {
        return flushRaw() || this._readStatement();
      } else if (
        this._cs.column === 0 &&
        this._cs.peekAfter(this._isWhitespace) === TokenStream.COMMENT
      ) {
        return flushRaw() || this._readComment();
      } else {
        raw += this._cs.next();
      }
    }

    return flushRaw() || null;
  }

  _skipShebang() {
    // Skip shebang
    this._cs.nextWhile((c) => c !== '\n');

    // Skip new line
    this._cs.next();
  }

  _missingInlineSpaceError(tag) {
    throw new TemplateSyntaxError(
      this._cs,
      `Inline tag: "${tag}" must be followed by a whitespace`
    );
  }

  _readInlineGeneric(tokenType, openTag, closeTag) {
    this._cs.next(openTag.length);

    // Check for mandatory space
    if (this._cs.peek() !== ' ') {
      this._missingInlineSpaceError(openTag);
    }
    this._cs.next();

    let value = '';
    while (!this._cs.eof() && !this._cs.isNext(closeTag)) {
      const char = this._cs.next();

      if (this._cs.isNext(closeTag)) {
        if (char !== ' ') {
          this._missingInlineSpaceError(closeTag);
        }
      } else {
        value += char;
      }
    }
    this._cs.next(closeTag.length);
    return {type: tokenType, value};
  }

  _readInline() {
    return this._readInlineGeneric(
      'inline',
      TokenStream.OPEN_INLINE,
      TokenStream.CLOSE_INLINE
    );
  }

  _readInlineUnescaped() {
    return this._readInlineGeneric(
      'inline_unescaped',
      TokenStream.OPEN_INLINE_UNESCAPED,
      TokenStream.CLOSE_INLINE_UNESCAPED
    );
  }

  _readInlineStatement() {
    return this._readInlineGeneric(
      'inline_statement',
      TokenStream.OPEN_INLINE_STATEMENT,
      TokenStream.CLOSE_INLINE_STATEMENT
    );
  }

  _readInlineStatementUnescaped() {
    return this._readInlineGeneric(
      'inline_statement_unescaped',
      TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED,
      TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED
    );
  }

  _missingFullLineSpaceError(tag) {
    throw new TemplateSyntaxError(
      this._cs,
      `Full-Line Tag: "${tag}" must be followed by a whitespace`
    );
  }

  _readFullLineGeneric(tokenType, tag) {
    // Skip whitespace
    this._cs.nextWhile(this._isWhitespace);

    // Skip tag
    this._cs.next(tag.length);

    // Check for mandatory space
    if (this._cs.peek() !== ' ') {
      this._missingFullLineSpaceError(tag);
    }
    
    // Consume space
    this._cs.next();

    // Remainder of line is value
    const value = this._cs.nextWhile((c) => c !== '\n');

    // Skip new line
    this._cs.next();

    return {type: tokenType, value};
  }

  _readStatement() {
    return this._readFullLineGeneric(
      'statement',
      TokenStream.STATEMENT
    );
  }

  _readComment() {
    return this._readFullLineGeneric(
      'comment',
      TokenStream.COMMENT
    );
  }

  // Utility methods
  _isWhitespace(c) {
    return /\s/.test(c);
  }
}