Template-Engine/src/token_stream.mjs

import { CharacterStream } from './character_stream.mjs';
import { TemplateSyntaxError } from './errors/template_syntax_error.mjs';

export class TokenStream {
  static ESCAPE = '\\';
  static OPEN_STATEMENT = '<{';
  static CLOSE_STATEMENT = '}>';
  static OPEN_INLINE = '<%';
  static CLOSE_INLINE = '%>';
  static OPEN_INLINE_UNESCAPED = '<!%';
  static CLOSE_INLINE_UNESCAPED = '%>';
  static OPEN_INLINE_STATEMENT = '<$%';
  static CLOSE_INLINE_STATEMENT = '%>';
  static OPEN_INLINE_STATEMENT_UNESCAPED = '<$!%';
  static CLOSE_INLINE_STATEMENT_UNESCAPED = '%>';
  static STATEMENT = '$';
  static COMMENT = '#';
  static SOURCE = '.INCLUDE';

  constructor() {
    this.reset();
  }

  reset() {
    this._cs = new CharacterStream();
    this._peekedToken = null;
  }

  loadFile(fileName) {
    this.reset();
    return this._cs.loadFile(fileName);
  }

  peek() {
    if (!this._peekedToken) this._peekedToken = this._readNext();
    return this._peekedToken;
  }

  next() {
    const token = this.peek();
    this._peekedToken = null;
    return token;
  }

  eof() {
    return this.peek() === null;
  }

  _readNext() {
    let raw = '';

    const flushRaw = () => {
      if (raw.length) {
        const value = raw;
        raw = '';
        return {type: 'raw', value};
      }
    }

    while (!this._cs.eof()) {
      if (
        this._cs.line === 0 &&
        this._cs.column === 0 &&
        this._cs.peek(2) === '#!'
      ) {
        // Skip shebang (can be used to execute templates directly with an
        // interpreter that first builds the template and then executes)
        this._skipShebang();
      } else if (this._isNextEscapedTag([
        // Allow escaping of open and full-line tags
        TokenStream.OPEN_STATEMENT,
        TokenStream.OPEN_INLINE,
        TokenStream.OPEN_INLINE_UNESCAPED,
        TokenStream.OPEN_INLINE_STATEMENT,
        TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED,
        TokenStream.STATEMENT,
        TokenStream.COMMENT,
        TokenStream.SOURCE
      ])) {
        // Skip escape
        this._cs.next(TokenStream.ESCAPE.length);
        // Treat everything as raw until next space
        raw += this._cs.nextWhile((c) => /\S/.test(c));
      } else if (this._cs.isNext(TokenStream.OPEN_STATEMENT)) {
        return flushRaw() || this._readBlockStatement();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE)) {
        return flushRaw() || this._readInline();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_UNESCAPED)) {
        return flushRaw() || this._readInlineUnescaped();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT)) {
        return flushRaw() || this._readInlineStatement();
      } else if (this._cs.isNext(TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED)) {
        return flushRaw() || this._readInlineStatementUnescaped();
      } else if (this._matchFullLine(TokenStream.STATEMENT)) {
        return flushRaw() || this._readStatement();
      } else if (this._matchFullLine(TokenStream.COMMENT)) {
        return flushRaw() || this._readComment();
      } else if (this._matchFullLine(TokenStream.SOURCE)) {
        return flushRaw() || this._readSource();
      } else {
        raw += this._cs.next();
      }
    }

    return flushRaw() || null;
  }

  _skipShebang() {
    // Skip shebang
    this._cs.nextWhile((c) => c !== '\n');

    // Skip new line
    this._cs.next();
  }

  _missingOpenSpaceError(tag) {
    throw new TemplateSyntaxError(
      this._cs,
      `Tag: "${tag}" must be followed by a whitespace or new line`
    );
  }

  _missingCloseSpaceError(tag) {
    throw new TemplateSyntaxError(
      this._cs,
      `Tag: "${tag}" must be preceded by a whitespace or new line`
    );
  }

  _readBlockGeneric(tokenType, openTag, closeTag) {
    this._cs.next(openTag.length);

    // Check for mandatory space
    if (/\S/.test(this._cs.peek())) {
      this._missingOpenSpaceError(openTag);
    }
    this._cs.next();

    let value = '';
    while (!this._cs.eof() && !this._cs.isNext(closeTag)) {
      // Handle any escaped closing tag
      if (this._isNextEscapedTag([closeTag])) {
        // Skip escape
        this._cs.next(TokenStream.ESCAPE.length);

        // Treat everything as raw until next space or eof
        value += this._cs.nextWhile((c) => /\S/.test(c));
      } else {
        const char = this._cs.next();

        if (this._cs.isNext(closeTag)) {
          if (/\S/.test(char)) {
            this._missingCloseSpaceError(closeTag);
          }
        } else {
          value += char;
        }
      }
    }
    this._cs.next(closeTag.length);
    return {type: tokenType, value};
  }

  _readBlockStatement() {
    return this._readBlockGeneric(
      'block_statement',
      TokenStream.OPEN_STATEMENT,
      TokenStream.CLOSE_STATEMENT
    );
  }

  _readInline() {
    return this._readBlockGeneric(
      'inline',
      TokenStream.OPEN_INLINE,
      TokenStream.CLOSE_INLINE
    );
  }

  _readInlineUnescaped() {
    return this._readBlockGeneric(
      'inline_unescaped',
      TokenStream.OPEN_INLINE_UNESCAPED,
      TokenStream.CLOSE_INLINE_UNESCAPED
    );
  }

  _readInlineStatement() {
    return this._readBlockGeneric(
      'inline_statement',
      TokenStream.OPEN_INLINE_STATEMENT,
      TokenStream.CLOSE_INLINE_STATEMENT
    );
  }

  _readInlineStatementUnescaped() {
    return this._readBlockGeneric(
      'inline_statement_unescaped',
      TokenStream.OPEN_INLINE_STATEMENT_UNESCAPED,
      TokenStream.CLOSE_INLINE_STATEMENT_UNESCAPED
    );
  }

  _missingFullLineSpaceError(tag) {
    throw new TemplateSyntaxError(
      this._cs,
      `Full-Line Tag: "${tag}" must be followed by a whitespace`
    );
  }

  _readFullLineGeneric(tokenType, tag) {
    // Skip whitespace
    this._cs.nextWhile(this._isSpace);

    // Skip tag
    this._cs.next(tag.length);

    // Check for mandatory space/tab/new line
    if (this._isSpace(this._cs.peek())) {
      // Consume space
      this._cs.next();
    } else if (/[^\r\n]/.test(this._cs.peek())) {
      this._missingFullLineSpaceError(tag);
    }

    // Remainder of line is value
    const value = this._cs.nextWhile((c) => c !== '\n');

    // Skip new line
    this._cs.next();

    return {type: tokenType, value};
  }

  _readStatement() {
    return this._readFullLineGeneric(
      'statement',
      TokenStream.STATEMENT
    );
  }

  _readComment() {
    return this._readFullLineGeneric(
      'comment',
      TokenStream.COMMENT
    );
  }

  _readSource() {
    return this._readFullLineGeneric(
      'source',
      TokenStream.SOURCE
    );
  }

  // Utility methods
  _isSpace(c) {
    return /[^\S\r\n]/.test(c);
  }

  _isNextEscapedTag(possibleTags=[]) {
    if (!this._cs.isNext(TokenStream.ESCAPE)) return false;

    return !!possibleTags.find(tag =>
      this._cs.isNext(TokenStream.ESCAPE + tag)
    );
  }

  _matchFullLine(tag) {
    return (
      this._cs.column === 0 &&
      this._cs.peekAfter(this._isSpace, tag.length) === tag
    );
  }
}