TokenStream.h 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159
  1. /*
  2. * Copyright (c) 2020-2021, the SerenityOS developers.
  3. * Copyright (c) 2021-2023, Sam Atkins <atkinssj@serenityos.org>
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #pragma once
  8. #include <AK/Format.h>
  9. #include <AK/Vector.h>
  10. #include <LibWeb/CSS/Parser/ComponentValue.h>
  11. #include <LibWeb/CSS/Parser/Tokenizer.h>
  12. namespace Web::CSS::Parser {
  13. template<typename T>
  14. class TokenStream {
  15. public:
  16. class StateTransaction {
  17. public:
  18. explicit StateTransaction(TokenStream<T>& token_stream)
  19. : m_token_stream(token_stream)
  20. , m_saved_iterator_offset(token_stream.m_iterator_offset)
  21. {
  22. }
  23. ~StateTransaction()
  24. {
  25. if (!m_commit)
  26. m_token_stream.m_iterator_offset = m_saved_iterator_offset;
  27. }
  28. StateTransaction create_child() { return StateTransaction(*this); }
  29. void commit()
  30. {
  31. m_commit = true;
  32. if (m_parent)
  33. m_parent->commit();
  34. }
  35. private:
  36. explicit StateTransaction(StateTransaction& parent)
  37. : m_parent(&parent)
  38. , m_token_stream(parent.m_token_stream)
  39. , m_saved_iterator_offset(parent.m_token_stream.m_iterator_offset)
  40. {
  41. }
  42. StateTransaction* m_parent { nullptr };
  43. TokenStream<T>& m_token_stream;
  44. int m_saved_iterator_offset { 0 };
  45. bool m_commit { false };
  46. };
  47. explicit TokenStream(Span<T const> tokens)
  48. : m_tokens(tokens)
  49. , m_eof(make_eof())
  50. {
  51. }
  52. explicit TokenStream(Vector<T> const& tokens)
  53. : m_tokens(tokens.span())
  54. , m_eof(make_eof())
  55. {
  56. }
  57. static TokenStream<T> of_single_token(T const& token)
  58. {
  59. return TokenStream(Span<T const> { &token, 1 });
  60. }
  61. TokenStream(TokenStream<T> const&) = delete;
  62. TokenStream(TokenStream<T>&&) = default;
  63. bool has_next_token()
  64. {
  65. return (size_t)(m_iterator_offset + 1) < m_tokens.size();
  66. }
  67. T const& next_token()
  68. {
  69. if (!has_next_token())
  70. return m_eof;
  71. ++m_iterator_offset;
  72. return m_tokens.at(m_iterator_offset);
  73. }
  74. T const& peek_token(int offset = 0)
  75. {
  76. if (!has_next_token())
  77. return m_eof;
  78. return m_tokens.at(m_iterator_offset + offset + 1);
  79. }
  80. T const& current_token()
  81. {
  82. if ((size_t)m_iterator_offset >= m_tokens.size())
  83. return m_eof;
  84. return m_tokens.at(m_iterator_offset);
  85. }
  86. void reconsume_current_input_token()
  87. {
  88. if (m_iterator_offset >= 0)
  89. --m_iterator_offset;
  90. }
  91. StateTransaction begin_transaction() { return StateTransaction(*this); }
  92. void skip_whitespace()
  93. {
  94. while (peek_token().is(Token::Type::Whitespace))
  95. next_token();
  96. }
  97. size_t token_count() const { return m_tokens.size(); }
  98. size_t remaining_token_count() const { return token_count() - m_iterator_offset - 1; }
  99. void dump_all_tokens()
  100. {
  101. dbgln("Dumping all tokens:");
  102. for (size_t i = 0; i < m_tokens.size(); ++i) {
  103. auto& token = m_tokens[i];
  104. if ((i - 1) == (size_t)m_iterator_offset)
  105. dbgln("-> {}", token.to_debug_string());
  106. else
  107. dbgln(" {}", token.to_debug_string());
  108. }
  109. }
  110. void copy_state(Badge<Parser>, TokenStream<T> const& other)
  111. {
  112. m_iterator_offset = other.m_iterator_offset;
  113. }
  114. private:
  115. Span<T const> m_tokens;
  116. int m_iterator_offset { -1 };
  117. T make_eof()
  118. {
  119. if constexpr (IsSame<T, Token>) {
  120. return Tokenizer::create_eof_token();
  121. }
  122. if constexpr (IsSame<T, ComponentValue>) {
  123. return ComponentValue(Tokenizer::create_eof_token());
  124. }
  125. }
  126. T m_eof;
  127. };
  128. }