Merge remote-tracking branch 'ricochet-import/master'HEADmaster

1 files changed, 119 insertions, 0 deletions

diff --git a/ricochet/src/parser/html_lexer.cc b/ricochet/src/parser/html_lexer.cc
new file mode 100644
index 0000000..4d70288
--- /dev/null
+++ b/ricochet/src/parser/html_lexer.cc
@@ -0,0 +1,119 @@
+/*
+ * Ricochet - A minimalist CLI web browser for fast, distraction-free navigation.
+ * Copyright (C) 2026  Richard Qin
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "ricochet/parser/html_lexer.hh"
+
+#include <cstdint>
+#include <string_view>
+
+namespace ricochet::parser {
+
+namespace {
+enum class LexerState : std::uint8_t
+{
+  Data,             // Reading plain text
+  TagOpen,          // Just encountered '<'
+  TagName,          // Reading the name of an opening tag (e.g., "div")
+  ClosingTagName,   // Reading the name of a closing tag (e.g., "/div")
+  SwallowAttributes // Ignore attributes mode (ignore until encountering '>')
+};
+
+void emit_text( std::string& buffer, std::vector<HtmlToken>& tokens )
+{
+  if ( buffer.empty() ) {
+    return;
+  }
+
+  bool all_whitespace = true;
+  for ( const char c : buffer ) {
+    if ( !std::isspace( static_cast<unsigned char>( c ) ) ) {
+      all_whitespace = false;
+      break;
+    }
+  }
+
+  if ( !all_whitespace ) {
+    tokens.emplace_back( TextToken { buffer } );
+  }
+  buffer.clear();
+}
+
+} // namespace
+
+std::vector<HtmlToken> HtmlLexer::tokenize( // NOLINT(readability-convert-member-functions-to-static)
+  std::string_view html ) const
+{
+  std::vector<HtmlToken> tokens;
+
+  LexerState state = LexerState::Data;
+  std::string buffer {};
+
+  for ( const char c : html ) {
+    switch ( state ) {
+      case LexerState::Data:
+        if ( c == '<' ) {
+          emit_text( buffer, tokens );
+          state = LexerState::TagOpen;
+        } else {
+          buffer += c;
+        }
+        break;
+      case LexerState::TagOpen:
+        if ( c == '/' ) {
+          state = LexerState::ClosingTagName;
+        } else if ( std::isalpha( static_cast<unsigned char>( c ) ) ) {
+          buffer.push_back( c );
+          state = LexerState::TagName;
+        } else {
+          buffer.push_back( '<' );
+          buffer.push_back( c );
+          state = LexerState::Data;
+        }
+        break;
+      case LexerState::TagName:
+        if ( c == '>' ) {
+          tokens.emplace_back( TagOpenToken { buffer } );
+          buffer.clear();
+          state = LexerState::Data;
+        } else {
+          buffer += c;
+        }
+        break;
+      case LexerState::ClosingTagName:
+        if ( c == '>' ) {
+          tokens.emplace_back( TagCloseToken { buffer } );
+          buffer.clear();
+          state = LexerState::Data;
+        } else {
+          buffer += static_cast<char>( std::tolower( static_cast<unsigned char>( c ) ) );
+        }
+        break;
+      case LexerState::SwallowAttributes:
+        if ( c == '>' ) {
+          state = LexerState::Data;
+        }
+        break;
+    }
+  }
+
+  emit_text( buffer, tokens );
+
+  return tokens;
+}
+
+} // namespace ricochet::parser
\ No newline at end of file