1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
/*
* Ricochet - A minimalist CLI web browser for fast, distraction-free navigation.
* Copyright (C) 2026 Richard Qin
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "text.hh"
#include <climits>
#include <cwchar>
#include <memory>
#include <stdexcept>
#include <unicode/ubrk.h>
#include <unicode/uchar.h>
#include <unicode/utext.h>
#include <utility>
namespace ricochet::tui::detail {
namespace {
using UnicodeText = std::unique_ptr<UText, decltype( &utext_close )>;
using Breaks = std::unique_ptr<UBreakIterator, decltype( &ubrk_close )>;
Breaks breaks_for( UBreakIteratorType type, UText& text )
{
UErrorCode status = U_ZERO_ERROR;
Breaks breaks( ubrk_open( type, "root", nullptr, 0, &status ), &ubrk_close );
if ( U_FAILURE( status ) ) {
throw std::runtime_error( u_errorName( status ) );
}
ubrk_setUText( breaks.get(), &text, &status );
if ( U_FAILURE( status ) ) {
throw std::runtime_error( u_errorName( status ) );
}
return breaks;
}
WordKind word_kind( UChar32 codepoint )
{
if ( u_isUWhiteSpace( codepoint ) ) {
return WordKind::Space;
}
if ( u_hasBinaryProperty( codepoint, UCHAR_IDEOGRAPHIC ) ) {
return WordKind::Ideograph;
}
if ( codepoint == '_' || u_isalnum( codepoint ) ) {
return WordKind::Word;
}
return WordKind::Punctuation;
}
} // namespace
TextService::TextService( std::size_t glyph_capacity ) : glyph_capacity_( glyph_capacity ) {}
TextAnalysis TextService::analyze( std::string_view utf8, bool line_breaks, bool tabs ) const
{
// ICU BreakIterator exposes int32_t native offsets even with a UTF-8 UText.
if ( utf8.size() > INT32_MAX ) {
throw std::length_error( "Text exceeds ICU break-iterator offset capacity" );
}
UErrorCode status = U_ZERO_ERROR;
UnicodeText text(
utext_openUTF8( nullptr, utf8.empty() ? "" : utf8.data(), static_cast<int64_t>( utf8.size() ), &status ),
&utext_close );
if ( U_FAILURE( status ) ) {
throw std::runtime_error( u_errorName( status ) );
}
auto characters = breaks_for( UBRK_CHARACTER, *text );
auto lines = breaks_for( UBRK_LINE, *text );
TextAnalysis result;
int32_t begin = ubrk_first( characters.get() );
int32_t line_end = ubrk_next( lines.get() );
WordKind previous = WordKind::Space;
for ( int32_t end = ubrk_next( characters.get() ); end != UBRK_DONE;
begin = end, end = ubrk_next( characters.get() ) ) {
while ( line_end != UBRK_DONE && line_end < end ) {
line_end = ubrk_next( lines.get() );
}
Grapheme glyph { .bytes = { static_cast<std::size_t>( begin ), static_cast<std::size_t>( end ) },
.break_after = line_end == end };
utext_setNativeIndex( text.get(), begin );
const auto first = utext_next32( text.get() );
glyph.word = word_kind( first );
const auto source = utf8.substr( glyph.bytes.start, glyph.bytes.end - glyph.bytes.start );
if ( line_breaks && ( source == "\n" || source == "\r\n" ) ) {
glyph.kind = TextUnitKind::LineBreak;
glyph.columns = 0;
} else if ( tabs && source == "\t" ) {
glyph.kind = TextUnitKind::Tab;
glyph.display = L" ";
} else {
const int width = ::wcwidth( static_cast<wchar_t>( first ) );
glyph.display.push_back( static_cast<wchar_t>( first ) );
bool representable = width > 0 && !u_hasBinaryProperty( first, UCHAR_DEFAULT_IGNORABLE_CODE_POINT )
&& !u_hasBinaryProperty( first, UCHAR_EMOJI_PRESENTATION );
while ( utext_getNativeIndex( text.get() ) < end ) {
const auto codepoint = utext_next32( text.get() );
glyph.display.push_back( static_cast<wchar_t>( codepoint ) );
// ncurses accepts one spacing character followed by nonspacing marks.
// Joiners, selectors and extra spacing characters use one visible fallback.
representable = representable && ::wcwidth( static_cast<wchar_t>( codepoint ) ) == 0
&& u_charType( codepoint ) == U_NON_SPACING_MARK
&& !u_hasBinaryProperty( codepoint, UCHAR_DEFAULT_IGNORABLE_CODE_POINT );
}
if ( !representable || glyph.display.size() > glyph_capacity_ ) {
glyph.display = L"?";
glyph.columns = 1;
} else {
glyph.columns = width;
}
}
if ( glyph.word != WordKind::Space ) {
if ( glyph.word == previous && glyph.word != WordKind::Ideograph ) {
result.words.back().end = glyph.bytes.end;
} else {
result.words.push_back( glyph.bytes );
}
}
previous = glyph.word;
result.graphemes.push_back( std::move( glyph ) );
}
return result;
}
} // namespace ricochet::tui::detail
|
