diff options
Diffstat (limited to 'src/net/http_client.cc')
| -rw-r--r-- | src/net/http_client.cc | 393 |
1 files changed, 68 insertions, 325 deletions
diff --git a/src/net/http_client.cc b/src/net/http_client.cc index 5f5b449..4a10c02 100644 --- a/src/net/http_client.cc +++ b/src/net/http_client.cc @@ -17,302 +17,30 @@ */ #include "ricochet/net/http_client.hh" -#include "ricochet/net/ssl_socket.hh" +#include "ricochet/net/config.hh" +#include "ricochet/net/cookie_jar.hh" +#include "ricochet/net/http_utils.hh" +#include "ricochet/net/ssl_socket.hh" #include "ricochet/net/sys/address.hh" #include "ricochet/net/sys/socket.hh" +#include "ricochet/net/url_parser.hh" #include <cstddef> -#include <cstdlib> #include <format> -#include <fstream> #include <string> #include <string_view> -#include <sys/stat.h> -#include <unordered_map> namespace ricochet::net { namespace { -struct ParsedUrl -{ - std::string host {}; - std::string path {}; -}; - -std::expected<ParsedUrl, std::string> parse_url( std::string_view url ) -{ - std::string_view prefix = "http://"; - if ( url.starts_with( "https://" ) ) { - prefix = "https://"; - } else if ( url.starts_with( "http://" ) ) { - prefix = "http://"; - } else { - return std::unexpected( "Only 'http://' or 'https://' URLs are supported." ); - } - - url.remove_prefix( prefix.length() ); - - const auto sep_pos = url.find_first_of( "/?#" ); - - ParsedUrl result; - if ( sep_pos == std::string_view::npos ) { - result.host = std::string( url ); - result.path = "/"; - } else { - result.host = std::string( url.substr( 0, sep_pos ) ); - result.path = std::string( url.substr( sep_pos ) ); - if ( result.path[0] != '/' ) { - result.path.insert( 0, "/" ); - } - } - return result; -} - -std::string dechunk( std::string_view body ) // NOLINT(readability-convert-member-functions-to-static) -{ - std::string result; - std::size_t pos = 0; - while ( pos < body.size() ) { - // find hex length end with \r\n - auto line_end = body.find( "\r\n", pos ); - if ( line_end == std::string_view::npos ) { - break; - } - - // parse hex length - const std::string len_str( body.substr( pos, line_end - pos ) ); - unsigned int chunk_size = 0; - try { - chunk_size = std::stoul( len_str, nullptr, 16 ); - } catch ( ... ) { - break; - } - - if ( chunk_size == 0 ) { - break; // end of stream - } - - // extract data block (skip \r\n) - pos = line_end + 2; - if ( pos + chunk_size > body.size() ) { - break; - } - result.append( body.substr( pos, chunk_size ) ); - - // skip data block's \r\n - pos += chunk_size + 2; - } - return result; -} - -std::string remove_html_comments( std::string html ) // NOLINT(readability-convert-member-functions-to-static) -{ - std::size_t start_pos = 0; - while ( ( start_pos = html.find( "<!--", start_pos ) ) != std::string::npos ) { - auto end_pos = html.find( "-->", start_pos ); - if ( end_pos != std::string::npos ) { - html.erase( start_pos, end_pos - start_pos + 3 ); - } else { - html.erase( start_pos ); - break; - } - } - return html; -} - -std::string remove_tag_blocks( std::string html, const std::string& tag_name ) // NOLINT -{ - const std::string open_tag = "<" + tag_name; - const std::string close_tag = "</" + tag_name + ">"; - std::size_t start_pos = 0; - - while ( ( start_pos = html.find( open_tag, start_pos ) ) != std::string::npos ) { - const std::size_t end_pos = html.find( close_tag, start_pos ); - if ( end_pos != std::string::npos ) { - html.erase( start_pos, end_pos - start_pos + close_tag.length() ); - } else { - html.erase( start_pos ); - break; - } - } - return html; -} - -std::string extract_redirect_url( const std::string& raw_response, bool is_https, const std::string& host ) -{ - auto loc_pos = raw_response.find( "Location: " ); - if ( loc_pos == std::string::npos ) { - loc_pos = raw_response.find( "location: " ); - } - if ( loc_pos != std::string::npos ) { - loc_pos += 10; - auto end_pos = raw_response.find( '\r', loc_pos ); - if ( end_pos == std::string::npos ) { - end_pos = raw_response.find( '\n', loc_pos ); - } - if ( end_pos != std::string::npos ) { - std::string next_url = raw_response.substr( loc_pos, end_pos - loc_pos ); - while ( !next_url.empty() && std::isspace( static_cast<unsigned char>( next_url.front() ) ) ) { - next_url.erase( 0, 1 ); - } - if ( next_url.starts_with( "/" ) ) { - next_url = ( is_https ? "https://" : "http://" ) + host + next_url; - } - return next_url; - } - } - return ""; -} - -std::string get_config_path( const std::string& filename ) -{ - const char* home = std::getenv( "HOME" ); // NOLINT(concurrency-mt-unsafe) - const std::string dir = ( home != nullptr ) ? std::string( home ) + "/.ricochet" : ".ricochet"; - static_cast<void>( mkdir( dir.c_str(), 0755 ) ); - return dir + "/" + filename; -} - -struct BrowserConfig -{ - std::string user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36"; - std::string homepage = "https://wikipedia.org"; -}; - -BrowserConfig& get_browser_config() -{ - static BrowserConfig config; - static bool loaded = false; - if ( !loaded ) { - std::ifstream file( get_config_path( "config.txt" ) ); - if ( file.is_open() ) { - std::string line; - while ( std::getline( file, line ) ) { - const std::size_t eq = line.find( '=' ); - if ( eq == std::string::npos ) { - continue; - } - const std::string key = line.substr( 0, eq ); - const std::string val = line.substr( eq + 1 ); - if ( key == "user_agent" ) { - config.user_agent = val; - } else if ( key == "homepage" ) { - config.homepage = val; - } - } - } - loaded = true; - } - return config; -} - -using CookieJar = std::unordered_map<std::string, std::unordered_map<std::string, std::string>>; - -void load_cookies_from_file( CookieJar& jar ) -{ - std::ifstream file( get_config_path( "cookies.txt" ) ); - if ( !file.is_open() ) { - return; - } - - std::string line; - while ( std::getline( file, line ) ) { - const std::size_t t1 = line.find( '\t' ); - if ( t1 == std::string::npos ) { - continue; - } - const std::size_t t2 = line.find( '\t', t1 + 1 ); - if ( t2 != std::string::npos ) { - jar[line.substr( 0, t1 )][line.substr( t1 + 1, t2 - t1 - 1 )] = line.substr( t2 + 1 ); - } - } -} - -CookieJar& get_cookie_jar() -{ - static CookieJar jar; - static bool initialized = false; - if ( !initialized ) { - load_cookies_from_file( jar ); - initialized = true; - } - return jar; -} - -void save_cookies_to_file( const CookieJar& jar ) -{ - std::ofstream file( get_config_path( "cookies.txt" ), std::ios::trunc ); - if ( !file.is_open() ) { - return; - } - - for ( const auto& [domain, cookies] : jar ) { - for ( const auto& [k, v] : cookies ) { - file << domain << '\t' << k << '\t' << v << '\n'; - } - } -} - -void update_cookies( const std::string& raw_response, // NOLINT(bugprone-easily-swappable-parameters) - const std::string& host ) -{ - std::size_t pos = 0; - bool updated = false; - while ( ( pos = raw_response.find( "Set-Cookie: ", pos ) ) != std::string::npos ) { - pos += 12; - const std::size_t end_pos = raw_response.find( '\r', pos ); - if ( end_pos == std::string::npos ) { - break; - } - - const std::string cookie_line = raw_response.substr( pos, end_pos - pos ); - const std::size_t semi_pos = cookie_line.find( ';' ); - const std::string cookie_pair - = ( semi_pos == std::string::npos ) ? cookie_line : cookie_line.substr( 0, semi_pos ); - - const std::size_t eq_pos = cookie_pair.find( '=' ); - if ( eq_pos != std::string::npos ) { - const std::string key = cookie_pair.substr( 0, eq_pos ); - const std::string val = cookie_pair.substr( eq_pos + 1 ); - get_cookie_jar()[host][key] = val; - updated = true; - } - } - if ( updated ) { - save_cookies_to_file( get_cookie_jar() ); - } -} - -std::string build_cookie_header( const std::string& host ) -{ - const auto& jar = get_cookie_jar(); - auto it = jar.find( host ); - if ( it == jar.end() || it->second.empty() ) { - return ""; - } - - std::string header = "Cookie: "; - bool first = true; - for ( const auto& [k, v] : it->second ) { - if ( !first ) { - header += "; "; - } - header += k; - header += "="; - header += v; - first = false; - } - header += "\r\n"; - return header; -} - struct FetchResult { std::string raw_response {}; std::string error {}; }; + FetchResult do_network_request( const std::string& host, const std::string& service, // NOLINT(bugprone-easily-swappable-parameters) const std::string& path, // NOLINT(bugprone-easily-swappable-parameters) @@ -373,11 +101,67 @@ FetchResult do_network_request( const std::string& host, return res; } +void parse_response_headers( const std::string& raw_response, // NOLINT(bugprone-easily-swappable-parameters) + const std::string& host, // NOLINT(bugprone-easily-swappable-parameters) + HttpResponse& response, + std::string& headers ) +{ + const auto header_end = raw_response.find( "\r\n\r\n" ); + headers = ( header_end != std::string::npos ) ? raw_response.substr( 0, header_end ) : raw_response; + update_cookies( headers, host ); + + if ( header_end != std::string::npos ) { + response.body = raw_response.substr( header_end + 4 ); + std::size_t ct_pos = headers.find( "Content-Type: " ); + if ( ct_pos == std::string::npos ) { + ct_pos = headers.find( "content-type: " ); + } + if ( ct_pos != std::string::npos ) { + ct_pos += 14; + const std::size_t end_pos = headers.find( '\r', ct_pos ); + if ( end_pos != std::string::npos ) { + response.content_type = headers.substr( ct_pos, end_pos - ct_pos ); + } + } + if ( raw_response.size() >= 12 ) { + try { + response.status_code = std::stoi( raw_response.substr( 9, 3 ) ); + } catch ( ... ) { + response.status_code = 0; + } + } + } else { + response.body = raw_response; + response.status_code = 0; + } +} + +void sanitize_response_body( HttpResponse& response, const std::string& raw_response ) +{ + if ( raw_response.contains( "Transfer-Encoding: chunked" ) ) { + response.body = dechunk( response.body ); + } + std::string ct_lower = response.content_type; + for ( char& c : ct_lower ) { + if ( c >= 'A' && c <= 'Z' ) { + c = static_cast<char>( c + 32 ); + } + } + + if ( ct_lower.starts_with( "text/" ) || ct_lower.contains( "xml" ) ) { + response.body = remove_html_comments( response.body ); + response.body = remove_tag_blocks( response.body, "script" ); + response.body = remove_tag_blocks( response.body, "style" ); + } +} + } // namespace -std::expected<HttpResponse, std::string> HttpClient::fetch( std::string_view url, // NOLINT - std::string_view method, - std::string_view body ) const +std::expected<HttpResponse, std::string> +HttpClient::fetch( // NOLINT(readability-convert-member-functions-to-static) + std::string_view url, // NOLINT(bugprone-easily-swappable-parameters) + std::string_view method, // NOLINT(bugprone-easily-swappable-parameters) + std::string_view body ) const // NOLINT(bugprone-easily-swappable-parameters) { std::string current_url( url ); std::string current_method( method ); @@ -403,35 +187,8 @@ std::expected<HttpResponse, std::string> HttpClient::fetch( std::string_view url const std::string& raw_response = net_res.raw_response; HttpResponse response; - const auto header_end = raw_response.find( "\r\n\r\n" ); - const std::string headers - = ( header_end != std::string::npos ) ? raw_response.substr( 0, header_end ) : raw_response; - update_cookies( headers, host ); - - if ( header_end != std::string::npos ) { - response.body = raw_response.substr( header_end + 4 ); - std::size_t ct_pos = headers.find( "Content-Type: " ); - if ( ct_pos == std::string::npos ) { - ct_pos = headers.find( "content-type: " ); - } - if ( ct_pos != std::string::npos ) { - ct_pos += 14; - const std::size_t end_pos = headers.find( '\r', ct_pos ); - if ( end_pos != std::string::npos ) { - response.content_type = headers.substr( ct_pos, end_pos - ct_pos ); - } - } - if ( raw_response.size() >= 12 ) { - try { - response.status_code = std::stoi( raw_response.substr( 9, 3 ) ); - } catch ( ... ) { - response.status_code = 0; - } - } - } else { - response.body = raw_response; - response.status_code = 0; - } + std::string headers; + parse_response_headers( raw_response, host, response, headers ); if ( response.status_code >= 300 && response.status_code < 400 ) { const std::string next_url = extract_redirect_url( headers, is_https, host ); @@ -445,21 +202,7 @@ std::expected<HttpResponse, std::string> HttpClient::fetch( std::string_view url } } - if ( raw_response.contains( "Transfer-Encoding: chunked" ) ) { - response.body = dechunk( response.body ); - } - std::string ct_lower = response.content_type; - for ( char& c : ct_lower ) { - if ( c >= 'A' && c <= 'Z' ) { - c = static_cast<char>( c + 32 ); - } - } - - if ( ct_lower.starts_with( "text/" ) || ct_lower.contains( "xml" ) ) { - response.body = remove_html_comments( response.body ); - response.body = remove_tag_blocks( response.body, "script" ); - response.body = remove_tag_blocks( response.body, "style" ); - } + sanitize_response_body( response, raw_response ); return response; } |