libzypp 17.31.23
request.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8----------------------------------------------------------------------*/
13#include <zypp-core/zyppng/base/EventDispatcher>
14#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15#include <zypp-core/zyppng/core/String>
16#include <zypp-core/fs/PathInfo.h>
18#include <zypp-curl/CurlConfig>
19#include <zypp-curl/auth/CurlAuthData>
20#include <zypp-media/MediaConfig>
21#include <zypp-core/base/String.h>
22#include <zypp-core/base/StringV.h>
23#include <zypp-core/Pathname.h>
24#include <curl/curl.h>
25#include <stdio.h>
26#include <fcntl.h>
27#include <sstream>
28#include <utility>
29
30#include <iostream>
31#include <boost/variant.hpp>
32#include <boost/variant/polymorphic_get.hpp>
33
34
35namespace zyppng {
36
37 namespace {
38 static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
39 if ( !userdata )
40 return 0;
41
42 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
43 return that->headerCallback( ptr, size, nmemb );
44 }
45 static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
46 if ( !userdata )
47 return 0;
48
49 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
50 return that->writeCallback( ptr, size, nmemb );
51 }
52
53 //helper for std::visit
54 template<class T> struct always_false : std::false_type {};
55 }
56
57 std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
58 {
59 if ( !fd )
60 return {};
61
62 fflush( fd );
63
64 std::vector<char> data( count + 1 , '\0' );
65
66 ssize_t l = -1;
67 while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
68 ;
69 if (l == -1)
70 return {};
71
72 return data;
73 }
74
75 NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
76 {
78 .start = start,
79 .len = len,
80 .bytesWritten = 0,
81 ._digest = std::move( digest ),
82 ._checksum = std::move( expectedChkSum ),
83 ._relevantDigestLen = std::move( digestCompareLen ),
84 ._chksumPad = std::move( dataBlockPadding ),
85 .userData = std::move( userData ),
86 ._rangeState = State::Pending
87 };
88 }
89
91 : _outFile( std::move(prevState._outFile) )
92 , _downloaded( prevState._downloaded )
93 , _rangeAttemptIdx( prevState._rangeAttemptIdx )
94 { }
95
97 : _requireStatusPartial( prevState._requireStatusPartial )
98 { }
99
101 : _outFile( std::move(prevState._outFile) )
102 , _requireStatusPartial( true )
103 , _downloaded( prevState._downloaded )
104 , _rangeAttemptIdx( prevState._rangeAttemptIdx )
105 { }
106
108 : BasePrivate(p)
109 , _url ( std::move(url) )
110 , _targetFile ( std::move( targetFile) )
111 , _fMode ( std::move(fMode) )
112 , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
113 { }
114
116 {
117 if ( _easyHandle ) {
118 //clean up for now, later we might reuse handles
119 curl_easy_cleanup( _easyHandle );
120 //reset in request but make sure the request was not enqueued again and got a new handle
121 _easyHandle = nullptr;
122 }
123 }
124
125 bool NetworkRequestPrivate::initialize( std::string &errBuf )
126 {
127 reset();
128
129 if ( _easyHandle )
130 //will reset to defaults but keep live connections, session ID and DNS caches
131 curl_easy_reset( _easyHandle );
132 else
133 _easyHandle = curl_easy_init();
134 return setupHandle ( errBuf );
135 }
136
137 bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
138 {
140 curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
141
142 const std::string urlScheme = _url.getScheme();
143 if ( urlScheme == "http" || urlScheme == "https" )
145
146 try {
147
148 setCurlOption( CURLOPT_PRIVATE, this );
150 setCurlOption( CURLOPT_XFERINFODATA, this );
151 setCurlOption( CURLOPT_NOPROGRESS, 0L);
152 setCurlOption( CURLOPT_FAILONERROR, 1L);
153 setCurlOption( CURLOPT_NOSIGNAL, 1L);
154
155 std::string urlBuffer( _url.asString() );
156 setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
157
158 setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
159 setCurlOption( CURLOPT_WRITEDATA, this );
160
162 setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
163 setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
164 }
166 // instead of returning no data with NOBODY, we return
167 // little data, that works with broken servers, and
168 // works for ftp as well, because retrieving only headers
169 // ftp will return always OK code ?
170 // See http://curl.haxx.se/docs/knownbugs.html #58
172 setCurlOption( CURLOPT_NOBODY, 1L );
173 else
174 setCurlOption( CURLOPT_RANGE, "0-1" );
175 }
176
178 if ( _requestedRanges.size() ) {
179 if ( ! prepareNextRangeBatch ( errBuf ))
180 return false;
181 } else {
182 std::visit( [&]( auto &arg ){
183 using T = std::decay_t<decltype(arg)>;
184 if constexpr ( std::is_same_v<T, pending_t> ) {
185 arg._requireStatusPartial = false;
186 } else {
187 DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
188 }
189 }, _runningMode );
192 }
193 }
194
195 //make a local copy of the settings, so headers are not added multiple times
197
198 if ( _dispatcher ) {
199 locSet.setUserAgentString( _dispatcher->agentString().c_str() );
200
201 // add custom headers as configured (bsc#955801)
202 const auto &cHeaders = _dispatcher->hostSpecificHeaders();
203 if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
204 for ( const auto &[key, value] : i->second ) {
206 "%s: %s", key.c_str(), value.c_str() )
207 ));
208 }
209 }
210 }
211
212 locSet.addHeader("Pragma:");
213
216 {
217 case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
218 case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
219 default: break;
220 }
221
222 setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
223 setCurlOption( CURLOPT_HEADERDATA, this );
224
228 setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
229 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
230 // just in case curl does not trigger its progress callback frequently
231 // enough.
232 if ( locSet.timeout() )
233 {
234 setCurlOption( CURLOPT_TIMEOUT, 3600L );
235 }
236
237 if ( urlScheme == "https" )
238 {
239#if CURLVERSION_AT_LEAST(7,19,4)
240 // restrict following of redirections from https to https only
241 if ( _url.getHost() == "download.opensuse.org" )
242 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
243 else
244 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
245#endif
246
247 if( locSet.verifyPeerEnabled() ||
248 locSet.verifyHostEnabled() )
249 {
250 setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
251 }
252
253 if( ! locSet.clientCertificatePath().empty() )
254 {
255 setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
256 }
257 if( ! locSet.clientKeyPath().empty() )
258 {
259 setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
260 }
261
262#ifdef CURLSSLOPT_ALLOW_BEAST
263 // see bnc#779177
264 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
265#endif
266 setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
267 setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
268 // bnc#903405 - POODLE: libzypp should only talk TLS
269 setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
270 }
271
272 // follow any Location: header that the server sends as part of
273 // an HTTP header (#113275)
274 setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
275 // 3 redirects seem to be too few in some cases (bnc #465532)
276 setCurlOption( CURLOPT_MAXREDIRS, 6L );
277
278 //set the user agent
279 setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
280
281
282 /*---------------------------------------------------------------*
283 CURLOPT_USERPWD: [user name]:[password]
284 Url::username/password -> CURLOPT_USERPWD
285 If not provided, anonymous FTP identification
286 *---------------------------------------------------------------*/
287 if ( locSet.userPassword().size() )
288 {
289 setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
290 std::string use_auth = _settings.authType();
291 if (use_auth.empty())
292 use_auth = "digest,basic"; // our default
294 if( auth != CURLAUTH_NONE)
295 {
296 DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
297 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
298 setCurlOption(CURLOPT_HTTPAUTH, auth);
299 }
300 }
301
302 if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
303 {
304 DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
305 setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
306 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
307
308 /*---------------------------------------------------------------*
309 * CURLOPT_PROXYUSERPWD: [user name]:[password]
310 *
311 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
312 * If not provided, $HOME/.curlrc is evaluated
313 *---------------------------------------------------------------*/
314
315 std::string proxyuserpwd = locSet.proxyUserPassword();
316
317 if ( proxyuserpwd.empty() )
318 {
320 zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
321 if ( curlconf.proxyuserpwd.empty() )
322 DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
323 else
324 {
325 proxyuserpwd = curlconf.proxyuserpwd;
326 DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
327 }
328 }
329 else
330 {
331 DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
332 }
333
334 if ( ! proxyuserpwd.empty() )
335 {
336 setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
337 }
338 }
339#if CURLVERSION_AT_LEAST(7,19,4)
340 else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
341 {
342 // Explicitly disabled in URL (see fillSettingsFromUrl()).
343 // This should also prevent libcurl from looking into the environment.
344 DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
345 setCurlOption(CURLOPT_NOPROXY, "*");
346 }
347
348#endif
349 // else: Proxy: not explicitly set; libcurl may look into the environment
350
352 if ( locSet.minDownloadSpeed() != 0 )
353 {
354 setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
355 // default to 10 seconds at low speed
356 setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
357 }
358
359#if CURLVERSION_AT_LEAST(7,15,5)
360 if ( locSet.maxDownloadSpeed() != 0 )
361 setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
362#endif
363
365 if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
366 setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
367 else
368 MIL << _easyHandle << " " << "No cookies requested" << std::endl;
369 setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
370
371#if CURLVERSION_AT_LEAST(7,18,0)
372 // bnc #306272
373 setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
374#endif
375
376 // Append settings custom headers to curl.
377 // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
378 for ( const auto &header : locSet.headers() ) {
379 if ( !z_func()->addRequestHeader( header.c_str() ) )
381 }
382
383 if ( _headers )
384 setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
385
386 return true;
387
388 } catch ( const zypp::Exception &excp ) {
389 ZYPP_CAUGHT(excp);
390 errBuf = excp.asString();
391 }
392 return false;
393 }
394
396 {
397 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
398 if ( !rmode ) {
399 DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
400 return false;
401 }
402 // if we have no open file create or open it
403 if ( !rmode->_outFile ) {
404 std::string openMode = "w+b";
406 openMode = "r+b";
407
408 rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
409
410 //if the file does not exist create a new one
411 if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
412 rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
413 }
414
415 if ( !rmode->_outFile ) {
417 ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
418 return false;
419 }
420 }
421
422 return true;
423 }
424
426 {
427 // We can recover from RangeFail errors if we have more batch sizes to try
428 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
429 if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
430 return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
431 return false;
432 }
433
435 {
436 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
437
438 if ( hasMoreWork() ) {
439 // go to the next range batch level if we are restarted due to a failed range request
440 if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
441 if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
442 errBuf = "No more range batch sizes available";
443 return false;
444 }
445 rmode->_rangeAttemptIdx++;
446 }
447
448 _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
449
450 // we reset the handle to default values. We do this to not run into
451 // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
452 // we cancel a connection because of a range error to request a smaller batch.
453 // The error will still happen but much less frequently than without resetting the handle.
454 //
455 // Note: Even creating a new handle will NOT fix the issue
456 curl_easy_reset( _easyHandle );
457 if ( !setupHandle (errBuf) )
458 return false;
459 return true;
460 }
461 errBuf = "Request has no more work";
462 return false;
463
464 }
465
467 {
468 if ( _requestedRanges.size() == 0 ) {
469 errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
470 return false;
471 }
472
473 std::string rangeDesc;
474 uint rangesAdded = 0;
475 if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
476 errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
477 return false;
478 }
479
480 // check if we have one big range convering the whole file
481 if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
482 if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
483 errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
484 return false;
485 }
486
488 std::get<pending_t>( _runningMode )._requireStatusPartial = false;
489
490 } else {
491 std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
492 return ( elem1.start < elem2.start );
493 });
494
495 if ( std::holds_alternative<pending_t>( _runningMode ) )
496 std::get<pending_t>( _runningMode )._requireStatusPartial = true;
497
498 auto maxRanges = _rangeAttempt[0];
499 if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
500 maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
501
502 // helper function to build up the request string for the range
503 auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
504 std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
505 if( range.second > 0 )
506 rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
507
508 if ( rangeDesc.size() )
509 rangeDesc.append(",").append( rangeD );
510 else
511 rangeDesc = std::move( rangeD );
512
513 rangesAdded++;
514 };
515
516 std::optional<std::pair<size_t, size_t>> currentZippedRange;
517 bool closedRange = true;
518 for ( auto &range : _requestedRanges ) {
519
520 if ( range._rangeState != NetworkRequest::Pending )
521 continue;
522
523 //reset the download results
524 range.bytesWritten = 0;
525
526 //when we have a open range in the list of ranges we will get from start of range to end of file,
527 //all following ranges would never be marked as valid, so we have to fail early
528 if ( !closedRange ) {
529 errBuf = "It is not supported to request more ranges after a open range.";
530 return false;
531 }
532
533 const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
534 closedRange = (rangeEnd > 0);
535
536 // remember this range was already requested
537 range._rangeState = NetworkRequest::Running;
538 range.bytesWritten = 0;
539 if ( range._digest )
540 range._digest->reset();
541
542 // we try to compress the requested ranges into as big chunks as possible for the request,
543 // when receiving we still track the original ranges so we can collect and test their checksums
544 if ( !currentZippedRange ) {
545 currentZippedRange = std::make_pair( range.start, rangeEnd );
546 } else {
547 //range is directly consecutive to the previous range
548 if ( currentZippedRange->second + 1 == range.start ) {
549 currentZippedRange->second = rangeEnd;
550 } else {
551 //this range does not directly follow the previous one, we build the string and start a new one
552 addRangeString( *currentZippedRange );
553 currentZippedRange = std::make_pair( range.start, rangeEnd );
554 }
555 }
556
557 if ( rangesAdded >= maxRanges ) {
558 MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
559 break;
560 }
561 }
562
563 // add the last range too
564 if ( currentZippedRange )
565 addRangeString( *currentZippedRange );
566
567 MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
568
569 setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
570 }
571
572 return true;
573 }
574
576 {
577 // check if we have ranges that have never been requested
578 return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
579 }
580
582 {
583 bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
584 if ( isRangeContinuation ) {
585 MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
586 _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
587 } else {
588 auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
589 if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
590 mode._currentRange = 0;
591
592 _runningMode = std::move(mode);
593 }
594
595 auto &m = std::get<running_t>( _runningMode );
596
597 if ( m._activityTimer ) {
598 DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
599 m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
600 m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
601 }
602
603 if ( !isRangeContinuation )
604 _sigStarted.emit( *z_func() );
605 }
606
608 {
609 if ( std::holds_alternative<running_t>(_runningMode) ) {
610 auto &rmode = std::get<running_t>( _runningMode );
611 // if we still have a current range set it valid by checking the checksum
612 if ( rmode._currentRange >= 0 ) {
613 auto &currR = _requestedRanges[rmode._currentRange];
614 rmode._currentRange = -1;
615 validateRange( currR );
616 }
617 }
618 }
619
621 {
622
623 finished_t resState;
624 resState._result = std::move(err);
625
626 if ( std::holds_alternative<running_t>(_runningMode) ) {
627
628 auto &rmode = std::get<running_t>( _runningMode );
629 rmode._outFile.reset();
630 resState._downloaded = rmode._downloaded;
631 resState._contentLenght = rmode._contentLenght;
632
634 //we have a successful download lets see if we got everything we needed
635 for ( const auto &r : _requestedRanges ) {
636 if ( r._rangeState != NetworkRequest::Finished ) {
637 if ( r.len > 0 && r.bytesWritten != r.len )
638 resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
639 else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
640 resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
641 } else {
643 }
644 //we only report the first error
645 break;
646 }
647 }
648 }
649 }
650
651 _runningMode = std::move( resState );
652 _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
653 }
654
656 {
658 _headers.reset( nullptr );
659 _errorBuf.fill( 0 );
661 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
662 range._rangeState = NetworkRequest::Pending;
663 });
664 }
665
667 {
668 auto &m = std::get<running_t>( _runningMode );
669
670 MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
671 std::map<std::string, boost::any> extraInfo;
672 extraInfo.insert( {"requestUrl", _url } );
673 extraInfo.insert( {"filepath", _targetFile } );
674 _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
675 }
676
678 {
679 if ( rng._digest && rng._checksum.size() ) {
680 auto bytesHashed = rng._digest->bytesHashed ();
681 if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
682 MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
683 zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
684 rng._digest->update( padding.data(), padding.size() );
685 }
686 auto digVec = rng._digest->digestVector();
687 if ( rng._relevantDigestLen ) {
688 digVec.resize( *rng._relevantDigestLen );
689 }
690 return ( digVec == rng._checksum );
691 }
692
693 // no checksum required
694 return true;
695 }
696
698 {
699 if ( rng._digest && rng._checksum.size() ) {
700 if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
702 else
704 } else {
705 if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
707 else
709 }
710 }
711
712 bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
713 { //content-range: bytes 10485760-19147879/19147880
714 static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
715
717 if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
718 DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
719 return false;
720 }
721
722 size_t s = zypp::str::strtonum<size_t>( what[1]);
723 size_t e = zypp::str::strtonum<size_t>( what[2]);
724 start = std::move(s);
725 len = ( e - s ) + 1;
726 return true;
727 }
728
729 bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
730 {
731 static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
732
734 if( zypp::str::regex_match( std::string(line), what, regex ) ) {
735 if ( what.size() >= 2 ) {
736 boundary = what[1];
737 return true;
738 }
739 }
740 return false;
741 }
742
744 {
745 return std::string( _errorBuf.data() );
746 }
747
749 {
750 if ( std::holds_alternative<running_t>( _runningMode ) ){
751 auto &rmode = std::get<running_t>( _runningMode );
752 if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
753 rmode._activityTimer->start();
754 }
755 }
756
757 int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
758 {
759 if ( !clientp )
760 return CURLE_OK;
761 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
762
763 if ( !std::holds_alternative<running_t>(that->_runningMode) ){
764 DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
765 return -1;
766 }
767
768 auto &rmode = std::get<running_t>( that->_runningMode );
769
770 //reset the timer
771 that->resetActivityTimer();
772
773 rmode._isInCallback = true;
774 if ( rmode._lastProgressNow != dlnow ) {
775 rmode._lastProgressNow = dlnow;
776 that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
777 }
778 rmode._isInCallback = false;
779
780 return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
781 }
782
783 size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
784 {
785 //it is valid to call this function with no data to write, just return OK
786 if ( size * nmemb == 0)
787 return 0;
788
790
792
793 std::string_view hdr( ptr, size*nmemb );
794
795 hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
796 const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
797 if ( lastNonWhitespace != hdr.npos )
798 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
799 else
800 hdr = std::string_view();
801
802 auto &rmode = std::get<running_t>( _runningMode );
803 if ( !hdr.size() ) {
804 return ( size * nmemb );
805 }
806 if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
807
808 long statuscode = 0;
809 (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
810
811 const auto &doRangeFail = [&](){
812 WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
813 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
814
815 // reset all ranges we requested to pending, we never got the data for them
816 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
817 if ( range._rangeState == NetworkRequest::Running )
818 range._rangeState = NetworkRequest::Pending;
819 });
820 return 0;
821 };
822
823 // if we have a status 204 we need to create a empty file
824 if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
826
827 if ( rmode._requireStatusPartial ) {
828 // ignore other status codes, maybe we are redirected etc.
829 if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
830 || statuscode == 416 ) {
831 return doRangeFail();
832 }
833 }
834
835 } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
836 _lastRedirect = hdr.substr( 9 );
837 DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
838
839 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
840 std::string sep;
841 if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
842 rmode._gotMultiRangeHeader = true;
843 rmode._seperatorString = "--"+sep;
844 }
845 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
847 if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
848 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
849 return 0;
850 }
851 DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
852 rmode._gotContentRangeHeader = true;
853 rmode._currentSrvRange = r;
854
855 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
856 auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
857 auto str = std::string ( lenStr.data(), lenStr.length() );
858 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
859 if ( len > 0 ) {
860 DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
861 rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
862 }
863 }
864 }
865
866 return ( size * nmemb );
867 }
868
869 size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
870 {
871 const auto max = ( size * nmemb );
872
874
875 //it is valid to call this function with no data to write, just return OK
876 if ( max == 0)
877 return 0;
878
879 //in case of a HEAD request, we do not write anything
881 return ( size * nmemb );
882 }
883
884 auto &rmode = std::get<running_t>( _runningMode );
885
886 auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
887
888 if ( rmode._currentRange < 0 ) {
889 DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
890 return 0;
891 }
892
893 // if we have no open file create or open it
894 if ( !assertOutputFile() )
895 return 0;
896
897 // seek to the given offset
898 if ( offset >= 0 ) {
899 if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
901 "Unable to set output file pointer." );
902 return 0;
903 }
904 }
905
906 auto &rng = _requestedRanges[ rmode._currentRange ];
907 const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
908
909 //make sure we do not write after the expected file size
910 if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
911 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
912 return 0;
913 }
914
915 auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
916 if ( written == 0 )
917 return 0;
918
919 if ( rng._digest && rng._checksum.size() ) {
920 if ( !rng._digest->update( data, written ) )
921 return 0;
922 }
923
924 rng.bytesWritten += written;
925 if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
926
927 if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
928 rmode._currentRange = -1;
929 validateRange( rng );
930 }
931
932 if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
933 rmode._currentSrvRange.reset();
934 // we ran out of data in the current chunk, reset the target range as well because next data will be
935 // a chunk header again
936 rmode._currentRange = -1;
937 }
938
939 // count the number of real bytes we have downloaded so far
940 rmode._downloaded += written;
941 _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
942
943 return written;
944 };
945
946 // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
947 size_t bytesWrittenSoFar = 0;
948
949 while ( bytesWrittenSoFar != max ) {
950
951 off_t seekTo = -1;
952
953 // this is called after all headers have been processed
954 if ( !rmode._allHeadersReceived ) {
955 rmode._allHeadersReceived = true;
956
957 // no ranges at all, must be a normal download
958 if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
959
960 if ( rmode._requireStatusPartial ) {
961 //we got a invalid response, the status code pointed to being partial but we got no range definition
963 "Invalid data from server, range respone was announced but there was no range definiton." );
964 return 0;
965 }
966
967 //we always download a range even if it is not explicitly requested
968 if ( _requestedRanges.empty() ) {
971 }
972
973 rmode._currentRange = 0;
974 seekTo = _requestedRanges[0].start;
975 }
976 }
977
978 if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
979 //if we enter this branch, we just have finished writing a requested chunk but
980 //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
981 //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
982 //Or we just parsed a Content-Lenght header and start a new block
983
984 std::optional<uint> foundRange;
985 const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
986 const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
987 auto currDist = ULONG_MAX;
988 for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
989 const auto &currR = _requestedRanges[i];
990
991 // do not allow double ranges
992 if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
993 continue;
994
995 // check if the range was already written
996 if ( currR.len == currR.bytesWritten )
997 continue;
998
999 const auto currRBegin = currR.start + currR.bytesWritten;
1000 if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
1001 continue;
1002
1003 // calculate the distance of the current ranges offset+data written to the range we got back from the server
1004 const auto newDist = currRBegin - beginRange;
1005
1006 if ( !foundRange ) {
1007 foundRange = i;
1008 currDist = newDist;
1009 } else {
1010 //pick the range with the closest distance
1011 if ( newDist < currDist ) {
1012 foundRange = i;
1013 currDist = newDist;
1014 }
1015 }
1016 }
1017 if ( !foundRange ) {
1019 , "Unable to find a matching range for data returned by the server." );
1020 return 0;
1021 }
1022
1023 //set the found range as the current one
1024 rmode._currentRange = *foundRange;
1025
1026 //continue writing where we stopped
1027 seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1028
1029 //if we skip bytes we need to advance our written bytecount
1030 const auto skipBytes = seekTo - beginRange;
1031 bytesWrittenSoFar += skipBytes;
1032 rmode._currentSrvRange->bytesWritten += skipBytes;
1033 }
1034
1035 if ( rmode._currentRange >= 0 ) {
1036 auto availableData = max - bytesWrittenSoFar;
1037 if ( rmode._currentSrvRange ) {
1038 availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1039 }
1040 auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1041 if ( bw <= 0 )
1042 return 0;
1043
1044 bytesWrittenSoFar += bw;
1045 }
1046
1047 if ( bytesWrittenSoFar == max )
1048 return max;
1049
1050 if ( rmode._currentRange == -1 ) {
1051
1052 // we still are inside the current range from the server
1053 if ( rmode._currentSrvRange )
1054 continue;
1055
1056 std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1057 auto hdrEnd = incoming.find("\r\n\r\n");
1058 if ( hdrEnd == incoming.npos ) {
1059 //no header end in the data yet, push to buffer and return
1060 rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1061 return max;
1062 }
1063
1064 //append the data of the current header to the buffer and parse it
1065 rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1066 bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1067
1068 std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1069 auto sepStrIndex = data.find( rmode._seperatorString );
1070 if ( sepStrIndex == data.npos ) {
1072 "Invalid multirange header format, seperator string missing." );
1073 return 0;
1074 }
1075
1076 auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1077 std::vector<std::string_view> lines;
1078 zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1079 for ( const auto &hdrLine : lines ) {
1080 if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1082 //if we can not parse the header the message must be broken
1083 if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1084 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1085 return 0;
1086 }
1087 rmode._currentSrvRange = r;
1088 break;
1089 }
1090 }
1091 //clear the buffer again
1092 rmode._rangePrefaceBuffer.clear();
1093 }
1094 }
1095 return bytesWrittenSoFar;
1096 }
1097
1099
1100 NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1101 : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1102 {
1103 }
1104
1106 {
1107 Z_D();
1108
1109 if ( d->_dispatcher )
1110 d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1111 }
1112
1114 {
1115 d_func()->_expectedFileSize = std::move( expectedFileSize );
1116 }
1117
1118 void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1119 {
1120 Z_D();
1121 d->_priority = prio;
1122 if ( state() == Pending && triggerReschedule && d->_dispatcher )
1123 d->_dispatcher->reschedule();
1124 }
1125
1127 {
1128 return d_func()->_priority;
1129 }
1130
1131 void NetworkRequest::setOptions( Options opt )
1132 {
1133 d_func()->_options = opt;
1134 }
1135
1136 NetworkRequest::Options NetworkRequest::options() const
1137 {
1138 return d_func()->_options;
1139 }
1140
1141 void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1142 {
1143 Z_D();
1144 if ( state() == Running )
1145 return;
1146
1147 d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1148 }
1149
1151 {
1152 Z_D();
1153 if ( state() == Running )
1154 return;
1155
1156 d->_requestedRanges.push_back( range );
1157 auto &rng = d->_requestedRanges.back();
1158 rng._rangeState = NetworkRequest::Pending;
1159 rng.bytesWritten = 0;
1160 if ( rng._digest )
1161 rng._digest->reset();
1162 }
1163
1165 {
1166 Z_D();
1167 if ( state() == Running )
1168 return;
1169 d->_requestedRanges.clear();
1170 }
1171
1172 std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1173 {
1174 const auto mystate = state();
1175 if ( mystate != Finished && mystate != Error )
1176 return {};
1177
1178 Z_D();
1179
1180 std::vector<Range> failed;
1181 for ( const auto &r : d->_requestedRanges ) {
1182 if ( r._rangeState != NetworkRequest::Finished )
1183 failed.push_back( r );
1184 }
1185 return failed;
1186 }
1187
1188 const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1189 {
1190 return d_func()->_requestedRanges;
1191 }
1192
1193 const std::string &NetworkRequest::lastRedirectInfo() const
1194 {
1195 return d_func()->_lastRedirect;
1196 }
1197
1199 {
1200 return d_func()->_easyHandle;
1201 }
1202
1203 std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1204 {
1205 const auto myerr = error();
1206 const auto mystate = state();
1207 if ( mystate != Finished )
1208 return {};
1209
1210 Timings t;
1211
1212 auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1213 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1214 double val = 0;
1215 const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1216 if ( CURLE_OK == res ) {
1217 target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1218 }
1219 };
1220
1221 getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1222 getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1223 getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1224 getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1225 getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1226 getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1227
1228 return t;
1229 }
1230
1231 std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1232 {
1233 Z_D();
1234
1235 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1236 return {};
1237
1238 const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1239 return peek_data_fd( rmode._outFile, offset, count );
1240 }
1241
1243 {
1244 return d_func()->_url;
1245 }
1246
1247 void NetworkRequest::setUrl(const Url &url)
1248 {
1249 Z_D();
1250 if ( state() == NetworkRequest::Running )
1251 return;
1252
1253 d->_url = url;
1254 }
1255
1257 {
1258 return d_func()->_targetFile;
1259 }
1260
1262 {
1263 Z_D();
1264 if ( state() == NetworkRequest::Running )
1265 return;
1266 d->_targetFile = path;
1267 }
1268
1270 {
1271 return d_func()->_fMode;
1272 }
1273
1275 {
1276 Z_D();
1277 if ( state() == NetworkRequest::Running )
1278 return;
1279 d->_fMode = std::move( mode );
1280 }
1281
1283 {
1284 char *ptr = NULL;
1285 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1286 return std::string(ptr);
1287 return std::string();
1288 }
1289
1291 {
1292 return std::visit([](auto& arg) -> zypp::ByteCount {
1293 using T = std::decay_t<decltype(arg)>;
1294 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1295 return zypp::ByteCount(0);
1296 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1297 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1298 return arg._contentLenght;
1299 else
1300 static_assert(always_false<T>::value, "Unhandled state type");
1301 }, d_func()->_runningMode);
1302 }
1303
1305 {
1306 return std::visit([](auto& arg) -> zypp::ByteCount {
1307 using T = std::decay_t<decltype(arg)>;
1308 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1309 return zypp::ByteCount();
1310 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1311 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1312 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1313 return arg._downloaded;
1314 else
1315 static_assert(always_false<T>::value, "Unhandled state type");
1316 }, d_func()->_runningMode);
1317 }
1318
1320 {
1321 return d_func()->_settings;
1322 }
1323
1325 {
1326 return std::visit([this](auto& arg) {
1327 using T = std::decay_t<decltype(arg)>;
1328 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1329 return Pending;
1330 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1331 return Running;
1332 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1333 if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1334 return Error;
1335 else
1336 return Finished;
1337 }
1338 else
1339 static_assert(always_false<T>::value, "Unhandled state type");
1340 }, d_func()->_runningMode);
1341 }
1342
1344 {
1345 const auto s = state();
1346 if ( s != Error && s != Finished )
1347 return NetworkRequestError();
1348 return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1349 }
1350
1352 {
1353 if ( !hasError() )
1354 return std::string();
1355
1356 return error().nativeErrorString();
1357 }
1358
1360 {
1361 return error().isError();
1362 }
1363
1364 bool NetworkRequest::addRequestHeader( const std::string &header )
1365 {
1366 Z_D();
1367
1368 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1369 if ( !res )
1370 return false;
1371
1372 if ( !d->_headers )
1373 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1374
1375 return true;
1376 }
1377
1379 {
1380 return d_func()->_sigStarted;
1381 }
1382
1384 {
1385 return d_func()->_sigBytesDownloaded;
1386 }
1387
1388 SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1389 {
1390 return d_func()->_sigProgress;
1391 }
1392
1394 {
1395 return d_func()->_sigFinished;
1396 }
1397
1398}
ZYppCommitResult & _result
Definition: TargetImpl.cc:1597
Store and operate with byte count.
Definition: ByteCount.h:31
static const Unit B
1 Byte
Definition: ByteCount.h:42
Unit::ValueType SizeType
Definition: ByteCount.h:37
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
Base class for Exception.
Definition: Exception.h:146
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
Holds transfer setting.
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
const std::string & authType() const
get the allowed authentication types
long timeout() const
transfer timeout
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
const Headers & headers() const
returns a list of all added headers (trimmed)
const std::string & proxy() const
proxy host
const Pathname & clientKeyPath() const
SSL client key file.
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & userAgentString() const
user agent string (trimmed)
bool headRequestsAllowed() const
whether HEAD requests are allowed
bool proxyEnabled() const
proxy is enabled
const std::string & proxyUsername() const
proxy auth username
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
Regular expression.
Definition: Regex.h:95
@ icase
Do not differentiate case.
Definition: Regex.h:99
@ rxdefault
These are enforced even if you don't pass them as flag argument.
Definition: Regex.h:103
Regular expression match result.
Definition: Regex.h:168
unsigned size() const
Definition: Regex.cc:106
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
The NetworkRequestError class Represents a error that occured in.
Type type() const
type Returns the type of the error
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
const std::string _currentCookieFile
Definition: request_p.h:126
zypp::Pathname _targetFile
Definition: request_p.h:116
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:729
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:133
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:129
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:757
std::string errorMessage() const
Definition: request.cc:743
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:132
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:211
bool initialize(std::string &errBuf)
Definition: request.cc:125
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:697
void onActivityTimeout(Timer &)
Definition: request.cc:666
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:134
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:869
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:125
NetworkRequest::Options _options
Definition: request_p.h:118
static constexpr int _rangeAttempt[]
Definition: request_p.h:148
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:434
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:466
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:783
void setResult(NetworkRequestError &&err)
Definition: request.cc:620
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
bool setupHandle(std::string &errBuf)
Definition: request.cc:137
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:107
TransferSettings _settings
Definition: request_p.h:117
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:135
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:141
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:677
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:712
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size,...
Definition: request.cc:1290
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1256
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1304
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1247
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1113
virtual ~NetworkRequest()
Definition: request.cc:1105
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1118
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1231
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1282
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1274
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
bool addRequestHeader(const std::string &header)
Definition: request.cc:1364
void setOptions(Options opt)
Definition: request.cc:1131
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1269
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1359
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1324
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1393
UByteArray CheckSumBytes
Definition: request.h:47
Options options() const
Definition: request.cc:1136
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1383
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1141
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1203
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1351
Priority priority() const
Definition: request.cc:1126
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1343
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1261
void * nativeHandle() const
Definition: request.cc:1198
std::vector< Range > failedRanges() const
Definition: request.cc:1172
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1188
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1378
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1388
TransferSettings & transferSettings()
Definition: request.cc:1319
const std::string & lastRedirectInfo() const
Definition: request.cc:1193
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
#define MIL_MEDIA
Definition: mediadebug_p.h:29
#define DBG_MEDIA
Definition: mediadebug_p.h:28
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
Definition: Arch.h:361
String related utilities and Regular expression matching.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
constexpr bool always_false
Definition: PathInfo.cc:544
@ TRIM
Definition: String.h:500
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition: Regex.h:70
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:57
ZYPP_IMPL_PRIVATE(Provide)
Structure holding values of curlrc options.
Definition: curlconfig.h:27
std::string proxyuserpwd
Definition: curlconfig.h:49
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:253
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:212
running_t(pending_t &&prevState)
Definition: request.cc:96
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:75
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::optional< size_t > _chksumPad
Definition: request.h:88
std::chrono::microseconds appconnect
Definition: request.h:99
std::chrono::microseconds redirect
Definition: request.h:102
std::chrono::microseconds pretransfer
Definition: request.h:100
std::chrono::microseconds total
Definition: request.h:101
std::chrono::microseconds namelookup
Definition: request.h:97
std::chrono::microseconds connect
Definition: request.h:98
#define nullptr
Definition: Easy.h:55
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define WAR
Definition: Logger.h:97