libzypp  17.28.5
CurlHelper.cc
Go to the documentation of this file.
1 #include "CurlHelper.h"
2 
3 #include <zypp/PathInfo.h>
4 #include <zypp/Pathname.h>
5 #include <zypp/Target.h>
6 #include <zypp/base/Logger.h>
7 #include <zypp/base/String.h>
8 #include <zypp/media/ProxyInfo.h>
11 #include <list>
12 
13 using std::endl;
14 using namespace zypp;
15 
16 namespace zypp
17 {
18  namespace env
19  {
20  namespace
21  {
22  inline int getZYPP_MEDIA_CURL_IPRESOLVE()
23  {
24  int ret = 0;
25  if ( const char * envp = getenv( "ZYPP_MEDIA_CURL_IPRESOLVE" ) )
26  {
27  WAR << "env set: $ZYPP_MEDIA_CURL_IPRESOLVE='" << envp << "'" << std::endl;
28  if ( strcmp( envp, "4" ) == 0 ) ret = 4;
29  else if ( strcmp( envp, "6" ) == 0 ) ret = 6;
30  }
31  return ret;
32  }
33  } //namespace
34 
36  {
37  static int _v = getZYPP_MEDIA_CURL_IPRESOLVE();
38  return _v;
39  }
40  } // namespace env
41 } // namespace zypp
42 
43 namespace internal
44 {
45 
47 {
48  // function-level static <=> std::call_once
49  static bool once __attribute__ ((__unused__)) = ( [] {
50  if ( curl_global_init( CURL_GLOBAL_ALL ) != 0 )
51  WAR << "curl global init failed" << std::endl;
52  } (), true );
53 }
54 
55 int log_curl(CURL *curl, curl_infotype info,
56  char *ptr, size_t len, void *max_lvl)
57 {
58  if ( max_lvl == nullptr )
59  return 0;
60 
61  long maxlvl = *((long *)max_lvl);
62 
63  char pfx = ' ';
64  switch( info )
65  {
66  case CURLINFO_TEXT: if ( maxlvl < 1 ) return 0; pfx = '*'; break;
67  case CURLINFO_HEADER_IN: if ( maxlvl < 2 ) return 0; pfx = '<'; break;
68  case CURLINFO_HEADER_OUT: if ( maxlvl < 2 ) return 0; pfx = '>'; break;
69  default:
70  return 0;
71  }
72 
73  std::vector<std::string> lines;
74  str::split( std::string(ptr,len), std::back_inserter(lines), "\r\n" );
75  for( const auto & line : lines )
76  {
77  if ( str::startsWith( line, "Authorization:" ) ) {
78  std::string::size_type pos { line.find( " ", 15 ) }; // Authorization: <type> <credentials>
79  if ( pos == std::string::npos )
80  pos = 15;
81  DBG << pfx << " " << line.substr( 0, pos ) << " <credentials removed>" << std::endl;
82  }
83  else
84  DBG << pfx << " " << line << std::endl;
85  }
86  return 0;
87 }
88 
89 size_t log_redirects_curl( char *ptr, size_t size, size_t nmemb, void *userdata)
90 {
91  //INT << "got header: " << std::string(ptr, ptr + size*nmemb) << endl;
92 
93  char * lstart = ptr, * lend = ptr;
94  size_t pos = 0;
95  size_t max = size * nmemb;
96  while (pos + 1 < max)
97  {
98  // get line
99  for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
100 
101  // look for "Location"
102  if ( strncasecmp( lstart, "Location:", 9 ) == 0 )
103  {
104  std::string line { lstart, *(lend-1)=='\r' ? lend-1 : lend };
105  DBG << "redirecting to " << line << std::endl;
106  if ( userdata ) {
107  *reinterpret_cast<std::string *>( userdata ) = line;
108  }
109  return max;
110  }
111 
112  // continue with the next line
113  if (pos + 1 < max)
114  {
115  ++lend;
116  ++pos;
117  }
118  else
119  break;
120  }
121 
122  return max;
123 }
124 
130 {
131  {
132  const std::string & param { url.getQueryParam("timeout") };
133  if( ! param.empty() )
134  {
135  long num = str::strtonum<long>(param);
136  if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX )
137  s.setTimeout( num );
138  }
139  }
140  {
141  std::string param { url.getUsername() };
142  if ( ! param.empty() )
143  {
144  s.setUsername( std::move(param) );
145  param = url.getPassword();
146  if ( ! param.empty() )
147  s.setPassword( std::move(param) );
148  }
149  else
150  {
151  // if there is no username, set anonymous auth
152  if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
153  s.setAnonymousAuth();
154  }
155  }
156  if ( url.getScheme() == "https" )
157  {
158  s.setVerifyPeerEnabled( false );
159  s.setVerifyHostEnabled( false );
160 
161  const std::string & verify { url.getQueryParam("ssl_verify") };
162  if( verify.empty() || verify == "yes" )
163  {
164  s.setVerifyPeerEnabled( true );
165  s.setVerifyHostEnabled( true );
166  }
167  else if ( verify == "no" )
168  {
169  s.setVerifyPeerEnabled( false );
170  s.setVerifyHostEnabled( false );
171  }
172  else
173  {
174  std::vector<std::string> flags;
175  str::split( verify, std::back_inserter(flags), "," );
176  for ( const auto & flag : flags )
177  {
178  if ( flag == "host" )
179  s.setVerifyHostEnabled( true );
180  else if ( flag == "peer" )
181  s.setVerifyPeerEnabled( true );
182  else
183  ZYPP_THROW( media::MediaBadUrlException(url, "Unknown ssl_verify flag "+flag) );
184  }
185  }
186  }
187  {
188  Pathname ca_path { url.getQueryParam("ssl_capath") };
189  if( ! ca_path.empty() )
190  {
191  if( ! PathInfo(ca_path).isDir() || ! ca_path.absolute() )
192  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_capath path"));
193  else
194  s.setCertificateAuthoritiesPath( std::move(ca_path) );
195  }
196  }
197  {
198  Pathname client_cert { url.getQueryParam("ssl_clientcert") };
199  if( ! client_cert.empty() )
200  {
201  if( ! PathInfo(client_cert).isFile() || ! client_cert.absolute() )
202  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientcert file"));
203  else
204  s.setClientCertificatePath( std::move(client_cert) );
205  }
206  }
207  {
208  Pathname client_key { url.getQueryParam("ssl_clientkey") };
209  if( ! client_key.empty() )
210  {
211  if( ! PathInfo(client_key).isFile() || ! client_key.absolute() )
212  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientkey file"));
213  else
214  s.setClientKeyPath( std::move(client_key) );
215  }
216  }
217  {
218  std::string param { url.getQueryParam( "proxy" ) };
219  if ( ! param.empty() )
220  {
221  if ( param == EXPLICITLY_NO_PROXY ) {
222  // Workaround TransferSettings shortcoming: With an
223  // empty proxy string, code will continue to look for
224  // valid proxy settings. So set proxy to some non-empty
225  // string, to indicate it has been explicitly disabled.
227  s.setProxyEnabled(false);
228  }
229  else {
230  const std::string & proxyport { url.getQueryParam( "proxyport" ) };
231  if ( ! proxyport.empty() ) {
232  param += ":";
233  param += proxyport;
234  }
235  s.setProxy( std::move(param) );
236  s.setProxyEnabled( true );
237  }
238  }
239  }
240  {
241  std::string param { url.getQueryParam( "proxyuser" ) };
242  if ( ! param.empty() )
243  {
244  s.setProxyUsername( std::move(param) );
245  s.setProxyPassword( url.getQueryParam( "proxypass" ) );
246  }
247  }
248  {
249  // HTTP authentication type
250  std::string param { url.getQueryParam("auth") };
251  if ( ! param.empty() && (url.getScheme() == "http" || url.getScheme() == "https") )
252  {
253  try
254  {
255  media::CurlAuthData::auth_type_str2long (param ); // check if we know it
256  }
257  catch ( const media::MediaException & ex_r )
258  {
259  DBG << "Rethrowing as MediaUnauthorizedException.";
260  ZYPP_THROW(media::MediaUnauthorizedException(url, ex_r.msg(), "", ""));
261  }
262  s.setAuthType( std::move(param) );
263  }
264  }
265  {
266  // workarounds
267  const std::string & param { url.getQueryParam("head_requests") };
268  if( ! param.empty() && param == "no" )
269  s.setHeadRequestsAllowed( false );
270  }
271 }
272 
278 {
279  media::ProxyInfo proxy_info;
280  if ( proxy_info.useProxyFor( url ) )
281  {
282  // We must extract any 'user:pass' from the proxy url
283  // otherwise they won't make it into curl (.curlrc wins).
284  try {
285  Url u( proxy_info.proxy( url ) );
287  // don't overwrite explicit auth settings
288  if ( s.proxyUsername().empty() )
289  {
290  s.setProxyUsername( u.getUsername( url::E_ENCODED ) );
291  s.setProxyPassword( u.getPassword( url::E_ENCODED ) );
292  }
293  s.setProxyEnabled( true );
294  }
295  catch (...) {} // no proxy if URL is malformed
296  }
297 }
298 
299 
300 const char * anonymousIdHeader()
301 {
302  // we need to add the release and identifier to the
303  // agent string.
304  // The target could be not initialized, and then this information
305  // is guessed.
306  static const std::string _value(
308  "X-ZYpp-AnonymousId: %s",
309  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str() ) )
310  );
311  return _value.c_str();
312 }
313 
315 {
316  // we need to add the release and identifier to the
317  // agent string.
318  // The target could be not initialized, and then this information
319  // is guessed.
320  static const std::string _value(
322  "X-ZYpp-DistributionFlavor: %s",
323  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str() ) )
324  );
325  return _value.c_str();
326 }
327 
328 const char * agentString()
329 {
330  // we need to add the release and identifier to the
331  // agent string.
332  // The target could be not initialized, and then this information
333  // is guessed.
334  static const std::string _value(
335  str::form(
336  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
337  , curl_version_info(CURLVERSION_NOW)->version
338  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
339  )
340  );
341  return _value.c_str();
342 }
343 
344 void curlEscape( std::string & str_r,
345  const char char_r, const std::string & escaped_r ) {
346  for ( std::string::size_type pos = str_r.find( char_r );
347  pos != std::string::npos; pos = str_r.find( char_r, pos ) ) {
348  str_r.replace( pos, 1, escaped_r );
349  }
350 }
351 
352 std::string curlEscapedPath( std::string path_r ) {
353  curlEscape( path_r, ' ', "%20" );
354  return path_r;
355 }
356 
357 std::string curlUnEscape( std::string text_r ) {
358  char * tmp = curl_unescape( text_r.c_str(), 0 );
359  std::string ret( tmp );
360  curl_free( tmp );
361  return ret;
362 }
363 
365 {
366  Url curlUrl (url);
367  curlUrl.setUsername( "" );
368  curlUrl.setPassword( "" );
369  curlUrl.setPathParams( "" );
370  curlUrl.setFragment( "" );
371  curlUrl.delQueryParam("cookies");
372  curlUrl.delQueryParam("proxy");
373  curlUrl.delQueryParam("proxyport");
374  curlUrl.delQueryParam("proxyuser");
375  curlUrl.delQueryParam("proxypass");
376  curlUrl.delQueryParam("ssl_capath");
377  curlUrl.delQueryParam("ssl_verify");
378  curlUrl.delQueryParam("ssl_clientcert");
379  curlUrl.delQueryParam("timeout");
380  curlUrl.delQueryParam("auth");
381  curlUrl.delQueryParam("username");
382  curlUrl.delQueryParam("password");
383  curlUrl.delQueryParam("mediahandler");
384  curlUrl.delQueryParam("credentials");
385  curlUrl.delQueryParam("head_requests");
386  return curlUrl;
387 }
388 
389 // bsc#933839: propagate proxy settings passed in the repo URL
390 zypp::Url propagateQueryParams( zypp::Url url_r, const zypp::Url & template_r )
391 {
392  for ( std::string param : { "proxy", "proxyport", "proxyuser", "proxypass"} )
393  {
394  const std::string & value( template_r.getQueryParam( param ) );
395  if ( ! value.empty() )
396  url_r.setQueryParam( param, value );
397  }
398  return url_r;
399 }
400 
401 ProgressData::ProgressData(CURL *_curl, time_t _timeout, const Url &_url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
402 : curl( _curl )
403 , url( _url )
404 , timeout( _timeout )
405 , reached( false )
406 , fileSizeExceeded ( false )
407 , report( _report )
408 , _expectedFileSize( expectedFileSize_r )
409 {}
410 
411 void ProgressData::updateStats(double dltotal, double dlnow)
412 {
413  time_t now = _timeNow = time(0);
414 
415  // If called without args (0.0), recompute based on the last values seen
416  if ( dltotal && dltotal != _dnlTotal )
417  _dnlTotal = dltotal;
418 
419  if ( dlnow && dlnow != _dnlNow )
420  {
421  _timeRcv = now;
422  _dnlNow = dlnow;
423  }
424  else if ( !_dnlNow && !_dnlTotal )
425  {
426  // Start time counting as soon as first data arrives.
427  // Skip the connection / redirection time at begin.
428  return;
429  }
430 
431  // init or reset if time jumps back
432  if ( !_timeStart || _timeStart > now )
433  _timeStart = _timeLast = _timeRcv = now;
434 
435  // timeout condition
436  if ( timeout )
437  reached = ( (now - _timeRcv) > timeout );
438 
439  // check if the downloaded data is already bigger than what we expected
440  fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
441 
442  // percentage:
443  if ( _dnlTotal )
444  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
445 
446  // download rates:
447  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
448 
449  if ( _timeLast < now )
450  {
451  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
452  // start new period
453  _timeLast = now;
454  _dnlLast = _dnlNow;
455  }
456  else if ( _timeStart == _timeLast )
458 }
459 
461 {
462  if ( fileSizeExceeded )
463  return 1;
464  if ( reached )
465  return 1; // no-data timeout
466  if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) )
467  return 1; // user requested abort
468  return 0;
469 }
470 
471 }
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:528
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition: Url.cc:734
void globalInitCurlOnce()
Definition: CurlHelper.cc:46
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
void setAuthType(std::string &&val_r)
set the allowed authentication types
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: CurlHelper.cc:89
void setQueryParam(const std::string &param, const std::string &value)
Set or add value for the specified query parameter.
Definition: Url.cc:833
ProgressData()
Ctor no range [0,0](0).
Definition: ProgressData.h:171
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
const char * anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header ...
Definition: CurlHelper.cc:300
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
Definition: CurlHelper.cc:390
Flag to request encoded string(s).
Definition: UrlUtils.h:53
void setUsername(std::string &&val_r)
sets the auth username
Store and operate with byte count.
Definition: ByteCount.h:30
Holds transfer setting.
#define TRANSFER_TIMEOUT_MAX
Definition: CurlHelper.h:23
int reportProgress() const
Definition: CurlHelper.cc:460
Url clearQueryString(const Url &url)
Definition: CurlHelper.cc:364
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition: ProxyInfo.cc:55
static const ViewOption WITH_SCHEME
Option to include scheme name in the URL string.
Definition: UrlBase.h:51
static const ViewOption WITH_HOST
Option to include hostname in the URL string.
Definition: UrlBase.h:74
void setPathParams(const std::string &params)
Set the path parameters.
Definition: Url.cc:786
std::string curlEscapedPath(std::string path_r)
Definition: CurlHelper.cc:352
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
time_t _timeNow
Now.
Definition: CurlHelper.h:69
std::string username() const
auth username
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition: Url.cc:725
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
const char * distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: CurlHelper.cc:314
void setClientCertificatePath(Pathname &&val_r)
Sets the SSL client certificate file.
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition: Url.cc:717
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: CurlHelper.cc:35
void curlEscape(std::string &str_r, const char char_r, const std::string &escaped_r)
Definition: CurlHelper.cc:344
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: CurlHelper.h:71
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:655
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void setProxyPassword(std::string &&val_r)
sets the proxy password
Just inherits Exception to separate media exceptions.
void setClientKeyPath(Pathname &&val_r)
Sets the SSL client key file.
#define WAR
Definition: Logger.h:97
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1085
double _dnlLast
Bytes downloaded at period start.
Definition: CurlHelper.h:72
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: CurlHelper.cc:129
void setTimeout(long t)
set the transfer timeout
std::string proxy(const Url &url) const
Definition: ProxyInfo.cc:43
void setProxyUsername(std::string &&val_r)
sets the proxy user
SolvableIdType size_type
Definition: PoolMember.h:126
zypp::ByteCount _expectedFileSize
Definition: CurlHelper.h:64
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: CurlHelper.h:75
void setProxy(std::string &&val_r)
proxy to use if it is enabled
double _drateLast
Download rate in last period.
Definition: CurlHelper.h:78
std::string curlUnEscape(std::string text_r)
Definition: CurlHelper.cc:357
time_t _timeStart
Start total stats.
Definition: CurlHelper.h:66
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
double _dnlNow
Bytes downloaded now.
Definition: CurlHelper.h:73
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:218
const char * agentString()
initialized only once, this gets the agent string which also includes the curl version ...
Definition: CurlHelper.cc:328
void updateStats(double dltotal=0.0, double dlnow=0.0)
Definition: CurlHelper.cc:411
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: CurlHelper.cc:277
double _drateTotal
Download rate so far.
Definition: CurlHelper.h:77
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
time_t _timeRcv
Start of no-data timeout.
Definition: CurlHelper.h:68
void setPassword(std::string &&val_r)
sets the auth password
std::string proxyUsername() const
proxy auth username
static const ViewOption WITH_PORT
Option to include port number in the URL string.
Definition: UrlBase.h:81
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: CurlHelper.cc:55
void setCertificateAuthoritiesPath(Pathname &&val_r)
Sets the SSL certificate authorities path.
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
time_t _timeLast
Start last period(~1sec)
Definition: CurlHelper.h:67
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition: Url.cc:575
const char * c_str() const
Definition: IdStringType.h:105
Convenience interface for handling authentication data of media user.
#define EXPLICITLY_NO_PROXY
Definition: CurlHelper.h:26
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
Url manipulation class.
Definition: Url.h:91
void setProxyEnabled(bool enabled)
whether the proxy is used or not
#define DBG
Definition: Logger.h:95
void delQueryParam(const std::string &param)
remove the specified query parameter.
Definition: Url.cc:840
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: CurlHelper.h:63
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:567
const std::string & msg() const
Return the message string provided to the ctor.
Definition: Exception.h:195