libzypp  17.28.5
Downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 
10 #include <fstream>
11 #include <solv/solvversion.h>
12 #include <zypp/base/String.h>
13 #include <zypp/base/LogTools.h>
14 #include <zypp/base/Function.h>
15 #include <zypp/ZConfig.h>
16 
17 #include "Downloader.h"
20 #include <zypp/parser/xml/Reader.h>
22 
23 using namespace zypp::xml;
24 using namespace zypp::parser::yum;
25 
26 namespace zypp
27 {
28 namespace repo
29 {
30 namespace yum
31 {
33  namespace
34  {
35  inline OnMediaLocation loc_with_path_prefix( OnMediaLocation loc_r, const Pathname & prefix_r )
36  {
37  if ( ! prefix_r.empty() && prefix_r != "/" )
38  loc_r.changeFilename( prefix_r / loc_r.filename() );
39  return loc_r;
40  }
41 
42  // search old repository file to run the delta algorithm on
43  Pathname search_deltafile( const Pathname & dir, const Pathname & file )
44  {
46  if ( ! PathInfo(dir).isDir() )
47  return deltafile;
48 
49  // Strip the checksum preceding the file stem so we can look for an
50  // old *-primary.xml which may contain some reusable blocks.
51  std::string base { file.basename() };
52  size_t hypoff = base.find( "-" );
53  if ( hypoff != std::string::npos )
54  base.replace( 0, hypoff + 1, "" );
55 
56  std::list<std::string> retlist;
57  if ( ! filesystem::readdir( retlist, dir, false ) )
58  {
59  for ( const auto & fn : retlist )
60  {
61  if ( str::endsWith( fn, base ) )
62  deltafile = fn;
63  }
64  }
65  return deltafile;
66  }
67  } // namespace
69 
84  {
85  NON_COPYABLE( Impl );
86  NON_MOVABLE( Impl );
87 
88  Impl( Downloader & downloader_r, MediaSetAccess & media_r, const Pathname & destDir_r )
89  : _downloader { downloader_r }
90  , _media { media_r }
91  , _destDir { destDir_r }
92  {
93  addWantedLocale( ZConfig::instance().textLocale() );
94  for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
95  addWantedLocale( it );
96  }
97 
102  bool operator()( const OnMediaLocation & loc_r, const std::string & typestr_r )
103  {
104  if ( str::endsWith( typestr_r, "_db" ) )
105  return true; // skip sqlitedb
106 
107  bool zchk { str::endsWith( typestr_r, "_zck" ) };
108 #if defined(LIBSOLVEXT_FEATURE_ZCHUNK_COMPRESSION)
109  const std::string & basetype { zchk ? typestr_r.substr( 0, typestr_r.size()-4 ) : typestr_r };
110 #else
111  if ( zchk )
112  return true; // skip zchunk if not supported by libsolv
113  const std::string & basetype { typestr_r };
114 #endif
115 
116  // filter well known resource types
117  if ( basetype == "other" || basetype == "filelists" )
118  return true; // skip it
119 
120  // filter localized susedata
121  if ( str::startsWith( basetype, "susedata." ) )
122  {
123  // susedata.LANG
124  if ( ! wantLocale( Locale(basetype.c_str()+9) ) )
125  return true; // skip it
126  }
127 
128  // may take it... (prefer zchnk)
129  if ( zchk || !_wantedFiles.count( basetype ) )
130  _wantedFiles[basetype] = loc_r;
131 
132  return true;
133  }
134 
135  void finalize()
136  {
137  // schedule fileS for download
138  for ( const auto & el : _wantedFiles )
139  {
140  const OnMediaLocation & loc { el.second };
141  const OnMediaLocation & loc_with_path { loc_with_path_prefix( loc, _downloader.repoInfo().path() ) };
142  _downloader.enqueueDigested( OnMediaLocation(loc_with_path).setDeltafile( search_deltafile( deltaDir()/"repodata", loc.filename() ) ), FileChecker() );
143  }
144  }
145 
146  private:
147  const Pathname & deltaDir() const
148  { return _downloader._deltaDir; }
149 
150  bool wantLocale( const Locale & locale_r ) const
151  { return _wantedLocales.count( locale_r ); }
152 
153  void addWantedLocale( Locale locale_r )
154  {
155  while ( locale_r )
156  {
157  _wantedLocales.insert( locale_r );
158  locale_r = locale_r.fallback();
159  }
160  }
161 
162  private:
166 
168  std::map<std::string,OnMediaLocation> _wantedFiles;
169  };
170 
172  //
173  // class Downloader
174  //
176 
177  Downloader::Downloader( const RepoInfo & info_r, const Pathname & deltaDir_r )
178  : repo::Downloader { info_r}
179  , _deltaDir { deltaDir_r }
180  {}
181 
182  void Downloader::download( MediaSetAccess & media_r, const Pathname & destDir_r, const ProgressData::ReceiverFnc & progress_r )
183  {
184  downloadMediaInfo( destDir_r, media_r );
185 
186  Pathname masterIndex { repoInfo().path() / "/repodata/repomd.xml" };
187  defaultDownloadMasterIndex( media_r, destDir_r, masterIndex );
188 
189  //enable precache
190  setMediaSetAccess( media_r );
191 
192  // setup parser
193  Impl pimpl( *this, media_r, destDir_r );
194  RepomdFileReader( destDir_r / masterIndex, std::ref(pimpl) );
195  pimpl.finalize();
196 
197  // ready, go!
198  start( destDir_r );
199  }
200 
202  {
203  RepoStatus ret { media_r.provideOptionalFile( repoInfo().path() / "/repodata/repomd.xml" ) };
204  if ( !ret.empty() ) // else: mandatory master index is missing
205  ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
206  // else: mandatory master index is missing -> stay empty
207  return ret;
208  }
209 } // namespace yum
210 } // namespace repo
211 } // namespace zypp
212 
213 
214 
Pathname path() const
Repository path.
Definition: RepoInfo.cc:721
void addWantedLocale(Locale locale_r)
Definition: Downloader.cc:153
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:141
Pathname deltafile
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Helper filtering the files offered by a RepomdFileReader.
Definition: Downloader.cc:83
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
What is known about a repository.
Definition: RepoInfo.h:71
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool operator()(const OnMediaLocation &loc_r, const std::string &typestr_r)
The callback invoked by the RepomdFileReader.
Definition: Downloader.cc:102
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
LocaleSet _wantedLocales
Locales do download.
Definition: Downloader.cc:167
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
std::map< std::string, OnMediaLocation > _wantedFiles
Definition: Downloader.cc:168
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
const Pathname & deltaDir() const
Definition: Downloader.cc:147
void download(MediaSetAccess &media_r, const Pathname &destDir_r, const ProgressData::ReceiverFnc &progress_r=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:182
Impl(Downloader &downloader_r, MediaSetAccess &media_r, const Pathname &destDir_r)
Definition: Downloader.cc:88
RepoStatus status(MediaSetAccess &media_r) override
Status of the remote repository.
Definition: Downloader.cc:201
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1085
const RepoInfo & repoInfo() const
Definition: Downloader.h:58
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
const Pathname & filename() const
The path to the resource on the medium.
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
bool wantLocale(const Locale &locale_r) const
Definition: Downloader.cc:150
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:49
int readdir(std::list< std::string > &retlist_r, const Pathname &path_r, bool dots_r)
Return content of directory via retlist.
Definition: PathInfo.cc:576
#define NON_MOVABLE(CLASS)
Delete move ctor and move assign.
Definition: Easy.h:69
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
#define NON_COPYABLE(CLASS)
Delete copy ctor and copy assign.
Definition: Easy.h:59
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:218
Interface of repomd.xml file reader.
Track changing files or directories.
Definition: RepoStatus.h:40
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
Downloader for YUM (rpm-nmd) repositories Encapsulates all the knowledge of which files have to be do...
Definition: Downloader.h:40
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
std::unordered_set< Locale > LocaleSet
Definition: Locale.h:27