30 #undef ZYPP_BASE_LOGGER_LOGGROUP 31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher" 67 if ( lhs->location.medianr() == rhs->location.medianr() )
68 return lhs->location.filename() < rhs->location.filename();
70 return lhs->location.medianr() < rhs->location.medianr();
118 return str << obj->location;
136 Fetcher::Options
options()
const;
154 static shared_ptr<Impl> _nullimpl(
new Impl );
196 void validate(
const Pathname & localfile_r,
const std::list<FileChecker> & checkers_r );
202 const Pathname &dest_dir, FetcherJob::Flags flags );
217 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
220 {
return new Impl( *
this ); }
223 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
237 job.reset(
new FetcherJob(resource, deltafile));
260 job->checkers.push_back(checker);
265 _resources.push_back(job);
275 job->checkers.push_back(checker);
281 _resources.push_back(job);
290 job->checkers.push_back(checker);
291 _resources.push_back(job);
296 MIL <<
"adding index " << resource << endl;
316 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
317 _caches.insert(cache_dir);
322 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
327 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
343 swap( ret, cacheLocation );
347 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
348 for(
const Pathname & cacheDir : _caches )
350 cacheLocation = cacheDir / resource_r.
filename();
353 MIL <<
"file " << resource_r.
filename() <<
" found in cache " << cacheDir << endl;
354 swap( ret, cacheLocation );
366 MIL <<
"Checking job [" << localfile_r <<
"] (" << checkers_r.size() <<
" checkers )" << endl;
371 chkfnc( localfile_r );
373 ERR <<
"Invalid checker for '" << localfile_r <<
"'" << endl;
396 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool 405 downloadAndReadIndexList( media, dest_dir );
411 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
415 fnc_addIfInContent(
"content" );
424 != _dircontent.end() )
428 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
437 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
444 const Pathname &dest_dir, FetcherJob::Flags flags )
448 MIL <<
"Adding directory " << resource.
filename() << endl;
451 getDirectoryContent(media, resource, content);
456 WAR <<
"Skiping subtree hidden at " << resource.
filename() << endl;
462 autoaddIndexes(content, media, resource, dest_dir);
464 for ( filesystem::DirContent::const_iterator it = content.begin();
480 if ( _checksums.find(filename.
asString()) != _checksums.end() )
483 chksm = _checksums[filename.
asString()];
487 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
497 addDirJobs(media, filename, dest_dir, flags);
512 scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard;
515 Pathname tmpFile = locateInCache( resource, destDir_r );
516 if ( tmpFile.
empty() )
518 MIL <<
"Not found in cache, retrieving..." << endl;
527 if ( tmpFile == destFullPath )
531 validate( tmpFile, jobp_r->checkers );
534 if ( tmpFile == destFullPath )
535 destFullPath.resetDispose();
538 if (
assert_dir( destFullPath->dirname() ) != 0 )
550 WAR <<
"optional resource " << resource <<
" could not be transferred." << endl;
570 { _repoindex = data_r; }
579 readChecksumsIndex(index, basedir);
580 else if ( index.
basename() ==
"content" )
581 readContentFileIndex(index, basedir);
583 WAR << index <<
": index file format not known" << endl;
591 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
595 _checksums[(basedir + it->first).
asString()] = it->second;
602 std::ifstream in( index.
c_str() );
606 while (
getline( in, buffer ) )
609 if ( buffer[0] ==
'#' )
616 if ( buffer.empty() )
618 WAR <<
"Missing filename in CHECKSUMS file: " << index.
asString() <<
" (" <<
checksum <<
")" << endl;
631 MIL <<
"downloading index " << resource << endl;
657 fetcher.
start( dest_dir, media );
666 fetcher.
start( dest_dir, media );
673 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
677 fetcher.
start( dest_dir, media );
688 if ( _indexes.empty() )
690 MIL <<
"No indexes to read." << endl;
694 for_( it_idx, _indexes.begin(), _indexes.end() )
696 if ( (*it_idx)->read )
698 DBG <<
"Already read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
703 downloadIndex( media, (*it_idx)->location, dest_dir );
705 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
707 MIL <<
"Remember read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
708 (*it_idx)->read =
true;
711 MIL <<
"done reading indexes" << endl;
721 progress.
sendTo(progress_receiver);
723 downloadAndReadIndexList(media, dest_dir);
730 addDirJobs(media, location, dest_dir, jobp->flags);
744 getDirectoryContent(media, jobp->location.filename().dirname(), content);
747 MIL <<
"Autodiscovering signed indexes on '" 748 << jobp->location.filename().dirname() <<
"' for '" 749 << jobp->location.filename() <<
"'" << endl;
751 autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
755 getDirectoryContent(media,
Pathname(
"/"), content);
758 MIL <<
"Autodiscovering signed indexes on '" 760 << jobp->location.filename() <<
"'" << endl;
762 autoaddIndexes(content, media,
Pathname(
"/"), dest_dir);
767 if ( jobp->location.checksum().empty() )
769 if ( _checksums.find(jobp->location.filename().asString())
770 != _checksums.end() )
772 CheckSum chksm = _checksums[jobp->location.filename().asString()];
774 jobp->checkers.push_back(digest_check);
784 jobp->checkers.push_back(digest_check);
792 jobp->checkers.push_back(digest_check);
797 provideToDest( media, dest_dir, jobp );
799 if ( ! progress.incr() )
807 for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
shared_ptr< FetcherIndex > FetcherIndex_Ptr
void setOptions(Options options)
Set the Fetcher options.
Listentry returned by readdir.
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Checks for the validity of a signature.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
parser::susetags::RepoIndex_Ptr _repoindex
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
const char * c_str() const
String representation.
String related utilities and Regular expression matching.
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
If a content file is found, it is downloaded and read.
std::string basename() const
Return the last component of this path.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void addCachePath(const Pathname &cache_dir)
OnMediaLocation location
Index localtion.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
std::list< FileChecker > checkers
static shared_ptr< Impl > nullimpl()
Offer default Impl.
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
void remember(const Exception &old_r)
Store an other Exception as history.
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
bool empty() const
Test for an empty path.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
std::string getline(std::istream &str)
Read one line from stream.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Fetcher::Options _options
std::ostream & operator<<(std::ostream &str, const Exception &obj)
int unlink(const Pathname &path)
Like 'unlink'.
const std::string & asString() const
String representation.
bool isExist() const
Return whether valid stat info exists.
void reset()
Reset the transfer (jobs) list.
Pathname dirname() const
Return all but the last component od this path.
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Parse repoindex part from a content file.
std::set ordering (less semantic)
Fetcher::Options options() const
Maintain [min,max] and counter (value) for progress counting.
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
std::list< DirEntry > DirContent
Returned by readdir.
DefaultIntegral< bool, false > read
Whether we read this index.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::map< std::string, filesystem::DirContent > _dircontent
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
std::list< FetcherJob_Ptr > _resources
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
ZYPP_DECLARE_FLAGS(Flags, Flag)
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
If a CHECKSUMS file is found, it is downloaded and read.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Class to encapsulate the OnMediaLocation object and the FileChecker together.
void setOptions(Fetcher::Options options)
FetcherIndex(const OnMediaLocation &loc)
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Base class for Exception.
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
std::map< std::string, CheckSum > _checksums
Wrapper class for ::stat/::lstat.
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
std::set< Pathname > _caches
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Options options() const
Get current options.
shared_ptr< FetcherJob > FetcherJob_Ptr
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Easy-to use interface to the ZYPP dependency resolver.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
void addIndex(const OnMediaLocation &resource)
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Impl * clone() const
clone for RWCOW_pointer