libzypp  17.37.5
Fetcher.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include <iostream>
13 #include <fstream>
14 #include <list>
15 #include <map>
16 
17 #include <zypp/base/Easy.h>
18 #include <zypp/base/LogControl.h>
19 #include <zypp/base/LogTools.h>
20 #include <zypp/base/PtrTypes.h>
21 #include <zypp-core/base/DefaultIntegral>
22 #include <zypp/base/String.h>
23 #include <zypp-media/MediaException>
24 #include <zypp/Fetcher.h>
25 #include <zypp/ZYppFactory.h>
26 #include <zypp/CheckSum.h>
27 #include <zypp-core/base/UserRequestException>
30 
31 #undef ZYPP_BASE_LOGGER_LOGGROUP
32 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
33 
35 namespace zypp
36 {
37 
43  struct FetcherIndex
44  {
46  : location(loc)
47  {}
52  };
53 
54  using FetcherIndex_Ptr = shared_ptr<FetcherIndex>;
55 
58  {
59  bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
60  {
61  if ( lhs == rhs )
62  return false; // incl. NULL == NULL
63  if ( ! lhs )
64  return true; // NULL < nonNULL
65  if ( ! rhs )
66  return false; // nonNULL > NULL
67  // both nonNULL ==> compare medianr and path
68  if ( lhs->location.medianr() == rhs->location.medianr() )
69  return lhs->location.filename() < rhs->location.filename();
70  //else
71  return lhs->location.medianr() < rhs->location.medianr();
72  }
73  };
74 
79  struct FetcherJob
80  {
81  enum Flag
82  {
83  None = 0x0000,
84  Directory = 0x0001,
85  Recursive = 0x0002,
87  // check checksums even if there is no such
88  // checksum (warns of no checksum)
90  };
91  ZYPP_DECLARE_FLAGS(Flags, Flag);
92 
93  FetcherJob(const FetcherJob &) = default;
94  FetcherJob(FetcherJob &&) = default;
95  FetcherJob &operator=(const FetcherJob &) = default;
96  FetcherJob &operator=(FetcherJob &&) = default;
97 
98  FetcherJob(const OnMediaLocation &loc) : location(loc), flags(None) {
99  //MIL << location << endl;
100  }
101 
103  {
104  //MIL << location << " | * " << checkers.size() << endl;
105  }
106 
108  //CompositeFileChecker checkers;
109  std::list<FileChecker> checkers;
110  Flags flags;
111  };
112 
113  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
114  using FetcherJob_Ptr = shared_ptr<FetcherJob>;
115 
116  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
117  {
118  return str << obj->location;
119  }
120 
122  //
123  // CLASS NAME : Fetcher::Impl
124  //
127  {
128  friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
129 
130  public:
131  Impl();
132 
133  Impl(const Impl &) = default;
134  Impl(Impl &&) = delete;
135  Impl &operator=(const Impl &) = delete;
136  Impl &operator=(Impl &&) = delete;
137 
138  ~Impl() {}
139 
140  struct CacheInfo{
143 
144  friend bool operator<( const CacheInfo &a, const CacheInfo & b) {
145  return a._pathName < b._pathName;
146  }
147  };
148 
149  void setOptions( Fetcher::Options options );
150  Fetcher::Options options() const;
151 
152  void addIndex( const OnMediaLocation &resource );
153 
154  void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
155  void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
156 
157  void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
158  void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
159  void addCachePath( const Pathname &cache_dir, Fetcher::CacheOptions options );
160  void reset();
161  void setMediaSetAccess ( MediaSetAccess &media );
162  void start( const Pathname &dest_dir,
163  const ProgressData::ReceiverFnc &progress );
164  void start( const Pathname &dest_dir,
165  MediaSetAccess &media,
166  const ProgressData::ReceiverFnc & progress_receiver );
167 
169  static shared_ptr<Impl> nullimpl()
170  {
171  static shared_ptr<Impl> _nullimpl( new Impl );
172  return _nullimpl;
173  }
174  private:
178  void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
179 
183  void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);
184 
192  void readIndex( const Pathname &index, const Pathname &basedir );
193 
195  void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
196 
198  void readContentFileIndex( const Pathname &index, const Pathname &basedir );
199 
201  void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
202 
208  ManagedFile locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r );
213  void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );
214 
218  void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
219  const Pathname &dest_dir, FetcherJob::Flags flags );
220 
224  void autoaddIndexes( const filesystem::DirContent &content,
225  MediaSetAccess &media,
226  const OnMediaLocation &resource,
227  const Pathname &dest_dir );
231  void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
232 
233  private:
234  friend Impl * rwcowClone<Impl>( const Impl * rhs );
236  Impl * clone() const
237  { return new Impl( *this ); }
238 
239  std::list<FetcherJob_Ptr> _resources;
240  std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
241  std::set<CacheInfo> _caches;
242  // checksums read from the indexes
243  std::map<std::string, CheckSum> _checksums;
244  // cache of dir contents
245  std::map<std::string, filesystem::DirContent> _dircontent;
246 
248 
249  Fetcher::Options _options;
250  };
252 
254  {
255  if ( _mediaSetAccess )
256  _mediaSetAccess->precacheFiles( {resource} );
257 
258  FetcherJob_Ptr job;
259  job.reset(new FetcherJob(resource));
260  job->flags |= FetcherJob:: AlwaysVerifyChecksum;
261  _resources.push_back(job);
262  }
263 
265  : _options(0)
266  {
267  }
268 
269  void Fetcher::Impl::setOptions( Fetcher::Options options )
270  { _options = options; }
271 
272  Fetcher::Options Fetcher::Impl::options() const
273  { return _options; }
274 
276  bool recursive,
277  const FileChecker &checker )
278  {
279  FetcherJob_Ptr job;
280  job.reset(new FetcherJob(resource));
281  if ( checker )
282  job->checkers.push_back(checker);
283  if ( recursive )
284  job->flags |= FetcherJob::Recursive;
285  job->flags |= FetcherJob::Directory;
286 
287  _resources.push_back(job);
288  }
289 
291  bool recursive,
292  const FileChecker &checker )
293  {
294  FetcherJob_Ptr job;
295  job.reset(new FetcherJob(resource));
296  if ( checker )
297  job->checkers.push_back(checker);
298  if ( recursive )
299  job->flags |= FetcherJob::Recursive;
300  job->flags |= FetcherJob::Directory;
301  job->flags |= FetcherJob::AlwaysVerifyChecksum;
302 
303  _resources.push_back(job);
304 
305  }
306 
307  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
308  {
309  if ( _mediaSetAccess )
310  _mediaSetAccess->precacheFiles( {resource} );
311 
312  FetcherJob_Ptr job;
313  job.reset(new FetcherJob(resource));
314  if ( checker )
315  job->checkers.push_back(checker);
316  _resources.push_back(job);
317  }
318 
319  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
320  {
321  MIL << "adding index " << resource << endl;
322  _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
323  }
324 
325 
327  {
328  _resources.clear();
329  _indexes.clear();
330  _checksums.clear();
331  _dircontent.clear();
332  }
333 
335  {
336  _mediaSetAccess = &media;
337  }
338 
340  {
341  PathInfo info(cache_dir);
342  if ( info.isExist() )
343  {
344  if ( info.isDir() )
345  {
346  DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
347  _caches.insert( { cache_dir, options } );
348  }
349  else
350  {
351  // don't add bad cache directory, just log the error
352  ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
353  }
354  }
355  else
356  {
357  ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
358  }
359 
360  }
361 
362  ManagedFile Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r )
363  {
364  // No checksum - no match
365  if ( resource_r.checksum().empty() )
366  return {};
367 
368  // first check in the destination directory
369  ManagedFile cacheLocation(destDir_r / resource_r.filename());
370  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) {
371  return cacheLocation;
372  }
373 
374  MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
375  for( const CacheInfo & cacheInfo : _caches ) {
376  cacheLocation = ManagedFile(cacheInfo._pathName / resource_r.filename());
377  if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
378  {
379  MIL << "file " << resource_r.filename() << " found in cache " << cacheInfo._pathName << endl;
380  if ( cacheInfo._options & Fetcher::CleanFiles )
381  cacheLocation.setDispose( filesystem::unlink );
382  return cacheLocation;
383  }
384  }
385 
386  return {};
387  }
388 
389  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
390  {
391  try
392  {
393  MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
394 
395  for ( const FileChecker & chkfnc : checkers_r )
396  {
397  if ( chkfnc )
398  chkfnc( localfile_r );
399  else
400  ERR << "Invalid checker for '" << localfile_r << "'" << endl;
401  }
402 
403  }
404  catch ( const FileCheckException &e )
405  {
406  throw; // let known exceptions bypass silently
407  }
408  catch ( const Exception &e )
409  {
410  throw; // slet known exceptions bypass silently
411  }
412  catch (...)
413  {
414  ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
415  }
416  }
417 
419  MediaSetAccess &media,
420  const OnMediaLocation &resource,
421  const Pathname &dest_dir )
422  {
423  auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
424  {
425  if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
426  return false;
427  // add the index of this directory
428  OnMediaLocation indexloc( resource );
429  indexloc.changeFilename( resource.filename() + index_r );
430  addIndex( indexloc );
431  // we need to read it now
432  downloadAndReadIndexList( media, dest_dir );
433  return true;
434  } );
435 
437  {
438  fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
439  }
441  {
442  fnc_addIfInContent( "content" );
443  }
444  }
445 
447  const OnMediaLocation &resource,
448  filesystem::DirContent &content )
449  {
450  if ( _dircontent.find(resource.filename().asString())
451  != _dircontent.end() )
452  {
453  filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
454 
455  std::copy(filled.begin(), filled.end(), std::back_inserter(content));
456  }
457  else
458  {
459  filesystem::DirContent tofill;
460  media.dirInfo( tofill,
461  resource.filename(),
462  false /* dots */,
463  resource.medianr());
464  std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
465  _dircontent[resource.filename().asString()] = tofill;
466  }
467  }
468 
470  const OnMediaLocation &resource,
471  const Pathname &dest_dir, FetcherJob::Flags flags )
472  {
473  // first get the content of the directory so we can add
474  // individual transfer jobs
475  MIL << "Adding directory " << resource.filename() << endl;
476  filesystem::DirContent content;
477  try {
478  getDirectoryContent(media, resource, content);
479  }
480  catch ( media::MediaFileNotFoundException & exception )
481  {
482  ZYPP_CAUGHT( exception );
483  WAR << "Skipping subtree hidden at " << resource.filename() << endl;
484  return;
485  }
486 
487  // this method test for the option flags so indexes are added
488  // only if the options are enabled
489  autoaddIndexes(content, media, resource, dest_dir);
490 
491  for ( filesystem::DirContent::const_iterator it = content.begin();
492  it != content.end();
493  ++it )
494  {
495  // skip CHECKSUMS* as they were already retrieved
496  if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
497  continue;
498 
499  Pathname filename = resource.filename() + it->name;
500 
501  switch ( it->type )
502  {
503  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
504  case filesystem::FT_FILE:
505  {
506  CheckSum chksm(resource.checksum());
507  if ( _checksums.find(filename.asString()) != _checksums.end() )
508  {
509  // the checksum can be replaced with the one in the index.
510  chksm = _checksums[filename.asString()];
511  //MIL << "resource " << filename << " has checksum in the index file." << endl;
512  }
513  else
514  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
515 
516  if ( flags & FetcherJob::AlwaysVerifyChecksum )
517  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
518  else
519  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
520  break;
521  }
522  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
523  if ( flags & FetcherJob::Recursive )
524  addDirJobs(media, filename, dest_dir, flags);
525  break;
526  default:
527  // don't provide devices, sockets, etc.
528  break;
529  }
530  }
531  }
532 
533  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
534  {
535  const OnMediaLocation & resource( jobp_r->location );
536 
537  try
538  {
539  scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
540 
541  // get cached file (by checksum) or provide from media
542  ManagedFile managedTmpFile = locateInCache( resource, destDir_r );
543 
544  Pathname tmpFile = managedTmpFile;
545  if ( tmpFile.empty() )
546  {
547  MIL << "Not found in cache, retrieving..." << endl;
549  releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
550  }
551 
552  // The final destination: locateInCache also checks destFullPath!
553  // If we find a cache match (by checksum) at destFullPath, take
554  // care it gets deleted, in case the validation fails.
555  ManagedFile destFullPath( destDir_r / resource.filename() );
556  if ( tmpFile == destFullPath )
557  destFullPath.setDispose( filesystem::unlink );
558 
559  // validate the file (throws if not valid)
560  validate( tmpFile, jobp_r->checkers );
561 
562  // move it to the final destination
563  if ( tmpFile == destFullPath )
564  destFullPath.resetDispose(); // keep it!
565  else
566  {
567  if ( assert_dir( destFullPath->dirname() ) != 0 )
568  ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) );
569 
570  if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 )
571  ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) );
572  }
573  }
574  catch ( Exception & excpt )
575  {
576  if ( resource.optional() )
577  {
578  ZYPP_CAUGHT( excpt );
579  WAR << "optional resource " << resource << " could not be transferred." << endl;
580  return;
581  }
582  else
583  {
584  excpt.remember( "Can't provide " + resource.filename().asString() );
585  ZYPP_RETHROW( excpt );
586  }
587  }
588  }
589 
590  // helper class to consume a content file
592  {
594  {
595  setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
596  }
597 
598  void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
599  { _repoindex = data_r; }
600 
601  parser::susetags::RepoIndex_Ptr _repoindex;
602  };
603 
604  // generic function for reading indexes
605  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
606  {
607  if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
608  readChecksumsIndex(index, basedir);
609  else if ( index.basename() == "content" )
610  readContentFileIndex(index, basedir);
611  else
612  WAR << index << ": index file format not known" << endl;
613  }
614 
615  // reads a content file index
616  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
617  {
618  ContentReaderHelper reader;
619  reader.parse(index);
620  MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
621  for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
622  {
623  // content file entries don't start with /
624  _checksums[(basedir + it->first).asString()] = it->second;
625  }
626  }
627 
628  // reads a CHECKSUMS (old SHA1SUMS) file index
629  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
630  {
631  std::ifstream in( index.c_str() );
632  if ( ! in.fail() )
633  {
634  std::string buffer;
635  while ( getline( in, buffer ) )
636  {
637 
638  if ( buffer[0] == '#' )
639  continue; // simple comment
640 
641  CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
642  if ( checksum.empty() )
643  continue; // empty line | unknown cheksum format
644 
645  if ( buffer.empty() )
646  {
647  WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
648  continue;
649  }
650 
651  _checksums[(basedir/buffer).asString()] = checksum;
652  }
653  }
654  else
655  ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
656  }
657 
658  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
659  {
660  MIL << "downloading index " << resource << endl;
661  // create a new fetcher with a different state to transfer the
662  // file containing checksums and its signature
663  Fetcher fetcher;
664  // signature checker for index. We havent got the signature from
665  // the nextwork yet.
666  SignatureFileChecker sigchecker;
667 
668  // build the name of the index and the signature
669  const OnMediaLocation& idxloc(resource);
670  OnMediaLocation sigloc(resource);
671  OnMediaLocation keyloc(resource);
672 
673  // we should not fail the download if those don't exists
674  // the checking will warn later
675  sigloc.setOptional(true);
676  keyloc.setOptional(true);
677 
678  // calculate signature and key name
679  sigloc.changeFilename( sigloc.filename().extend(".asc") );
680  keyloc.changeFilename( keyloc.filename().extend(".key") );
681 
682  //assert_dir(dest_dir + idxloc.filename().dirname());
683 
684  // transfer the signature
685  fetcher.enqueue(sigloc);
686  fetcher.start( dest_dir, media );
687  // if we get the signature, update the checker
688  if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
689  sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
690 
691  fetcher.reset();
692 
693  // now the key
694  fetcher.enqueue(keyloc);
695  fetcher.start( dest_dir, media );
696  fetcher.reset();
697 
698  // try to import the key
699  if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
700  getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
701  else
702  WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
703 
704  // now the index itself
705  fetcher.enqueue( idxloc, FileChecker(sigchecker) );
706  fetcher.start( dest_dir, media );
707  fetcher.reset();
708  }
709 
710  // this method takes all the user pointed indexes, gets them and also tries to
711  // download their signature, and verify them. After that, it parses each one
712  // to fill the checksum cache.
714  {
715  // if there is no indexes, then just return to avoid
716  // the directory listing
717  if ( _indexes.empty() )
718  {
719  MIL << "No indexes to read." << endl;
720  return;
721  }
722 
723  for_( it_idx, _indexes.begin(), _indexes.end() )
724  {
725  if ( (*it_idx)->read )
726  {
727  DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
728  }
729  else
730  {
731  // base::LogControl::TmpLineWriter shutUp;
732  downloadIndex( media, (*it_idx)->location, dest_dir );
733  // now we have the indexes in dest_dir
734  readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
735  // Take care we don't process it again
736  MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
737  (*it_idx)->read = true;
738  }
739  }
740  MIL << "done reading indexes" << endl;
741  }
742 
743  void Fetcher::Impl::start( const Pathname &dest_dir,
744  const ProgressData::ReceiverFnc & progress )
745  {
746  if ( !_mediaSetAccess )
747  ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
748  start( dest_dir, *_mediaSetAccess, progress );
749  }
750 
751  // start processing all fetcher jobs.
752  // it processes any user pointed index first
753  void Fetcher::Impl::start( const Pathname &dest_dir,
754  MediaSetAccess &media,
755  const ProgressData::ReceiverFnc & progress_receiver )
756  {
757  _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
758 
759  ProgressData progress(_resources.size());
760  progress.sendTo(progress_receiver);
761 
762  downloadAndReadIndexList(media, dest_dir);
763 
764  for ( const FetcherJob_Ptr & jobp : _resources )
765  {
766  if ( jobp->flags & FetcherJob::Directory )
767  {
768  const OnMediaLocation location(jobp->location);
769  addDirJobs(media, location, dest_dir, jobp->flags);
770  continue;
771  }
772 
773  // may be this code can be factored out
774  // together with the autodiscovery of indexes
775  // of addDirJobs
776  if ( ( _options & AutoAddChecksumsIndexes ) ||
778  {
779  // if auto indexing is enabled, then we need to read the
780  // index for each file. We look only in the directory
781  // where the file is. this is expensive of course.
782  filesystem::DirContent content;
783  getDirectoryContent(media, jobp->location.filename().dirname(), content);
784  // this method test for the option flags so indexes are added
785  // only if the options are enabled
786  MIL << "Autodiscovering signed indexes on '"
787  << jobp->location.filename().dirname() << "' for '"
788  << jobp->location.filename() << "'" << endl;
789 
790  autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
791 
792  // also look in the root of the media
793  content.clear();
794  getDirectoryContent(media, Pathname("/"), content);
795  // this method test for the option flags so indexes are added
796  // only if the options are enabled
797  MIL << "Autodiscovering signed indexes on '"
798  << "/" << "' for '"
799  << jobp->location.filename() << "'" << endl;
800 
801  autoaddIndexes(content, media, Pathname("/"), dest_dir);
802  }
803 
804  // if the checksum is empty, but the checksum is in one of the
805  // indexes checksum, then add a checker
806  if ( jobp->location.checksum().empty() )
807  {
808  if ( _checksums.find(jobp->location.filename().asString())
809  != _checksums.end() )
810  {
811  CheckSum chksm = _checksums[jobp->location.filename().asString()];
812  ChecksumFileChecker digest_check(chksm);
813  jobp->checkers.push_back(digest_check);
814  }
815  else
816  {
817  // if the index checksum is empty too, we only add the checker
818  // if the AlwaysVerifyChecksum option is set on
819  if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
820  {
821  // add the checker with the empty checksum
822  ChecksumFileChecker digest_check(jobp->location.checksum());
823  jobp->checkers.push_back(digest_check);
824  }
825  }
826  }
827  else
828  {
829  // checksum is not empty, so add a checksum checker
830  ChecksumFileChecker digest_check(jobp->location.checksum());
831  jobp->checkers.push_back(digest_check);
832  }
833 
834  // Provide and validate the file. If the file was not transferred
835  // and no exception was thrown, it was an optional file.
836  provideToDest( media, dest_dir, jobp );
837 
838  if ( ! progress.incr() )
839  ZYPP_THROW(AbortRequestException());
840  } // for each job
841  }
842 
844  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
845  {
846  for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
847  {
848  str << *it_res;
849  }
850  return str;
851  }
852 
854  : _pimpl( new Impl() )
855  {}
856 
858  {}
859 
860  void Fetcher::setOptions( Fetcher::Options options )
861  {
863  }
864 
865  Fetcher::Options Fetcher::options() const
866  {
867  return _pimpl->options();
868  }
869 
870  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
871  {
872  enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
873  }
874 
875  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
876  {
877  _pimpl->enqueueDigested( resource, checker );
878  }
879 
880  void Fetcher::enqueueDir( const OnMediaLocation &resource,
881  bool recursive,
882  const FileChecker &checker )
883  {
884  _pimpl->enqueueDir(resource, recursive, checker);
885  }
886 
888  bool recursive,
889  const FileChecker &checker )
890  {
891  _pimpl->enqueueDigestedDir(resource, recursive, checker);
892  }
893 
894 
895  void Fetcher::addIndex( const OnMediaLocation &resource )
896  {
897  _pimpl->addIndex(resource);
898  }
899 
900 
901  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
902  {
903  _pimpl->enqueue(resource, checker);
904  }
905 
906 
907  void Fetcher::addCachePath( const Pathname &cache_dir )
908  {
909  _pimpl->addCachePath(cache_dir, Fetcher::Default);
910  }
911 
912  void Fetcher::addCachePath(const Pathname &cache_dir, CacheOptions options)
913  {
914  _pimpl->addCachePath(cache_dir, options);
915  }
916 
918  {
919  _pimpl->reset();
920  }
921 
923  {
924  _pimpl->setMediaSetAccess( media );
925  }
926 
928  {
929  _pimpl->start( dest_dir, progress );
930  }
931 
932  void Fetcher::start( const Pathname &dest_dir,
933  MediaSetAccess &media,
934  const ProgressData::ReceiverFnc & progress_receiver )
935  {
936  _pimpl->start(dest_dir, media, progress_receiver);
937  }
938 
939  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
940  {
941  return str << *obj._pimpl;
942  }
943 
945 } // namespace zypp
947 
std::string asString(const Patch::Category &obj)
Definition: Patch.cc:122
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Definition: Fetcher.cc:598
#define MIL
Definition: Logger.h:100
std::set< CacheInfo > _caches
Definition: Fetcher.cc:241
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
Definition: Fetcher.cc:616
Pathname deltafile
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
void setOptions(Options options)
Set the Fetcher options.
Definition: Fetcher.cc:860
Listentry returned by readdir.
Definition: PathInfo.h:509
Pathname provideFile(const OnMediaLocation &resource, ProvideFileOptions options=PROVIDE_DEFAULT)
Provides a file from a media location.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:253
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
Definition: progressdata.h:229
void precacheFiles(const std::vector< OnMediaLocation > &files)
Tries to fetch the given files and precaches them.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:459
Describes a resource file located on a medium.
unsigned short b
Checks for the validity of a signature.
Definition: FileChecker.h:70
FetcherJob & operator=(const FetcherJob &)=default
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Definition: Fetcher.cc:895
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Fetcher.cc:601
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:290
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
ZYPP_DECLARE_OPERATORS_FOR_FLAGS(DiskUsageCounter::MountPoint::HintFlags)
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
const char * c_str() const
String representation.
Definition: Pathname.h:112
String related utilities and Regular expression matching.
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
Definition: Fetcher.cc:533
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:307
std::string getline(std::istream &str)
Read one line from stream.
Definition: IOStream.cc:33
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
Definition: SerialNumber.cc:52
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
Definition: Fetcher.cc:907
If a content file is found, it is downloaded and read.
Definition: Fetcher.h:123
const CheckSum & checksum() const
The checksum of the resource on the server.
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
Definition: Fetcher.cc:887
OnMediaLocation location
Index localtion.
Definition: Fetcher.cc:49
zypp::RepoManager::RefreshServiceOptions _options
Definition: serviceswf.cc:748
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:526
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:888
std::list< FileChecker > checkers
Definition: Fetcher.cc:109
#define ERR
Definition: Logger.h:102
shared_ptr< FetcherJob > FetcherJob_Ptr
Definition: Fetcher.cc:114
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
unsigned medianr() const
The media number the resource is located on.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
Definition: Fetcher.cc:169
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
Definition: Fetcher.h:357
void remember(const Exception &old_r)
Store an other Exception as history.
Definition: Exception.cc:154
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:922
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:927
Impl & operator=(const Impl &)=delete
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Definition: Fetcher.cc:59
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
Definition: Fetcher.cc:240
bool empty() const
Test for an empty path.
Definition: Pathname.h:116
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:479
void addCachePath(const Pathname &cache_dir, Fetcher::CacheOptions options)
Definition: Fetcher.cc:339
FetcherJob(const FetcherJob &)=default
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Definition: String.cc:266
Fetcher::Options _options
Definition: Fetcher.cc:249
const std::string & asString() const
String representation.
Definition: Pathname.h:93
void setMediaSetAccess(MediaSetAccess &media)
Definition: Fetcher.cc:334
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:286
void dirInfo(filesystem::DirContent &retlist, const Pathname &dirname, bool dots=true, unsigned media_nr=1)
Fills retlist with directory information.
friend bool operator<(const CacheInfo &a, const CacheInfo &b)
Definition: Fetcher.cc:144
void reset()
Reset the transfer (jobs) list.
Definition: Fetcher.cc:917
MediaSetAccess * _mediaSetAccess
Definition: Fetcher.cc:247
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
#define WAR
Definition: Logger.h:101
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Definition: Fetcher.cc:605
Parse repoindex part from a content file.
FetcherJob(const OnMediaLocation &loc)
Definition: Fetcher.cc:98
std::set ordering (less semantic)
Definition: Fetcher.cc:57
Fetcher::Options options() const
Definition: Fetcher.cc:272
Maintain [min,max] and counter (value) for progress counting.
Definition: progressdata.h:131
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Definition: Fetcher.cc:275
DefaultIntegral< bool, false > read
Whether we read this index.
Definition: Fetcher.cc:51
std::map< std::string, filesystem::DirContent > _dircontent
Definition: Fetcher.cc:245
OnMediaLocation location
Definition: Fetcher.cc:107
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:875
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:658
const Pathname & filename() const
The path to the resource on the medium.
std::list< FetcherJob_Ptr > _resources
Definition: Fetcher.cc:239
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
Definition: Fetcher.cc:743
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
Fetcher()
Default ctor.
Definition: Fetcher.cc:853
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
Definition: Fetcher.cc:880
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Definition: Fetcher.cc:389
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
Definition: IOStream.h:51
ZYPP_DECLARE_FLAGS(Flags, Flag)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
Definition: Fetcher.cc:54
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
Definition: Fetcher.cc:446
If a CHECKSUMS file is found, it is downloaded and read.
Definition: Fetcher.h:128
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Definition: AutoDispose.h:167
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:475
Class to encapsulate the OnMediaLocation object and the FileChecker together.
Definition: Fetcher.cc:79
void setOptions(Fetcher::Options options)
Definition: Fetcher.cc:269
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition: susetags.cc:324
FetcherIndex(const OnMediaLocation &loc)
Definition: Fetcher.cc:45
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
Definition: Fetcher.cc:629
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
Definition: Fetcher.cc:713
Fetcher implementation.
Definition: Fetcher.cc:126
Base class for Exception.
Definition: Exception.h:152
Built in file checkers.
Definition: FileChecker.h:47
bool empty() const
Definition: CheckSum.cc:173
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Definition: PathInfo.cc:1056
unsigned short a
std::map< std::string, CheckSum > _checksums
Definition: Fetcher.cc:243
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
ManagedFile locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory)...
Definition: Fetcher.cc:362
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
Definition: Fetcher.cc:43
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Definition: Fetcher.cc:844
Options options() const
Get current options.
Definition: Fetcher.cc:865
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:901
virtual ~Fetcher()
Dtor.
Definition: Fetcher.cc:857
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
OnMediaLocation & setOptional(bool val)
Set whether the resource is optional.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
Definition: Fetcher.cc:418
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1098
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
bool optional() const
Whether this is an optional resource.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
Definition: Fetcher.h:105
Fetcher::CacheOptions _options
Definition: Fetcher.cc:142
void addIndex(const OnMediaLocation &resource)
Definition: Fetcher.cc:319
#define DBG
Definition: Logger.h:99
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1068
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Definition: Fetcher.cc:469
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Impl * clone() const
clone for RWCOW_pointer
Definition: Fetcher.cc:236
The user is not asked anything, and the error exception is just propagated.