Fetcher.cc

Go to the documentation of this file.

/*---------------------------------------------------------------------\
|                          ____ _   __ __ ___                          |
|                         |__  / \ / / . \ . \                         |
|                           / / \ V /|  _/  _/                         |
|                          / /__ | | | | | |                           |
|                         /_____||_| |_| |_|                           |
|                                                                      |
\---------------------------------------------------------------------*/
#include <iostream>
#include <fstream>
#include <list>
#include <map>
 
#include <zypp-core/base/Easy.h>
#include <zypp-core/base/LogControl.h>
#include <zypp-core/base/LogTools.h>
#include <zypp-core/base/PtrTypes.h>
#include <zypp-core/base/DefaultIntegral>
#include <zypp-core/base/String.h>
#include <zypp-media/MediaException>
#include <zypp/Fetcher.h>
#include <zypp/ZYppFactory.h>
#include <zypp/CheckSum.h>
#include <zypp-core/base/UserRequestException>
#include <zypp/parser/susetags/ContentFileReader.h>
#include <zypp/parser/susetags/RepoIndex.h>
 
#undef ZYPP_BASE_LOGGER_LOGGROUP
#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"

namespace zypp
{ 
 
  namespace {
    // Translate leading "./(../)+" to "./(%2E%2E/)+" for Fetcher::mapToCachePath
    inline void fixup( Pathname & path_r )
    {
      const std::string & s { path_r.asString() };
 
      // 1. Quick exit if string is too short or doesn't start with ./
      if ( s.size() < 4 || s.compare(0, 2, "./" ) != 0 )
        return;
 
      size_t count = 0;
      size_t pos   = 2;
 
      // 2. Count consecutive "../" sequences
      while ( pos + 3 <= s.size() && s.compare(pos, 3, "../") == 0 ) {
        ++count;
        pos += 3;
      }
 
      if (count == 0)
        return;
 
      {
        std::string s { path_r.asString() };
 
        // 3. Resize string once to avoid multiple reallocations
        size_t original_size = s.size();
        size_t added_space = count * 4; // ".."(2) becomes "%2E%2E"(6)
        s.resize(original_size + added_space);
 
        // 4. Move the 'tail' of the string to the new end
        size_t tail_start = 2 + (count * 3);
        size_t tail_length = original_size - tail_start;
        if (tail_length > 0) {
          std::copy_backward(s.begin() + tail_start, s.begin() + original_size, s.end());
        }
 
        // 5. Fill the gap with %2E%2E/
        size_t write_pos = 2;
        for (size_t i = 0; i < count; ++i) {
          s.replace(write_pos, 7, "%2E%2E/");
          write_pos += 7;
        }
 
        path_r = s;
      }
    }
  } // namespace

  struct FetcherIndex
  {
    FetcherIndex( const OnMediaLocation &loc )
      : location(loc)
    {}

    OnMediaLocation             location;
    DefaultIntegral<bool,false> read;
  };
 
  using FetcherIndex_Ptr = shared_ptr<FetcherIndex>;

  struct SameFetcherIndex
  {
    bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const
    {
      if ( lhs == rhs )
        return false; // incl. NULL == NULL
      if ( ! lhs )
        return true;  // NULL < nonNULL
      if ( ! rhs )
        return false; // nonNULL > NULL
      // both nonNULL ==> compare medianr and path
      if ( lhs->location.medianr() == rhs->location.medianr() )
        return lhs->location.filename() < rhs->location.filename();
      //else
      return lhs->location.medianr() < rhs->location.medianr();
    }
  };

  struct FetcherJob
  {
    enum Flag
    {
        None      = 0x0000,
        Directory = 0x0001,
        Recursive = 0x0002,
        RecursiveDirectory = Directory | Recursive,
        // check checksums even if there is no such
        // checksum (warns of no checksum)
        AlwaysVerifyChecksum = 0x0004,
    };
    ZYPP_DECLARE_FLAGS(Flags, Flag);
 
    FetcherJob(const FetcherJob &) = default;
    FetcherJob(FetcherJob &&) = default;
    FetcherJob &operator=(const FetcherJob &) = default;
    FetcherJob &operator=(FetcherJob &&) = default;
 
    FetcherJob(const OnMediaLocation &loc) : location(loc), flags(None) {
      //MIL << location << endl;
    }
 
    ~FetcherJob()
    {
      //MIL << location << " | * " << checkers.size() << endl;
    }
 
    OnMediaLocation location;
    //CompositeFileChecker checkers;
    std::list<FileChecker> checkers;
    Flags flags;
  };
 
  ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
  using FetcherJob_Ptr = shared_ptr<FetcherJob>;
 
  std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
  {
    return str << obj->location;
  }

  //
  //    CLASS NAME : Fetcher::Impl
  //
  class Fetcher::Impl
  {
    friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
 
  public:
    Impl();
 
    Impl(const Impl &) = default;
    Impl(Impl &&) = delete;
    Impl &operator=(const Impl &) = delete;
    Impl &operator=(Impl &&) = delete;
 
    ~Impl() {}
 
    struct CacheInfo{
      zypp::Pathname _pathName;
      Fetcher::CacheOptions _options = Fetcher::Default;
 
      friend bool operator<( const CacheInfo &a, const CacheInfo & b) {
        return a._pathName < b._pathName;
      }
    };
 
    void setOptions( Fetcher::Options options );
    Fetcher::Options options() const;
 
    void addIndex( const OnMediaLocation &resource );
 
    void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
    void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
 
    void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker()  );
    void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
    void addCachePath( const Pathname &cache_dir, Fetcher::CacheOptions options );
    void reset();
    void setMediaSetAccess ( MediaSetAccess &media );
    void start( const Pathname &dest_dir,
                const ProgressData::ReceiverFnc &progress );
    void start( const Pathname &dest_dir,
                MediaSetAccess &media,
                const ProgressData::ReceiverFnc & progress_receiver );

    static shared_ptr<Impl> nullimpl()
    {
      static shared_ptr<Impl> _nullimpl( new Impl );
      return _nullimpl;
    }
  private:
      void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);

      void downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir);

      void readIndex( const Pathname &index, const Pathname &basedir );

      void readChecksumsIndex( const Pathname &index, const Pathname &basedir );

      void readContentFileIndex( const Pathname &index, const Pathname &basedir );

      void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );

      ManagedFile locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r, const Pathname & inCachePath_r );
      void validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r );

       void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
                        const Pathname &dest_dir, FetcherJob::Flags flags );

      void autoaddIndexes( const filesystem::DirContent &content,
                           MediaSetAccess &media,
                           const OnMediaLocation &resource,
                           const Pathname &dest_dir );
      void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r );
 
  private:
    friend Impl * rwcowClone<Impl>( const Impl * rhs );
    Impl * clone() const
    { return new Impl( *this ); }
 
    std::list<FetcherJob_Ptr>   _resources;
    std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
    std::set<CacheInfo> _caches;
    // checksums read from the indexes
    std::map<std::string, CheckSum> _checksums;
    // cache of dir contents
    std::map<std::string, filesystem::DirContent> _dircontent;
 
    MediaSetAccess * _mediaSetAccess = nullptr;
 
    Fetcher::Options _options;
  };

 
  void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker & )
  {
    if ( _mediaSetAccess )
      _mediaSetAccess->precacheFiles( {resource} );
 
    FetcherJob_Ptr job;
    job.reset(new FetcherJob(resource));
    job->flags |= FetcherJob:: AlwaysVerifyChecksum;
    _resources.push_back(job);
  }
 
  Fetcher::Impl::Impl()
      : _options(0)
  {
  }
 
  void Fetcher::Impl::setOptions( Fetcher::Options options )
  { _options = options; }
 
  Fetcher::Options Fetcher::Impl::options() const
  { return _options; }
 
  void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
                                  bool recursive,
                                  const FileChecker &checker )
  {
    FetcherJob_Ptr job;
    job.reset(new FetcherJob(resource));
    if ( checker )
        job->checkers.push_back(checker);
    if ( recursive )
        job->flags |= FetcherJob::Recursive;
    job->flags |= FetcherJob::Directory;
 
    _resources.push_back(job);
  }
 
  void Fetcher::Impl::enqueueDigestedDir( const OnMediaLocation &resource,
                                          bool recursive,
                                          const FileChecker &checker )
  {
    FetcherJob_Ptr job;
    job.reset(new FetcherJob(resource));
    if ( checker )
        job->checkers.push_back(checker);
    if ( recursive )
        job->flags |= FetcherJob::Recursive;
    job->flags |= FetcherJob::Directory;
    job->flags |= FetcherJob::AlwaysVerifyChecksum;
 
    _resources.push_back(job);
 
  }
 
  void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
  {
    if ( _mediaSetAccess )
      _mediaSetAccess->precacheFiles( {resource} );
 
    FetcherJob_Ptr job;
    job.reset(new FetcherJob(resource));
    if ( checker )
      job->checkers.push_back(checker);
    _resources.push_back(job);
  }
 
  void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
  {
    MIL << "adding index " << resource << endl;
    _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
  }
 
 
  void Fetcher::Impl::reset()
  {
    _resources.clear();
    _indexes.clear();
    _checksums.clear();
    _dircontent.clear();
  }
 
  void Fetcher::Impl::setMediaSetAccess( MediaSetAccess &media )
  {
    _mediaSetAccess = &media;
  }
 
  void Fetcher::Impl::addCachePath( const Pathname &cache_dir, Fetcher::CacheOptions options )
  {
    PathInfo info(cache_dir);
    if ( info.isExist() )
    {
      if ( info.isDir() )
      {
        DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
        _caches.insert( { cache_dir, options } );
      }
      else
      {
        // don't add bad cache directory, just log the error
        ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
      }
    }
    else
    {
        WAR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
    }
 
  }
 
  ManagedFile Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r, const Pathname & inCachePath_r )
  {
    // No checksum - no match
    if ( resource_r.checksum().empty() )
      return {};
 
    // first check in the final destination
    ManagedFile cacheLocation { destDir_r / inCachePath_r };
    if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) {
      pMIL( "file", inCachePath_r, "found at", destDir_r );
      return cacheLocation;
    }
 
    for( const CacheInfo & cacheInfo : _caches ) {
      cacheLocation = ManagedFile( cacheInfo._pathName / inCachePath_r );
      if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) )
      {
        pMIL( "file", inCachePath_r, "found in cache", cacheInfo._pathName );
        if ( cacheInfo._options & Fetcher::CleanFiles )
          cacheLocation.setDispose( filesystem::unlink );
        return cacheLocation;
      }
    }
 
    return {};
  }
 
  void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list<FileChecker> & checkers_r )
  {
    try
    {
      MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl;
 
      for ( const FileChecker & chkfnc : checkers_r )
      {
        if ( chkfnc )
          chkfnc( localfile_r );
        else
          ERR << "Invalid checker for '" << localfile_r << "'" << endl;
      }
 
    }
    catch ( const FileCheckException &e )
    {
      throw;  // let known exceptions bypass  silently
    }
    catch ( const Exception &e )
    {
      throw; // slet known exceptions bypass  silently
    }
    catch (...)
    {
      ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString()));
    }
  }
 
  void Fetcher::Impl::autoaddIndexes( const filesystem::DirContent &content,
                                      MediaSetAccess &media,
                                      const OnMediaLocation &resource,
                                      const Pathname &dest_dir )
  {
      auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
      {
        if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
          return false;
        // add the index of this directory
        OnMediaLocation indexloc( resource );
        indexloc.changeFilename( resource.filename() + index_r );
        addIndex( indexloc );
        // we need to read it now
        downloadAndReadIndexList( media, dest_dir );
        return true;
      } );
 
      if ( _options & AutoAddChecksumsIndexes )
      {
        fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
      }
      if ( _options & AutoAddContentFileIndexes )
      {
        fnc_addIfInContent( "content" );
      }
  }
 
  void Fetcher::Impl::getDirectoryContent( MediaSetAccess &media,
                                           const OnMediaLocation &resource,
                                           filesystem::DirContent &content )
  {
      if ( _dircontent.find(resource.filename().asString())
           != _dircontent.end() )
      {
          filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
 
          std::copy(filled.begin(), filled.end(), std::back_inserter(content));
      }
      else
      {
          filesystem::DirContent tofill;
          media.dirInfo( tofill,
                         resource.filename(),
                         false /* dots */,
                         resource.medianr());
          std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
          _dircontent[resource.filename().asString()] = tofill;
      }
  }
 
  void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
                                  const OnMediaLocation &resource,
                                  const Pathname &dest_dir, FetcherJob::Flags flags  )
  {
      // first get the content of the directory so we can add
      // individual transfer jobs
      MIL << "Adding directory " << resource.filename() << endl;
      filesystem::DirContent content;
      try {
        getDirectoryContent(media, resource, content);
      }
      catch ( media::MediaFileNotFoundException & exception )
      {
        ZYPP_CAUGHT( exception );
        WAR << "Skipping subtree hidden at " << resource.filename() << endl;
        return;
      }
 
      // this method test for the option flags so indexes are added
      // only if the options are enabled
      autoaddIndexes(content, media, resource, dest_dir);
 
      for ( filesystem::DirContent::const_iterator it = content.begin();
            it != content.end();
            ++it )
      {
          // skip CHECKSUMS* as they were already retrieved
          if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
              continue;
 
          Pathname filename = resource.filename() + it->name;
 
          switch ( it->type )
          {
          case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
          case filesystem::FT_FILE:
          {
              CheckSum chksm(resource.checksum());
              if ( _checksums.find(filename.asString()) != _checksums.end() )
              {
                  // the checksum can be replaced with the one in the index.
                  chksm = _checksums[filename.asString()];
                  //MIL << "resource " << filename << " has checksum in the index file." << endl;
              }
              else
                  WAR << "Resource " << filename << " has no checksum in the index either." << endl;
 
              if ( flags & FetcherJob::AlwaysVerifyChecksum )
                  enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
              else
                  enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
              break;
          }
          case filesystem::FT_DIR: // newer directory.yast contain at least directory info
              if ( flags & FetcherJob::Recursive )
                  addDirJobs(media, filename, dest_dir, flags);
              break;
          default:
              // don't provide devices, sockets, etc.
              break;
          }
      }
  }
 
  void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r )
  {
    const OnMediaLocation & resource  { jobp_r->location };
    Pathname inCachePath              { mapToCachePath( resource ) }; // Map remote filename to a local path below destDir_r.
    Pathname finalDestination         { destDir_r / inCachePath };    // The full destination path.
 
    try
    {
      scoped_ptr<MediaSetAccess::ReleaseFileGuard> releaseFileGuard; // will take care provided files get released
      Pathname candidate;
 
      // Try to get cached file (by checksum) or provide it from media.
      ManagedFile cachedFile = locateInCache( resource, destDir_r, inCachePath );
      if ( (*cachedFile).empty() ) {
        pMIL( "file", inCachePath, "not found in cache, retrieving..." );
        candidate = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT );
        releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block
      } else {
        candidate = cachedFile;
      }
 
      if ( candidate == finalDestination ) {
        // If we have a match at destFullPath, take care
        // it gets deleted, in case the validation fails.
        cachedFile.setDispose( filesystem::unlink );
      }
 
      // Validate the file (throws if not valid)
      validate( candidate, jobp_r->checkers );
 
      // move it to the final destination
      if ( candidate == finalDestination )
        cachedFile.resetDispose();      // keep it!
      else
      {
        if ( assert_dir( finalDestination.dirname() ) != 0 )
          ZYPP_THROW( Exception( str::sprint( "Can't create", finalDestination.dirname() ) ) );
 
        if ( filesystem::hardlinkCopy( candidate, finalDestination ) != 0 )
          ZYPP_THROW( Exception( str::sprint( "Can't hardlink/copy", candidate, "to", finalDestination ) ) );
      }
    }
    catch ( Exception & excpt )
    {
      if ( resource.optional() )
      {
        ZYPP_CAUGHT( excpt );
        WAR << "Optional resource " << resource << " could not be transferred." << endl;
        return;
      }
      else
      {
        excpt.remember( str::sprint("Can't provide", resource.filename() ) );
        ZYPP_RETHROW( excpt );
      }
    }
  }
 
  // helper class to consume a content file
  struct ContentReaderHelper : public parser::susetags::ContentFileReader
  {
    ContentReaderHelper()
    {
      setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
    }
 
    void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
    { _repoindex = data_r; }
 
    parser::susetags::RepoIndex_Ptr _repoindex;
  };
 
  // generic function for reading indexes
  void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
  {
    if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
      readChecksumsIndex(index, basedir);
    else if ( index.basename() == "content" )
      readContentFileIndex(index, basedir);
    else
      WAR << index << ": index file format not known" << endl;
  }
 
  // reads a content file index
  void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
  {
      ContentReaderHelper reader;
      reader.parse(index);
      MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
      for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
      {
          // content file entries don't start with /
          _checksums[(basedir + it->first).asString()] = it->second;
      }
  }
 
  // reads a CHECKSUMS (old SHA1SUMS) file index
  void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
  {
      std::ifstream in( index.c_str() );
      if ( ! in.fail() )
      {
          std::string buffer;
          while ( getline( in, buffer ) )
          {
 
              if ( buffer[0] == '#' )
                continue;       // simple comment
 
              CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
              if ( checksum.empty() )
                continue;       // empty line | unknown cheksum format
 
              if ( buffer.empty() )
              {
                WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
                continue;
              }
 
              _checksums[(basedir/buffer).asString()] = checksum;
          }
      }
      else
          ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
  }
 
  void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
  {
    MIL << "downloading index " << resource << endl;
    // create a new fetcher with a different state to transfer the
    // file containing checksums and its signature
    Fetcher fetcher;
    // signature checker for index. We havent got the signature from
    // the nextwork yet.
    SignatureFileChecker sigchecker;
 
    // build the name of the index and the signature
    const OnMediaLocation& idxloc(resource);
    OnMediaLocation sigloc(resource);
    OnMediaLocation keyloc(resource);
 
    // we should not fail the download if those don't exists
    // the checking will warn later
    sigloc.setOptional(true);
    keyloc.setOptional(true);
 
    // calculate signature and key name
    sigloc.changeFilename( sigloc.filename().extend(".asc") );
    keyloc.changeFilename( keyloc.filename().extend(".key") );
 
    //assert_dir(dest_dir + idxloc.filename().dirname());
 
    // transfer the signature
    fetcher.enqueue(sigloc);
    fetcher.start( dest_dir, media );
    // if we get the signature, update the checker
    if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
        sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
 
    fetcher.reset();
 
    // now the key
    fetcher.enqueue(keyloc);
    fetcher.start( dest_dir, media );
    fetcher.reset();
 
    // try to import the key
    if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
        getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
    else
        WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
 
    // now the index itself
    fetcher.enqueue( idxloc, FileChecker(sigchecker) );
    fetcher.start( dest_dir, media );
    fetcher.reset();
 }
 
  // this method takes all the user pointed indexes, gets them and also tries to
  // download their signature, and verify them. After that, it parses each one
  // to fill the checksum cache.
  void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir)
  {
      // if there is no indexes, then just return to avoid
      // the directory listing
      if ( _indexes.empty() )
      {
          MIL << "No indexes to read." << endl;
          return;
      }
 
      for_( it_idx, _indexes.begin(), _indexes.end() )
      {
        if ( (*it_idx)->read )
        {
          DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
        }
        else
        {
          // base::LogControl::TmpLineWriter shutUp;
          downloadIndex( media, (*it_idx)->location, dest_dir );
          // now we have the indexes in dest_dir
          readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
          // Take care we don't process it again
          MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
          (*it_idx)->read = true;
        }
      }
      MIL << "done reading indexes" << endl;
  }
 
  void Fetcher::Impl::start( const Pathname &dest_dir,
                             const ProgressData::ReceiverFnc & progress )
  {
    if ( !_mediaSetAccess )
      ZYPP_THROW( zypp::Exception("Called Fetcher::start without setting MediaSetAccess before.") );
    start( dest_dir, *_mediaSetAccess, progress );
  }
 
  // start processing all fetcher jobs.
  // it processes any user pointed index first
  void Fetcher::Impl::start( const Pathname &dest_dir,
                             MediaSetAccess &media,
                             const ProgressData::ReceiverFnc & progress_receiver )
  {
    _mediaSetAccess = nullptr; //reset the internally stored MediaSetAccess
 
    ProgressData progress(_resources.size());
    progress.sendTo(progress_receiver);
 
    downloadAndReadIndexList(media, dest_dir);
 
    for ( const FetcherJob_Ptr & jobp : _resources )
    {
      if ( jobp->flags & FetcherJob::Directory )
      {
          const OnMediaLocation location(jobp->location);
          addDirJobs(media, location, dest_dir, jobp->flags);
          continue;
      }
 
      // may be this code can be factored out
      // together with the autodiscovery of indexes
      // of addDirJobs
      if ( ( _options & AutoAddChecksumsIndexes ) ||
           ( _options & AutoAddContentFileIndexes ) )
      {
          // if auto indexing is enabled, then we need to read the
          // index for each file. We look only in the directory
          // where the file is. this is expensive of course.
          filesystem::DirContent content;
          getDirectoryContent(media, jobp->location.filename().dirname(), content);
          // this method test for the option flags so indexes are added
          // only if the options are enabled
          MIL << "Autodiscovering signed indexes on '"
              << jobp->location.filename().dirname() << "' for '"
              << jobp->location.filename() << "'" << endl;
 
          autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir);
 
          // also look in the root of the media
          content.clear();
          getDirectoryContent(media, Pathname("/"), content);
          // this method test for the option flags so indexes are added
          // only if the options are enabled
          MIL << "Autodiscovering signed indexes on '"
              << "/" << "' for '"
              << jobp->location.filename() << "'" << endl;
 
          autoaddIndexes(content, media, Pathname("/"), dest_dir);
      }
 
      // if the checksum is empty, but the checksum is in one of the
      // indexes checksum, then add a checker
      if ( jobp->location.checksum().empty() )
      {
          if ( _checksums.find(jobp->location.filename().asString())
               != _checksums.end() )
          {
              CheckSum chksm = _checksums[jobp->location.filename().asString()];
              ChecksumFileChecker digest_check(chksm);
              jobp->checkers.push_back(digest_check);
          }
          else
          {
              // if the index checksum is empty too, we only add the checker
              // if the  AlwaysVerifyChecksum option is set on
              if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum )
              {
                  // add the checker with the empty checksum
                  ChecksumFileChecker digest_check(jobp->location.checksum());
                  jobp->checkers.push_back(digest_check);
              }
          }
      }
      else
      {
          // checksum is not empty, so add a checksum checker
          ChecksumFileChecker digest_check(jobp->location.checksum());
          jobp->checkers.push_back(digest_check);
      }
 
      // Provide and validate the file. If the file was not transferred
      // and no exception was thrown, it was an optional file.
      provideToDest( media, dest_dir, jobp );
 
      if ( ! progress.incr() )
        ZYPP_THROW(AbortRequestException());
    } // for each job
  }

  inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
  {
      for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
      {
          str << *it_res;
      }
      return str;
  }
 
  Fetcher::Fetcher()
  : _pimpl( new Impl() )
  {}
 
  Fetcher::~Fetcher()
  {}
 
  void Fetcher::setOptions( Fetcher::Options options )
  {
    _pimpl->setOptions(options);
  }
 
  Fetcher::Options Fetcher::options() const
  {
    return _pimpl->options();
  }
 
  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
  {
    enqueueDigested( OnMediaLocation(resource).setDeltafile(deltafile), checker );
  }
 
  void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
  {
    _pimpl->enqueueDigested( resource, checker );
  }
 
  void Fetcher::enqueueDir( const OnMediaLocation &resource,
                            bool recursive,
                            const FileChecker &checker )
  {
      _pimpl->enqueueDir(resource, recursive, checker);
  }
 
  void Fetcher::enqueueDigestedDir( const OnMediaLocation &resource,
                                    bool recursive,
                                    const FileChecker &checker )
  {
      _pimpl->enqueueDigestedDir(resource, recursive, checker);
  }
 
 
  void Fetcher::addIndex( const OnMediaLocation &resource )
  {
    _pimpl->addIndex(resource);
  }
 
 
  void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker  )
  {
    _pimpl->enqueue(resource, checker);
  }
 
 
  void Fetcher::addCachePath( const Pathname &cache_dir )
  {
    _pimpl->addCachePath(cache_dir, Fetcher::Default);
  }
 
  void Fetcher::addCachePath(const Pathname &cache_dir, CacheOptions options)
  {
    _pimpl->addCachePath(cache_dir, options);
  }
 
  void Fetcher::reset()
  {
    _pimpl->reset();
  }
 
  void Fetcher::setMediaSetAccess( MediaSetAccess &media )
  {
    _pimpl->setMediaSetAccess( media );
  }
 
  void Fetcher::start(const zypp::filesystem::Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
  {
    _pimpl->start( dest_dir, progress );
  }
 
  void Fetcher::start( const Pathname &dest_dir,
                       MediaSetAccess &media,
                       const ProgressData::ReceiverFnc & progress_receiver )
  {
    _pimpl->start(dest_dir, media, progress_receiver);
  }
 
  Pathname Fetcher::mapToCachePath( Pathname remotePath_r )
  {
    fixup( remotePath_r );
    return remotePath_r;
  }
 
  std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
  {
    return str << *obj._pimpl;
  }

} // namespace zypp