qgsabstractcontentcache.h

Go to the documentation of this file.

/***************************************************************************
                         qgsabstractcontentcache.h
                         ---------------
    begin                : December 2018
    copyright            : (C) 2018 by Nyall Dawson
    email                : nyall dot dawson at gmail dot com
 ***************************************************************************/
 
/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
 
#ifndef QGSABSTRACTCONTENTCACHE_H
#define QGSABSTRACTCONTENTCACHE_H
 
#include "qgis_core.h"
#include "qgis_sip.h"
#include "qgslogger.h"
#include "qgsmessagelog.h"
#include "qgsapplication.h"
#include "qgsnetworkaccessmanager.h"
#include "qgsnetworkcontentfetchertask.h"
#include "qgsvariantutils.h"
 
#include <QObject>
#include <QRecursiveMutex>
#include <QCache>
#include <QSet>
#include <QDateTime>
#include <QList>
#include <QFile>
#include <QNetworkReply>
#include <QFileInfo>
#include <QUrl>
 
class CORE_EXPORT QgsAbstractContentCacheEntry
{
  public:
 
    QgsAbstractContentCacheEntry( const QString &path ) ;
 
    virtual ~QgsAbstractContentCacheEntry() = default;
 
    QgsAbstractContentCacheEntry( const QgsAbstractContentCacheEntry &rh ) = delete;
    QgsAbstractContentCacheEntry &operator=( const QgsAbstractContentCacheEntry &rh ) = delete;
 
    QString path;
 
    QDateTime fileModified;
 
    QElapsedTimer fileModifiedLastCheckTimer;
 
    int mFileModifiedCheckTimeout = 30000;
 
    QgsAbstractContentCacheEntry *nextEntry = nullptr;
 
    QgsAbstractContentCacheEntry *previousEntry = nullptr;
 
    bool operator==( const QgsAbstractContentCacheEntry &other ) const
    {
      return other.path == path;
    }
 
    virtual int dataSize() const = 0;
 
    virtual void dump() const = 0;
 
  protected:
 
    virtual bool isEqual( const QgsAbstractContentCacheEntry *other ) const = 0;
 
  private:
#ifdef SIP_RUN
    QgsAbstractContentCacheEntry( const QgsAbstractContentCacheEntry &rh );
#endif
 
};
 
class CORE_EXPORT QgsAbstractContentCacheBase: public QObject
{
    Q_OBJECT
 
  public:
 
    QgsAbstractContentCacheBase( QObject *parent );
 
  signals:
 
    void remoteContentFetched( const QString &url );
 
  protected:
 
    virtual bool checkReply( QNetworkReply *reply, const QString &path ) const
    {
      Q_UNUSED( reply )
      Q_UNUSED( path )
      return true;
    }
 
  protected slots:
 
    virtual void onRemoteContentFetched( const QString &url, bool success );
 
};
 
#ifndef SIP_RUN
 
template<class T>
class CORE_EXPORT QgsAbstractContentCache : public QgsAbstractContentCacheBase
{
 
  public:
 
    QgsAbstractContentCache( QObject *parent SIP_TRANSFERTHIS = nullptr,
                             const QString &typeString = QString(),
                             long maxCacheSize = 20000000,
                             int fileModifiedCheckTimeout = 30000 )
      : QgsAbstractContentCacheBase( parent )
      , mMaxCacheSize( maxCacheSize )
      , mFileModifiedCheckTimeout( fileModifiedCheckTimeout )
      , mTypeString( typeString.isEmpty() ? QObject::tr( "Content" ) : typeString )
    {
    }
 
    ~QgsAbstractContentCache() override
    {
      qDeleteAll( mEntryLookup );
    }
 
  protected:
 
    void trimToMaximumSize()
    {
      //only one entry in cache
      if ( mLeastRecentEntry == mMostRecentEntry )
      {
        return;
      }
      T *entry = mLeastRecentEntry;
      while ( entry && ( mTotalSize > mMaxCacheSize ) )
      {
        T *bkEntry = entry;
        entry = static_cast< T * >( entry->nextEntry );
 
        takeEntryFromList( bkEntry );
        mEntryLookup.remove( bkEntry->path, bkEntry );
        mTotalSize -= bkEntry->dataSize();
        delete bkEntry;
      }
    }
 
    QByteArray getContent( const QString &path, const QByteArray &missingContent, const QByteArray &fetchingContent, bool blocking = false ) const
    {
      // is it a path to local file?
      QFile file( path );
      if ( file.exists() )
      {
        if ( file.open( QIODevice::ReadOnly ) )
        {
          return file.readAll();
        }
        else
        {
          return missingContent;
        }
      }
 
      // maybe it's an embedded base64 string
      if ( path.startsWith( QLatin1String( "base64:" ), Qt::CaseInsensitive ) )
      {
        const QByteArray base64 = path.mid( 7 ).toLocal8Bit(); // strip 'base64:' prefix
        return QByteArray::fromBase64( base64, QByteArray::OmitTrailingEquals );
      }
 
      // maybe it's a url...
      if ( !path.contains( QLatin1String( "://" ) ) ) // otherwise short, relative SVG paths might be considered URLs
      {
        return missingContent;
      }
 
      const QUrl url( path );
      if ( !url.isValid() )
      {
        return missingContent;
      }
 
      // check whether it's a url pointing to a local file
      if ( url.scheme().compare( QLatin1String( "file" ), Qt::CaseInsensitive ) == 0 )
      {
        file.setFileName( url.toLocalFile() );
        if ( file.exists() )
        {
          if ( file.open( QIODevice::ReadOnly ) )
          {
            return file.readAll();
          }
        }
 
        // not found...
        return missingContent;
      }
 
      const QMutexLocker locker( &mMutex );
 
      // already a request in progress for this url
      if ( mPendingRemoteUrls.contains( path ) )
      {
        // it's a non blocking request so return fetching content
        if ( !blocking )
        {
          return fetchingContent;
        }
 
        // it's a blocking request so try to find the task and wait for task finished
        const auto constActiveTasks = QgsApplication::taskManager()->activeTasks();
        for ( QgsTask *task : constActiveTasks )
        {
          // the network content fetcher task's description ends with the path
          if ( !task->description().endsWith( path ) )
          {
            continue;
          }
 
          // cast task to network content fetcher task
          QgsNetworkContentFetcherTask *ncfTask = qobject_cast<QgsNetworkContentFetcherTask *>( task );
          if ( ncfTask )
          {
            // wait for task finished
            if ( waitForTaskFinished( ncfTask ) )
            {
              if ( mRemoteContentCache.contains( path ) )
              {
                // We got the file!
                return *mRemoteContentCache[ path ];
              }
            }
          }
          // task found, no needs to continue
          break;
        }
        // if no content returns the content is probably in remote content cache
        // or a new task will be created
      }
 
      if ( mRemoteContentCache.contains( path ) )
      {
        // already fetched this content - phew. Just return what we already got.
        return *mRemoteContentCache[ path ];
      }
 
      mPendingRemoteUrls.insert( path );
      //fire up task to fetch content in background
      QNetworkRequest request( url );
      QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsAbstractContentCache<%1>" ).arg( mTypeString ) );
      request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
      request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
 
      QgsNetworkContentFetcherTask *task = new QgsNetworkContentFetcherTask( request );
      connect( task, &QgsNetworkContentFetcherTask::fetched, this, [this, task, path, missingContent]
      {
        const QMutexLocker locker( &mMutex );
 
        QNetworkReply *reply = task->reply();
        if ( !reply )
        {
          // canceled
          QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, false ) );
          return;
        }
 
        if ( reply->error() != QNetworkReply::NoError )
        {
          QgsMessageLog::logMessage( tr( "%3 request failed [error: %1 - url: %2]" ).arg( reply->errorString(), path, mTypeString ), mTypeString );
          return;
        }
 
        bool ok = true;
 
        const QVariant status = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute );
        if ( !QgsVariantUtils::isNull( status ) && status.toInt() >= 400 )
        {
          const QVariant phrase = reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute );
          QgsMessageLog::logMessage( tr( "%4 request error [status: %1 - reason phrase: %2] for %3" ).arg( status.toInt() ).arg( phrase.toString(), path, mTypeString ), mTypeString );
          mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
          ok = false;
        }
 
        if ( !checkReply( reply, path ) )
        {
          mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
          ok = false;
        }
 
        if ( ok )
        {
          // read the content data
          const QByteArray ba = reply->readAll();
 
          // because of the fragility listed below in waitForTaskFinished, this slot may get called twice. In that case
          // the second time will have an empty reply (we've already read it all...)
          if ( !ba.isEmpty() )
            mRemoteContentCache.insert( path, new QByteArray( ba ) );
        }
        QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, true ) );
      } );
 
      QgsApplication::taskManager()->addTask( task );
 
      // if blocking, wait for finished
      if ( blocking )
      {
        if ( waitForTaskFinished( task ) )
        {
          if ( mRemoteContentCache.contains( path ) )
          {
            // We got the file!
            return *mRemoteContentCache[ path ];
          }
        }
      }
      return fetchingContent;
    }
 
    void onRemoteContentFetched( const QString &url, bool success ) override
    {
      const QMutexLocker locker( &mMutex );
      mPendingRemoteUrls.remove( url );
 
      T *nextEntry = mLeastRecentEntry;
      while ( T *entry = nextEntry )
      {
        nextEntry = static_cast< T * >( entry->nextEntry );
        if ( entry->path == url )
        {
          takeEntryFromList( entry );
          mEntryLookup.remove( entry->path, entry );
          mTotalSize -= entry->dataSize();
          delete entry;
        }
      }
 
      if ( success )
        emit remoteContentFetched( url );
    }
 
    bool waitForTaskFinished( QgsNetworkContentFetcherTask *task ) const
    {
      // Wait up to timeout seconds for task finished
      if ( task->waitForFinished( QgsNetworkAccessManager::timeout() ) )
      {
        // The wait did not time out
        // Third step, check status as complete
        if ( task->status() == QgsTask::Complete )
        {
          // Fourth step, force the signal fetched to be sure reply has been checked
 
          // ARGH this is BAD BAD BAD. The connection will get called twice as a result!!!
          task->fetched();
          return true;
        }
      }
      return false;
    }
 
    T *findExistingEntry( T *entryTemplate )
    {
      //search entries in mEntryLookup
      const QString path = entryTemplate->path;
      T *currentEntry = nullptr;
      const QList<T *> entries = mEntryLookup.values( path );
      QDateTime modified;
      for ( T *cacheEntry : entries )
      {
        if ( cacheEntry->isEqual( entryTemplate ) )
        {
          if ( mFileModifiedCheckTimeout <= 0 || cacheEntry->fileModifiedLastCheckTimer.hasExpired( mFileModifiedCheckTimeout ) )
          {
            if ( !modified.isValid() )
              modified = QFileInfo( path ).lastModified();
 
            if ( cacheEntry->fileModified != modified )
              continue;
            else
              cacheEntry->fileModifiedLastCheckTimer.restart();
          }
          currentEntry = cacheEntry;
          break;
        }
      }
 
      //if not found: insert entryTemplate as a new entry
      if ( !currentEntry )
      {
        currentEntry = insertCacheEntry( entryTemplate );
      }
      else
      {
        delete entryTemplate;
        entryTemplate = nullptr;
        ( void )entryTemplate;
        takeEntryFromList( currentEntry );
        if ( !mMostRecentEntry ) //list is empty
        {
          mMostRecentEntry = currentEntry;
          mLeastRecentEntry = currentEntry;
        }
        else
        {
          mMostRecentEntry->nextEntry = currentEntry;
          currentEntry->previousEntry = mMostRecentEntry;
          currentEntry->nextEntry = nullptr;
          mMostRecentEntry = currentEntry;
        }
      }
 
      //debugging
      //printEntryList();
 
      return currentEntry;
    }
    mutable QRecursiveMutex mMutex;
 
    long mTotalSize = 0;
 
    long mMaxCacheSize = 20000000;
 
  private:
 
    T *insertCacheEntry( T *entry )
    {
      entry->mFileModifiedCheckTimeout = mFileModifiedCheckTimeout;
 
      if ( !entry->path.startsWith( QLatin1String( "base64:" ) ) )
      {
        entry->fileModified = QFileInfo( entry->path ).lastModified();
        entry->fileModifiedLastCheckTimer.start();
      }
 
      mEntryLookup.insert( entry->path, entry );
 
      //insert to most recent place in entry list
      if ( !mMostRecentEntry ) //inserting first entry
      {
        mLeastRecentEntry = entry;
        mMostRecentEntry = entry;
        entry->previousEntry = nullptr;
        entry->nextEntry = nullptr;
      }
      else
      {
        entry->previousEntry = mMostRecentEntry;
        entry->nextEntry = nullptr;
        mMostRecentEntry->nextEntry = entry;
        mMostRecentEntry = entry;
      }
 
      trimToMaximumSize();
      return entry;
    }
 
 
    void takeEntryFromList( T *entry )
    {
      if ( !entry )
      {
        return;
      }
 
      if ( entry->previousEntry )
      {
        entry->previousEntry->nextEntry = entry->nextEntry;
      }
      else
      {
        mLeastRecentEntry = static_cast< T * >( entry->nextEntry );
      }
      if ( entry->nextEntry )
      {
        entry->nextEntry->previousEntry = entry->previousEntry;
      }
      else
      {
        mMostRecentEntry = static_cast< T * >( entry->previousEntry );
      }
    }
 
    void printEntryList()
    {
      QgsDebugMsgLevel( QStringLiteral( "****************cache entry list*************************" ), 1 );
      QgsDebugMsgLevel( "Cache size: " + QString::number( mTotalSize ), 1 );
      T *entry = mLeastRecentEntry;
      while ( entry )
      {
        QgsDebugMsgLevel( QStringLiteral( "***Entry:" ), 1 );
        entry->dump();
        entry = entry->nextEntry;
      }
    }
 
    QMultiHash< QString, T * > mEntryLookup;
 
    int mFileModifiedCheckTimeout = 30000;
 
    //The content cache keeps the entries on a double connected list, moving the current entry to the front.
    //That way, removing entries for more space can start with the least used objects.
    T *mLeastRecentEntry = nullptr;
    T *mMostRecentEntry = nullptr;
 
    mutable QCache< QString, QByteArray > mRemoteContentCache;
    mutable QSet< QString > mPendingRemoteUrls;
 
    QString mTypeString;
 
    friend class TestQgsSvgCache;
    friend class TestQgsImageCache;
};
 
#endif
 
#endif // QGSABSTRACTCONTENTCACHE_H