qgsvectorlayercache.cpp

Go to the documentation of this file.

/***************************************************************************
  qgsvectorlayercache.cpp
  Cache features of a vector layer
  -------------------
         begin                : January 2013
         copyright            : (C) Matthias Kuhn
         email                : matthias at opengis dot ch
 
 ***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
 
#include "qgsvectorlayercache.h"
 
#include "qgscachedfeatureiterator.h"
#include "qgscacheindex.h"
#include "qgsthreadingutils.h"
#include "qgsvectorlayer.h"
#include "qgsvectorlayerjoinbuffer.h"
#include "qgsvectorlayerjoininfo.h"
 
#include <QElapsedTimer>
 
#include "moc_qgsvectorlayercache.cpp"
 
QgsVectorLayerCache::QgsVectorLayerCache( QgsVectorLayer *layer, int cacheSize, QObject *parent )
  : QObject( parent )
  , mLayer( layer )
{
  QGIS_CHECK_QOBJECT_THREAD_EQUALITY( layer )
 
  mCache.setMaxCost( cacheSize );
 
  connect( mLayer, &QgsVectorLayer::featureDeleted, this, &QgsVectorLayerCache::onFeatureDeleted );
  connect( mLayer, &QgsVectorLayer::featureAdded, this, &QgsVectorLayerCache::onFeatureAdded );
  connect( mLayer, &QgsVectorLayer::destroyed, this, &QgsVectorLayerCache::layerDeleted );
 
  setCacheGeometry( true );
  setCacheSubsetOfAttributes( mLayer->attributeList() );
  setCacheAddedAttributes( true );
 
  connect( mLayer, &QgsVectorLayer::attributeDeleted, this, &QgsVectorLayerCache::attributeDeleted );
  connect( mLayer, &QgsVectorLayer::updatedFields, this, &QgsVectorLayerCache::invalidate );
  connect( mLayer, &QgsVectorLayer::dataChanged, this, &QgsVectorLayerCache::invalidate );
  connect( mLayer, &QgsVectorLayer::attributeValueChanged, this, &QgsVectorLayerCache::onAttributeValueChanged );
 
  connectJoinedLayers();
}
 
QgsVectorLayerCache::~QgsVectorLayerCache()
{
  qDeleteAll( mCacheIndices );
  mCacheIndices.clear();
}
 
void QgsVectorLayerCache::setCacheSize( long long cacheSize )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  mCache.setMaxCost( cacheSize );
}
 
long long QgsVectorLayerCache::cacheSize() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mCache.maxCost();
}
 
void QgsVectorLayerCache::setCacheGeometry( bool cacheGeometry )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  bool shouldCacheGeometry = cacheGeometry && mLayer->isSpatial();
  bool mustInvalidate = shouldCacheGeometry && !mCacheGeometry; // going from no geometry -> geometry, so have to clear existing cache entries
  mCacheGeometry = shouldCacheGeometry;
  if ( cacheGeometry )
  {
    connect( mLayer, &QgsVectorLayer::geometryChanged, this, &QgsVectorLayerCache::geometryChanged, Qt::UniqueConnection );
  }
  else
  {
    disconnect( mLayer, &QgsVectorLayer::geometryChanged, this, &QgsVectorLayerCache::geometryChanged );
  }
  if ( mustInvalidate )
  {
    invalidate();
  }
}
 
void QgsVectorLayerCache::setCacheSubsetOfAttributes( const QgsAttributeList &attributes )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  if ( !mCache.isEmpty() && !QSet<int>( mCachedAttributes.cbegin(), mCachedAttributes.cend() ).contains( QSet<int>( attributes.cbegin(), attributes.cend() ) ) )
    invalidate();
  mCachedAttributes = attributes;
}
 
QgsAttributeList QgsVectorLayerCache::cacheSubsetOfAttributes() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mCachedAttributes;
}
 
void QgsVectorLayerCache::setFullCache( bool fullCache )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  mFullCache = fullCache;
 
  if ( mFullCache )
  {
    // Add a little more than necessary...
    setCacheSize( mLayer->featureCount() + 100 );
 
    // Initialize the cache...
    QgsFeatureIterator it(
      new QgsCachedFeatureWriterIterator( this, QgsFeatureRequest().setSubsetOfAttributes( mCachedAttributes ).setFlags( mCacheGeometry ? Qgis::FeatureRequestFlag::NoFlags : Qgis::FeatureRequestFlag::NoGeometry ) )
    );
 
    int i = 0;
 
    QElapsedTimer t;
    t.start();
 
    QgsFeature f;
    while ( it.nextFeature( f ) )
    {
      ++i;
 
      if ( t.elapsed() > 1000 )
      {
        bool cancel = false;
        emit progress( i, cancel );
        if ( cancel )
          break;
 
        t.restart();
      }
    }
 
    it.close();
 
    emit finished();
  }
}
 
void QgsVectorLayerCache::addCacheIndex( QgsAbstractCacheIndex *cacheIndex )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  mCacheIndices.append( cacheIndex );
}
 
void QgsVectorLayerCache::setCacheAddedAttributes( bool cacheAddedAttributes )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  if ( cacheAddedAttributes == mCacheAddedAttributes )
    return;
 
  mCacheAddedAttributes = cacheAddedAttributes;
  if ( mCacheAddedAttributes )
  {
    connect( mLayer, &QgsVectorLayer::attributeAdded, this, &QgsVectorLayerCache::attributeAdded );
  }
  else
  {
    disconnect( mLayer, &QgsVectorLayer::attributeAdded, this, &QgsVectorLayerCache::attributeAdded );
  }
}
 
bool QgsVectorLayerCache::featureAtId( QgsFeatureId featureId, QgsFeature &feature, bool skipCache )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  bool featureFound = false;
 
  QgsCachedFeature *cachedFeature = nullptr;
 
  if ( !skipCache )
  {
    cachedFeature = mCache[featureId];
  }
 
  if ( cachedFeature )
  {
    feature = cachedFeature->feature();
    featureFound = true;
  }
  else
  {
    QgsFeatureRequest request { featureId };
    const bool allAttrsFetched { mCachedAttributes.count() == mLayer->fields().count() };
    if ( !allAttrsFetched )
    {
      request.setSubsetOfAttributes( mCachedAttributes );
    }
    if ( !mCacheGeometry )
    {
      request.setFlags( request.flags().setFlag( Qgis::FeatureRequestFlag::NoGeometry ) );
    }
    if ( mLayer->getFeatures( request ).nextFeature( feature ) )
    {
      cacheFeature( feature, allAttrsFetched );
      featureFound = true;
    }
  }
 
  return featureFound;
}
 
bool QgsVectorLayerCache::featureAtIdWithAllAttributes( QgsFeatureId featureId, QgsFeature &feature, bool skipCache )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  bool featureFound = false;
 
  QgsCachedFeature *cachedFeature = nullptr;
 
  if ( !skipCache )
  {
    cachedFeature = mCache[featureId];
  }
 
  if ( cachedFeature && cachedFeature->allAttributesFetched() )
  {
    feature = cachedFeature->feature();
    featureFound = true;
  }
  else if ( mLayer->getFeatures( QgsFeatureRequest().setFilterFid( featureId ).setFlags( !mCacheGeometry ? Qgis::FeatureRequestFlag::NoGeometry : Qgis::FeatureRequestFlags() ) ).nextFeature( feature ) )
  {
    cacheFeature( feature, true );
    featureFound = true;
  }
 
  return featureFound;
}
 
bool QgsVectorLayerCache::completeFeatureAtId( QgsFeatureId featureId, QgsFeature &feature, bool skipCache )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  bool featureFound = false;
 
  QgsCachedFeature *cachedFeature = nullptr;
 
  if ( !skipCache )
  {
    cachedFeature = mCache[featureId];
  }
 
  if ( cachedFeature && cachedFeature->allAttributesFetched() && cachedFeature->geometryFetched() )
  {
    feature = cachedFeature->feature();
    featureFound = true;
  }
  else if ( mLayer->getFeatures( QgsFeatureRequest().setFilterFid( featureId ) ).nextFeature( feature ) )
  {
    cacheFeature( feature, true, true );
    featureFound = true;
  }
 
  return featureFound;
}
 
bool QgsVectorLayerCache::removeCachedFeature( QgsFeatureId fid )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  bool removed = mCache.remove( fid );
  if ( removed )
  {
    if ( auto unorderedIt = std::find( mCacheUnorderedKeys.begin(), mCacheUnorderedKeys.end(), fid ); unorderedIt != mCacheUnorderedKeys.end() )
    {
      mCacheUnorderedKeys.erase( unorderedIt );
 
      if ( auto orderedIt = std::find( mCacheOrderedKeys.begin(), mCacheOrderedKeys.end(), fid ); orderedIt != mCacheOrderedKeys.end() )
        mCacheOrderedKeys.erase( orderedIt );
    }
  }
  return removed;
}
 
QgsVectorLayer *QgsVectorLayerCache::layer()
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mLayer;
}
 
QgsCoordinateReferenceSystem QgsVectorLayerCache::sourceCrs() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mLayer->crs();
}
 
Qgis::WkbType QgsVectorLayerCache::wkbType() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mLayer->wkbType();
}
 
QgsFields QgsVectorLayerCache::fields() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mLayer->fields();
}
 
long long QgsVectorLayerCache::featureCount() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mLayer->featureCount();
}
 
void QgsVectorLayerCache::requestCompleted( const QgsFeatureRequest &featureRequest, const QgsFeatureIds &fids )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  // If a request is too large for the cache don't notify to prevent from indexing incomplete requests
  if ( fids.count() <= mCache.maxCost() )
  {
    for ( const auto &idx : std::as_const( mCacheIndices ) )
    {
      idx->requestCompleted( featureRequest, fids );
    }
    if ( featureRequest.filterType() == Qgis::FeatureRequestFilterType::NoFilter
         && ( featureRequest.spatialFilterType() == Qgis::SpatialFilterType::NoFilter || featureRequest.filterRect().contains( mLayer->extent() ) ) )
    {
      mFullCache = true;
    }
  }
}
 
void QgsVectorLayerCache::featureRemoved( QgsFeatureId fid )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  const auto constMCacheIndices = mCacheIndices;
  for ( QgsAbstractCacheIndex *idx : constMCacheIndices )
  {
    idx->flushFeature( fid );
  }
}
 
void QgsVectorLayerCache::onAttributeValueChanged( QgsFeatureId fid, int field, const QVariant &value )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  QgsCachedFeature *cachedFeat = mCache[fid];
 
  if ( cachedFeat )
  {
    cachedFeat->mFeature.setAttribute( field, value );
  }
 
  emit attributeValueChanged( fid, field, value );
}
 
void QgsVectorLayerCache::onJoinAttributeValueChanged( QgsFeatureId fid, int field, const QVariant &value )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  const QgsVectorLayer *joinLayer = qobject_cast<const QgsVectorLayer *>( sender() );
 
  const auto constVectorJoins = mLayer->vectorJoins();
  for ( const QgsVectorLayerJoinInfo &info : constVectorJoins )
  {
    if ( joinLayer == info.joinLayer() )
    {
      const QgsFeature feature = mLayer->joinBuffer()->targetedFeatureOf( &info, joinLayer->getFeature( fid ) );
 
      const QString fieldName = info.prefixedFieldName( joinLayer->fields().field( field ) );
      const int fieldIndex = mLayer->fields().indexFromName( fieldName );
 
      if ( feature.isValid() && fieldIndex != -1 )
      {
        onAttributeValueChanged( feature.id(), fieldIndex, value );
        return;
      }
    }
  }
}
 
void QgsVectorLayerCache::onFeatureDeleted( QgsFeatureId fid )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  emit featureAboutToBeDeleted( fid );
  mCache.remove( fid );
 
  if ( auto it = mCacheUnorderedKeys.find( fid ); it != mCacheUnorderedKeys.end() )
  {
    mCacheUnorderedKeys.erase( it );
    if ( auto orderedIt = std::find( mCacheOrderedKeys.begin(), mCacheOrderedKeys.end(), fid ); orderedIt != mCacheOrderedKeys.end() )
      mCacheOrderedKeys.erase( orderedIt );
  }
  emit featureDeleted( fid );
}
 
void QgsVectorLayerCache::onFeatureAdded( QgsFeatureId fid )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  emit featureAboutToBeAdded( fid );
 
  if ( mFullCache )
  {
    if ( cacheSize() <= mLayer->featureCount() )
    {
      setCacheSize( mLayer->featureCount() + 100 );
    }
 
    QgsFeature feat;
    featureAtId( fid, feat );
  }
  emit featureAdded( fid );
}
 
void QgsVectorLayerCache::attributeAdded( int field )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  if ( !mCachedAttributes.contains( field ) )
  {
    mCachedAttributes.append( field );
  }
  invalidate();
}
 
void QgsVectorLayerCache::attributeDeleted( int field )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  const QgsAttributeList attrs = mCachedAttributes;
  mCachedAttributes.clear();
 
  for ( int attr : attrs )
  {
    if ( attr < field )
      mCachedAttributes << attr;
    else if ( attr > field )
      mCachedAttributes << attr - 1;
  }
  invalidate();
}
 
void QgsVectorLayerCache::geometryChanged( QgsFeatureId fid, const QgsGeometry &geom )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  QgsCachedFeature *cachedFeat = mCache[fid];
 
  if ( cachedFeat )
  {
    cachedFeat->mFeature.setGeometry( geom );
  }
}
 
void QgsVectorLayerCache::layerDeleted()
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  emit cachedLayerDeleted();
  mLayer = nullptr;
}
 
void QgsVectorLayerCache::invalidate()
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  if ( !mCache.isEmpty() )
  {
    mCache.clear();
    mCacheOrderedKeys.clear();
    mCacheUnorderedKeys.clear();
    mFullCache = false;
    emit invalidated();
  }
}
 
bool QgsVectorLayerCache::canUseCacheForRequest( const QgsFeatureRequest &featureRequest, QgsFeatureIterator &it )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  // check first for available indices
  const auto constMCacheIndices = mCacheIndices;
  for ( QgsAbstractCacheIndex *idx : constMCacheIndices )
  {
    if ( idx->getCacheIterator( it, featureRequest ) )
    {
      return true;
    }
  }
 
  // no indexes available, but maybe we have already cached all required features anyway?
  switch ( featureRequest.filterType() )
  {
    case Qgis::FeatureRequestFilterType::Fid:
    {
      if ( mCache.contains( featureRequest.filterFid() ) )
      {
        it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
        return true;
      }
      break;
    }
    case Qgis::FeatureRequestFilterType::Fids:
    {
      if ( cachedFeatureIds().contains( featureRequest.filterFids() ) )
      {
        it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
        return true;
      }
      break;
    }
    case Qgis::FeatureRequestFilterType::NoFilter:
    case Qgis::FeatureRequestFilterType::Expression:
    {
      if ( mFullCache )
      {
        it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
        return true;
      }
      break;
    }
  }
  return false;
}
 
QgsFeatureIterator QgsVectorLayerCache::getFeatures( const QgsFeatureRequest &featureRequest )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  QgsFeatureIterator it;
  bool requiresWriterIt = true; // If a not yet cached, but cacheable request is made, this stays true.
 
  if ( checkInformationCovered( featureRequest ) )
  {
    // If we have a full cache available, run on this
    if ( mFullCache )
    {
      it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
      requiresWriterIt = false;
    }
    else
    {
      // may still be able to satisfy request using cache
      requiresWriterIt = !canUseCacheForRequest( featureRequest, it );
    }
  }
  else
  {
    // Let the layer answer the request, so no caching of requests
    // we don't want to cache is done
    requiresWriterIt = false;
    it = mLayer->getFeatures( featureRequest );
  }
 
  if ( requiresWriterIt && mLayer->dataProvider() )
  {
    // No index was able to satisfy the request
    QgsFeatureRequest myRequest = QgsFeatureRequest( featureRequest );
 
    // Make sure if we cache the geometry, it gets fetched
    if ( mCacheGeometry && mLayer->isSpatial() )
      myRequest.setFlags( featureRequest.flags() & ~( static_cast< int >( Qgis::FeatureRequestFlag::NoGeometry ) ) );
 
    // Make sure all the cached attributes are requested as well if requesting a subset
    if ( myRequest.flags().testFlag( Qgis::FeatureRequestFlag::SubsetOfAttributes ) )
    {
      if ( mCachedAttributes.count() != mLayer->fields().count() )
      {
        const QgsAttributeList requestSubset = featureRequest.subsetOfAttributes();
        QSet<int> attrs( requestSubset.begin(), requestSubset.end() );
        for ( int attr : std::as_const( mCachedAttributes ) )
          attrs.insert( attr );
        myRequest.setSubsetOfAttributes( attrs.values() );
      }
      else // we are already caching all attributes
      {
        myRequest.setSubsetOfAttributes( QgsAttributeList() );
        myRequest.setFlags( myRequest.flags().setFlag( Qgis::FeatureRequestFlag::SubsetOfAttributes, false ) );
      }
    }
 
    it = QgsFeatureIterator( new QgsCachedFeatureWriterIterator( this, myRequest ) );
  }
 
  return it;
}
 
bool QgsVectorLayerCache::isFidCached( const QgsFeatureId fid ) const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  return mCache.contains( fid );
}
 
QgsFeatureIds QgsVectorLayerCache::cachedFeatureIds() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  const QList< QgsFeatureId > keys = mCache.keys();
  return QgsFeatureIds( keys.begin(), keys.end() );
}
 
bool QgsVectorLayerCache::checkInformationCovered( const QgsFeatureRequest &featureRequest )
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  QgsAttributeList requestedAttributes;
 
  if ( !featureRequest.flags().testFlag( Qgis::FeatureRequestFlag::SubsetOfAttributes ) )
  {
    requestedAttributes = mLayer->attributeList();
  }
  else
  {
    requestedAttributes = featureRequest.subsetOfAttributes();
  }
 
  // Check if we even cache the information requested
  const auto constRequestedAttributes = requestedAttributes;
  for ( int attr : constRequestedAttributes )
  {
    if ( !mCachedAttributes.contains( attr ) )
    {
      return false;
    }
  }
 
  // If the request needs geometry but we don't cache this...
  return !( !featureRequest.flags().testFlag( Qgis::FeatureRequestFlag::NoGeometry ) && !mCacheGeometry );
}
 
void QgsVectorLayerCache::connectJoinedLayers() const
{
  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
  const auto constVectorJoins = mLayer->vectorJoins();
  for ( const QgsVectorLayerJoinInfo &info : constVectorJoins )
  {
    const QgsVectorLayer *vl = info.joinLayer();
    if ( vl )
      connect( vl, &QgsVectorLayer::attributeValueChanged, this, &QgsVectorLayerCache::onJoinAttributeValueChanged );
  }
}
 
bool QgsVectorLayerCache::QgsCachedFeature::allAttributesFetched() const
{
  return mAllAttributesFetched;
}
 
bool QgsVectorLayerCache::QgsCachedFeature::geometryFetched() const
{
  return mGeometryFetched;
}