qgsfilebaseddataitemprovider.cpp

Go to the documentation of this file.

/***************************************************************************
  qgsfilebaseddataitemprovider.cpp
  --------------------------------------
  Date                 : July 2021
  Copyright            : (C) 2021 by Nyall Dawson
  Email                : nyall dot dawson at gmail dot com
 ***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
 
#include "qgsfilebaseddataitemprovider.h"
#include "qgsdataprovider.h"
#include "qgsproviderregistry.h"
#include "qgslogger.h"
#include "qgssettings.h"
#include "qgsogrproviderutils.h"
#include "qgsstyle.h"
#include "qgsgeopackagedataitems.h"
#include "qgsprovidersublayerdetails.h"
#include "qgsfieldsitem.h"
#include "qgsfielddomainsitem.h"
#include "qgsrelationshipsitem.h"
#include "qgsproviderutils.h"
#include "qgsmbtiles.h"
#include "qgsvectortiledataitems.h"
#include "qgsprovidermetadata.h"
#include <QUrlQuery>
 
//
// QgsProviderSublayerItem
//
 
QgsProviderSublayerItem::QgsProviderSublayerItem( QgsDataItem *parent, const QString &name,
    const QgsProviderSublayerDetails &details, const QString &filePath )
  : QgsLayerItem( parent, name, filePath.isEmpty() ? details.uri() : filePath, details.uri(), layerTypeFromSublayer( details ), details.providerKey() )
  , mDetails( details )
{
  mToolTip = details.uri();
 
  // no children, except for vector layers, which will show the fields item
  setState( details.type() == Qgis::LayerType::Vector ? Qgis::BrowserItemState::NotPopulated : Qgis::BrowserItemState::Populated );
}
 
QVector<QgsDataItem *> QgsProviderSublayerItem::createChildren()
{
  QVector<QgsDataItem *> children;
 
  if ( mDetails.type() == Qgis::LayerType::Vector )
  {
    // sqlite gets special handling because it delegates to the dedicated spatialite provider
    if ( mDetails.driverName() == QLatin1String( "SQLite" ) )
    {
      children.push_back( new QgsFieldsItem( this,
                                             path() + QStringLiteral( "/columns/ " ),
                                             QStringLiteral( R"(dbname="%1")" ).arg( parent()->path().replace( '"', QLatin1String( R"(\")" ) ) ),
                                             QStringLiteral( "spatialite" ), QString(), name() ) );
    }
    else if ( mDetails.providerKey() == QLatin1String( "ogr" ) )
    {
      // otherwise we use the default OGR database connection approach, which is the generic way to handle this
      // for all OGR layer types
      children.push_back( new QgsFieldsItem( this,
                                             path() + QStringLiteral( "/columns/ " ),
                                             path(),
                                             QStringLiteral( "ogr" ), QString(), name() ) );
 
      std::unique_ptr<QgsAbstractDatabaseProviderConnection> conn( databaseConnection() );
      if ( conn && ( conn->capabilities() & QgsAbstractDatabaseProviderConnection::Capability::RetrieveRelationships ) )
      {
        QString relationError;
        QList< QgsWeakRelation > relations;
        try
        {
          relations = conn->relationships( QString(), mDetails.name() );
        }
        catch ( QgsProviderConnectionException &ex )
        {
          relationError = ex.what();
        }
 
        if ( !relations.empty() || !relationError.isEmpty() )
        {
          std::unique_ptr< QgsRelationshipsItem > relationsItem = std::make_unique< QgsRelationshipsItem >( this, mPath + "/relations", conn->uri(), QStringLiteral( "ogr" ), QString(), mDetails.name() );
          // force this item to appear last by setting a maximum string value for the sort key
          relationsItem->setSortKey( QString( QChar( 0x11FFFF ) ) );
          children.append( relationsItem.release() );
        }
      }
    }
  }
  return children;
}
 
QgsProviderSublayerDetails QgsProviderSublayerItem::sublayerDetails() const
{
  return mDetails;
}
 
QgsAbstractDatabaseProviderConnection *QgsProviderSublayerItem::databaseConnection() const
{
  if ( parent() )
  {
    if ( QgsAbstractDatabaseProviderConnection *connection = parent()->databaseConnection() )
      return connection;
  }
 
  if ( mDetails.providerKey() == QLatin1String( "ogr" ) )
  {
    if ( QgsProviderMetadata *md = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "ogr" ) ) )
    {
      QVariantMap parts;
      parts.insert( QStringLiteral( "path" ), path() );
      return static_cast<QgsAbstractDatabaseProviderConnection *>( md->createConnection( md->encodeUri( parts ), {} ) );
    }
  }
 
  return nullptr;
}
 
Qgis::BrowserLayerType QgsProviderSublayerItem::layerTypeFromSublayer( const QgsProviderSublayerDetails &sublayer )
{
  switch ( sublayer.type() )
  {
    case Qgis::LayerType::Vector:
    {
      switch ( QgsWkbTypes::geometryType( sublayer.wkbType() ) )
      {
        case Qgis::GeometryType::Point:
          return Qgis::BrowserLayerType::Point;
 
        case Qgis::GeometryType::Line:
          return Qgis::BrowserLayerType::Line;
 
        case Qgis::GeometryType::Polygon:
          return Qgis::BrowserLayerType::Polygon;
 
        case Qgis::GeometryType::Null:
          return Qgis::BrowserLayerType::TableLayer;
 
        case Qgis::GeometryType::Unknown:
          return Qgis::BrowserLayerType::Vector;
      }
 
      break;
    }
    case Qgis::LayerType::Raster:
      return Qgis::BrowserLayerType::Raster;
 
    case Qgis::LayerType::Plugin:
      return Qgis::BrowserLayerType::Plugin;
 
    case Qgis::LayerType::Mesh:
      return Qgis::BrowserLayerType::Mesh;
 
    case Qgis::LayerType::VectorTile:
      return Qgis::BrowserLayerType::VectorTile;
 
    case Qgis::LayerType::PointCloud:
      return Qgis::BrowserLayerType::PointCloud;
 
    case Qgis::LayerType::Annotation:
    case Qgis::LayerType::Group:
      break;
  }
  return Qgis::BrowserLayerType::NoType;
}
 
QString QgsProviderSublayerItem::layerName() const
{
  return mDetails.name();
}
 
//
// QgsFileDataCollectionGroupItem
//
QgsFileDataCollectionGroupItem::QgsFileDataCollectionGroupItem( QgsDataItem *parent, const QString &groupName, const QString &path )
  : QgsDataCollectionItem( parent, groupName, path )
{
  mCapabilities = Qgis::BrowserItemCapability::RefreshChildrenWhenItemIsRefreshed;
  mIconName = QStringLiteral( "mIconDbSchema.svg" );
}
 
void QgsFileDataCollectionGroupItem::appendSublayer( const QgsProviderSublayerDetails &sublayer )
{
  mSublayers.append( sublayer );
}
 
bool QgsFileDataCollectionGroupItem::hasDragEnabled() const
{
  return true;
}
 
QgsMimeDataUtils::UriList QgsFileDataCollectionGroupItem::mimeUris() const
{
  QgsMimeDataUtils::UriList res;
  res.reserve( mSublayers.size() );
 
  for ( const QgsProviderSublayerDetails &sublayer : mSublayers )
  {
    res << sublayer.toMimeUri();
  }
  return res;
}
 
//
// QgsFileDataCollectionItem
//
 
QgsFileDataCollectionItem::QgsFileDataCollectionItem( QgsDataItem *parent, const QString &name, const QString &path, const QList<QgsProviderSublayerDetails> &sublayers )
  : QgsDataCollectionItem( parent, name, path )
  , mSublayers( sublayers )
{
  if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) )
    setCapabilities( Qgis::BrowserItemCapability::Fertile );
  else
    setCapabilities( Qgis::BrowserItemCapability::Fast | Qgis::BrowserItemCapability::Fertile );
 
  if ( !qgsVsiPrefix( path ).isEmpty() )
  {
    mIconName = QStringLiteral( "/mIconZip.svg" );
  }
}
 
QVector<QgsDataItem *> QgsFileDataCollectionItem::createChildren()
{
  QList< QgsProviderSublayerDetails> sublayers;
  if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount )
       || mSublayers.empty() )
  {
    sublayers = QgsProviderRegistry::instance()->querySublayers( path(), Qgis::SublayerQueryFlag::ResolveGeometryType );
  }
  else
  {
    sublayers = mSublayers;
  }
  // only ever use the initial sublayers for first population -- after that we requery when asked to create children,
  // or the item won't "refresh" and update its sublayers when the actual file changes
  mSublayers.clear();
  // remove the fast flag -- after the first population we need to requery the dataset
  setCapabilities( capabilities2() & ~static_cast< int >( Qgis::BrowserItemCapability::Fast ) );
 
  QVector<QgsDataItem *> children;
  children.reserve( sublayers.size() );
  QMap< QStringList, QgsFileDataCollectionGroupItem * > groupItems;
  for ( const QgsProviderSublayerDetails &sublayer : std::as_const( sublayers ) )
  {
    QgsProviderSublayerItem *item = new QgsProviderSublayerItem( nullptr, sublayer.name(), sublayer, QString() );
 
    if ( !sublayer.path().isEmpty() )
    {
      QStringList currentPath;
      QStringList remainingPaths = sublayer.path();
      QgsFileDataCollectionGroupItem *groupItem = nullptr;
 
      while ( !remainingPaths.empty() )
      {
        currentPath << remainingPaths.takeAt( 0 );
 
        auto it = groupItems.constFind( currentPath );
        if ( it == groupItems.constEnd() )
        {
          QgsFileDataCollectionGroupItem *newGroupItem = new QgsFileDataCollectionGroupItem( this, currentPath.constLast(), path() + '/' + currentPath.join( ',' ) );
          newGroupItem->setState( Qgis::BrowserItemState::Populated );
          groupItems.insert( currentPath, newGroupItem );
          if ( groupItem )
            groupItem->addChildItem( newGroupItem );
          else
            children.append( newGroupItem );
          groupItem = newGroupItem;
        }
        else
        {
          groupItem = it.value();
        }
 
        if ( groupItem )
          groupItem->appendSublayer( sublayer );
      }
 
      if ( groupItem )
        groupItem->addChildItem( item );
    }
    else
    {
      children.append( item );
    }
  }
 
  std::unique_ptr<QgsAbstractDatabaseProviderConnection> conn( databaseConnection() );
  if ( conn && ( conn->capabilities() & QgsAbstractDatabaseProviderConnection::Capability::ListFieldDomains ) )
  {
    QString domainError;
    QStringList fieldDomains;
    try
    {
      fieldDomains = conn->fieldDomainNames();
    }
    catch ( QgsProviderConnectionException &ex )
    {
      domainError = ex.what();
    }
 
    if ( !fieldDomains.empty() || !domainError.isEmpty() )
    {
      std::unique_ptr< QgsFieldDomainsItem > domainsItem = std::make_unique< QgsFieldDomainsItem >( this, mPath + "/domains", conn->uri(), QStringLiteral( "ogr" ) );
      // force this item to appear last by setting a maximum string value for the sort key
      domainsItem->setSortKey( QString( QChar( 0x10FFFF ) ) );
      children.append( domainsItem.release() );
    }
  }
  if ( conn && ( conn->capabilities() & QgsAbstractDatabaseProviderConnection::Capability::RetrieveRelationships ) )
  {
    QString relationError;
    QList< QgsWeakRelation > relations;
    try
    {
      relations = conn->relationships();
    }
    catch ( QgsProviderConnectionException &ex )
    {
      relationError = ex.what();
    }
 
    if ( !relations.empty() || !relationError.isEmpty() )
    {
      std::unique_ptr< QgsRelationshipsItem > relationsItem = std::make_unique< QgsRelationshipsItem >( this, mPath + "/relations", conn->uri(), QStringLiteral( "ogr" ) );
      // force this item to appear last by setting a maximum string value for the sort key
      relationsItem->setSortKey( QString( QChar( 0x11FFFF ) ) );
      children.append( relationsItem.release() );
    }
  }
 
  return children;
}
 
bool QgsFileDataCollectionItem::hasDragEnabled() const
{
  return true;
}
 
QgsMimeDataUtils::UriList QgsFileDataCollectionItem::mimeUris() const
{
  QgsMimeDataUtils::Uri collectionUri;
  collectionUri.uri = path();
  collectionUri.layerType = QStringLiteral( "collection" );
  collectionUri.filePath = path();
  return { collectionUri };
}
 
QgsAbstractDatabaseProviderConnection *QgsFileDataCollectionItem::databaseConnection() const
{
  // test that file is valid with OGR
  if ( OGRGetDriverCount() == 0 )
  {
    OGRRegisterAll();
  }
  // do not print errors, but write to debug
  CPLPushErrorHandler( CPLQuietErrorHandler );
  CPLErrorReset();
  GDALDriverH hDriver = GDALIdentifyDriverEx( path().toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr );
  CPLPopErrorHandler();
 
  if ( ! hDriver )
  {
    QgsDebugMsgLevel( QStringLiteral( "GDALIdentifyDriverEx error # %1 : %2 on %3" ).arg( CPLGetLastErrorNo() ).arg( CPLGetLastErrorMsg() ).arg( path() ), 2 );
    return nullptr;
  }
 
  const QString driverName = GDALGetDriverShortName( hDriver );
  if ( driverName == QLatin1String( "PDF" ) )
  {
    // unwanted drivers -- it's slow to create connections for these, and we don't really want
    // to expose database capabilities for them (even though they kind of are database formats)
    return nullptr;
  }
 
  QgsAbstractDatabaseProviderConnection *conn = nullptr;
  if ( driverName == QLatin1String( "SQLite" ) )
  {
    // sqlite gets special handling, as we delegate to the native spatialite provider
    if ( QgsProviderMetadata *md = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "spatialite" ) ) )
    {
      QgsDataSourceUri uri;
      uri.setDatabase( path( ) );
      conn = static_cast<QgsAbstractDatabaseProviderConnection *>( md->createConnection( uri.uri(), {} ) );
    }
  }
  else
  {
    // for all other vector types we use the generic OGR provider
    if ( QgsProviderMetadata *md = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "ogr" ) ) )
    {
      QVariantMap parts;
      parts.insert( QStringLiteral( "path" ), path() );
      conn = static_cast<QgsAbstractDatabaseProviderConnection *>( md->createConnection( md->encodeUri( parts ), {} ) );
    }
  }
  return conn;
}
 
//
// QgsFileBasedDataItemProvider
//
 
QString QgsFileBasedDataItemProvider::name()
{
  return QStringLiteral( "files" );
}
 
int QgsFileBasedDataItemProvider::capabilities() const
{
  return QgsDataProvider::File | QgsDataProvider::Dir;
}
 
QgsDataItem *QgsFileBasedDataItemProvider::createDataItem( const QString &path, QgsDataItem *parentItem )
{
  if ( path.isEmpty() )
    return nullptr;
 
  const QFileInfo info( path );
  QString suffix = info.suffix().toLower();
  const QString name = info.fileName();
 
  // special handling for some suffixes
  if ( suffix.compare( QLatin1String( "gpkg" ), Qt::CaseInsensitive ) == 0 )
  {
    // Geopackage is special -- it gets a dedicated collection item type
    QgsGeoPackageCollectionItem *item = new QgsGeoPackageCollectionItem( parentItem, name, path );
    item->setCapabilities( item->capabilities2() | Qgis::BrowserItemCapability::ItemRepresentsFile );
    return item;
  }
  else if ( suffix == QLatin1String( "txt" ) )
  {
    // never ever show .txt files as datasets in browser -- they are only used for geospatial data in extremely rare cases
    // and are predominantly just noise in the browser
    return nullptr;
  }
  // If a .tab exists, then the corresponding .map/.dat is very likely a
  // side-car file of the .tab
  else if ( suffix == QLatin1String( "map" ) || suffix == QLatin1String( "dat" ) )
  {
    if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".tab" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".TAB" ) ) )
      return nullptr;
  }
  // .dbf and .shx should only appear if .shp is not present
  else if ( suffix == QLatin1String( "dbf" ) || suffix == QLatin1String( "shx" ) )
  {
    if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".shp" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".SHP" ) ) )
      return nullptr;
  }
  // skip QGIS style xml files
  else if ( suffix == QLatin1String( "xml" ) && QgsStyle::isXmlStyleFile( path ) )
  {
    return nullptr;
  }
  // GDAL 3.1 Shapefile driver directly handles .shp.zip files
  else if ( path.endsWith( QLatin1String( ".shp.zip" ), Qt::CaseInsensitive ) &&
            GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr ) )
  {
    suffix = QStringLiteral( "shp.zip" );
  }
  // special handling for mbtiles files
  else if ( suffix == QLatin1String( "mbtiles" ) )
  {
    QgsMbTiles reader( path );
    if ( reader.open() )
    {
      if ( reader.metadataValue( QStringLiteral( "format" ) ) == QLatin1String( "pbf" ) )
      {
        // these are vector tiles
        QUrlQuery uq;
        uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) );
        uq.addQueryItem( QStringLiteral( "url" ), path );
        const QString encodedUri = uq.toString();
        QgsVectorTileLayerItem *item = new QgsVectorTileLayerItem( parentItem, name, path, encodedUri );
        item->setCapabilities( item->capabilities2() | Qgis::BrowserItemCapability::ItemRepresentsFile );
        return item;
      }
      else
      {
        // handled by WMS provider
        QUrlQuery uq;
        uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) );
        uq.addQueryItem( QStringLiteral( "url" ), QUrl::fromLocalFile( path ).toString() );
        const QString encodedUri = uq.toString();
        QgsLayerItem *item = new QgsLayerItem( parentItem, name, path, encodedUri, Qgis::BrowserLayerType::Raster, QStringLiteral( "wms" ) );
        item->setState( Qgis::BrowserItemState::Populated );
        item->setCapabilities( item->capabilities2() | Qgis::BrowserItemCapability::ItemRepresentsFile );
        return item;
      }
    }
  }
 
  // hide blocklisted URIs, such as .aux.xml files
  if ( QgsProviderRegistry::instance()->uriIsBlocklisted( path ) )
    return nullptr;
 
  QgsSettings settings;
 
  Qgis::SublayerQueryFlags queryFlags = Qgis::SublayerQueryFlags();
 
  // should we fast scan only?
  if ( ( settings.value( QStringLiteral( "qgis/scanItemsInBrowser2" ),
                         "extension" ).toString() == QLatin1String( "extension" ) ) ||
       ( parentItem && settings.value( QStringLiteral( "qgis/scanItemsFastScanUris" ),
                                       QStringList() ).toStringList().contains( parentItem->path() ) ) )
  {
    queryFlags |= Qgis::SublayerQueryFlag::FastScan;
  }
 
  const QList<QgsProviderSublayerDetails> sublayers = QgsProviderRegistry::instance()->querySublayers( path, queryFlags );
 
  if ( sublayers.size() == 1
       && ( ( ( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount | QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownGeometryType ) )
            || ( !( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) ) )
     )
  {
    QgsProviderSublayerItem *item = new QgsProviderSublayerItem( parentItem, name, sublayers.at( 0 ), path );
    item->setCapabilities( item->capabilities2() | Qgis::BrowserItemCapability::ItemRepresentsFile );
    return item;
  }
  else if ( !sublayers.empty() )
  {
    QgsFileDataCollectionItem *item = new QgsFileDataCollectionItem( parentItem, name, path, sublayers );
    item->setCapabilities( item->capabilities2() | Qgis::BrowserItemCapability::ItemRepresentsFile );
    return item;
  }
  else
  {
    return nullptr;
  }
}
 
bool QgsFileBasedDataItemProvider::handlesDirectoryPath( const QString &path )
{
  QFileInfo info( path );
  QString suffix = info.suffix().toLower();
 
  QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions();
  return dirExtensions.contains( suffix );
}