23#include <QJsonDocument> 
   28#include <QNetworkRequest> 
   42#include "qgspointcloudexpression.h" 
   48#define PROVIDER_KEY QStringLiteral( "ept" ) 
   49#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" ) 
   51QgsEptPointCloudIndex::QgsEptPointCloudIndex()
 
   56QgsEptPointCloudIndex::~QgsEptPointCloudIndex() = 
default;
 
   58void QgsEptPointCloudIndex::load( 
const QString &urlString )
 
   62  if ( url.isValid() && ( url.scheme() == 
"http" || url.scheme() == 
"https" ) )
 
   68  QStringList splitUrl = mUri.split( 
'/' );
 
   70  mUrlDirectoryPart = splitUrl.join( 
'/' );
 
   75    QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
 
   91    if ( !f.open( QIODevice::ReadOnly ) )
 
   93      mError = QObject::tr( 
"Unable to open %1 for reading" ).arg( mUri );
 
   97    content = f.readAll();
 
  100  bool success = loadSchema( content );
 
  104    const QString manifestPath = mUrlDirectoryPart + QStringLiteral( 
"/ept-sources/manifest.json" );
 
  105    QByteArray manifestJson;
 
  108      QUrl manifestUrl( manifestPath );
 
  110      QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
 
  118      QFile manifestFile( manifestPath );
 
  119      if ( manifestFile.open( QIODevice::ReadOnly ) )
 
  120        manifestJson = manifestFile.readAll();
 
  123    if ( !manifestJson.isEmpty() )
 
  124      loadManifest( manifestJson );
 
  129    QgsDebugError( QStringLiteral( 
"Failed to load root EPT node" ) );
 
  136void QgsEptPointCloudIndex::loadManifest( 
const QByteArray &manifestJson )
 
  140  const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
 
  141  if ( err.error != QJsonParseError::NoError )
 
  144  const QJsonArray manifestArray = manifestDoc.array();
 
  145  if ( manifestArray.empty() )
 
  149  const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
 
  150  const QString metadataPath = sourceObject.value( QStringLiteral( 
"metadataPath" ) ).toString();
 
  151  const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral( 
"/ept-sources/" ) + metadataPath;
 
  153  QByteArray metadataJson;
 
  156    QUrl metadataUrl( fullMetadataPath );
 
  157    QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
 
  166    QFile metadataFile( fullMetadataPath );
 
  167    if ( ! metadataFile.open( QIODevice::ReadOnly ) )
 
  169    metadataJson = metadataFile.readAll();
 
  172  const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
 
  173  if ( err.error != QJsonParseError::NoError )
 
  176  const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( 
"metadata" ) ).toObject();
 
  177  if ( metadataObject.empty() )
 
  180  const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
 
  181  mOriginalMetadata = sourceMetadata.toVariantMap();
 
  184bool QgsEptPointCloudIndex::loadSchema( 
const QByteArray &dataJson )
 
  187  const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
 
  188  if ( err.error != QJsonParseError::NoError )
 
  190  const QJsonObject result = doc.object();
 
  191  mDataType = result.value( QLatin1String( 
"dataType" ) ).toString();  
 
  192  if ( mDataType != QLatin1String( 
"laszip" ) && mDataType != QLatin1String( 
"binary" ) && mDataType != QLatin1String( 
"zstandard" ) )
 
  195  const QString hierarchyType = result.value( QLatin1String( 
"hierarchyType" ) ).toString();  
 
  196  if ( hierarchyType != QLatin1String( 
"json" ) )
 
  199  mSpan = result.value( QLatin1String( 
"span" ) ).toInt();
 
  200  mPointCount = result.value( QLatin1String( 
"points" ) ).toDouble();
 
  203  const QJsonObject srs = result.value( QLatin1String( 
"srs" ) ).toObject();
 
  204  mWkt = srs.value( QLatin1String( 
"wkt" ) ).toString();
 
  207  const QJsonArray bounds = result.value( QLatin1String( 
"bounds" ) ).toArray();
 
  208  if ( bounds.size() != 6 )
 
  211  const QJsonArray boundsConforming = result.value( QLatin1String( 
"boundsConforming" ) ).toArray();
 
  212  if ( boundsConforming.size() != 6 )
 
  214  mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
 
  215               boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
 
  216  mZMin = boundsConforming[2].toDouble();
 
  217  mZMax = boundsConforming[5].toDouble();
 
  219  const QJsonArray schemaArray = result.value( QLatin1String( 
"schema" ) ).toArray();
 
  222  for ( 
const QJsonValue &schemaItem : schemaArray )
 
  224    const QJsonObject schemaObj = schemaItem.toObject();
 
  225    const QString name = schemaObj.value( QLatin1String( 
"name" ) ).toString();
 
  226    const QString type = schemaObj.value( QLatin1String( 
"type" ) ).toString();
 
  228    const int size = schemaObj.value( QLatin1String( 
"size" ) ).toInt();
 
  230    if ( name == QLatin1String( 
"ClassFlags" ) && size == 1 )
 
  237    else if ( type == QLatin1String( 
"float" ) && ( size == 4 ) )
 
  241    else if ( type == QLatin1String( 
"float" ) && ( size == 8 ) )
 
  245    else if ( size == 1 )
 
  249    else if ( type == QLatin1String( 
"unsigned" ) && size == 2 )
 
  253    else if ( size == 2 )
 
  257    else if ( size == 4 )
 
  268    if ( schemaObj.contains( QLatin1String( 
"scale" ) ) )
 
  269      scale = schemaObj.value( QLatin1String( 
"scale" ) ).toDouble();
 
  272    if ( schemaObj.contains( QLatin1String( 
"offset" ) ) )
 
  273      offset = schemaObj.value( QLatin1String( 
"offset" ) ).toDouble();
 
  275    if ( name == QLatin1String( 
"X" ) )
 
  277      mOffset.set( offset, mOffset.y(), mOffset.z() );
 
  278      mScale.set( scale, mScale.y(), mScale.z() );
 
  280    else if ( name == QLatin1String( 
"Y" ) )
 
  282      mOffset.set( mOffset.x(), offset, mOffset.z() );
 
  283      mScale.set( mScale.x(), scale, mScale.z() );
 
  285    else if ( name == QLatin1String( 
"Z" ) )
 
  287      mOffset.set( mOffset.x(), mOffset.y(), offset );
 
  288      mScale.set( mScale.x(), mScale.y(), scale );
 
  292    AttributeStatistics stats;
 
  293    bool foundStats = 
false;
 
  294    if ( schemaObj.contains( QLatin1String( 
"count" ) ) )
 
  296      stats.count = schemaObj.value( QLatin1String( 
"count" ) ).toInt();
 
  299    if ( schemaObj.contains( QLatin1String( 
"minimum" ) ) )
 
  301      stats.minimum = schemaObj.value( QLatin1String( 
"minimum" ) ).toDouble();
 
  304    if ( schemaObj.contains( QLatin1String( 
"maximum" ) ) )
 
  306      stats.maximum = schemaObj.value( QLatin1String( 
"maximum" ) ).toDouble();
 
  309    if ( schemaObj.contains( QLatin1String( 
"count" ) ) )
 
  311      stats.mean = schemaObj.value( QLatin1String( 
"mean" ) ).toDouble();
 
  314    if ( schemaObj.contains( QLatin1String( 
"stddev" ) ) )
 
  316      stats.stDev = schemaObj.value( QLatin1String( 
"stddev" ) ).toDouble();
 
  319    if ( schemaObj.contains( QLatin1String( 
"variance" ) ) )
 
  321      stats.variance = schemaObj.value( QLatin1String( 
"variance" ) ).toDouble();
 
  325      mMetadataStats.insert( name, stats );
 
  327    if ( schemaObj.contains( QLatin1String( 
"counts" ) ) )
 
  329      QMap< int, int >  classCounts;
 
  330      const QJsonArray counts = schemaObj.value( QLatin1String( 
"counts" ) ).toArray();
 
  331      for ( 
const QJsonValue &count : counts )
 
  333        const QJsonObject countObj = count.toObject();
 
  334        classCounts.insert( countObj.value( QLatin1String( 
"value" ) ).toInt(), countObj.value( QLatin1String( 
"count" ) ).toInt() );
 
  336      mAttributeClasses.insert( name, classCounts );
 
  339  setAttributes( attributes );
 
  344  const double xmin = bounds[0].toDouble();
 
  345  const double ymin = bounds[1].toDouble();
 
  346  const double zmin = bounds[2].toDouble();
 
  347  const double xmax = bounds[3].toDouble();
 
  348  const double ymax = bounds[4].toDouble();
 
  349  const double zmax = bounds[5].toDouble();
 
  351  mRootBounds = 
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
 
  354  double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
 
  355  QgsDebugMsgLevel( QStringLiteral( 
"lvl0 node size in CRS units: %1 %2 %3" ).arg( dx ).arg( dy ).arg( dz ), 2 );    
 
  356  QgsDebugMsgLevel( QStringLiteral( 
"res at lvl0 %1" ).arg( dx / mSpan ), 2 );
 
  357  QgsDebugMsgLevel( QStringLiteral( 
"res at lvl1 %1" ).arg( dx / mSpan / 2 ), 2 );
 
  358  QgsDebugMsgLevel( QStringLiteral( 
"res at lvl2 %1 with node size %2" ).arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
 
  368    return std::unique_ptr<QgsPointCloudBlock>( cached );
 
  371  std::unique_ptr<QgsPointCloudBlock> block;
 
  374    std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
 
  382    block = blockRequest->takeBlock();
 
  385      QgsDebugError( QStringLiteral( 
"Error downloading node %1 data, error : %2 " ).arg( n.
toString(), blockRequest->errorStr() ) );
 
  393    QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
 
  395    requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
 
  398    if ( mDataType == QLatin1String( 
"binary" ) )
 
  400      const QString filename = QStringLiteral( 
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
 
  401      block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
 
  403    else if ( mDataType == QLatin1String( 
"zstandard" ) )
 
  405      const QString filename = QStringLiteral( 
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
 
  406      block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
 
  408    else if ( mDataType == QLatin1String( 
"laszip" ) )
 
  410      const QString filename = QStringLiteral( 
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
 
  411      block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
 
  415  storeNodeDataToCache( block.get(), n, request );
 
  424           scale(), offset(), mFilterExpression, request.
filterRect() );
 
  430  if ( !loadNodeHierarchy( n ) )
 
  434  if ( mDataType == QLatin1String( 
"binary" ) )
 
  436    fileUrl = QStringLiteral( 
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
 
  438  else if ( mDataType == QLatin1String( 
"zstandard" ) )
 
  440    fileUrl = QStringLiteral( 
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
 
  442  else if ( mDataType == QLatin1String( 
"laszip" ) )
 
  444    fileUrl = QStringLiteral( 
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
 
  454  QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
 
  456  requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
 
  462  return loadNodeHierarchy( n );
 
  470qint64 QgsEptPointCloudIndex::pointCount()
 const 
  485  QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( 
id );
 
  486  for ( 
int i = pathToRoot.size() - 1; i >= 0; --i )
 
  488    loadSingleNodeHierarchy( pathToRoot[i] );
 
  490    QMutexLocker locker( &mHierarchyMutex );
 
  491    qint64 pointCount = mHierarchy.value( 
id, -1 );
 
  492    if ( pointCount != -1 )
 
  502  QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
 
  505    QString name = attribute.name();
 
  506    const AttributeStatistics &stats = mMetadataStats[ name ];
 
  507    if ( !stats.minimum.isValid() )
 
  510    s.
minimum = stats.minimum.toDouble();
 
  511    s.
maximum = stats.maximum.toDouble();
 
  513    s.
stDev = stats.stDev;
 
  514    s.
count = stats.count;
 
  518    statsMap[ name ] = std::move( s );
 
  523bool QgsEptPointCloudIndex::loadSingleNodeHierarchy( 
const QgsPointCloudNodeId &nodeId )
 const 
  525  mHierarchyMutex.lock();
 
  526  const bool foundInHierarchy = mHierarchy.contains( nodeId );
 
  527  const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
 
  528  mHierarchyMutex.unlock();
 
  530  if ( foundInHierarchy )
 
  533  if ( !foundInHierarchyNodes )
 
  536  const QString filePath = QStringLiteral( 
"%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.
toString() );
 
  538  QByteArray dataJsonH;
 
  541    QNetworkRequest nr( filePath );
 
  543    nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
 
  544    nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, 
true );
 
  552    if ( reply->error() != QNetworkReply::NoError )
 
  554      QgsDebugError( QStringLiteral( 
"Request failed: " ) + filePath );
 
  558    dataJsonH = reply->data();
 
  562    QFile file( filePath );
 
  563    if ( ! file.open( QIODevice::ReadOnly ) )
 
  565      QgsDebugError( QStringLiteral( 
"Loading file failed: " ) + filePath );
 
  568    dataJsonH = file.readAll();
 
  571  QJsonParseError errH;
 
  572  const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
 
  573  if ( errH.error != QJsonParseError::NoError )
 
  575    QgsDebugMsgLevel( QStringLiteral( 
"QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 );
 
  579  QMutexLocker locker( &mHierarchyMutex );
 
  580  const QJsonObject rootHObj = docH.object();
 
  581  for ( 
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
 
  583    const QString nodeIdStr = it.key();
 
  584    const int nodePointCount = it.value().toInt();
 
  586    if ( nodePointCount >= 0 )
 
  587      mHierarchy[nodeId] = nodePointCount;
 
  588    else if ( nodePointCount == -1 )
 
  589      mHierarchyNodes.insert( nodeId );
 
  595QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot( 
const QgsPointCloudNodeId &nodeId )
 const 
  597  QVector<QgsPointCloudNodeId> path;
 
  601    path.push_back( currentNode );
 
  604  while ( currentNode.
d() >= 0 );
 
  613    QMutexLocker lock( &mHierarchyMutex );
 
  614    found = mHierarchy.contains( nodeId );
 
  619  QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
 
  620  for ( 
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
 
  623    if ( !loadSingleNodeHierarchy( node ) )
 
  628    QMutexLocker lock( &mHierarchyMutex );
 
  629    found = mHierarchy.contains( nodeId );
 
  636bool QgsEptPointCloudIndex::isValid()
 const 
  647#undef PROVIDER_DESCRIPTION 
PointCloudAccessType
The access type of the data, local is for local files and remote for remote files (over HTTP).
 
@ Local
Local means the source is a local file on the machine.
 
@ Remote
Remote means it's loaded through a protocol like HTTP.
 
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
 
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
 
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
 
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
 
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
 
@ NoError
No error was encountered.
 
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
 
A 3-dimensional box composed of x, y, z coordinates.
 
Class for handling a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
 
This class represents a coordinate reference system (CRS).
 
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
 
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
 
QByteArray content() const
Returns the reply content.
 
Collection of point cloud attributes.
 
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
 
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
 
Attribute for point cloud data pair of name and size in bytes.
 
@ UShort
Unsigned short int 2 bytes.
 
@ Short
Short int 2 bytes.
 
@ UChar
Unsigned char 1 byte.
 
Base class for handling loading QgsPointCloudBlock asynchronously.
 
void finished()
Emitted when the request processing has finished.
 
Base class for storing raw data from point cloud nodes.
 
Represents a indexed point cloud node's position in octree.
 
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
 
QString toString() const
Encode node to string.
 
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
 
Keeps metadata for indexed point cloud node.
 
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
 
qint64 pointCount() const
Returns number of points contained in node data.
 
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
 
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
 
Point cloud data request.
 
bool ignoreIndexFilterEnabled() const
Returns whether the request will ignore the point cloud index's filter expression,...
 
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
 
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
 
Class used to store statistics of a point cloud dataset.
 
A rectangle specified with double values.
 
void finished()
Emitted when the reply has finished (either with a success or with a failure)
 
#define QgsDebugMsgLevel(str, level)
 
#define QgsDebugError(str)
 
#define QgsSetRequestInitiatorClass(request, _class)
 
Class used to store statistics of one attribute of a point cloud dataset.
 
QMap< int, int > classCount