23#include <QJsonDocument>
28#include <QNetworkRequest>
41#include "qgspointcloudexpression.h"
47#define PROVIDER_KEY QStringLiteral( "ept" )
48#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )
50QgsEptPointCloudIndex::QgsEptPointCloudIndex()
55QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
57std::unique_ptr<QgsPointCloudIndex> QgsEptPointCloudIndex::clone()
const
59 QgsEptPointCloudIndex *clone =
new QgsEptPointCloudIndex;
60 QMutexLocker locker( &mHierarchyMutex );
61 copyCommonProperties( clone );
62 return std::unique_ptr<QgsPointCloudIndex>( clone );
65void QgsEptPointCloudIndex::load(
const QString &urlString )
69 if ( url.isValid() && ( url.scheme() ==
"http" || url.scheme() ==
"https" ) )
75 QStringList splitUrl = mUri.split(
'/' );
77 mUrlDirectoryPart = splitUrl.join(
'/' );
80 if ( mAccessType == Remote )
82 QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
98 if ( !f.open( QIODevice::ReadOnly ) )
100 mError = QObject::tr(
"Unable to open %1 for reading" ).arg( mUri );
104 content = f.readAll();
107 bool success = loadSchema( content );
111 const QString manifestPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/manifest.json" );
112 QByteArray manifestJson;
113 if ( mAccessType == Remote )
115 QUrl manifestUrl( manifestPath );
117 QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
125 QFile manifestFile( manifestPath );
126 if ( manifestFile.open( QIODevice::ReadOnly ) )
127 manifestJson = manifestFile.readAll();
130 if ( !manifestJson.isEmpty() )
131 loadManifest( manifestJson );
136 QgsDebugError( QStringLiteral(
"Failed to load root EPT node" ) );
143void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
147 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
148 if ( err.error != QJsonParseError::NoError )
151 const QJsonArray manifestArray = manifestDoc.array();
152 if ( manifestArray.empty() )
156 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
157 const QString metadataPath = sourceObject.value( QStringLiteral(
"metadataPath" ) ).toString();
158 const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/" ) + metadataPath;
160 QByteArray metadataJson;
161 if ( mAccessType == Remote )
163 QUrl metadataUrl( fullMetadataPath );
164 QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
173 QFile metadataFile( fullMetadataPath );
174 if ( ! metadataFile.open( QIODevice::ReadOnly ) )
176 metadataJson = metadataFile.readAll();
179 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
180 if ( err.error != QJsonParseError::NoError )
183 const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral(
"metadata" ) ).toObject();
184 if ( metadataObject.empty() )
187 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
188 mOriginalMetadata = sourceMetadata.toVariantMap();
191bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
194 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
195 if ( err.error != QJsonParseError::NoError )
197 const QJsonObject result = doc.object();
198 mDataType = result.value( QLatin1String(
"dataType" ) ).toString();
199 if ( mDataType != QLatin1String(
"laszip" ) && mDataType != QLatin1String(
"binary" ) && mDataType != QLatin1String(
"zstandard" ) )
202 const QString hierarchyType = result.value( QLatin1String(
"hierarchyType" ) ).toString();
203 if ( hierarchyType != QLatin1String(
"json" ) )
206 mSpan = result.value( QLatin1String(
"span" ) ).toInt();
207 mPointCount = result.value( QLatin1String(
"points" ) ).toDouble();
210 const QJsonObject srs = result.value( QLatin1String(
"srs" ) ).toObject();
211 mWkt = srs.value( QLatin1String(
"wkt" ) ).toString();
214 const QJsonArray bounds = result.value( QLatin1String(
"bounds" ) ).toArray();
215 if ( bounds.size() != 6 )
218 const QJsonArray boundsConforming = result.value( QLatin1String(
"boundsConforming" ) ).toArray();
219 if ( boundsConforming.size() != 6 )
221 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
222 boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
223 mZMin = boundsConforming[2].toDouble();
224 mZMax = boundsConforming[5].toDouble();
226 const QJsonArray schemaArray = result.value( QLatin1String(
"schema" ) ).toArray();
229 for (
const QJsonValue &schemaItem : schemaArray )
231 const QJsonObject schemaObj = schemaItem.toObject();
232 const QString name = schemaObj.value( QLatin1String(
"name" ) ).toString();
233 const QString type = schemaObj.value( QLatin1String(
"type" ) ).toString();
235 const int size = schemaObj.value( QLatin1String(
"size" ) ).toInt();
237 if ( name == QLatin1String(
"ClassFlags" ) && size == 1 )
244 else if ( type == QLatin1String(
"float" ) && ( size == 4 ) )
248 else if ( type == QLatin1String(
"float" ) && ( size == 8 ) )
252 else if ( size == 1 )
256 else if ( type == QLatin1String(
"unsigned" ) && size == 2 )
260 else if ( size == 2 )
264 else if ( size == 4 )
275 if ( schemaObj.contains( QLatin1String(
"scale" ) ) )
276 scale = schemaObj.value( QLatin1String(
"scale" ) ).toDouble();
279 if ( schemaObj.contains( QLatin1String(
"offset" ) ) )
280 offset = schemaObj.value( QLatin1String(
"offset" ) ).toDouble();
282 if ( name == QLatin1String(
"X" ) )
284 mOffset.set( offset, mOffset.y(), mOffset.z() );
285 mScale.set( scale, mScale.y(), mScale.z() );
287 else if ( name == QLatin1String(
"Y" ) )
289 mOffset.set( mOffset.x(), offset, mOffset.z() );
290 mScale.set( mScale.x(), scale, mScale.z() );
292 else if ( name == QLatin1String(
"Z" ) )
294 mOffset.set( mOffset.x(), mOffset.y(), offset );
295 mScale.set( mScale.x(), mScale.y(), scale );
299 AttributeStatistics stats;
300 bool foundStats =
false;
301 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
303 stats.count = schemaObj.value( QLatin1String(
"count" ) ).toInt();
306 if ( schemaObj.contains( QLatin1String(
"minimum" ) ) )
308 stats.minimum = schemaObj.value( QLatin1String(
"minimum" ) ).toDouble();
311 if ( schemaObj.contains( QLatin1String(
"maximum" ) ) )
313 stats.maximum = schemaObj.value( QLatin1String(
"maximum" ) ).toDouble();
316 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
318 stats.mean = schemaObj.value( QLatin1String(
"mean" ) ).toDouble();
321 if ( schemaObj.contains( QLatin1String(
"stddev" ) ) )
323 stats.stDev = schemaObj.value( QLatin1String(
"stddev" ) ).toDouble();
326 if ( schemaObj.contains( QLatin1String(
"variance" ) ) )
328 stats.variance = schemaObj.value( QLatin1String(
"variance" ) ).toDouble();
332 mMetadataStats.insert( name, stats );
334 if ( schemaObj.contains( QLatin1String(
"counts" ) ) )
336 QMap< int, int > classCounts;
337 const QJsonArray counts = schemaObj.value( QLatin1String(
"counts" ) ).toArray();
338 for (
const QJsonValue &count : counts )
340 const QJsonObject countObj = count.toObject();
341 classCounts.insert( countObj.value( QLatin1String(
"value" ) ).toInt(), countObj.value( QLatin1String(
"count" ) ).toInt() );
343 mAttributeClasses.insert( name, classCounts );
346 setAttributes( attributes );
351 const double xmin = bounds[0].toDouble();
352 const double ymin = bounds[1].toDouble();
353 const double zmin = bounds[2].toDouble();
354 const double xmax = bounds[3].toDouble();
355 const double ymax = bounds[4].toDouble();
356 const double zmax = bounds[5].toDouble();
358 mRootBounds =
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
361 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
362 QgsDebugMsgLevel( QStringLiteral(
"lvl0 node size in CRS units: %1 %2 %3" ).arg( dx ).arg( dy ).arg( dz ), 2 );
363 QgsDebugMsgLevel( QStringLiteral(
"res at lvl0 %1" ).arg( dx / mSpan ), 2 );
364 QgsDebugMsgLevel( QStringLiteral(
"res at lvl1 %1" ).arg( dx / mSpan / 2 ), 2 );
365 QgsDebugMsgLevel( QStringLiteral(
"res at lvl2 %1 with node size %2" ).arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
375 return std::unique_ptr<QgsPointCloudBlock>( cached );
378 std::unique_ptr<QgsPointCloudBlock> block;
379 if ( mAccessType == Remote )
381 std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
389 block = blockRequest->takeBlock();
392 QgsDebugError( QStringLiteral(
"Error downloading node %1 data, error : %2 " ).arg( n.
toString(), blockRequest->errorStr() ) );
400 QgsPointCloudExpression filterExpression = mFilterExpression;
402 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
405 if ( mDataType == QLatin1String(
"binary" ) )
407 const QString filename = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
408 block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
410 else if ( mDataType == QLatin1String(
"zstandard" ) )
412 const QString filename = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
413 block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
415 else if ( mDataType == QLatin1String(
"laszip" ) )
417 const QString filename = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
418 block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
422 storeNodeDataToCache( block.get(), n, request );
431 scale(), offset(), mFilterExpression, request.
filterRect() );
434 if ( mAccessType != Remote )
437 if ( !loadNodeHierarchy( n ) )
441 if ( mDataType == QLatin1String(
"binary" ) )
443 fileUrl = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
445 else if ( mDataType == QLatin1String(
"zstandard" ) )
447 fileUrl = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
449 else if ( mDataType == QLatin1String(
"laszip" ) )
451 fileUrl = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
461 QgsPointCloudExpression filterExpression = mFilterExpression;
463 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
469 return loadNodeHierarchy( n );
477qint64 QgsEptPointCloudIndex::pointCount()
const
492 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot(
id );
493 for (
int i = pathToRoot.size() - 1; i >= 0; --i )
495 loadSingleNodeHierarchy( pathToRoot[i] );
497 QMutexLocker locker( &mHierarchyMutex );
498 qint64 pointCount = mHierarchy.value(
id, -1 );
499 if ( pointCount != -1 )
509 QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
512 QString name = attribute.name();
513 const AttributeStatistics &stats = mMetadataStats[ name ];
514 if ( !stats.minimum.isValid() )
517 s.
minimum = stats.minimum.toDouble();
518 s.
maximum = stats.maximum.toDouble();
520 s.
stDev = stats.stDev;
521 s.
count = stats.count;
525 statsMap[ name ] = s;
530bool QgsEptPointCloudIndex::loadSingleNodeHierarchy(
const QgsPointCloudNodeId &nodeId )
const
532 mHierarchyMutex.lock();
533 const bool foundInHierarchy = mHierarchy.contains( nodeId );
534 const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
535 mHierarchyMutex.unlock();
537 if ( foundInHierarchy )
540 if ( !foundInHierarchyNodes )
543 const QString filePath = QStringLiteral(
"%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.
toString() );
545 QByteArray dataJsonH;
546 if ( mAccessType == Remote )
548 QNetworkRequest nr( filePath );
550 nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
551 nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute,
true );
559 if ( reply->error() != QNetworkReply::NoError )
561 QgsDebugError( QStringLiteral(
"Request failed: " ) + filePath );
565 dataJsonH = reply->data();
569 QFile file( filePath );
570 if ( ! file.open( QIODevice::ReadOnly ) )
572 QgsDebugError( QStringLiteral(
"Loading file failed: " ) + filePath );
575 dataJsonH = file.readAll();
578 QJsonParseError errH;
579 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
580 if ( errH.error != QJsonParseError::NoError )
582 QgsDebugMsgLevel( QStringLiteral(
"QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 );
586 QMutexLocker locker( &mHierarchyMutex );
587 const QJsonObject rootHObj = docH.object();
588 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
590 const QString nodeIdStr = it.key();
591 const int nodePointCount = it.value().toInt();
593 if ( nodePointCount >= 0 )
594 mHierarchy[nodeId] = nodePointCount;
595 else if ( nodePointCount == -1 )
596 mHierarchyNodes.insert( nodeId );
602QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot(
const QgsPointCloudNodeId &nodeId )
const
604 QVector<QgsPointCloudNodeId> path;
608 path.push_back( currentNode );
611 while ( currentNode.
d() >= 0 );
620 QMutexLocker lock( &mHierarchyMutex );
621 found = mHierarchy.contains( nodeId );
626 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
627 for (
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
630 if ( !loadSingleNodeHierarchy( node ) )
635 QMutexLocker lock( &mHierarchyMutex );
636 found = mHierarchy.contains( nodeId );
643bool QgsEptPointCloudIndex::isValid()
const
653void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination )
const
658 destination->mIsValid = mIsValid;
659 destination->mDataType = mDataType;
660 destination->mUrlDirectoryPart = mUrlDirectoryPart;
661 destination->mWkt = mWkt;
662 destination->mHierarchyNodes = mHierarchyNodes;
663 destination->mPointCount = mPointCount;
664 destination->mMetadataStats = mMetadataStats;
665 destination->mAttributeClasses = mAttributeClasses;
666 destination->mOriginalMetadata = mOriginalMetadata;
670#undef PROVIDER_DESCRIPTION
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
A 3-dimensional box composed of x, y, z coordinates.
Class for handling a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
This class represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
QByteArray content() const
Returns the reply content.
Collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
void finished()
Emitted when the request processing has finished.
Base class for storing raw data from point cloud nodes.
AccessType
The access type of the data, local is for local files and remote for remote files (over HTTP)
void copyCommonProperties(QgsPointCloudIndex *destination) const
Copies common properties to the destination index.
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
Represents a indexed point cloud node's position in octree.
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
Keeps metadata for indexed point cloud node.
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
qint64 pointCount() const
Returns number of points contained in node data.
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
Point cloud data request.
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
Class used to store statistics of a point cloud dataset.
A rectangle specified with double values.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
#define QgsDebugMsgLevel(str, level)
#define QgsDebugError(str)
#define QgsSetRequestInitiatorClass(request, _class)
Class used to store statistics of one attribute of a point cloud dataset.
QMap< int, int > classCount