19#include "moc_qgseptpointcloudindex.cpp"
24#include <QJsonDocument>
29#include <QNetworkRequest>
42#include "qgspointcloudexpression.h"
48#define PROVIDER_KEY QStringLiteral( "ept" )
49#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )
51QgsEptPointCloudIndex::QgsEptPointCloudIndex()
56QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
58std::unique_ptr<QgsPointCloudIndex> QgsEptPointCloudIndex::clone()
const
60 QgsEptPointCloudIndex *clone =
new QgsEptPointCloudIndex;
61 QMutexLocker locker( &mHierarchyMutex );
62 copyCommonProperties( clone );
63 return std::unique_ptr<QgsPointCloudIndex>( clone );
66void QgsEptPointCloudIndex::load(
const QString &urlString )
70 if ( url.isValid() && ( url.scheme() ==
"http" || url.scheme() ==
"https" ) )
76 QStringList splitUrl = mUri.split(
'/' );
78 mUrlDirectoryPart = splitUrl.join(
'/' );
81 if ( mAccessType == Remote )
83 QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
99 if ( !f.open( QIODevice::ReadOnly ) )
101 mError = QObject::tr(
"Unable to open %1 for reading" ).arg( mUri );
105 content = f.readAll();
108 bool success = loadSchema( content );
112 const QString manifestPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/manifest.json" );
113 QByteArray manifestJson;
114 if ( mAccessType == Remote )
116 QUrl manifestUrl( manifestPath );
118 QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
126 QFile manifestFile( manifestPath );
127 if ( manifestFile.open( QIODevice::ReadOnly ) )
128 manifestJson = manifestFile.readAll();
131 if ( !manifestJson.isEmpty() )
132 loadManifest( manifestJson );
137 QgsDebugError( QStringLiteral(
"Failed to load root EPT node" ) );
144void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
148 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
149 if ( err.error != QJsonParseError::NoError )
152 const QJsonArray manifestArray = manifestDoc.array();
153 if ( manifestArray.empty() )
157 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
158 const QString metadataPath = sourceObject.value( QStringLiteral(
"metadataPath" ) ).toString();
159 const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/" ) + metadataPath;
161 QByteArray metadataJson;
162 if ( mAccessType == Remote )
164 QUrl metadataUrl( fullMetadataPath );
165 QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
174 QFile metadataFile( fullMetadataPath );
175 if ( ! metadataFile.open( QIODevice::ReadOnly ) )
177 metadataJson = metadataFile.readAll();
180 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
181 if ( err.error != QJsonParseError::NoError )
184 const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral(
"metadata" ) ).toObject();
185 if ( metadataObject.empty() )
188 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
189 mOriginalMetadata = sourceMetadata.toVariantMap();
192bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
195 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
196 if ( err.error != QJsonParseError::NoError )
198 const QJsonObject result = doc.object();
199 mDataType = result.value( QLatin1String(
"dataType" ) ).toString();
200 if ( mDataType != QLatin1String(
"laszip" ) && mDataType != QLatin1String(
"binary" ) && mDataType != QLatin1String(
"zstandard" ) )
203 const QString hierarchyType = result.value( QLatin1String(
"hierarchyType" ) ).toString();
204 if ( hierarchyType != QLatin1String(
"json" ) )
207 mSpan = result.value( QLatin1String(
"span" ) ).toInt();
208 mPointCount = result.value( QLatin1String(
"points" ) ).toDouble();
211 const QJsonObject srs = result.value( QLatin1String(
"srs" ) ).toObject();
212 mWkt = srs.value( QLatin1String(
"wkt" ) ).toString();
215 const QJsonArray bounds = result.value( QLatin1String(
"bounds" ) ).toArray();
216 if ( bounds.size() != 6 )
219 const QJsonArray boundsConforming = result.value( QLatin1String(
"boundsConforming" ) ).toArray();
220 if ( boundsConforming.size() != 6 )
222 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
223 boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
224 mZMin = boundsConforming[2].toDouble();
225 mZMax = boundsConforming[5].toDouble();
227 const QJsonArray schemaArray = result.value( QLatin1String(
"schema" ) ).toArray();
230 for (
const QJsonValue &schemaItem : schemaArray )
232 const QJsonObject schemaObj = schemaItem.toObject();
233 const QString name = schemaObj.value( QLatin1String(
"name" ) ).toString();
234 const QString type = schemaObj.value( QLatin1String(
"type" ) ).toString();
236 const int size = schemaObj.value( QLatin1String(
"size" ) ).toInt();
238 if ( name == QLatin1String(
"ClassFlags" ) && size == 1 )
245 else if ( type == QLatin1String(
"float" ) && ( size == 4 ) )
249 else if ( type == QLatin1String(
"float" ) && ( size == 8 ) )
253 else if ( size == 1 )
257 else if ( type == QLatin1String(
"unsigned" ) && size == 2 )
261 else if ( size == 2 )
265 else if ( size == 4 )
276 if ( schemaObj.contains( QLatin1String(
"scale" ) ) )
277 scale = schemaObj.value( QLatin1String(
"scale" ) ).toDouble();
280 if ( schemaObj.contains( QLatin1String(
"offset" ) ) )
281 offset = schemaObj.value( QLatin1String(
"offset" ) ).toDouble();
283 if ( name == QLatin1String(
"X" ) )
285 mOffset.set( offset, mOffset.y(), mOffset.z() );
286 mScale.set( scale, mScale.y(), mScale.z() );
288 else if ( name == QLatin1String(
"Y" ) )
290 mOffset.set( mOffset.x(), offset, mOffset.z() );
291 mScale.set( mScale.x(), scale, mScale.z() );
293 else if ( name == QLatin1String(
"Z" ) )
295 mOffset.set( mOffset.x(), mOffset.y(), offset );
296 mScale.set( mScale.x(), mScale.y(), scale );
300 AttributeStatistics stats;
301 bool foundStats =
false;
302 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
304 stats.count = schemaObj.value( QLatin1String(
"count" ) ).toInt();
307 if ( schemaObj.contains( QLatin1String(
"minimum" ) ) )
309 stats.minimum = schemaObj.value( QLatin1String(
"minimum" ) ).toDouble();
312 if ( schemaObj.contains( QLatin1String(
"maximum" ) ) )
314 stats.maximum = schemaObj.value( QLatin1String(
"maximum" ) ).toDouble();
317 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
319 stats.mean = schemaObj.value( QLatin1String(
"mean" ) ).toDouble();
322 if ( schemaObj.contains( QLatin1String(
"stddev" ) ) )
324 stats.stDev = schemaObj.value( QLatin1String(
"stddev" ) ).toDouble();
327 if ( schemaObj.contains( QLatin1String(
"variance" ) ) )
329 stats.variance = schemaObj.value( QLatin1String(
"variance" ) ).toDouble();
333 mMetadataStats.insert( name, stats );
335 if ( schemaObj.contains( QLatin1String(
"counts" ) ) )
337 QMap< int, int > classCounts;
338 const QJsonArray counts = schemaObj.value( QLatin1String(
"counts" ) ).toArray();
339 for (
const QJsonValue &count : counts )
341 const QJsonObject countObj = count.toObject();
342 classCounts.insert( countObj.value( QLatin1String(
"value" ) ).toInt(), countObj.value( QLatin1String(
"count" ) ).toInt() );
344 mAttributeClasses.insert( name, classCounts );
347 setAttributes( attributes );
352 const double xmin = bounds[0].toDouble();
353 const double ymin = bounds[1].toDouble();
354 const double zmin = bounds[2].toDouble();
355 const double xmax = bounds[3].toDouble();
356 const double ymax = bounds[4].toDouble();
357 const double zmax = bounds[5].toDouble();
359 mRootBounds =
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
362 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
363 QgsDebugMsgLevel( QStringLiteral(
"lvl0 node size in CRS units: %1 %2 %3" ).arg( dx ).arg( dy ).arg( dz ), 2 );
364 QgsDebugMsgLevel( QStringLiteral(
"res at lvl0 %1" ).arg( dx / mSpan ), 2 );
365 QgsDebugMsgLevel( QStringLiteral(
"res at lvl1 %1" ).arg( dx / mSpan / 2 ), 2 );
366 QgsDebugMsgLevel( QStringLiteral(
"res at lvl2 %1 with node size %2" ).arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
376 return std::unique_ptr<QgsPointCloudBlock>( cached );
379 std::unique_ptr<QgsPointCloudBlock> block;
380 if ( mAccessType == Remote )
382 std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
390 block = blockRequest->takeBlock();
393 QgsDebugError( QStringLiteral(
"Error downloading node %1 data, error : %2 " ).arg( n.
toString(), blockRequest->errorStr() ) );
401 QgsPointCloudExpression filterExpression = mFilterExpression;
403 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
406 if ( mDataType == QLatin1String(
"binary" ) )
408 const QString filename = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
409 block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
411 else if ( mDataType == QLatin1String(
"zstandard" ) )
413 const QString filename = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
414 block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
416 else if ( mDataType == QLatin1String(
"laszip" ) )
418 const QString filename = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
419 block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
423 storeNodeDataToCache( block.get(), n, request );
432 scale(), offset(), mFilterExpression, request.
filterRect() );
435 if ( mAccessType != Remote )
438 if ( !loadNodeHierarchy( n ) )
442 if ( mDataType == QLatin1String(
"binary" ) )
444 fileUrl = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
446 else if ( mDataType == QLatin1String(
"zstandard" ) )
448 fileUrl = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
450 else if ( mDataType == QLatin1String(
"laszip" ) )
452 fileUrl = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
462 QgsPointCloudExpression filterExpression = mFilterExpression;
464 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
470 return loadNodeHierarchy( n );
478qint64 QgsEptPointCloudIndex::pointCount()
const
493 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot(
id );
494 for (
int i = pathToRoot.size() - 1; i >= 0; --i )
496 loadSingleNodeHierarchy( pathToRoot[i] );
498 QMutexLocker locker( &mHierarchyMutex );
499 qint64 pointCount = mHierarchy.value(
id, -1 );
500 if ( pointCount != -1 )
510 QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
513 QString name = attribute.name();
514 const AttributeStatistics &stats = mMetadataStats[ name ];
515 if ( !stats.minimum.isValid() )
518 s.
minimum = stats.minimum.toDouble();
519 s.
maximum = stats.maximum.toDouble();
521 s.
stDev = stats.stDev;
522 s.
count = stats.count;
526 statsMap[ name ] = s;
531bool QgsEptPointCloudIndex::loadSingleNodeHierarchy(
const QgsPointCloudNodeId &nodeId )
const
533 mHierarchyMutex.lock();
534 const bool foundInHierarchy = mHierarchy.contains( nodeId );
535 const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
536 mHierarchyMutex.unlock();
538 if ( foundInHierarchy )
541 if ( !foundInHierarchyNodes )
544 const QString filePath = QStringLiteral(
"%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.
toString() );
546 QByteArray dataJsonH;
547 if ( mAccessType == Remote )
549 QNetworkRequest nr( filePath );
551 nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
552 nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute,
true );
560 if ( reply->error() != QNetworkReply::NoError )
562 QgsDebugError( QStringLiteral(
"Request failed: " ) + filePath );
566 dataJsonH = reply->data();
570 QFile file( filePath );
571 if ( ! file.open( QIODevice::ReadOnly ) )
573 QgsDebugError( QStringLiteral(
"Loading file failed: " ) + filePath );
576 dataJsonH = file.readAll();
579 QJsonParseError errH;
580 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
581 if ( errH.error != QJsonParseError::NoError )
583 QgsDebugMsgLevel( QStringLiteral(
"QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 );
587 QMutexLocker locker( &mHierarchyMutex );
588 const QJsonObject rootHObj = docH.object();
589 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
591 const QString nodeIdStr = it.key();
592 const int nodePointCount = it.value().toInt();
594 if ( nodePointCount >= 0 )
595 mHierarchy[nodeId] = nodePointCount;
596 else if ( nodePointCount == -1 )
597 mHierarchyNodes.insert( nodeId );
603QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot(
const QgsPointCloudNodeId &nodeId )
const
605 QVector<QgsPointCloudNodeId> path;
609 path.push_back( currentNode );
612 while ( currentNode.
d() >= 0 );
621 QMutexLocker lock( &mHierarchyMutex );
622 found = mHierarchy.contains( nodeId );
627 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
628 for (
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
631 if ( !loadSingleNodeHierarchy( node ) )
636 QMutexLocker lock( &mHierarchyMutex );
637 found = mHierarchy.contains( nodeId );
644bool QgsEptPointCloudIndex::isValid()
const
654void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination )
const
659 destination->mIsValid = mIsValid;
660 destination->mDataType = mDataType;
661 destination->mUrlDirectoryPart = mUrlDirectoryPart;
662 destination->mWkt = mWkt;
663 destination->mHierarchyNodes = mHierarchyNodes;
664 destination->mPointCount = mPointCount;
665 destination->mMetadataStats = mMetadataStats;
666 destination->mAttributeClasses = mAttributeClasses;
667 destination->mOriginalMetadata = mOriginalMetadata;
671#undef PROVIDER_DESCRIPTION
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
A 3-dimensional box composed of x, y, z coordinates.
Class for handling a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
This class represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
QByteArray content() const
Returns the reply content.
Collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
void finished()
Emitted when the request processing has finished.
Base class for storing raw data from point cloud nodes.
AccessType
The access type of the data, local is for local files and remote for remote files (over HTTP)
void copyCommonProperties(QgsPointCloudIndex *destination) const
Copies common properties to the destination index.
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
Represents a indexed point cloud node's position in octree.
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
Keeps metadata for indexed point cloud node.
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
qint64 pointCount() const
Returns number of points contained in node data.
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
Point cloud data request.
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
Class used to store statistics of a point cloud dataset.
A rectangle specified with double values.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
#define QgsDebugMsgLevel(str, level)
#define QgsDebugError(str)
#define QgsSetRequestInitiatorClass(request, _class)
Class used to store statistics of one attribute of a point cloud dataset.
QMap< int, int > classCount