31#include "qgspointcloudexpression.h"
41#include <QJsonDocument>
43#include <QNetworkRequest>
49using namespace Qt::StringLiterals;
53#define PROVIDER_KEY u"ept"_s
54#define PROVIDER_DESCRIPTION u"EPT point cloud provider"_s
56QgsEptPointCloudIndex::QgsEptPointCloudIndex()
61QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
63void QgsEptPointCloudIndex::load(
const QString &urlString,
const QString &authcfg )
67 if ( url.isValid() && ( url.scheme() ==
"http" || url.scheme() ==
"https" ) )
73 QStringList splitUrl = mUri.split(
'/' );
75 mUrlDirectoryPart = splitUrl.join(
'/' );
81 QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
98 if ( !f.open( QIODevice::ReadOnly ) )
100 mError = QObject::tr(
"Unable to open %1 for reading" ).arg( mUri );
104 content = f.readAll();
107 bool success = loadSchema( content );
111 const QString manifestPath = mUrlDirectoryPart + u
"/ept-sources/manifest.json"_s;
112 QByteArray manifestJson;
115 QUrl manifestUrl( manifestPath );
117 QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
126 QFile manifestFile( manifestPath );
127 if ( manifestFile.open( QIODevice::ReadOnly ) )
128 manifestJson = manifestFile.readAll();
131 if ( !manifestJson.isEmpty() )
132 loadManifest( manifestJson );
144void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
148 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
149 if ( err.error != QJsonParseError::NoError )
152 const QJsonArray manifestArray = manifestDoc.array();
153 if ( manifestArray.empty() )
157 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
158 const QString metadataPath = sourceObject.value( u
"metadataPath"_s ).toString();
159 const QString fullMetadataPath = mUrlDirectoryPart + u
"/ept-sources/"_s + metadataPath;
161 QByteArray metadataJson;
164 QUrl metadataUrl( fullMetadataPath );
165 QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
175 QFile metadataFile( fullMetadataPath );
176 if ( ! metadataFile.open( QIODevice::ReadOnly ) )
178 metadataJson = metadataFile.readAll();
181 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
182 if ( err.error != QJsonParseError::NoError )
185 const QJsonObject metadataObject = metadataDoc.object().value( u
"metadata"_s ).toObject();
186 if ( metadataObject.empty() )
189 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
190 mOriginalMetadata = sourceMetadata.toVariantMap();
193bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
196 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
197 if ( err.error != QJsonParseError::NoError )
199 const QJsonObject result = doc.object();
200 mDataType = result.value(
"dataType"_L1 ).toString();
201 if ( mDataType !=
"laszip"_L1 && mDataType !=
"binary"_L1 && mDataType !=
"zstandard"_L1 )
204 const QString hierarchyType = result.value(
"hierarchyType"_L1 ).toString();
205 if ( hierarchyType !=
"json"_L1 )
208 mSpan = result.value(
"span"_L1 ).toInt();
209 mPointCount = result.value(
"points"_L1 ).toDouble();
212 const QJsonObject srs = result.value(
"srs"_L1 ).toObject();
213 mWkt = srs.value(
"wkt"_L1 ).toString();
216 const QJsonArray bounds = result.value(
"bounds"_L1 ).toArray();
217 if ( bounds.size() != 6 )
220 const QJsonArray boundsConforming = result.value(
"boundsConforming"_L1 ).toArray();
221 if ( boundsConforming.size() != 6 )
223 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
224 boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
225 mZMin = boundsConforming[2].toDouble();
226 mZMax = boundsConforming[5].toDouble();
228 const QJsonArray schemaArray = result.value(
"schema"_L1 ).toArray();
231 for (
const QJsonValue &schemaItem : schemaArray )
233 const QJsonObject schemaObj = schemaItem.toObject();
234 const QString name = schemaObj.value(
"name"_L1 ).toString();
235 const QString type = schemaObj.value(
"type"_L1 ).toString();
237 const int size = schemaObj.value(
"size"_L1 ).toInt();
239 if ( name ==
"ClassFlags"_L1 && size == 1 )
246 else if ( type ==
"float"_L1 && ( size == 4 ) )
250 else if ( type ==
"float"_L1 && ( size == 8 ) )
254 else if ( size == 1 )
258 else if ( type ==
"unsigned"_L1 && size == 2 )
262 else if ( size == 2 )
266 else if ( size == 4 )
277 if ( schemaObj.contains(
"scale"_L1 ) )
278 scale = schemaObj.value(
"scale"_L1 ).toDouble();
281 if ( schemaObj.contains(
"offset"_L1 ) )
282 offset = schemaObj.value(
"offset"_L1 ).toDouble();
284 if ( name ==
"X"_L1 )
286 mOffset.set( offset, mOffset.y(), mOffset.z() );
287 mScale.set( scale, mScale.y(), mScale.z() );
289 else if ( name ==
"Y"_L1 )
291 mOffset.set( mOffset.x(), offset, mOffset.z() );
292 mScale.set( mScale.x(), scale, mScale.z() );
294 else if ( name ==
"Z"_L1 )
296 mOffset.set( mOffset.x(), mOffset.y(), offset );
297 mScale.set( mScale.x(), mScale.y(), scale );
301 AttributeStatistics stats;
302 bool foundStats =
false;
303 if ( schemaObj.contains(
"count"_L1 ) )
305 stats.count = schemaObj.value(
"count"_L1 ).toInt();
308 if ( schemaObj.contains(
"minimum"_L1 ) )
310 stats.minimum = schemaObj.value(
"minimum"_L1 ).toDouble();
313 if ( schemaObj.contains(
"maximum"_L1 ) )
315 stats.maximum = schemaObj.value(
"maximum"_L1 ).toDouble();
318 if ( schemaObj.contains(
"count"_L1 ) )
320 stats.mean = schemaObj.value(
"mean"_L1 ).toDouble();
323 if ( schemaObj.contains(
"stddev"_L1 ) )
325 stats.stDev = schemaObj.value(
"stddev"_L1 ).toDouble();
328 if ( schemaObj.contains(
"variance"_L1 ) )
330 stats.variance = schemaObj.value(
"variance"_L1 ).toDouble();
334 mMetadataStats.insert( name, stats );
336 if ( schemaObj.contains(
"counts"_L1 ) )
338 QMap< int, int > classCounts;
339 const QJsonArray counts = schemaObj.value(
"counts"_L1 ).toArray();
340 for (
const QJsonValue &count : counts )
342 const QJsonObject countObj = count.toObject();
343 classCounts.insert( countObj.value(
"value"_L1 ).toInt(), countObj.value(
"count"_L1 ).toInt() );
345 mAttributeClasses.insert( name, classCounts );
348 setAttributes( attributes );
353 const double xmin = bounds[0].toDouble();
354 const double ymin = bounds[1].toDouble();
355 const double zmin = bounds[2].toDouble();
356 const double xmax = bounds[3].toDouble();
357 const double ymax = bounds[4].toDouble();
358 const double zmax = bounds[5].toDouble();
360 mRootBounds =
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
363 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
364 QgsDebugMsgLevel( u
"lvl0 node size in CRS units: %1 %2 %3"_s.arg( dx ).arg( dy ).arg( dz ), 2 );
367 QgsDebugMsgLevel( u
"res at lvl2 %1 with node size %2"_s.arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
377 return std::unique_ptr<QgsPointCloudBlock>( cached );
380 std::unique_ptr<QgsPointCloudBlock> block;
383 std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
391 block = blockRequest->takeBlock();
394 QgsDebugError( u
"Error downloading node %1 data, error : %2 "_s.arg( n.
toString(), blockRequest->errorStr() ) );
402 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
404 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
407 if ( mDataType ==
"binary"_L1 )
409 const QString filename = u
"%1/ept-data/%2.bin"_s.arg( mUrlDirectoryPart, n.
toString() );
410 block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
412 else if ( mDataType ==
"zstandard"_L1 )
414 const QString filename = u
"%1/ept-data/%2.zst"_s.arg( mUrlDirectoryPart, n.
toString() );
415 block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
417 else if ( mDataType ==
"laszip"_L1 )
419 const QString filename = u
"%1/ept-data/%2.laz"_s.arg( mUrlDirectoryPart, n.
toString() );
420 block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
424 storeNodeDataToCache( block.get(), n, request );
433 scale(), offset(), mFilterExpression, request.
filterRect() );
439 if ( !loadNodeHierarchy( n ) )
443 if ( mDataType ==
"binary"_L1 )
445 fileUrl = u
"%1/ept-data/%2.bin"_s.arg( mUrlDirectoryPart, n.
toString() );
447 else if ( mDataType ==
"zstandard"_L1 )
449 fileUrl = u
"%1/ept-data/%2.zst"_s.arg( mUrlDirectoryPart, n.
toString() );
451 else if ( mDataType ==
"laszip"_L1 )
453 fileUrl = u
"%1/ept-data/%2.laz"_s.arg( mUrlDirectoryPart, n.
toString() );
463 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
465 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
471 return loadNodeHierarchy( n );
479qint64 QgsEptPointCloudIndex::pointCount()
const
494 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot(
id );
495 for (
int i = pathToRoot.size() - 1; i >= 0; --i )
497 loadSingleNodeHierarchy( pathToRoot[i] );
499 QMutexLocker locker( &mHierarchyMutex );
500 qint64 pointCount = mHierarchy.value(
id, -1 );
501 if ( pointCount != -1 )
511 QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
514 QString name = attribute.name();
515 const AttributeStatistics &stats = mMetadataStats[ name ];
516 if ( !stats.minimum.isValid() )
519 s.
minimum = stats.minimum.toDouble();
520 s.
maximum = stats.maximum.toDouble();
522 s.
stDev = stats.stDev;
523 s.
count = stats.count;
527 statsMap[ name ] = std::move( s );
532bool QgsEptPointCloudIndex::loadSingleNodeHierarchy(
const QgsPointCloudNodeId &nodeId )
const
534 mHierarchyMutex.lock();
535 const bool foundInHierarchy = mHierarchy.contains( nodeId );
536 const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
537 mHierarchyMutex.unlock();
539 if ( foundInHierarchy )
542 if ( !foundInHierarchyNodes )
545 const QString filePath = u
"%1/ept-hierarchy/%2.json"_s.arg( mUrlDirectoryPart, nodeId.
toString() );
547 QByteArray dataJsonH;
550 QNetworkRequest nr( filePath );
552 nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
553 nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute,
true );
557 QgsDebugError( u
"Network request update failed for authcfg: %1"_s.arg( mAuthCfg ) );
567 if ( reply->error() != QNetworkReply::NoError )
573 dataJsonH = reply->data();
577 QFile file( filePath );
578 if ( ! file.open( QIODevice::ReadOnly ) )
583 dataJsonH = file.readAll();
586 QJsonParseError errH;
587 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
588 if ( errH.error != QJsonParseError::NoError )
590 QgsDebugMsgLevel( u
"QJsonParseError when reading hierarchy from file %1"_s.arg( filePath ), 2 );
594 QMutexLocker locker( &mHierarchyMutex );
595 const QJsonObject rootHObj = docH.object();
596 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
598 const QString nodeIdStr = it.key();
599 const int nodePointCount = it.value().toInt();
601 if ( nodePointCount >= 0 )
602 mHierarchy[nodeId] = nodePointCount;
603 else if ( nodePointCount == -1 )
604 mHierarchyNodes.insert( nodeId );
610QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot(
const QgsPointCloudNodeId &nodeId )
const
612 QVector<QgsPointCloudNodeId> path;
616 path.push_back( currentNode );
619 while ( currentNode.
d() >= 0 );
628 QMutexLocker lock( &mHierarchyMutex );
629 found = mHierarchy.contains( nodeId );
634 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
635 for (
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
638 if ( !loadSingleNodeHierarchy( node ) )
643 QMutexLocker lock( &mHierarchyMutex );
644 found = mHierarchy.contains( nodeId );
651bool QgsEptPointCloudIndex::isValid()
const
662#undef PROVIDER_DESCRIPTION
PointCloudAccessType
The access type of the data, local is for local files and remote for remote files (over HTTP).
@ Local
Local means the source is a local file on the machine.
@ Remote
Remote means it's loaded through a protocol like HTTP.
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
A 3-dimensional box composed of x, y, z coordinates.
Handles a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
Represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
QByteArray content() const
Returns the reply content.
A collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
void finished()
Emitted when the request processing has finished.
Base class for storing raw data from point cloud nodes.
Represents an indexed point cloud node's position in octree.
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
Keeps metadata for an indexed point cloud node.
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
qint64 pointCount() const
Returns number of points contained in node data.
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
Point cloud data request.
bool ignoreIndexFilterEnabled() const
Returns whether the request will ignore the point cloud index's filter expression,...
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
Used to store statistics of a point cloud dataset.
A rectangle specified with double values.
void finished()
Emitted when the reply has finished (either with a success or with a failure).
#define QgsDebugMsgLevel(str, level)
#define QgsDebugError(str)
#define QgsSetRequestInitiatorClass(request, _class)
Stores statistics of one attribute of a point cloud dataset.
QMap< int, int > classCount