31#include "qgspointcloudexpression.h"
41#include <QJsonDocument>
43#include <QNetworkRequest>
49using namespace Qt::StringLiterals;
53#define PROVIDER_KEY u"ept"_s
54#define PROVIDER_DESCRIPTION u"EPT point cloud provider"_s
56QgsEptPointCloudIndex::QgsEptPointCloudIndex()
61QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
63void QgsEptPointCloudIndex::load(
const QString &urlString,
const QString &authcfg )
67 if ( url.isValid() && ( url.scheme() ==
"http" || url.scheme() ==
"https" ) )
73 QStringList splitUrl = mUri.split(
'/' );
75 mUrlDirectoryPart = splitUrl.join(
'/' );
81 QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
98 if ( !f.open( QIODevice::ReadOnly ) )
100 mError = QObject::tr(
"Unable to open %1 for reading" ).arg( mUri );
104 content = f.readAll();
107 bool success = loadSchema( content );
111 const QString manifestPath = mUrlDirectoryPart + u
"/ept-sources/manifest.json"_s;
112 QByteArray manifestJson;
115 QUrl manifestUrl( manifestPath );
117 QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
126 QFile manifestFile( manifestPath );
127 if ( manifestFile.open( QIODevice::ReadOnly ) )
128 manifestJson = manifestFile.readAll();
131 if ( !manifestJson.isEmpty() )
132 loadManifest( manifestJson );
144void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
148 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
149 if ( err.error != QJsonParseError::NoError )
152 const QJsonArray manifestArray = manifestDoc.array();
153 if ( manifestArray.empty() )
157 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
158 const QString metadataPath = sourceObject.value( u
"metadataPath"_s ).toString();
159 const QString fullMetadataPath = mUrlDirectoryPart + u
"/ept-sources/"_s + metadataPath;
161 QByteArray metadataJson;
164 QUrl metadataUrl( fullMetadataPath );
165 QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
175 QFile metadataFile( fullMetadataPath );
176 if ( !metadataFile.open( QIODevice::ReadOnly ) )
178 metadataJson = metadataFile.readAll();
181 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
182 if ( err.error != QJsonParseError::NoError )
185 const QJsonObject metadataObject = metadataDoc.object().value( u
"metadata"_s ).toObject();
186 if ( metadataObject.empty() )
189 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
190 mOriginalMetadata = sourceMetadata.toVariantMap();
193bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
196 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
197 if ( err.error != QJsonParseError::NoError )
199 const QJsonObject result = doc.object();
200 mDataType = result.value(
"dataType"_L1 ).toString();
201 if ( mDataType !=
"laszip"_L1 && mDataType !=
"binary"_L1 && mDataType !=
"zstandard"_L1 )
204 const QString hierarchyType = result.value(
"hierarchyType"_L1 ).toString();
205 if ( hierarchyType !=
"json"_L1 )
208 mSpan = result.value(
"span"_L1 ).toInt();
209 mPointCount = result.value(
"points"_L1 ).toDouble();
212 const QJsonObject srs = result.value(
"srs"_L1 ).toObject();
213 mWkt = srs.value(
"wkt"_L1 ).toString();
216 const QJsonArray bounds = result.value(
"bounds"_L1 ).toArray();
217 if ( bounds.size() != 6 )
220 const QJsonArray boundsConforming = result.value(
"boundsConforming"_L1 ).toArray();
221 if ( boundsConforming.size() != 6 )
223 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(), boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
224 mZMin = boundsConforming[2].toDouble();
225 mZMax = boundsConforming[5].toDouble();
227 const QJsonArray schemaArray = result.value(
"schema"_L1 ).toArray();
230 for (
const QJsonValue &schemaItem : schemaArray )
232 const QJsonObject schemaObj = schemaItem.toObject();
233 const QString name = schemaObj.value(
"name"_L1 ).toString();
234 const QString type = schemaObj.value(
"type"_L1 ).toString();
236 const int size = schemaObj.value(
"size"_L1 ).toInt();
238 if ( name ==
"ClassFlags"_L1 && size == 1 )
245 else if ( type ==
"float"_L1 && ( size == 4 ) )
249 else if ( type ==
"float"_L1 && ( size == 8 ) )
253 else if ( size == 1 )
257 else if ( type ==
"unsigned"_L1 && size == 2 )
261 else if ( size == 2 )
265 else if ( size == 4 )
276 if ( schemaObj.contains(
"scale"_L1 ) )
277 scale = schemaObj.value(
"scale"_L1 ).toDouble();
280 if ( schemaObj.contains(
"offset"_L1 ) )
281 offset = schemaObj.value(
"offset"_L1 ).toDouble();
283 if ( name ==
"X"_L1 )
285 mOffset.set( offset, mOffset.y(), mOffset.z() );
286 mScale.set( scale, mScale.y(), mScale.z() );
288 else if ( name ==
"Y"_L1 )
290 mOffset.set( mOffset.x(), offset, mOffset.z() );
291 mScale.set( mScale.x(), scale, mScale.z() );
293 else if ( name ==
"Z"_L1 )
295 mOffset.set( mOffset.x(), mOffset.y(), offset );
296 mScale.set( mScale.x(), mScale.y(), scale );
300 AttributeStatistics stats;
301 bool foundStats =
false;
302 if ( schemaObj.contains(
"count"_L1 ) )
304 stats.count = schemaObj.value(
"count"_L1 ).toInt();
307 if ( schemaObj.contains(
"minimum"_L1 ) )
309 stats.minimum = schemaObj.value(
"minimum"_L1 ).toDouble();
312 if ( schemaObj.contains(
"maximum"_L1 ) )
314 stats.maximum = schemaObj.value(
"maximum"_L1 ).toDouble();
317 if ( schemaObj.contains(
"count"_L1 ) )
319 stats.mean = schemaObj.value(
"mean"_L1 ).toDouble();
322 if ( schemaObj.contains(
"stddev"_L1 ) )
324 stats.stDev = schemaObj.value(
"stddev"_L1 ).toDouble();
327 if ( schemaObj.contains(
"variance"_L1 ) )
329 stats.variance = schemaObj.value(
"variance"_L1 ).toDouble();
333 mMetadataStats.insert( name, stats );
335 if ( schemaObj.contains(
"counts"_L1 ) )
337 QMap< int, int > classCounts;
338 const QJsonArray counts = schemaObj.value(
"counts"_L1 ).toArray();
339 for (
const QJsonValue &count : counts )
341 const QJsonObject countObj = count.toObject();
342 classCounts.insert( countObj.value(
"value"_L1 ).toInt(), countObj.value(
"count"_L1 ).toInt() );
344 mAttributeClasses.insert( name, classCounts );
347 setAttributes( attributes );
352 const double xmin = bounds[0].toDouble();
353 const double ymin = bounds[1].toDouble();
354 const double zmin = bounds[2].toDouble();
355 const double xmax = bounds[3].toDouble();
356 const double ymax = bounds[4].toDouble();
357 const double zmax = bounds[5].toDouble();
359 mRootBounds =
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
362 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
363 QgsDebugMsgLevel( u
"lvl0 node size in CRS units: %1 %2 %3"_s.arg( dx ).arg( dy ).arg( dz ), 2 );
366 QgsDebugMsgLevel( u
"res at lvl2 %1 with node size %2"_s.arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
376 return std::unique_ptr<QgsPointCloudBlock>( cached );
379 std::unique_ptr<QgsPointCloudBlock> block;
382 std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
390 block = blockRequest->takeBlock();
393 QgsDebugError( u
"Error downloading node %1 data, error : %2 "_s.arg( n.
toString(), blockRequest->errorStr() ) );
401 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
403 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
406 if ( mDataType ==
"binary"_L1 )
408 const QString filename = u
"%1/ept-data/%2.bin"_s.arg( mUrlDirectoryPart, n.
toString() );
409 block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
411 else if ( mDataType ==
"zstandard"_L1 )
413 const QString filename = u
"%1/ept-data/%2.zst"_s.arg( mUrlDirectoryPart, n.
toString() );
414 block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
416 else if ( mDataType ==
"laszip"_L1 )
418 const QString filename = u
"%1/ept-data/%2.laz"_s.arg( mUrlDirectoryPart, n.
toString() );
419 block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
423 storeNodeDataToCache( block.get(), n, request );
437 if ( !loadNodeHierarchy( n ) )
441 if ( mDataType ==
"binary"_L1 )
443 fileUrl = u
"%1/ept-data/%2.bin"_s.arg( mUrlDirectoryPart, n.
toString() );
445 else if ( mDataType ==
"zstandard"_L1 )
447 fileUrl = u
"%1/ept-data/%2.zst"_s.arg( mUrlDirectoryPart, n.
toString() );
449 else if ( mDataType ==
"laszip"_L1 )
451 fileUrl = u
"%1/ept-data/%2.laz"_s.arg( mUrlDirectoryPart, n.
toString() );
461 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
463 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
469 return loadNodeHierarchy( n );
477qint64 QgsEptPointCloudIndex::pointCount()
const
492 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot(
id );
493 for (
int i = pathToRoot.size() - 1; i >= 0; --i )
495 loadSingleNodeHierarchy( pathToRoot[i] );
497 QMutexLocker locker( &mHierarchyMutex );
498 qint64 pointCount = mHierarchy.value(
id, -1 );
499 if ( pointCount != -1 )
509 QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
512 QString name = attribute.name();
513 const AttributeStatistics &stats = mMetadataStats[name];
514 if ( !stats.minimum.isValid() )
517 s.
minimum = stats.minimum.toDouble();
518 s.
maximum = stats.maximum.toDouble();
520 s.
stDev = stats.stDev;
521 s.
count = stats.count;
525 statsMap[name] = std::move( s );
530bool QgsEptPointCloudIndex::loadSingleNodeHierarchy(
const QgsPointCloudNodeId &nodeId )
const
532 mHierarchyMutex.lock();
533 const bool foundInHierarchy = mHierarchy.contains( nodeId );
534 const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
535 mHierarchyMutex.unlock();
537 if ( foundInHierarchy )
540 if ( !foundInHierarchyNodes )
543 const QString filePath = u
"%1/ept-hierarchy/%2.json"_s.arg( mUrlDirectoryPart, nodeId.
toString() );
545 QByteArray dataJsonH;
548 QNetworkRequest nr( filePath );
550 nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
551 nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute,
true );
555 QgsDebugError( u
"Network request update failed for authcfg: %1"_s.arg( mAuthCfg ) );
565 if ( reply->error() != QNetworkReply::NoError )
571 dataJsonH = reply->data();
575 QFile file( filePath );
576 if ( !file.open( QIODevice::ReadOnly ) )
581 dataJsonH = file.readAll();
584 QJsonParseError errH;
585 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
586 if ( errH.error != QJsonParseError::NoError )
588 QgsDebugMsgLevel( u
"QJsonParseError when reading hierarchy from file %1"_s.arg( filePath ), 2 );
592 QMutexLocker locker( &mHierarchyMutex );
593 const QJsonObject rootHObj = docH.object();
594 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
596 const QString nodeIdStr = it.key();
597 const int nodePointCount = it.value().toInt();
599 if ( nodePointCount >= 0 )
600 mHierarchy[nodeId] = nodePointCount;
601 else if ( nodePointCount == -1 )
602 mHierarchyNodes.insert( nodeId );
608QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot(
const QgsPointCloudNodeId &nodeId )
const
610 QVector<QgsPointCloudNodeId> path;
614 path.push_back( currentNode );
616 }
while ( currentNode.
d() >= 0 );
625 QMutexLocker lock( &mHierarchyMutex );
626 found = mHierarchy.contains( nodeId );
631 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
632 for (
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
635 if ( !loadSingleNodeHierarchy( node ) )
640 QMutexLocker lock( &mHierarchyMutex );
641 found = mHierarchy.contains( nodeId );
648bool QgsEptPointCloudIndex::isValid()
const
659#undef PROVIDER_DESCRIPTION
PointCloudAccessType
The access type of the data, local is for local files and remote for remote files (over HTTP).
@ Local
Local means the source is a local file on the machine.
@ Remote
Remote means it's loaded through a protocol like HTTP.
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
A 3-dimensional box composed of x, y, z coordinates.
Handles a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
Represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
QByteArray content() const
Returns the reply content.
A collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
void finished()
Emitted when the request processing has finished.
Base class for storing raw data from point cloud nodes.
Represents an indexed point cloud node's position in octree.
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
Keeps metadata for an indexed point cloud node.
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
qint64 pointCount() const
Returns number of points contained in node data.
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
Point cloud data request.
bool ignoreIndexFilterEnabled() const
Returns whether the request will ignore the point cloud index's filter expression,...
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
Used to store statistics of a point cloud dataset.
A rectangle specified with double values.
void finished()
Emitted when the reply has finished (either with a success or with a failure).
#define QgsDebugMsgLevel(str, level)
#define QgsDebugError(str)
#define QgsSetRequestInitiatorClass(request, _class)
Stores statistics of one attribute of a point cloud dataset.
QMap< int, int > classCount