23#include <QJsonDocument>
35#include "qgspointcloudexpression.h"
39#define PROVIDER_KEY QStringLiteral( "ept" )
40#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )
42QgsEptPointCloudIndex::QgsEptPointCloudIndex() =
default;
44QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
46std::unique_ptr<QgsPointCloudIndex> QgsEptPointCloudIndex::clone()
const
48 QgsEptPointCloudIndex *clone =
new QgsEptPointCloudIndex;
49 QMutexLocker locker( &mHierarchyMutex );
50 copyCommonProperties( clone );
51 return std::unique_ptr<QgsPointCloudIndex>( clone );
54void QgsEptPointCloudIndex::load(
const QString &fileName )
58 if ( !f.open( QIODevice::ReadOnly ) )
60 mError = tr(
"Unable to open %1 for reading" ).arg( fileName );
65 const QDir directory = QFileInfo( fileName ).absoluteDir();
66 mDirectory = directory.absolutePath();
68 const QByteArray dataJson = f.readAll();
69 bool success = loadSchema( dataJson );
74 QFile manifestFile( mDirectory + QStringLiteral(
"/ept-sources/manifest.json" ) );
75 if ( manifestFile.open( QIODevice::ReadOnly ) )
77 const QByteArray manifestJson = manifestFile.readAll();
78 loadManifest( manifestJson );
84 success = loadHierarchy();
90void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
94 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
95 if ( err.error == QJsonParseError::NoError )
97 const QJsonArray manifestArray = manifestDoc.array();
99 if ( ! manifestArray.empty() )
101 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
102 const QString metadataPath = sourceObject.value( QStringLiteral(
"metadataPath" ) ).toString();
103 QFile metadataFile( mDirectory + QStringLiteral(
"/ept-sources/" ) + metadataPath );
104 if ( metadataFile.open( QIODevice::ReadOnly ) )
106 const QByteArray metadataJson = metadataFile.readAll();
107 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
108 if ( err.error == QJsonParseError::NoError )
110 const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral(
"metadata" ) ).toObject();
111 if ( !metadataObject.empty() )
113 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
114 mOriginalMetadata = sourceMetadata.toVariantMap();
122bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
125 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
126 if ( err.error != QJsonParseError::NoError )
128 const QJsonObject result = doc.object();
129 mDataType = result.value( QLatin1String(
"dataType" ) ).toString();
130 if ( mDataType != QLatin1String(
"laszip" ) && mDataType != QLatin1String(
"binary" ) && mDataType != QLatin1String(
"zstandard" ) )
133 const QString hierarchyType = result.value( QLatin1String(
"hierarchyType" ) ).toString();
134 if ( hierarchyType != QLatin1String(
"json" ) )
137 mSpan = result.value( QLatin1String(
"span" ) ).toInt();
138 mPointCount = result.value( QLatin1String(
"points" ) ).toDouble();
141 const QJsonObject srs = result.value( QLatin1String(
"srs" ) ).toObject();
142 mWkt = srs.value( QLatin1String(
"wkt" ) ).toString();
145 const QJsonArray bounds = result.value( QLatin1String(
"bounds" ) ).toArray();
146 if ( bounds.size() != 6 )
149 const QJsonArray boundsConforming = result.value( QLatin1String(
"boundsConforming" ) ).toArray();
150 if ( boundsConforming.size() != 6 )
152 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
153 boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
154 mZMin = boundsConforming[2].toDouble();
155 mZMax = boundsConforming[5].toDouble();
157 const QJsonArray schemaArray = result.value( QLatin1String(
"schema" ) ).toArray();
160 for (
const QJsonValue &schemaItem : schemaArray )
162 const QJsonObject schemaObj = schemaItem.toObject();
163 const QString name = schemaObj.value( QLatin1String(
"name" ) ).toString();
164 const QString type = schemaObj.value( QLatin1String(
"type" ) ).toString();
166 const int size = schemaObj.value( QLatin1String(
"size" ) ).toInt();
168 if ( name == QLatin1String(
"ClassFlags" ) && size == 1 )
175 else if ( type == QLatin1String(
"float" ) && ( size == 4 ) )
179 else if ( type == QLatin1String(
"float" ) && ( size == 8 ) )
183 else if ( size == 1 )
187 else if ( type == QLatin1String(
"unsigned" ) && size == 2 )
191 else if ( size == 2 )
195 else if ( size == 4 )
206 if ( schemaObj.contains( QLatin1String(
"scale" ) ) )
207 scale = schemaObj.value( QLatin1String(
"scale" ) ).toDouble();
210 if ( schemaObj.contains( QLatin1String(
"offset" ) ) )
211 offset = schemaObj.value( QLatin1String(
"offset" ) ).toDouble();
213 if ( name == QLatin1String(
"X" ) )
215 mOffset.set( offset, mOffset.y(), mOffset.z() );
216 mScale.set( scale, mScale.y(), mScale.z() );
218 else if ( name == QLatin1String(
"Y" ) )
220 mOffset.set( mOffset.x(), offset, mOffset.z() );
221 mScale.set( mScale.x(), scale, mScale.z() );
223 else if ( name == QLatin1String(
"Z" ) )
225 mOffset.set( mOffset.x(), mOffset.y(), offset );
226 mScale.set( mScale.x(), mScale.y(), scale );
230 AttributeStatistics stats;
231 bool foundStats =
false;
232 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
234 stats.count = schemaObj.value( QLatin1String(
"count" ) ).toInt();
237 if ( schemaObj.contains( QLatin1String(
"minimum" ) ) )
239 stats.minimum = schemaObj.value( QLatin1String(
"minimum" ) ).toDouble();
242 if ( schemaObj.contains( QLatin1String(
"maximum" ) ) )
244 stats.maximum = schemaObj.value( QLatin1String(
"maximum" ) ).toDouble();
247 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
249 stats.mean = schemaObj.value( QLatin1String(
"mean" ) ).toDouble();
252 if ( schemaObj.contains( QLatin1String(
"stddev" ) ) )
254 stats.stDev = schemaObj.value( QLatin1String(
"stddev" ) ).toDouble();
257 if ( schemaObj.contains( QLatin1String(
"variance" ) ) )
259 stats.variance = schemaObj.value( QLatin1String(
"variance" ) ).toDouble();
263 mMetadataStats.insert( name, stats );
265 if ( schemaObj.contains( QLatin1String(
"counts" ) ) )
267 QMap< int, int > classCounts;
268 const QJsonArray counts = schemaObj.value( QLatin1String(
"counts" ) ).toArray();
269 for (
const QJsonValue &count : counts )
271 const QJsonObject countObj = count.toObject();
272 classCounts.insert( countObj.value( QLatin1String(
"value" ) ).toInt(), countObj.value( QLatin1String(
"count" ) ).toInt() );
274 mAttributeClasses.insert( name, classCounts );
277 setAttributes( attributes );
282 const double xmin = bounds[0].toDouble();
283 const double ymin = bounds[1].toDouble();
284 const double zmin = bounds[2].toDouble();
285 const double xmax = bounds[3].toDouble();
286 const double ymax = bounds[4].toDouble();
287 const double zmax = bounds[5].toDouble();
290 ( xmin - mOffset.x() ) / mScale.x(),
291 ( ymin - mOffset.y() ) / mScale.y(),
292 ( zmin - mOffset.z() ) / mScale.z(),
293 ( xmax - mOffset.x() ) / mScale.x(),
294 ( ymax - mOffset.y() ) / mScale.y(),
295 ( zmax - mOffset.z() ) / mScale.z()
300 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
301 QgsDebugMsgLevel( QStringLiteral(
"lvl0 node size in CRS units: %1 %2 %3" ).arg( dx ).arg( dy ).arg( dz ), 2 );
302 QgsDebugMsgLevel( QStringLiteral(
"res at lvl0 %1" ).arg( dx / mSpan ), 2 );
303 QgsDebugMsgLevel( QStringLiteral(
"res at lvl1 %1" ).arg( dx / mSpan / 2 ), 2 );
304 QgsDebugMsgLevel( QStringLiteral(
"res at lvl2 %1 with node size %2" ).arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
314 return std::unique_ptr<QgsPointCloudBlock>( cached );
317 mHierarchyMutex.lock();
318 const bool found = mHierarchy.contains( n );
319 mHierarchyMutex.unlock();
326 QgsPointCloudExpression filterExpression = mFilterExpression;
328 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
331 std::unique_ptr<QgsPointCloudBlock> decoded;
332 if ( mDataType == QLatin1String(
"binary" ) )
334 const QString filename = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mDirectory, n.
toString() );
335 decoded = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
337 else if ( mDataType == QLatin1String(
"zstandard" ) )
339 const QString filename = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mDirectory, n.
toString() );
340 decoded = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
342 else if ( mDataType == QLatin1String(
"laszip" ) )
344 const QString filename = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mDirectory, n.
toString() );
345 decoded = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
348 storeNodeDataToCache( decoded.get(), n, request );
365qint64 QgsEptPointCloudIndex::pointCount()
const
370bool QgsEptPointCloudIndex::hasStatisticsMetadata()
const
372 return !mMetadataStats.isEmpty();
375QVariant QgsEptPointCloudIndex::metadataStatistic(
const QString &attribute,
Qgis::Statistic statistic )
const
377 if ( !mMetadataStats.contains( attribute ) )
380 const AttributeStatistics &stats = mMetadataStats[ attribute ];
384 return stats.count >= 0 ? QVariant( stats.count ) : QVariant();
387 return std::isnan( stats.mean ) ? QVariant() : QVariant( stats.mean );
390 return std::isnan( stats.stDev ) ? QVariant() : QVariant( stats.stDev );
393 return stats.minimum;
396 return stats.maximum;
399 return stats.minimum.isValid() && stats.maximum.isValid() ? QVariant( stats.maximum.toDouble() - stats.minimum.toDouble() ) : QVariant();
419QVariantList QgsEptPointCloudIndex::metadataClasses(
const QString &attribute )
const
421 QVariantList classes;
422 const QMap< int, int > values = mAttributeClasses.value( attribute );
423 for (
auto it = values.constBegin(); it != values.constEnd(); ++it )
430QVariant QgsEptPointCloudIndex::metadataClassStatistic(
const QString &attribute,
const QVariant &value,
Qgis::Statistic statistic )
const
435 const QMap< int, int > values = mAttributeClasses.value( attribute );
436 if ( !values.contains( value.toInt() ) )
438 return values.value( value.toInt() );
441bool QgsEptPointCloudIndex::loadHierarchy()
443 QQueue<QString> queue;
444 queue.enqueue( QStringLiteral(
"0-0-0-0" ) );
445 while ( !queue.isEmpty() )
447 const QString filename = QStringLiteral(
"%1/ept-hierarchy/%2.json" ).arg( mDirectory, queue.dequeue() );
448 QFile fH( filename );
449 if ( !fH.open( QIODevice::ReadOnly ) )
451 QgsDebugMsgLevel( QStringLiteral(
"unable to read hierarchy from file %1" ).arg( filename ), 2 );
452 mError = QStringLiteral(
"unable to read hierarchy from file %1" ).arg( filename );
456 const QByteArray dataJsonH = fH.readAll();
457 QJsonParseError errH;
458 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
459 if ( errH.error != QJsonParseError::NoError )
461 QgsDebugMsgLevel( QStringLiteral(
"QJsonParseError when reading hierarchy from file %1" ).arg( filename ), 2 );
462 mError = QStringLiteral(
"QJsonParseError when reading hierarchy from file %1" ).arg( filename );
466 const QJsonObject rootHObj = docH.object();
467 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
469 const QString nodeIdStr = it.key();
470 const int nodePointCount = it.value().toInt();
471 if ( nodePointCount < 0 )
473 queue.enqueue( nodeIdStr );
478 mHierarchyMutex.lock();
479 mHierarchy[nodeId] = nodePointCount;
480 mHierarchyMutex.unlock();
487bool QgsEptPointCloudIndex::isValid()
const
492void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination )
const
497 destination->mIsValid = mIsValid;
498 destination->mDataType = mDataType;
499 destination->mDirectory = mDirectory;
500 destination->mWkt = mWkt;
501 destination->mPointCount = mPointCount;
502 destination->mMetadataStats = mMetadataStats;
503 destination->mAttributeClasses = mAttributeClasses;
504 destination->mOriginalMetadata = mOriginalMetadata;
Represents a indexed point cloud node in octree.
static IndexedPointCloudNode fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
Statistic
Available generic statistics.
@ StDev
Standard deviation of values.
@ FirstQuartile
First quartile.
@ Median
Median of values.
@ Range
Range of values (max - min)
@ Minority
Minority of values.
@ CountMissing
Number of missing (null) values.
@ Majority
Majority of values.
@ Variety
Variety (count of distinct) values.
@ StDevSample
Sample standard deviation of values.
@ ThirdQuartile
Third quartile.
@ InterQuartileRange
Inter quartile range (IQR)
This class represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
Base class for storing raw data from point cloud nodes.
Represents packaged data bounds.
void copyCommonProperties(QgsPointCloudIndex *destination) const
Copies common properties to the destination index.
Point cloud data request.
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
A rectangle specified with double values.
#define QgsDebugMsgLevel(str, level)