18#include <nlohmann/json.hpp>
20#include "nanoarrow/nanoarrow.hpp"
24#define QGIS_NANOARROW_THROW_NOT_OK_ERR( expr, err ) \
27 const int ec = ( expr ); \
28 if ( ec != NANOARROW_OK ) \
30 throw QgsException( QStringLiteral( "nanoarrow error (%1): %2" ).arg( ec ).arg( QString::fromUtf8( ( err )->message ) ) ); \
34#define QGIS_NANOARROW_THROW_NOT_OK( expr ) \
37 const int ec = ( expr ); \
38 if ( ec != NANOARROW_OK ) \
40 throw QgsException( QStringLiteral( "nanoarrow error (%1)" ).arg( ec ) ); \
55 return mGeometryColumnName;
64 mGeometryColumnIndex = other.mGeometryColumnIndex;
69 if ( mSchema.release )
71 ArrowSchemaRelease( &mSchema );
74 mGeometryColumnIndex = other.mGeometryColumnIndex;
80 if ( mSchema.release )
82 ArrowSchemaRelease( &mSchema );
98 return reinterpret_cast<unsigned long long>( &mSchema );
103 struct ArrowSchema *otherArrowSchema =
reinterpret_cast<struct ArrowSchema *
>( otherAddress );
109 return mSchema.release;
118 if ( mArray.release )
120 ArrowArrayRelease( &mArray );
123 ArrowArrayMove( other.array(), &mArray );
128 if (
this != &other )
130 ArrowArrayMove( other.array(), &mArray );
138 if ( mArray.release )
140 ArrowArrayRelease( &mArray );
156 return reinterpret_cast<unsigned long long>( &mArray );
161 struct ArrowArray *otherArrowArray =
reinterpret_cast<struct ArrowArray *
>( otherAddress );
162 ArrowArrayMove( &mArray, otherArrowArray );
167 return mArray.release;
180 std::string geoArrowMetadata;
181 if ( crsString.empty() )
183 geoArrowMetadata =
"{}";
187 geoArrowMetadata = R
"({"crs":)" + crsString + R"(})";
190 nanoarrow::UniqueBuffer metadataKv;
192 QGIS_NANOARROW_THROW_NOT_OK( ArrowMetadataBuilderAppend( metadataKv.get(), ArrowCharView(
"ARROW:extension:name" ), ArrowCharView(
"geoarrow.wkb" ) ) );
193 QGIS_NANOARROW_THROW_NOT_OK( ArrowMetadataBuilderAppend( metadataKv.get(), ArrowCharView(
"ARROW:extension:metadata" ), ArrowCharView( geoArrowMetadata.c_str() ) ) );
197 void appendGeometry(
const QgsFeature &feature,
struct ArrowArray *col )
206 struct ArrowBufferView v;
207 v.data.data = wkb.data();
208 v.size_bytes =
static_cast<int64_t
>( wkb.size() );
212 void inferMetaType(
const QMetaType::Type metaType,
struct ArrowSchema *col,
const QString &fieldName )
216 case QMetaType::Bool:
219 case QMetaType::QChar:
220 case QMetaType::SChar:
223 case QMetaType::UChar:
226 case QMetaType::Short:
229 case QMetaType::UShort:
235 case QMetaType::UInt:
238 case QMetaType::Long:
239 case QMetaType::LongLong:
242 case QMetaType::ULong:
243 case QMetaType::ULongLong:
246 case QMetaType::Float:
249 case QMetaType::Double:
252 case QMetaType::QString:
255 case QMetaType::QByteArray:
258 case QMetaType::QDate:
261 case QMetaType::QTime:
264 case QMetaType::QDateTime:
267 case QMetaType::QStringList:
272 throw QgsException( QStringLiteral(
"QgsArrowIterator can't infer field type '%1' for field '%2'" ).arg( QMetaType::typeName( metaType ) ).arg( fieldName ) );
276 void inferField(
const QgsField &field,
struct ArrowSchema *col )
279 switch ( field.
type() )
281 case QMetaType::QVariantList:
283 inferMetaType( field.
subType(), col->children[0], field.
name() );
286 inferMetaType( field.
type(), col, field.
name() );
291 void appendVariant(
const QVariant &v,
struct ArrowArray *col,
struct ArrowSchemaView &columnTypeView,
struct ArrowSchemaView &columnListTypeView )
299 switch ( columnTypeView.type )
301 case NANOARROW_TYPE_BOOL:
302 if ( v.canConvert( QMetaType::Bool ) )
308 case NANOARROW_TYPE_UINT8:
309 case NANOARROW_TYPE_UINT16:
310 case NANOARROW_TYPE_UINT32:
311 case NANOARROW_TYPE_UINT64:
312 if ( v.canConvert( QMetaType::ULongLong ) )
318 case NANOARROW_TYPE_INT8:
319 case NANOARROW_TYPE_INT16:
320 case NANOARROW_TYPE_INT32:
321 case NANOARROW_TYPE_INT64:
322 if ( v.canConvert( QMetaType::LongLong ) )
328 case NANOARROW_TYPE_HALF_FLOAT:
329 case NANOARROW_TYPE_FLOAT:
330 case NANOARROW_TYPE_DOUBLE:
331 if ( v.canConvert( QMetaType::Double ) )
337 case NANOARROW_TYPE_STRING:
338 case NANOARROW_TYPE_LARGE_STRING:
339 case NANOARROW_TYPE_STRING_VIEW:
341 if ( v.canConvert( QMetaType::QString ) )
343 const QByteArray
string = v.toString().toUtf8();
344 struct ArrowBufferView bytesView;
345 bytesView.data.data =
string.constData();
346 bytesView.size_bytes =
static_cast<int64_t
>(
string.size() );
353 case NANOARROW_TYPE_BINARY:
354 case NANOARROW_TYPE_LARGE_BINARY:
355 case NANOARROW_TYPE_BINARY_VIEW:
356 case NANOARROW_TYPE_FIXED_SIZE_BINARY:
358 if ( v.canConvert( QMetaType::QByteArray ) )
360 const QByteArray bytes = v.toByteArray();
361 struct ArrowBufferView bytesView;
362 bytesView.data.data = bytes.data();
363 bytesView.size_bytes =
static_cast<int64_t
>( bytes.size() );
370 case NANOARROW_TYPE_DATE32:
372 if ( v.canConvert( QMetaType::QDate ) )
374 static QDate epoch = QDate( 1970, 1, 1 );
375 int64_t daysSinceEpoch = epoch.daysTo( v.toDate() );
382 case NANOARROW_TYPE_DATE64:
384 if ( v.canConvert( QMetaType::QDate ) )
386 static QDate epoch = QDate( 1970, 1, 1 );
387 int64_t daysSinceEpoch = epoch.daysTo( v.toDate() );
388 int64_t msSinceEpoch = daysSinceEpoch * 24 * 60 * 60 * 1000;
395 case NANOARROW_TYPE_TIMESTAMP:
397 if ( v.canConvert( QMetaType::QDateTime ) )
399 const QDateTime dateTime = v.toDateTime().toUTC();
400 switch ( columnTypeView.time_unit )
402 case NANOARROW_TIME_UNIT_SECOND:
405 case NANOARROW_TIME_UNIT_MILLI:
408 case NANOARROW_TIME_UNIT_MICRO:
411 case NANOARROW_TIME_UNIT_NANO:
419 case NANOARROW_TYPE_TIME32:
420 case NANOARROW_TYPE_TIME64:
422 if ( v.canConvert( QMetaType::QTime ) )
424 const QTime time = v.toTime();
425 switch ( columnTypeView.time_unit )
427 case NANOARROW_TIME_UNIT_SECOND:
430 case NANOARROW_TIME_UNIT_MILLI:
433 case NANOARROW_TIME_UNIT_MICRO:
436 case NANOARROW_TIME_UNIT_NANO:
445 case NANOARROW_TYPE_LIST:
446 case NANOARROW_TYPE_FIXED_SIZE_LIST:
447 case NANOARROW_TYPE_LARGE_LIST:
448 case NANOARROW_TYPE_LIST_VIEW:
449 case NANOARROW_TYPE_LARGE_LIST_VIEW:
451 if ( v.canConvert( QMetaType::QVariantList ) )
453 const QVariantList variantList = v.toList();
454 struct ArrowSchemaView dummyListType {};
455 for (
const QVariant &item : variantList )
457 appendVariant( item, col->children[0], columnListTypeView, dummyListType );
469 throw QgsException( QStringLiteral(
"Can't convert variant of type '%1' to Arrow type '%2'" ).arg( v.typeName() ).arg( ArrowTypeString( columnTypeView.type ) ) );
475 : mFeatureIterator( featureIterator )
483 throw QgsException( QStringLiteral(
"Invalid or null ArrowSchema provided" ) );
494 throw QgsException( QStringLiteral(
"QgsArrowIterator can't iterate over less than one feature" ) );
497 if ( !mSchema.isValid() )
499 throw QgsException( QStringLiteral(
"QgsArrowIterator schema not set" ) );
505 const struct ArrowSchema *schema = mSchema.schema();
507 struct ArrowError error {};
510 struct ArrowSchemaView schemaView;
512 if ( schemaView.type != NANOARROW_TYPE_STRUCT )
514 throw QgsException( QStringLiteral(
"QgsArrowIterator expected requested schema as struct but got '%1'" ).arg( ArrowTypeString( schemaView.type ) ) );
517 std::vector<QString> columnNames( schema->n_children );
518 std::vector<struct ArrowSchemaView> colTypeViews( schema->n_children );
519 std::vector<struct ArrowSchemaView> colListTypeViews( schema->n_children );
520 for ( int64_t i = 0; i < schema->n_children; i++ )
523 columnNames[i] = QString( schema->children[i]->name !=
nullptr ? schema->children[i]->name : QString() );
527 switch ( colTypeViews[i].type )
529 case NANOARROW_TYPE_LIST:
530 case NANOARROW_TYPE_FIXED_SIZE_LIST:
531 case NANOARROW_TYPE_LARGE_LIST:
532 case NANOARROW_TYPE_LIST_VIEW:
533 case NANOARROW_TYPE_LARGE_LIST_VIEW:
535 struct ArrowSchemaView childView;
537 colListTypeViews[i] = std::move( childView );
541 colListTypeViews[i] = ArrowSchemaView {};
547 nanoarrow::UniqueArray tmp;
553 std::vector<int> featureAttributeIndex;
555 while ( n > 0 && mFeatureIterator.nextFeature( feature ) )
560 if ( featureAttributeIndex.empty() )
562 for ( int64_t i = 0; i < schema->n_children; i++ )
564 featureAttributeIndex.push_back( feature.
fieldNameIndex( columnNames[i] ) );
570 for ( int64_t i = 0; i < schema->n_children; i++ )
572 int attributeIndex = featureAttributeIndex[i];
573 struct ArrowArray *columnArray = tmp->children[i];
575 if ( i == mSchema.geometryColumnIndex() )
577 appendGeometry( feature, columnArray );
579 else if ( attributeIndex >= 0 && attributeIndex < feature.
attributeCount() )
581 appendVariant( feature.
attribute( attributeIndex ), columnArray, colTypeViews[i], colListTypeViews[i] );
595 if ( tmp->length > 0 )
597 ArrowArrayMove( tmp.get(), out.
array() );
604 bool layerHasGeometry = layer.
isSpatial();
621 QgisPrivateArrowSchemaInit( out.
schema() );
623 for (
int i = 0; i < fields.
count(); i++ )
625 inferField( fields.
field( i ), out.
schema()->children[i] );
631 if ( geometryColumnName.isEmpty() )
633 geometryColumnName = QStringLiteral(
"geometry" );
636 inferGeometry( out.
schema()->children[fields.
count()], geometryColumnName, crs );
@ FlagExportTrianglesAsPolygons
Triangles should be exported as polygon geometries.
Wrapper around an ArrowArray.
QgsArrowArray()=default
Construct invalid array holder.
struct ArrowArray * array()
Access the underlying ArrowArray from C++.
QgsArrowArray & operator=(QgsArrowArray &other)=delete
bool isValid() const
Returns true if this wrapper object holds a valid ArrowArray.
void exportToAddress(unsigned long long otherAddress)
Export this array to the address of an empty ArrowArray for export across boundaries.
unsigned long long cArrayAddress() const
Returns the address of the underlying ArrowArray for import or export across boundaries.
Options for inferring an ArrowSchema from a feature source.
void setGeometryColumnName(const QString &geometryColumnName)
Set the name that should be used to refer to the geometry column.
QgsArrowInferSchemaOptions()
Construct default options.
QString geometryColumnName() const
The name that should be used for a layer's geometry column.
static QgsArrowSchema inferSchema(const QgsVectorLayer &layer, const QgsArrowInferSchemaOptions &options=QgsArrowInferSchemaOptions())
Infer the QgsArrowSchema for a given QgsVectorLayer.
QgsArrowArray nextFeatures(int n)
Build an ArrowArray using the next n features (or fewer depending on the number of features remaining...
QgsArrowIterator()=default
Construct invalid iterator.
void setSchema(const QgsArrowSchema &schema)
Set the ArrowSchema for the output of all future batches.
Wrapper around an ArrowSchema.
int geometryColumnIndex() const
Returns the index of the column in this schema that should be populated with a feature geometry.
bool isValid() const
Returns true if this wrapper object holds a valid ArrowSchema.
unsigned long long cSchemaAddress() const
Returns the address of the underlying ArrowSchema for import or export across boundaries.
struct ArrowSchema * schema()
Access the underlying ArrowSchema from C++.
QgsArrowSchema & operator=(const QgsArrowSchema &other)
Assignment operator.
void exportToAddress(unsigned long long otherAddress)
Export this array to the address of an empty ArrowSchema for export across boundaries.
QgsArrowSchema()
Construct invalid schema holder.
void setGeometryColumnIndex(int geometryColumnIndex)
Set the index of the column in this schema that should be populated with a feature geometry.
Represents a coordinate reference system (CRS).
std::string toJsonString(bool multiline=false, int indentationWidth=4, const QString &schema=QString()) const
Returns a JSON string representation of this CRS.
Defines a QGIS exception class.
Wrapper for iterator of features from vector data provider or vector layer.
The feature class encapsulates a single feature including its unique ID, geometry and a list of field...
int fieldNameIndex(const QString &fieldName) const
Utility method to get attribute index from name.
int attributeCount() const
Returns the number of attributes attached to the feature.
bool hasGeometry() const
Returns true if the feature has an associated geometry.
Q_INVOKABLE QVariant attribute(const QString &name) const
Lookup attribute value by attribute name.
Encapsulate a field in an attribute table or data source.
QMetaType::Type subType() const
If the field is a collection, gets its element's type.
Container of fields for a vector layer.
QgsField field(int fieldIdx) const
Returns the field at particular index (must be in range 0..N-1).
QByteArray asWkb(QgsAbstractGeometry::WkbFlags flags=QgsAbstractGeometry::WkbFlags()) const
Export the geometry to WKB.
QgsCoordinateReferenceSystem crs
static bool isNull(const QVariant &variant, bool silenceNullWarnings=false)
Returns true if the specified variant should be considered a NULL value.
virtual QString geometryColumnName() const
Returns the name of the column storing geometry, if applicable.
Represents a vector layer which manages a vector based dataset.
bool isSpatial() const final
Returns true if this is a geometry layer and false in case of NoGeometry (table only) or UnknownGeome...
QgsVectorDataProvider * dataProvider() final
Returns the layer's data provider, it may be nullptr.
#define QGIS_NANOARROW_THROW_NOT_OK_ERR(expr, err)
#define QGIS_NANOARROW_THROW_NOT_OK(expr)