QGIS API Documentation  3.14.0-Pi (9f7028fd23)
qgsvectortilewriter.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsvectortilewriter.cpp
3  --------------------------------------
4  Date : April 2020
5  Copyright : (C) 2020 by Martin Dobias
6  Email : wonder dot sk at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
16 #include "qgsvectortilewriter.h"
17 
18 #include "qgsdatasourceuri.h"
19 #include "qgsfeedback.h"
20 #include "qgsjsonutils.h"
21 #include "qgslogger.h"
22 #include "qgsmbtiles.h"
23 #include "qgstiles.h"
24 #include "qgsvectorlayer.h"
26 #include "qgsvectortileutils.h"
27 
28 #include <nlohmann/json.hpp>
29 
30 #include <QDir>
31 #include <QFile>
32 #include <QFileInfo>
33 #include <QUrl>
34 
35 
37 {
38 }
39 
40 
42 {
43  if ( mMinZoom < 0 )
44  {
45  mErrorMessage = tr( "Invalid min. zoom level" );
46  return false;
47  }
48  if ( mMaxZoom > 24 )
49  {
50  mErrorMessage = tr( "Invalid max. zoom level" );
51  return false;
52  }
53 
54  std::unique_ptr<QgsMbTiles> mbtiles;
55 
56  QgsDataSourceUri dsUri;
57  dsUri.setEncodedUri( mDestinationUri );
58 
59  QString sourceType = dsUri.param( QStringLiteral( "type" ) );
60  QString sourcePath = dsUri.param( QStringLiteral( "url" ) );
61  if ( sourceType == QStringLiteral( "xyz" ) )
62  {
63  // remove the initial file:// scheme
64  sourcePath = QUrl( sourcePath ).toLocalFile();
65 
66  if ( !QgsVectorTileUtils::checkXYZUrlTemplate( sourcePath ) )
67  {
68  mErrorMessage = tr( "Invalid template for XYZ: " ) + sourcePath;
69  return false;
70  }
71  }
72  else if ( sourceType == QStringLiteral( "mbtiles" ) )
73  {
74  mbtiles.reset( new QgsMbTiles( sourcePath ) );
75  }
76  else
77  {
78  mErrorMessage = tr( "Unsupported source type for writing: " ) + sourceType;
79  return false;
80  }
81 
82  QgsRectangle outputExtent = mExtent;
83  if ( outputExtent.isEmpty() )
84  {
85  outputExtent = fullExtent();
86  if ( outputExtent.isEmpty() )
87  {
88  mErrorMessage = tr( "Failed to calculate output extent" );
89  return false;
90  }
91  }
92 
93  // figure out how many tiles we will need to do
94  int tilesToCreate = 0;
95  for ( int zoomLevel = mMinZoom; zoomLevel <= mMaxZoom; ++zoomLevel )
96  {
97  QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
98 
99  QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
100  tilesToCreate += ( tileRange.endRow() - tileRange.startRow() + 1 ) *
101  ( tileRange.endColumn() - tileRange.startColumn() + 1 );
102  }
103 
104  if ( tilesToCreate == 0 )
105  {
106  mErrorMessage = tr( "No tiles to generate" );
107  return false;
108  }
109 
110  if ( mbtiles )
111  {
112  if ( !mbtiles->create() )
113  {
114  mErrorMessage = tr( "Failed to create MBTiles file: " ) + sourcePath;
115  return false;
116  }
117 
118  // required metadata
119  mbtiles->setMetadataValue( "format", "pbf" );
120  mbtiles->setMetadataValue( "json", mbtilesJsonSchema() );
121 
122  // metadata specified by the client
123  const QStringList metaKeys = mMetadata.keys();
124  for ( const QString &key : metaKeys )
125  {
126  mbtiles->setMetadataValue( key, mMetadata[key].toString() );
127  }
128 
129  // default metadata that we always write (if not written by the client)
130  if ( !mMetadata.contains( "name" ) )
131  mbtiles->setMetadataValue( "name", "unnamed" ); // required by the spec
132  if ( !mMetadata.contains( "minzoom" ) )
133  mbtiles->setMetadataValue( "minzoom", QString::number( mMinZoom ) );
134  if ( !mMetadata.contains( "maxzoom" ) )
135  mbtiles->setMetadataValue( "maxzoom", QString::number( mMaxZoom ) );
136  if ( !mMetadata.contains( "bounds" ) )
137  {
138  try
139  {
140  QgsCoordinateTransform ct( QgsCoordinateReferenceSystem( "EPSG:3857" ), QgsCoordinateReferenceSystem( "EPSG:4326" ), mTransformContext );
141  QgsRectangle wgsExtent = ct.transform( outputExtent );
142  QString boundsStr = QString( "%1,%2,%3,%4" )
143  .arg( wgsExtent.xMinimum() ).arg( wgsExtent.yMinimum() )
144  .arg( wgsExtent.xMaximum() ).arg( wgsExtent.yMaximum() );
145  mbtiles->setMetadataValue( "bounds", boundsStr );
146  }
147  catch ( const QgsCsException & )
148  {
149  // bounds won't be written (not a problem - it is an optional value)
150  }
151  }
152  }
153 
154  int tilesCreated = 0;
155  for ( int zoomLevel = mMinZoom; zoomLevel <= mMaxZoom; ++zoomLevel )
156  {
157  QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
158 
159  QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
160  for ( int row = tileRange.startRow(); row <= tileRange.endRow(); ++row )
161  {
162  for ( int col = tileRange.startColumn(); col <= tileRange.endColumn(); ++col )
163  {
164  QgsTileXYZ tileID( col, row, zoomLevel );
165  QgsVectorTileMVTEncoder encoder( tileID );
166  encoder.setTransformContext( mTransformContext );
167 
168  for ( const Layer &layer : qgis::as_const( mLayers ) )
169  {
170  if ( ( layer.minZoom() >= 0 && zoomLevel < layer.minZoom() ) ||
171  ( layer.maxZoom() >= 0 && zoomLevel > layer.maxZoom() ) )
172  continue;
173 
174  encoder.addLayer( layer.layer(), feedback, layer.filterExpression(), layer.layerName() );
175  }
176 
177  if ( feedback && feedback->isCanceled() )
178  {
179  mErrorMessage = tr( "Operation has been canceled" );
180  return false;
181  }
182 
183  QByteArray tileData = encoder.encode();
184 
185  ++tilesCreated;
186  if ( feedback )
187  {
188  feedback->setProgress( static_cast<double>( tilesCreated ) / tilesToCreate * 100 );
189  }
190 
191  if ( tileData.isEmpty() )
192  {
193  // skipping empty tile - no need to write it
194  continue;
195  }
196 
197  if ( sourceType == QStringLiteral( "xyz" ) )
198  {
199  if ( !writeTileFileXYZ( sourcePath, tileID, tileMatrix, tileData ) )
200  return false; // error message already set
201  }
202  else // mbtiles
203  {
204  QByteArray gzipTileData;
205  QgsMbTiles::encodeGzip( tileData, gzipTileData );
206  int rowTMS = pow( 2, tileID.zoomLevel() ) - tileID.row() - 1;
207  mbtiles->setTileData( tileID.zoomLevel(), tileID.column(), rowTMS, gzipTileData );
208  }
209  }
210  }
211  }
212 
213  return true;
214 }
215 
217 {
218  QgsRectangle extent;
219  QgsCoordinateReferenceSystem destCrs( "EPSG:3857" );
220 
221  for ( const Layer &layer : mLayers )
222  {
223  QgsVectorLayer *vl = layer.layer();
224  QgsCoordinateTransform ct( vl->crs(), destCrs, mTransformContext );
225  try
226  {
227  QgsRectangle r = ct.transformBoundingBox( vl->extent() );
228  extent.combineExtentWith( r );
229  }
230  catch ( const QgsCsException & )
231  {
232  QgsDebugMsg( "Failed to reproject layer extent to destination CRS" );
233  }
234  }
235  return extent;
236 }
237 
238 bool QgsVectorTileWriter::writeTileFileXYZ( const QString &sourcePath, QgsTileXYZ tileID, const QgsTileMatrix &tileMatrix, const QByteArray &tileData )
239 {
240  QString filePath = QgsVectorTileUtils::formatXYZUrlTemplate( sourcePath, tileID, tileMatrix );
241 
242  // make dirs if needed
243  QFileInfo fi( filePath );
244  QDir fileDir = fi.dir();
245  if ( !fileDir.exists() )
246  {
247  if ( !fileDir.mkpath( "." ) )
248  {
249  mErrorMessage = tr( "Cannot create directory " ) + fileDir.path();
250  return false;
251  }
252  }
253 
254  QFile f( filePath );
255  if ( !f.open( QIODevice::WriteOnly ) )
256  {
257  mErrorMessage = tr( "Cannot open file for writing " ) + filePath;
258  return false;
259  }
260 
261  f.write( tileData );
262  f.close();
263  return true;
264 }
265 
266 
267 QString QgsVectorTileWriter::mbtilesJsonSchema()
268 {
269  QVariantList arrayLayers;
270  for ( const Layer &layer : qgis::as_const( mLayers ) )
271  {
272  QgsVectorLayer *vl = layer.layer();
273  const QgsFields fields = vl->fields();
274 
275  QVariantMap fieldsObj;
276  for ( const QgsField &field : fields )
277  {
278  QString fieldTypeStr;
279  if ( field.type() == QVariant::Bool )
280  fieldTypeStr = QStringLiteral( "Boolean" );
281  else if ( field.type() == QVariant::Int || field.type() == QVariant::Double )
282  fieldTypeStr = QStringLiteral( "Number" );
283  else
284  fieldTypeStr = QStringLiteral( "String" );
285 
286  fieldsObj[field.name()] = fieldTypeStr;
287  }
288 
289  QVariantMap layerObj;
290  layerObj["id"] = vl->name();
291  layerObj["fields"] = fieldsObj;
292  arrayLayers.append( layerObj );
293  }
294 
295  QVariantMap rootObj;
296  rootObj["vector_layers"] = arrayLayers;
297  return QString::fromStdString( QgsJsonUtils::jsonFromVariant( rootObj ).dump() );
298 }
QgsMapLayer::crs
QgsCoordinateReferenceSystem crs
Definition: qgsmaplayer.h:88
QgsFeedback::setProgress
void setProgress(double progress)
Sets the current progress for the feedback object.
Definition: qgsfeedback.h:75
QgsDataSourceUri
Definition: qgsdatasourceuri.h:35
QgsRectangle::combineExtentWith
void combineExtentWith(const QgsRectangle &rect)
Expands the rectangle so that it covers both the original rectangle and the given rectangle.
Definition: qgsrectangle.h:359
QgsTileXYZ
Definition: qgstiles.h:32
QgsTileRange
Definition: qgstiles.h:65
QgsRectangle::xMaximum
double xMaximum() const
Returns the x maximum value (right side of rectangle).
Definition: qgsrectangle.h:162
QgsFields
Definition: qgsfields.h:44
QgsVectorTileMVTEncoder::encode
QByteArray encode() const
Encodes MVT using data stored previously with addLayer() calls.
Definition: qgsvectortilemvtencoder.cpp:369
QgsVectorTileMVTEncoder::addLayer
void addLayer(QgsVectorLayer *layer, QgsFeedback *feedback=nullptr, QString filterExpression=QString(), QString layerName=QString())
Fetches data from vector layer for the given tile, does reprojection and clipping.
Definition: qgsvectortilemvtencoder.cpp:150
QgsDebugMsg
#define QgsDebugMsg(str)
Definition: qgslogger.h:38
QgsCoordinateTransform::transform
QgsPointXY transform(const QgsPointXY &point, TransformDirection direction=ForwardTransform) const SIP_THROW(QgsCsException)
Transform the point from the source CRS to the destination CRS.
Definition: qgscoordinatetransform.cpp:239
QgsRectangle
Definition: qgsrectangle.h:41
QgsCoordinateTransform::transformBoundingBox
QgsRectangle transformBoundingBox(const QgsRectangle &rectangle, TransformDirection direction=ForwardTransform, bool handle180Crossover=false) const SIP_THROW(QgsCsException)
Transforms a rectangle from the source CRS to the destination CRS.
Definition: qgscoordinatetransform.cpp:511
qgsmbtiles.h
QgsDataSourceUri::param
QString param(const QString &key) const
Returns a generic parameter value corresponding to the specified key.
Definition: qgsdatasourceuri.cpp:823
QgsMbTiles::encodeGzip
static bool encodeGzip(const QByteArray &bytesIn, QByteArray &bytesOut)
Encodes gzip byte stream, returns true on success. Useful for writing vector tiles.
Definition: qgsmbtiles.cpp:266
QgsVectorLayer::fields
QgsFields fields() const FINAL
Returns the list of fields of this layer.
Definition: qgsvectorlayer.cpp:3280
QgsTileRange::endRow
int endRow() const
Returns index of the last row in the range.
Definition: qgstiles.h:82
QgsTileMatrix
Definition: qgstiles.h:102
QgsMbTiles
Definition: qgsmbtiles.h:38
QgsCsException
Definition: qgsexception.h:65
QgsVectorTileUtils::checkXYZUrlTemplate
static bool checkXYZUrlTemplate(const QString &url)
Checks whether the URL template string is correct (contains {x}, {y} / {-y}, {z} placeholders)
Definition: qgsvectortileutils.cpp:136
qgsdatasourceuri.h
qgsvectortilemvtencoder.h
QgsFeedback
Definition: qgsfeedback.h:43
QgsVectorTileMVTEncoder::setTransformContext
void setTransformContext(const QgsCoordinateTransformContext &transformContext)
Sets coordinate transform context for transforms between layers and tile matrix CRS.
Definition: qgsvectortilemvtencoder.h:57
QgsVectorTileWriter::QgsVectorTileWriter
QgsVectorTileWriter()
Definition: qgsvectortilewriter.cpp:36
QgsTileRange::endColumn
int endColumn() const
Returns index of the last column in the range.
Definition: qgstiles.h:78
QgsTileXYZ::zoomLevel
int zoomLevel() const
Returns tile's zoom level (Z)
Definition: qgstiles.h:59
QgsCoordinateReferenceSystem
Definition: qgscoordinatereferencesystem.h:206
QgsRectangle::yMaximum
double yMaximum() const
Returns the y maximum value (top side of rectangle).
Definition: qgsrectangle.h:172
QgsVectorLayer::extent
QgsRectangle extent() const FINAL
Returns the extent of the layer.
Definition: qgsvectorlayer.cpp:833
qgsvectorlayer.h
QgsFeedback::isCanceled
bool isCanceled() const
Tells whether the operation has been canceled already.
Definition: qgsfeedback.h:66
qgsvectortilewriter.h
QgsTileRange::startRow
int startRow() const
Returns index of the first row in the range.
Definition: qgstiles.h:80
QgsVectorTileUtils::formatXYZUrlTemplate
static QString formatXYZUrlTemplate(const QString &url, QgsTileXYZ tile, const QgsTileMatrix &tileMatrix)
Returns formatted tile URL string replacing {x}, {y}, {z} placeholders (or {-y} instead of {y} for TM...
Definition: qgsvectortileutils.cpp:119
qgstiles.h
QgsDataSourceUri::setEncodedUri
void setEncodedUri(const QByteArray &uri)
Sets the complete encoded uri.
Definition: qgsdatasourceuri.cpp:630
QgsTileXYZ::row
int row() const
Returns tile's row index (Y)
Definition: qgstiles.h:57
QgsVectorLayer
Definition: qgsvectorlayer.h:385
QgsVectorTileMVTEncoder
Definition: qgsvectortilemvtencoder.h:40
QgsVectorTileWriter::Layer
Definition: qgsvectortilewriter.h:84
QgsMapLayer::name
QString name
Definition: qgsmaplayer.h:85
QgsRectangle::yMinimum
double yMinimum() const
Returns the y minimum value (bottom side of rectangle).
Definition: qgsrectangle.h:177
QgsTileMatrix::fromWebMercator
static QgsTileMatrix fromWebMercator(int mZoomLevel)
Returns a tile matrix for the usual web mercator.
Definition: qgstiles.cpp:20
QgsVectorTileWriter::writeTiles
bool writeTiles(QgsFeedback *feedback=nullptr)
Writes vector tiles according to the configuration.
Definition: qgsvectortilewriter.cpp:41
qgsjsonutils.h
qgslogger.h
qgsvectortileutils.h
QgsCoordinateTransform
Definition: qgscoordinatetransform.h:52
qgsfeedback.h
QgsRectangle::isEmpty
bool isEmpty() const
Returns true if the rectangle is empty.
Definition: qgsrectangle.h:437
QgsJsonUtils::jsonFromVariant
static json jsonFromVariant(const QVariant &v)
Converts a QVariant v to a json object.
Definition: qgsjsonutils.cpp:395
QgsTileMatrix::tileRangeFromExtent
QgsTileRange tileRangeFromExtent(const QgsRectangle &mExtent)
Returns tile range that fully covers the given extent.
Definition: qgstiles.cpp:54
QgsRectangle::xMinimum
double xMinimum() const
Returns the x minimum value (left side of rectangle).
Definition: qgsrectangle.h:167
QgsTileRange::startColumn
int startColumn() const
Returns index of the first column in the range.
Definition: qgstiles.h:76
QgsVectorTileWriter::fullExtent
QgsRectangle fullExtent() const
Returns calculated extent that combines extent of all input layers.
Definition: qgsvectortilewriter.cpp:216
QgsTileXYZ::column
int column() const
Returns tile's column index (X)
Definition: qgstiles.h:55
QgsField
Definition: qgsfield.h:49