QGIS API Documentation  3.27.0-Master (aef1b1ec20)
qgsvectortileloader.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsvectortileloader.cpp
3  --------------------------------------
4  Date : March 2020
5  Copyright : (C) 2020 by Martin Dobias
6  Email : wonder dot sk at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
16 #include "qgsvectortileloader.h"
17 
18 #include <QEventLoop>
19 
21 #include "qgslogger.h"
22 #include "qgsmbtiles.h"
23 #include "qgsvtpktiles.h"
25 #include "qgsvectortileutils.h"
26 #include "qgsapplication.h"
27 #include "qgsauthmanager.h"
28 #include "qgsmessagelog.h"
29 #include "qgsziputils.h"
30 
31 #include "qgstiledownloadmanager.h"
32 
33 QgsVectorTileLoader::QgsVectorTileLoader( const QString &uri, const QgsTileMatrix &tileMatrix, const QgsTileRange &range, const QPointF &viewCenter, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
34  : mEventLoop( new QEventLoop )
35  , mFeedback( feedback )
36  , mAuthCfg( authid )
37  , mHeaders( headers )
38 {
39  if ( feedback )
40  {
41  connect( feedback, &QgsFeedback::canceled, this, &QgsVectorTileLoader::canceled, Qt::QueuedConnection );
42 
43  // rendering could have been canceled before we started to listen to canceled() signal
44  // so let's check before doing the download and maybe quit prematurely
45  if ( feedback->isCanceled() )
46  return;
47  }
48 
49  QgsDebugMsgLevel( QStringLiteral( "Starting network loader" ), 2 );
50  QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
52  for ( QgsTileXYZ id : std::as_const( tiles ) )
53  {
54  loadFromNetworkAsync( id, tileMatrix, uri );
55  }
56 }
57 
59 {
60  QgsDebugMsgLevel( QStringLiteral( "Terminating network loader" ), 2 );
61 
62  if ( !mReplies.isEmpty() )
63  {
64  // this can happen when the loader is terminated without getting requests finalized
65  // (e.g. downloadBlocking() was not called)
66  canceled();
67  }
68 }
69 
71 {
72  if ( mFeedback && mFeedback->isCanceled() )
73  {
74  QgsDebugMsgLevel( QStringLiteral( "downloadBlocking - not staring event loop - canceled" ), 2 );
75  return; // nothing to do
76  }
77 
78  QgsDebugMsgLevel( QStringLiteral( "Starting event loop with %1 requests" ).arg( mReplies.count() ), 2 );
79 
80  mEventLoop->exec( QEventLoop::ExcludeUserInputEvents );
81 
82  QgsDebugMsgLevel( QStringLiteral( "downloadBlocking finished" ), 2 );
83 
84  Q_ASSERT( mReplies.isEmpty() );
85 }
86 
87 void QgsVectorTileLoader::loadFromNetworkAsync( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl )
88 {
89  QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
90  QNetworkRequest request( url );
91  QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsVectorTileLoader" ) );
92  QgsSetRequestInitiatorId( request, id.toString() );
93 
94  request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ), id.column() );
95  request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ), id.row() );
96  request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ), id.zoomLevel() );
97 
98  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
99  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
100 
101  mHeaders.updateNetworkRequest( request );
102 
103  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
104  {
105  QgsMessageLog::logMessage( tr( "network request update failed for authentication config" ), tr( "Network" ) );
106  }
107 
109  connect( reply, &QgsTileDownloadManagerReply::finished, this, &QgsVectorTileLoader::tileReplyFinished );
110  mReplies << reply;
111 }
112 
113 void QgsVectorTileLoader::tileReplyFinished()
114 {
115  QgsTileDownloadManagerReply *reply = qobject_cast<QgsTileDownloadManagerReply *>( sender() );
116 
117  int reqX = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ) ).toInt();
118  int reqY = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ) ).toInt();
119  int reqZ = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ) ).toInt();
120  QgsTileXYZ tileID( reqX, reqY, reqZ );
121 
122  if ( reply->error() == QNetworkReply::NoError )
123  {
124  // TODO: handle redirections?
125 
126  QgsDebugMsgLevel( QStringLiteral( "Tile download successful: " ) + tileID.toString(), 2 );
127  QByteArray rawData = reply->data();
128  mReplies.removeOne( reply );
129  reply->deleteLater();
130 
131  emit tileRequestFinished( QgsVectorTileRawData( tileID, rawData ) );
132  }
133  else
134  {
135  if ( reply->error() == QNetworkReply::ContentAccessDenied )
136  {
137  if ( reply->data().isEmpty() )
138  mError = tr( "Access denied" );
139  else
140  mError = tr( "Access denied: %1" ).arg( QString( reply->data() ) );
141  }
142 
143  QgsDebugMsg( QStringLiteral( "Tile download failed! " ) + reply->errorString() );
144  mReplies.removeOne( reply );
145  reply->deleteLater();
146 
147  emit tileRequestFinished( QgsVectorTileRawData( tileID, QByteArray() ) );
148  }
149 
150  if ( mReplies.isEmpty() )
151  {
152  // exist the event loop
153  QMetaObject::invokeMethod( mEventLoop.get(), "quit", Qt::QueuedConnection );
154  }
155 }
156 
157 void QgsVectorTileLoader::canceled()
158 {
159  QgsDebugMsgLevel( QStringLiteral( "Canceling %1 pending requests" ).arg( mReplies.count() ), 2 );
160  qDeleteAll( mReplies );
161  mReplies.clear();
162 
163  // stop blocking download
164  mEventLoop->quit();
165 
166 }
167 
169 {
170  return mError;
171 }
172 
174 
175 QList<QgsVectorTileRawData> QgsVectorTileLoader::blockingFetchTileRawData( const QString &sourceType, const QString &sourcePath, const QgsTileMatrix &tileMatrix, const QPointF &viewCenter, const QgsTileRange &range, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
176 {
177  QList<QgsVectorTileRawData> rawTiles;
178 
179  std::unique_ptr< QgsMbTiles > mbReader;
180  std::unique_ptr< QgsVtpkTiles > vtpkReader;
181  bool isUrl = ( sourceType == QLatin1String( "xyz" ) );
182  if ( sourceType == QLatin1String( "vtpk" ) )
183  {
184  vtpkReader = std::make_unique< QgsVtpkTiles >( sourcePath );
185  vtpkReader->open();
186  }
187  else if ( !isUrl )
188  {
189  mbReader = std::make_unique< QgsMbTiles >( sourcePath );
190  bool res = mbReader->open();
191  Q_UNUSED( res )
192  Q_ASSERT( res );
193  }
194 
195  if ( feedback && feedback->isCanceled() )
196  return {};
197 
198  QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
199 
200  // if a tile matrix results in a HUGE number of tile requests, we skip the sort -- it can be expensive
201  if ( tiles.size() < 10000 )
203 
204  rawTiles.reserve( tiles.size() );
205  for ( QgsTileXYZ id : std::as_const( tiles ) )
206  {
207  if ( feedback && feedback->isCanceled() )
208  return rawTiles;
209 
210  QByteArray rawData = isUrl ? loadFromNetwork( id, tileMatrix, sourcePath, authid, headers, feedback )
211  : ( mbReader ? loadFromMBTiles( id, *mbReader ) : loadFromVtpk( id, *vtpkReader ) );
212  if ( !rawData.isEmpty() )
213  {
214  rawTiles.append( QgsVectorTileRawData( id, rawData ) );
215  }
216  }
217  return rawTiles;
218 }
219 
220 QByteArray QgsVectorTileLoader::loadFromNetwork( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
221 {
222  QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
223  QNetworkRequest nr;
224  nr.setUrl( QUrl( url ) );
225 
226  headers.updateNetworkRequest( nr );
227 
229  req.setAuthCfg( authid );
230  QgsDebugMsgLevel( QStringLiteral( "Blocking request: " ) + url, 2 );
231  QgsBlockingNetworkRequest::ErrorCode errCode = req.get( nr, false, feedback );
232  if ( errCode != QgsBlockingNetworkRequest::NoError )
233  {
234  QgsDebugMsg( QStringLiteral( "Request failed: " ) + url );
235  return QByteArray();
236  }
237  QgsNetworkReplyContent reply = req.reply();
238  QgsDebugMsgLevel( QStringLiteral( "Request successful, content size %1" ).arg( reply.content().size() ), 2 );
239  return reply.content();
240 }
241 
242 
243 QByteArray QgsVectorTileLoader::loadFromMBTiles( const QgsTileXYZ &id, QgsMbTiles &mbTileReader, QgsFeedback *feedback )
244 {
245  // MBTiles uses TMS specs with Y starting at the bottom while XYZ uses Y starting at the top
246  int rowTMS = pow( 2, id.zoomLevel() ) - id.row() - 1;
247  QByteArray gzippedTileData = mbTileReader.tileData( id.zoomLevel(), id.column(), rowTMS );
248  if ( gzippedTileData.isEmpty() )
249  {
250  return QByteArray();
251  }
252 
253  if ( feedback && feedback->isCanceled() )
254  return QByteArray();
255 
256  QByteArray data;
257  if ( !QgsZipUtils::decodeGzip( gzippedTileData, data ) )
258  {
259  QgsDebugMsg( QStringLiteral( "Failed to decompress tile " ) + id.toString() );
260  return QByteArray();
261  }
262 
263  QgsDebugMsgLevel( QStringLiteral( "Tile blob size %1 -> uncompressed size %2" ).arg( gzippedTileData.size() ).arg( data.size() ), 2 );
264  return data;
265 }
266 
267 QByteArray QgsVectorTileLoader::loadFromVtpk( const QgsTileXYZ &id, QgsVtpkTiles &vtpkTileReader )
268 {
269  QByteArray tileData = vtpkTileReader.tileData( id.zoomLevel(), id.column(), id.row() );
270  if ( tileData.isEmpty() )
271  {
272  return QByteArray();
273  }
274  return tileData;
275 }
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
bool updateNetworkRequest(QNetworkRequest &request, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkRequest with an authentication config.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "get" operation on the specified request.
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
Base class for feedback objects to be used for cancellation of something running in a worker thread.
Definition: qgsfeedback.h:45
void canceled()
Internal routines can connect to this signal if they use event loop.
bool isCanceled() const SIP_HOLDGIL
Tells whether the operation has been canceled already.
Definition: qgsfeedback.h:54
This class implements simple http header management.
bool updateNetworkRequest(QNetworkRequest &request) const
Updates a request by adding all the HTTP headers.
Utility class for reading and writing MBTiles files (which are SQLite3 databases).
Definition: qgsmbtiles.h:39
QByteArray tileData(int z, int x, int y) const
Returns raw tile data for given tile.
Definition: qgsmbtiles.cpp:143
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between...
QByteArray content() const
Returns the reply content.
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
QNetworkRequest request() const
Returns the original request for this reply object.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
void finished()
Emitted when the reply has finished (either with a success or with a failure)
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
Defines a matrix of tiles for a single zoom level: it is defined by its size (width *.
Definition: qgstiles.h:108
int zoomLevel() const
Returns the zoom level of the tile matrix.
Definition: qgstiles.h:146
Range of tiles in a tile matrix to be rendered.
Definition: qgstiles.h:71
Stores coordinates of a tile in a tile matrix set.
Definition: qgstiles.h:38
void tileRequestFinished(const QgsVectorTileRawData &rawTile)
Emitted when a tile request has finished. If a tile request has failed, the returned raw tile byte ar...
static QList< QgsVectorTileRawData > blockingFetchTileRawData(const QString &sourceType, const QString &sourcePath, const QgsTileMatrix &tileMatrix, const QPointF &viewCenter, const QgsTileRange &range, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback=nullptr)
Returns raw tile data for the specified range of tiles. Blocks the caller until all tiles are fetched...
QString error() const
Returns a eventual error that occurred during loading, void if no error.
static QByteArray loadFromNetwork(const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback=nullptr)
Returns raw tile data for a single tile, doing a HTTP request. Block the caller until tile data are d...
static QByteArray loadFromMBTiles(const QgsTileXYZ &id, QgsMbTiles &mbTileReader, QgsFeedback *feedback=nullptr)
Returns raw tile data for a single tile loaded from MBTiles file.
void downloadBlocking()
Blocks the caller until all asynchronous requests are finished (with a success or a failure)
QgsVectorTileLoader(const QString &uri, const QgsTileMatrix &tileMatrix, const QgsTileRange &range, const QPointF &viewCenter, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback)
Constructs tile loader for doing asynchronous requests and starts network requests.
static QByteArray loadFromVtpk(const QgsTileXYZ &id, QgsVtpkTiles &vtpkTileReader)
Returns raw tile data for a single tile loaded from VTPK file.
Keeps track of raw tile data that need to be decoded.
static QString formatXYZUrlTemplate(const QString &url, QgsTileXYZ tile, const QgsTileMatrix &tileMatrix)
Returns formatted tile URL string replacing {x}, {y}, {z} placeholders (or {-y} instead of {y} for TM...
static QVector< QgsTileXYZ > tilesInRange(QgsTileRange range, int zoomLevel)
Returns a list of tiles in the given tile range.
static void sortTilesByDistanceFromCenter(QVector< QgsTileXYZ > &tiles, QPointF center)
Orders tile requests according to the distance from view center (given in tile matrix coords)
Utility class for reading and writing ESRI VTPK files.
Definition: qgsvtpktiles.h:38
QByteArray tileData(int z, int x, int y)
Returns the raw tile data for the matching tile.
CORE_EXPORT bool decodeGzip(const QByteArray &bytesIn, QByteArray &bytesOut)
Decodes gzip byte stream, returns true on success.
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39
#define QgsDebugMsg(str)
Definition: qgslogger.h:38
#define QgsSetRequestInitiatorClass(request, _class)
#define QgsSetRequestInitiatorId(request, str)