QGIS API Documentation 3.31.0-Master (1f25b41d79)
qgsvectortileloader.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgsvectortileloader.cpp
3 --------------------------------------
4 Date : March 2020
5 Copyright : (C) 2020 by Martin Dobias
6 Email : wonder dot sk at gmail dot com
7 ***************************************************************************
8 * *
9 * This program is free software; you can redistribute it and/or modify *
10 * it under the terms of the GNU General Public License as published by *
11 * the Free Software Foundation; either version 2 of the License, or *
12 * (at your option) any later version. *
13 * *
14 ***************************************************************************/
15
16#include "qgsvectortileloader.h"
17
18#include <QEventLoop>
19
21#include "qgslogger.h"
22#include "qgsmbtiles.h"
23#include "qgsvtpktiles.h"
25#include "qgsvectortileutils.h"
26#include "qgsapplication.h"
27#include "qgsauthmanager.h"
28#include "qgsmessagelog.h"
29#include "qgsziputils.h"
30
32
33QgsVectorTileLoader::QgsVectorTileLoader( const QString &uri, const QgsTileMatrix &tileMatrix, const QgsTileRange &range, const QPointF &viewCenter, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
34 : mEventLoop( new QEventLoop )
35 , mFeedback( feedback )
36 , mAuthCfg( authid )
37 , mHeaders( headers )
38{
39 if ( feedback )
40 {
41 connect( feedback, &QgsFeedback::canceled, this, &QgsVectorTileLoader::canceled, Qt::QueuedConnection );
42
43 // rendering could have been canceled before we started to listen to canceled() signal
44 // so let's check before doing the download and maybe quit prematurely
45 if ( feedback->isCanceled() )
46 return;
47 }
48
49 QgsDebugMsgLevel( QStringLiteral( "Starting network loader" ), 2 );
50 QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
52 for ( QgsTileXYZ id : std::as_const( tiles ) )
53 {
54 loadFromNetworkAsync( id, tileMatrix, uri );
55 }
56}
57
59{
60 QgsDebugMsgLevel( QStringLiteral( "Terminating network loader" ), 2 );
61
62 if ( !mReplies.isEmpty() )
63 {
64 // this can happen when the loader is terminated without getting requests finalized
65 // (e.g. downloadBlocking() was not called)
66 canceled();
67 }
68}
69
71{
72 if ( mFeedback && mFeedback->isCanceled() )
73 {
74 QgsDebugMsgLevel( QStringLiteral( "downloadBlocking - not staring event loop - canceled" ), 2 );
75 return; // nothing to do
76 }
77
78 QgsDebugMsgLevel( QStringLiteral( "Starting event loop with %1 requests" ).arg( mReplies.count() ), 2 );
79
80 mEventLoop->exec( QEventLoop::ExcludeUserInputEvents );
81
82 QgsDebugMsgLevel( QStringLiteral( "downloadBlocking finished" ), 2 );
83
84 Q_ASSERT( mReplies.isEmpty() );
85}
86
87void QgsVectorTileLoader::loadFromNetworkAsync( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl )
88{
89 QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
90 QNetworkRequest request( url );
91 QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsVectorTileLoader" ) );
92 QgsSetRequestInitiatorId( request, id.toString() );
93
94 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ), id.column() );
95 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ), id.row() );
96 request.setAttribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ), id.zoomLevel() );
97
98 request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
99 request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
100
101 mHeaders.updateNetworkRequest( request );
102
103 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
104 {
105 QgsMessageLog::logMessage( tr( "network request update failed for authentication config" ), tr( "Network" ) );
106 }
107
109 connect( reply, &QgsTileDownloadManagerReply::finished, this, &QgsVectorTileLoader::tileReplyFinished );
110 mReplies << reply;
111}
112
113void QgsVectorTileLoader::tileReplyFinished()
114{
115 QgsTileDownloadManagerReply *reply = qobject_cast<QgsTileDownloadManagerReply *>( sender() );
116
117 int reqX = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ) ).toInt();
118 int reqY = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ) ).toInt();
119 int reqZ = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ) ).toInt();
120 QgsTileXYZ tileID( reqX, reqY, reqZ );
121
122 if ( reply->error() == QNetworkReply::NoError )
123 {
124 // TODO: handle redirections?
125
126 QgsDebugMsgLevel( QStringLiteral( "Tile download successful: " ) + tileID.toString(), 2 );
127 QByteArray rawData = reply->data();
128 mReplies.removeOne( reply );
129 reply->deleteLater();
130
131 emit tileRequestFinished( QgsVectorTileRawData( tileID, rawData ) );
132 }
133 else
134 {
135 if ( reply->error() == QNetworkReply::ContentAccessDenied )
136 {
137 if ( reply->data().isEmpty() )
138 mError = tr( "Access denied" );
139 else
140 mError = tr( "Access denied: %1" ).arg( QString( reply->data() ) );
141 }
142
143 QgsDebugMsg( QStringLiteral( "Tile download failed! " ) + reply->errorString() );
144 mReplies.removeOne( reply );
145 reply->deleteLater();
146
147 emit tileRequestFinished( QgsVectorTileRawData( tileID, QByteArray() ) );
148 }
149
150 if ( mReplies.isEmpty() )
151 {
152 // exist the event loop
153 QMetaObject::invokeMethod( mEventLoop.get(), "quit", Qt::QueuedConnection );
154 }
155}
156
157void QgsVectorTileLoader::canceled()
158{
159 QgsDebugMsgLevel( QStringLiteral( "Canceling %1 pending requests" ).arg( mReplies.count() ), 2 );
160 qDeleteAll( mReplies );
161 mReplies.clear();
162
163 // stop blocking download
164 mEventLoop->quit();
165
166}
167
169{
170 return mError;
171}
172
174
175QList<QgsVectorTileRawData> QgsVectorTileLoader::blockingFetchTileRawData( const QString &sourceType, const QString &sourcePath, const QgsTileMatrix &tileMatrix, const QPointF &viewCenter, const QgsTileRange &range, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
176{
177 QList<QgsVectorTileRawData> rawTiles;
178
179 std::unique_ptr< QgsMbTiles > mbReader;
180 std::unique_ptr< QgsVtpkTiles > vtpkReader;
181 bool isUrl = ( sourceType == QLatin1String( "xyz" ) );
182 if ( sourceType == QLatin1String( "vtpk" ) )
183 {
184 vtpkReader = std::make_unique< QgsVtpkTiles >( sourcePath );
185 vtpkReader->open();
186 }
187 else if ( !isUrl )
188 {
189 mbReader = std::make_unique< QgsMbTiles >( sourcePath );
190 bool res = mbReader->open();
191 Q_UNUSED( res )
192 Q_ASSERT( res );
193 }
194
195 if ( feedback && feedback->isCanceled() )
196 return {};
197
198 QVector<QgsTileXYZ> tiles = QgsVectorTileUtils::tilesInRange( range, tileMatrix.zoomLevel() );
199
200 // if a tile matrix results in a HUGE number of tile requests, we skip the sort -- it can be expensive
201 if ( tiles.size() < 10000 )
203
204 rawTiles.reserve( tiles.size() );
205 for ( QgsTileXYZ id : std::as_const( tiles ) )
206 {
207 if ( feedback && feedback->isCanceled() )
208 return rawTiles;
209
210 QByteArray rawData = isUrl ? loadFromNetwork( id, tileMatrix, sourcePath, authid, headers, feedback )
211 : ( mbReader ? loadFromMBTiles( id, *mbReader ) : loadFromVtpk( id, *vtpkReader ) );
212 if ( !rawData.isEmpty() )
213 {
214 rawTiles.append( QgsVectorTileRawData( id, rawData ) );
215 }
216 }
217 return rawTiles;
218}
219
220QByteArray QgsVectorTileLoader::loadFromNetwork( const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback )
221{
222 QString url = QgsVectorTileUtils::formatXYZUrlTemplate( requestUrl, id, tileMatrix );
223 QNetworkRequest nr;
224 nr.setUrl( QUrl( url ) );
225
226 headers.updateNetworkRequest( nr );
227
229 req.setAuthCfg( authid );
230 QgsDebugMsgLevel( QStringLiteral( "Blocking request: " ) + url, 2 );
231 QgsBlockingNetworkRequest::ErrorCode errCode = req.get( nr, false, feedback );
232 if ( errCode != QgsBlockingNetworkRequest::NoError )
233 {
234 QgsDebugMsg( QStringLiteral( "Request failed: " ) + url );
235 return QByteArray();
236 }
237 QgsNetworkReplyContent reply = req.reply();
238 QgsDebugMsgLevel( QStringLiteral( "Request successful, content size %1" ).arg( reply.content().size() ), 2 );
239 return reply.content();
240}
241
242
243QByteArray QgsVectorTileLoader::loadFromMBTiles( const QgsTileXYZ &id, QgsMbTiles &mbTileReader, QgsFeedback *feedback )
244{
245 // MBTiles uses TMS specs with Y starting at the bottom while XYZ uses Y starting at the top
246 int rowTMS = pow( 2, id.zoomLevel() ) - id.row() - 1;
247 QByteArray gzippedTileData = mbTileReader.tileData( id.zoomLevel(), id.column(), rowTMS );
248 if ( gzippedTileData.isEmpty() )
249 {
250 return QByteArray();
251 }
252
253 if ( feedback && feedback->isCanceled() )
254 return QByteArray();
255
256 QByteArray data;
257 if ( !QgsZipUtils::decodeGzip( gzippedTileData, data ) )
258 {
259 QgsDebugMsg( QStringLiteral( "Failed to decompress tile " ) + id.toString() );
260 return QByteArray();
261 }
262
263 QgsDebugMsgLevel( QStringLiteral( "Tile blob size %1 -> uncompressed size %2" ).arg( gzippedTileData.size() ).arg( data.size() ), 2 );
264 return data;
265}
266
267QByteArray QgsVectorTileLoader::loadFromVtpk( const QgsTileXYZ &id, QgsVtpkTiles &vtpkTileReader )
268{
269 QByteArray tileData = vtpkTileReader.tileData( id.zoomLevel(), id.column(), id.row() );
270 if ( tileData.isEmpty() )
271 {
272 return QByteArray();
273 }
274 return tileData;
275}
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
bool updateNetworkRequest(QNetworkRequest &request, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkRequest with an authentication config.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "get" operation on the specified request.
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
Base class for feedback objects to be used for cancellation of something running in a worker thread.
Definition: qgsfeedback.h:45
void canceled()
Internal routines can connect to this signal if they use event loop.
bool isCanceled() const SIP_HOLDGIL
Tells whether the operation has been canceled already.
Definition: qgsfeedback.h:54
This class implements simple http header management.
bool updateNetworkRequest(QNetworkRequest &request) const
Updates a request by adding all the HTTP headers.
Utility class for reading and writing MBTiles files (which are SQLite3 databases).
Definition: qgsmbtiles.h:39
QByteArray tileData(int z, int x, int y) const
Returns raw tile data for given tile.
Definition: qgsmbtiles.cpp:143
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between...
QByteArray content() const
Returns the reply content.
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
QNetworkRequest request() const
Returns the original request for this reply object.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
void finished()
Emitted when the reply has finished (either with a success or with a failure)
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
Defines a matrix of tiles for a single zoom level: it is defined by its size (width *.
Definition: qgstiles.h:108
int zoomLevel() const
Returns the zoom level of the tile matrix.
Definition: qgstiles.h:146
Range of tiles in a tile matrix to be rendered.
Definition: qgstiles.h:71
Stores coordinates of a tile in a tile matrix set.
Definition: qgstiles.h:38
void tileRequestFinished(const QgsVectorTileRawData &rawTile)
Emitted when a tile request has finished. If a tile request has failed, the returned raw tile byte ar...
static QList< QgsVectorTileRawData > blockingFetchTileRawData(const QString &sourceType, const QString &sourcePath, const QgsTileMatrix &tileMatrix, const QPointF &viewCenter, const QgsTileRange &range, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback=nullptr)
Returns raw tile data for the specified range of tiles. Blocks the caller until all tiles are fetched...
QString error() const
Returns a eventual error that occurred during loading, void if no error.
static QByteArray loadFromNetwork(const QgsTileXYZ &id, const QgsTileMatrix &tileMatrix, const QString &requestUrl, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback=nullptr)
Returns raw tile data for a single tile, doing a HTTP request. Block the caller until tile data are d...
static QByteArray loadFromMBTiles(const QgsTileXYZ &id, QgsMbTiles &mbTileReader, QgsFeedback *feedback=nullptr)
Returns raw tile data for a single tile loaded from MBTiles file.
void downloadBlocking()
Blocks the caller until all asynchronous requests are finished (with a success or a failure)
QgsVectorTileLoader(const QString &uri, const QgsTileMatrix &tileMatrix, const QgsTileRange &range, const QPointF &viewCenter, const QString &authid, const QgsHttpHeaders &headers, QgsFeedback *feedback)
Constructs tile loader for doing asynchronous requests and starts network requests.
static QByteArray loadFromVtpk(const QgsTileXYZ &id, QgsVtpkTiles &vtpkTileReader)
Returns raw tile data for a single tile loaded from VTPK file.
Keeps track of raw tile data that need to be decoded.
static QString formatXYZUrlTemplate(const QString &url, QgsTileXYZ tile, const QgsTileMatrix &tileMatrix)
Returns formatted tile URL string replacing {x}, {y}, {z} placeholders (or {-y} instead of {y} for TM...
static QVector< QgsTileXYZ > tilesInRange(QgsTileRange range, int zoomLevel)
Returns a list of tiles in the given tile range.
static void sortTilesByDistanceFromCenter(QVector< QgsTileXYZ > &tiles, QPointF center)
Orders tile requests according to the distance from view center (given in tile matrix coords)
Utility class for reading and writing ESRI VTPK files.
Definition: qgsvtpktiles.h:38
QByteArray tileData(int z, int x, int y)
Returns the raw tile data for the matching tile.
CORE_EXPORT bool decodeGzip(const QByteArray &bytesIn, QByteArray &bytesOut)
Decodes gzip byte stream, returns true on success.
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39
#define QgsDebugMsg(str)
Definition: qgslogger.h:38
#define QgsSetRequestInitiatorClass(request, _class)
#define QgsSetRequestInitiatorId(request, str)