QGIS API Documentation 3.37.0-Master (fa0c88a90b0)
qgstiledownloadmanager.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgstiledownloadmanager.cpp
3 --------------------------
4 begin : January 2021
5 copyright : (C) 2021 by Martin Dobias
6 email : wonder dot sk at gmail dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
19
20#include "qgslogger.h"
23#include "qgssettings.h"
24
25#include <QElapsedTimer>
26#include <QNetworkReply>
27#include <QStandardPaths>
28#include <QRegularExpression>
29
31
32QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker( QgsTileDownloadManager *manager, QObject *parent )
33 : QObject( parent )
34 , mManager( manager )
35 , mIdleTimer( this )
36{
37 connect( &mIdleTimer, &QTimer::timeout, this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
38}
39
40void QgsTileDownloadManagerWorker::startIdleTimer()
41{
42 if ( !mIdleTimer.isActive() )
43 {
44 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
45 }
46}
47
48void QgsTileDownloadManagerWorker::queueUpdated()
49{
50 const QMutexLocker locker( &mManager->mMutex );
51
52 if ( mManager->mShuttingDown )
53 {
54 // here we HAVE to build up a list of replies from the queue before do anything
55 // with them. Otherwise we can hit the situation where aborting the replies
56 // triggers immediately their removal from the queue, and we'll be modifying
57 // mQueue elsewhere while still trying to iterate over it here => crash
58 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
59 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
60 std::vector< QNetworkReply * > replies;
61 replies.reserve( mManager->mQueue.size() );
62 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
63 {
64 replies.emplace_back( it->networkReply );
65 }
66 // now abort all replies
67 for ( QNetworkReply *reply : replies )
68 {
69 reply->abort();
70 }
71
72 quitThread();
73 return;
74 }
75
76 if ( mIdleTimer.isActive() && !mManager->mQueue.empty() )
77 {
78 // if timer to kill thread is running: stop the timer, we have work to do
79 mIdleTimer.stop();
80 }
81
82 // There's a potential race here -- if a reply finishes while we're still in the middle of iterating over the queue,
83 // then the associated queue entry would get removed while we're iterating over the queue here.
84 // So instead defer the actual queue removal until we've finished iterating over the queue.
85 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
86 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
87 mManager->mStageQueueRemovals = true;
88 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
89 {
90 if ( !it->networkReply )
91 {
92 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
93 // start entries which are not in progress
94
95 it->networkReply = QgsNetworkAccessManager::instance()->get( it->request );
96 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
97
98 ++mManager->mStats.networkRequestsStarted;
99 }
100 }
101 mManager->mStageQueueRemovals = false;
102 mManager->processStagedEntryRemovals();
103}
104
105void QgsTileDownloadManagerWorker::quitThread()
106{
107 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: stopping worker thread" ), 2 );
108
109 mManager->mWorker->deleteLater();
110 mManager->mWorker = nullptr;
111 // we signal to our worker thread it's time to go. Its finished() signal is connected
112 // to deleteLater() call, so it will get deleted automatically
113 mManager->mWorkerThread->quit();
114 mManager->mWorkerThread = nullptr;
115 mManager->mShuttingDown = false;
116}
117
118void QgsTileDownloadManagerWorker::idleTimerTimeout()
119{
120 const QMutexLocker locker( &mManager->mMutex );
121 Q_ASSERT( mManager->mQueue.empty() );
122 quitThread();
123}
124
125
127
128
129void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
130{
131 const QMutexLocker locker( &mManager->mMutex );
132
133 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
134
135 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
136 QByteArray data;
137
138 if ( reply->error() == QNetworkReply::NoError )
139 {
140 ++mManager->mStats.networkRequestsOk;
141 data = reply->readAll();
142 }
143 else
144 {
145 ++mManager->mStats.networkRequestsFailed;
146 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
147 if ( contentType.startsWith( QLatin1String( "text/plain" ) ) )
148 data = reply->readAll();
149 }
150
151 QMap<QNetworkRequest::Attribute, QVariant> attributes;
152 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
153 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
154 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
155 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
156
157 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
158 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
159
160 // Save loaded data to cache
161 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
162 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
163 {
164 mManager->mRangesCache->registerEntry( mRequest, data );
165 }
166
167 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
168
169 reply->deleteLater();
170
171 // kill the worker obj
172 deleteLater();
173
174 mManager->removeEntry( mRequest );
175
176 if ( mManager->mQueue.empty() )
177 {
178 // if this was the last thing in the queue, start a timer to kill thread after X seconds
179 mManager->mWorker->startIdleTimer();
180 }
181}
182
184
186
187
189{
190 mRangesCache.reset( new QgsRangeRequestCache );
191
192 const QgsSettings settings;
193 QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString();
194 if ( cacheDirectory.isEmpty() )
195 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
196 if ( !cacheDirectory.endsWith( QDir::separator() ) )
197 {
198 cacheDirectory.push_back( QDir::separator() );
199 }
200 cacheDirectory += QLatin1String( "http-ranges" );
201 const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong();
202
203 mRangesCache->setCacheDirectory( cacheDirectory );
204 mRangesCache->setCacheSize( cacheSize );
205}
206
208{
209 // make sure the worker thread is gone and any pending requests are canceled
210 shutdown();
211}
212
214{
215 const QMutexLocker locker( &mMutex );
216
217 if ( isCachedRangeRequest( request ) )
218 {
219 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
220 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
221 return reply;
222 }
223
224 if ( !mWorker )
225 {
226 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting worker thread" ), 2 );
227 mWorkerThread = new QThread;
228 mWorker = new QgsTileDownloadManagerWorker( this );
229 mWorker->moveToThread( mWorkerThread );
230 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
231 mWorkerThread->start();
232 }
233
234 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
235
236 ++mStats.requestsTotal;
237
238 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
239 if ( !entry.isValid() )
240 {
241 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
242 // create a new entry and add it to queue
243 entry.request = request;
244 entry.objWorker = new QgsTileDownloadManagerReplyWorkerObject( this, request );
245 entry.objWorker->moveToThread( mWorkerThread );
246
247 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
248
249 addEntry( entry );
250 }
251 else
252 {
253 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
254
255 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
256
257 ++mStats.requestsMerged;
258 }
259
260 signalQueueModified();
261
262 return reply;
263}
264
266{
267 const QMutexLocker locker( &mMutex );
268
269 return !mQueue.empty();
270}
271
273{
274 QElapsedTimer t;
275 t.start();
276
277 while ( msec == -1 || t.elapsed() < msec )
278 {
279 {
280 const QMutexLocker locker( &mMutex );
281 if ( mQueue.empty() )
282 return true;
283 }
284 QThread::usleep( 1000 );
285 }
286
287 return false;
288}
289
291{
292 {
293 const QMutexLocker locker( &mMutex );
294 if ( !mWorkerThread )
295 return; // nothing to stop
296
297 // let's signal to the thread
298 mShuttingDown = true;
299 signalQueueModified();
300 }
301
302 // wait until the thread is gone
303 while ( 1 )
304 {
305 {
306 const QMutexLocker locker( &mMutex );
307 if ( !mWorkerThread )
308 return; // the thread has stopped
309 }
310
311 QThread::usleep( 1000 );
312 }
313}
314
316{
317 return mWorkerThread && mWorkerThread->isRunning();
318}
319
321{
322 const QMutexLocker locker( &mMutex );
324}
325
326QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest( const QNetworkRequest &request )
327{
328 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
329 {
330 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
331 return *it;
332 }
333 return QgsTileDownloadManager::QueueEntry();
334}
335
336void QgsTileDownloadManager::addEntry( const QgsTileDownloadManager::QueueEntry &entry )
337{
338 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
339 {
340 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader( "Range" ) != it->request.rawHeader( "Range" ) );
341 }
342
343 mQueue.emplace_back( entry );
344}
345
346void QgsTileDownloadManager::updateEntry( const QgsTileDownloadManager::QueueEntry &entry )
347{
348 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
349 {
350 if ( entry.request.url() == it->request.url() && entry.request.rawHeader( "Range" ) == it->request.rawHeader( "Range" ) )
351 {
352 *it = entry;
353 return;
354 }
355 }
356 Q_ASSERT( false );
357}
358
359void QgsTileDownloadManager::removeEntry( const QNetworkRequest &request )
360{
361 if ( mStageQueueRemovals )
362 {
363 mStagedQueueRemovals.emplace_back( request );
364 }
365 else
366 {
367 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
368 {
369 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
370 {
371 mQueue.erase( it );
372 return;
373 }
374 }
375 Q_ASSERT( false );
376 }
377}
378
379void QgsTileDownloadManager::processStagedEntryRemovals()
380{
381 Q_ASSERT( !mStageQueueRemovals );
382 for ( const QNetworkRequest &request : mStagedQueueRemovals )
383 {
384 removeEntry( request );
385 }
386 mStagedQueueRemovals.clear();
387}
388
389void QgsTileDownloadManager::signalQueueModified()
390{
391 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
392}
393
394bool QgsTileDownloadManager::isRangeRequest( const QNetworkRequest &request )
395{
396 if ( request.rawHeader( "Range" ).isEmpty() )
397 return false;
398 const thread_local QRegularExpression regex( "^bytes=\\d+-\\d+$" );
399 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader( "Range" ) ) );
400 return match.hasMatch();
401}
402
403bool QgsTileDownloadManager::isCachedRangeRequest( const QNetworkRequest &request )
404{
405 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
406 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
407 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
408}
409
411
412
413QgsTileDownloadManagerReply::QgsTileDownloadManagerReply( QgsTileDownloadManager *manager, const QNetworkRequest &request )
414 : mManager( manager )
415 , mRequest( request )
416{
417}
418
420{
421 const QMutexLocker locker( &mManager->mMutex );
422
423 if ( !mHasFinished )
424 {
425 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
426
427 ++mManager->mStats.requestsEarlyDeleted;
428 }
429}
430
431void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url, const QMap<QNetworkRequest::Attribute, QVariant> &attributes, const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers, const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error, const QString &errorString )
432{
433 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
434
435 mHasFinished = true;
436 mData = data;
437 mUrl = url;
438 mAttributes = attributes;
439 mHeaders = headers;
440 mRawHeaderPairs = rawHeaderPairs;
441 mError = error;
442 mErrorString = errorString;
443 emit finished();
444}
445
446void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
447{
448 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
449 mHasFinished = true;
450 mData = mManager->mRangesCache->entry( mRequest );
451 mUrl = mRequest.url();
452 emit finished();
453}
454
455QVariant QgsTileDownloadManagerReply::attribute( QNetworkRequest::Attribute code )
456{
457 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
458}
459
460QVariant QgsTileDownloadManagerReply::header( QNetworkRequest::KnownHeaders header )
461{
462 return mHeaders.contains( header ) ? mHeaders.value( header ) : QVariant();
463}
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
This class is a composition of two QSettings instances:
Definition: qgssettings.h:64
QVariant value(const QString &key, const QVariant &defaultValue=QVariant(), Section section=NoSection) const
Returns the value for setting key.
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1) const
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39