QGIS API Documentation 3.41.0-Master (3440c17df1d)
Loading...
Searching...
No Matches
qgstiledownloadmanager.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgstiledownloadmanager.cpp
3 --------------------------
4 begin : January 2021
5 copyright : (C) 2021 by Martin Dobias
6 email : wonder dot sk at gmail dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
19#include "moc_qgstiledownloadmanager.cpp"
20
21#include "qgslogger.h"
24#include "qgssettings.h"
27
28#include <QElapsedTimer>
29#include <QNetworkReply>
30#include <QStandardPaths>
31#include <QRegularExpression>
32
34
35QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker( QgsTileDownloadManager *manager, QObject *parent )
36 : QObject( parent )
37 , mManager( manager )
38 , mIdleTimer( this )
39{
40 connect( &mIdleTimer, &QTimer::timeout, this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
41}
42
43void QgsTileDownloadManagerWorker::startIdleTimer()
44{
45 if ( !mIdleTimer.isActive() )
46 {
47 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
48 }
49}
50
51void QgsTileDownloadManagerWorker::queueUpdated()
52{
53 const QMutexLocker locker( &mManager->mMutex );
54
55 if ( mManager->mShuttingDown )
56 {
57 // here we HAVE to build up a list of replies from the queue before do anything
58 // with them. Otherwise we can hit the situation where aborting the replies
59 // triggers immediately their removal from the queue, and we'll be modifying
60 // mQueue elsewhere while still trying to iterate over it here => crash
61 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
62 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
63 std::vector< QNetworkReply * > replies;
64 replies.reserve( mManager->mQueue.size() );
65 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
66 {
67 replies.emplace_back( it->networkReply );
68 }
69 // now abort all replies
70 for ( QNetworkReply *reply : replies )
71 {
72 reply->abort();
73 }
74
75 quitThread();
76 return;
77 }
78
79 if ( mIdleTimer.isActive() && !mManager->mQueue.empty() )
80 {
81 // if timer to kill thread is running: stop the timer, we have work to do
82 mIdleTimer.stop();
83 }
84
85 // There's a potential race here -- if a reply finishes while we're still in the middle of iterating over the queue,
86 // then the associated queue entry would get removed while we're iterating over the queue here.
87 // So instead defer the actual queue removal until we've finished iterating over the queue.
88 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
89 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
90 mManager->mStageQueueRemovals = true;
91 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
92 {
93 if ( !it->networkReply )
94 {
95 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
96 // start entries which are not in progress
97
98 it->networkReply = QgsNetworkAccessManager::instance()->get( it->request );
99 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
100
101 ++mManager->mStats.networkRequestsStarted;
102 }
103 }
104 mManager->mStageQueueRemovals = false;
105 mManager->processStagedEntryRemovals();
106}
107
108void QgsTileDownloadManagerWorker::quitThread()
109{
110 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: stopping worker thread" ), 2 );
111
112 mManager->mWorker->deleteLater();
113 mManager->mWorker = nullptr;
114 // we signal to our worker thread it's time to go. Its finished() signal is connected
115 // to deleteLater() call, so it will get deleted automatically
116 mManager->mWorkerThread->quit();
117 mManager->mWorkerThread = nullptr;
118 mManager->mShuttingDown = false;
119}
120
121void QgsTileDownloadManagerWorker::idleTimerTimeout()
122{
123 const QMutexLocker locker( &mManager->mMutex );
124 Q_ASSERT( mManager->mQueue.empty() );
125 quitThread();
126}
127
128
130
131
132void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
133{
134 const QMutexLocker locker( &mManager->mMutex );
135
136 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
137
138 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
139 QByteArray data;
140
141 if ( reply->error() == QNetworkReply::NoError )
142 {
143 ++mManager->mStats.networkRequestsOk;
144 data = reply->readAll();
145 }
146 else
147 {
148 ++mManager->mStats.networkRequestsFailed;
149 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
150 if ( contentType.startsWith( QLatin1String( "text/plain" ) ) )
151 data = reply->readAll();
152 }
153
154 QMap<QNetworkRequest::Attribute, QVariant> attributes;
155 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
156 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
157 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
158 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
159
160 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
161 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
162
163 // Save loaded data to cache
164 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
165 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
166 {
167 mManager->mRangesCache->registerEntry( mRequest, data );
168 }
169
170 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
171
172 reply->deleteLater();
173
174 // kill the worker obj
175 deleteLater();
176
177 mManager->removeEntry( mRequest );
178
179 if ( mManager->mQueue.empty() )
180 {
181 // if this was the last thing in the queue, start a timer to kill thread after X seconds
182 mManager->mWorker->startIdleTimer();
183 }
184}
185
187
189
190
192{
193 mRangesCache.reset( new QgsRangeRequestCache );
194
195 const QgsSettings settings;
197 if ( cacheDirectory.isEmpty() )
198 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
199 if ( !cacheDirectory.endsWith( QDir::separator() ) )
200 {
201 cacheDirectory.push_back( QDir::separator() );
202 }
203 cacheDirectory += QLatin1String( "http-ranges" );
204 mRangesCache->setCacheDirectory( cacheDirectory );
206 mRangesCache->setCacheSize( cacheSize );
207}
208
210{
211 // make sure the worker thread is gone and any pending requests are canceled
212 shutdown();
213}
214
216{
217 const QMutexLocker locker( &mMutex );
218
219 if ( isCachedRangeRequest( request ) )
220 {
221 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
222 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
223 return reply;
224 }
225
226 if ( !mWorker )
227 {
228 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting worker thread" ), 2 );
229 mWorkerThread = new QThread;
230 mWorker = new QgsTileDownloadManagerWorker( this );
231 mWorker->moveToThread( mWorkerThread );
232 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
233 mWorkerThread->start();
234 }
235
236 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
237
238 ++mStats.requestsTotal;
239
240 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
241 if ( !entry.isValid() )
242 {
243 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
244 // create a new entry and add it to queue
245 entry.request = request;
246 entry.objWorker = new QgsTileDownloadManagerReplyWorkerObject( this, request );
247 entry.objWorker->moveToThread( mWorkerThread );
248
249 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
250
251 addEntry( entry );
252 }
253 else
254 {
255 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
256
257 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
258
259 ++mStats.requestsMerged;
260 }
261
262 signalQueueModified();
263
264 return reply;
265}
266
268{
269 const QMutexLocker locker( &mMutex );
270
271 return !mQueue.empty();
272}
273
275{
276 QElapsedTimer t;
277 t.start();
278
279 while ( msec == -1 || t.elapsed() < msec )
280 {
281 {
282 const QMutexLocker locker( &mMutex );
283 if ( mQueue.empty() )
284 return true;
285 }
286 QThread::usleep( 1000 );
287 }
288
289 return false;
290}
291
293{
294 {
295 const QMutexLocker locker( &mMutex );
296 if ( !mWorkerThread )
297 return; // nothing to stop
298
299 // let's signal to the thread
300 mShuttingDown = true;
301 signalQueueModified();
302 }
303
304 // wait until the thread is gone
305 while ( 1 )
306 {
307 {
308 const QMutexLocker locker( &mMutex );
309 if ( !mWorkerThread )
310 return; // the thread has stopped
311 }
312
313 QThread::usleep( 1000 );
314 }
315}
316
318{
319 return mWorkerThread && mWorkerThread->isRunning();
320}
321
323{
324 const QMutexLocker locker( &mMutex );
326}
327
328QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest( const QNetworkRequest &request )
329{
330 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
331 {
332 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
333 return *it;
334 }
335 return QgsTileDownloadManager::QueueEntry();
336}
337
338void QgsTileDownloadManager::addEntry( const QgsTileDownloadManager::QueueEntry &entry )
339{
340 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
341 {
342 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader( "Range" ) != it->request.rawHeader( "Range" ) );
343 }
344
345 mQueue.emplace_back( entry );
346}
347
348void QgsTileDownloadManager::updateEntry( const QgsTileDownloadManager::QueueEntry &entry )
349{
350 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
351 {
352 if ( entry.request.url() == it->request.url() && entry.request.rawHeader( "Range" ) == it->request.rawHeader( "Range" ) )
353 {
354 *it = entry;
355 return;
356 }
357 }
358 Q_ASSERT( false );
359}
360
361void QgsTileDownloadManager::removeEntry( const QNetworkRequest &request )
362{
363 if ( mStageQueueRemovals )
364 {
365 mStagedQueueRemovals.emplace_back( request );
366 }
367 else
368 {
369 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
370 {
371 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
372 {
373 mQueue.erase( it );
374 return;
375 }
376 }
377 Q_ASSERT( false );
378 }
379}
380
381void QgsTileDownloadManager::processStagedEntryRemovals()
382{
383 Q_ASSERT( !mStageQueueRemovals );
384 for ( const QNetworkRequest &request : mStagedQueueRemovals )
385 {
386 removeEntry( request );
387 }
388 mStagedQueueRemovals.clear();
389}
390
391void QgsTileDownloadManager::signalQueueModified()
392{
393 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
394}
395
396bool QgsTileDownloadManager::isRangeRequest( const QNetworkRequest &request )
397{
398 if ( request.rawHeader( "Range" ).isEmpty() )
399 return false;
400 const thread_local QRegularExpression regex( "^bytes=\\d+-\\d+$" );
401 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader( "Range" ) ) );
402 return match.hasMatch();
403}
404
405bool QgsTileDownloadManager::isCachedRangeRequest( const QNetworkRequest &request )
406{
407 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
408 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
409 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
410}
411
413
414
415QgsTileDownloadManagerReply::QgsTileDownloadManagerReply( QgsTileDownloadManager *manager, const QNetworkRequest &request )
416 : mManager( manager )
417 , mRequest( request )
418{
419}
420
422{
423 const QMutexLocker locker( &mManager->mMutex );
424
425 if ( !mHasFinished )
426 {
427 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
428
429 ++mManager->mStats.requestsEarlyDeleted;
430 }
431}
432
433void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url, const QMap<QNetworkRequest::Attribute, QVariant> &attributes, const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers, const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error, const QString &errorString )
434{
435 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
436
437 mHasFinished = true;
438 mData = data;
439 mUrl = url;
440 mAttributes = attributes;
441 mHeaders = headers;
442 mRawHeaderPairs = rawHeaderPairs;
443 mError = error;
444 mErrorString = errorString;
445 emit finished();
446}
447
448void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
449{
450 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
451 mHasFinished = true;
452 mData = mManager->mRangesCache->entry( mRequest );
453 mUrl = mRequest.url();
454 emit finished();
455}
456
457QVariant QgsTileDownloadManagerReply::attribute( QNetworkRequest::Attribute code )
458{
459 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
460}
461
462QVariant QgsTileDownloadManagerReply::header( QNetworkRequest::KnownHeaders header )
463{
464 return mHeaders.contains( header ) ? mHeaders.value( header ) : QVariant();
465}
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
T value(const QString &dynamicKeyPart=QString()) const
Returns settings value.
static const QgsSettingsEntryInteger64 * settingsNetworkCacheSize
Settings entry network cache directory.
static const QgsSettingsEntryString * settingsNetworkCacheDirectory
Settings entry network cache directory.
This class is a composition of two QSettings instances:
Definition qgssettings.h:64
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1) const
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)
Definition qgslogger.h:39