QGIS API Documentation 3.40.0-Bratislava (b56115d8743)
Loading...
Searching...
No Matches
qgstiledownloadmanager.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgstiledownloadmanager.cpp
3 --------------------------
4 begin : January 2021
5 copyright : (C) 2021 by Martin Dobias
6 email : wonder dot sk at gmail dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
19
20#include "qgslogger.h"
23#include "qgssettings.h"
26
27#include <QElapsedTimer>
28#include <QNetworkReply>
29#include <QStandardPaths>
30#include <QRegularExpression>
31
33
34QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker( QgsTileDownloadManager *manager, QObject *parent )
35 : QObject( parent )
36 , mManager( manager )
37 , mIdleTimer( this )
38{
39 connect( &mIdleTimer, &QTimer::timeout, this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
40}
41
42void QgsTileDownloadManagerWorker::startIdleTimer()
43{
44 if ( !mIdleTimer.isActive() )
45 {
46 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
47 }
48}
49
50void QgsTileDownloadManagerWorker::queueUpdated()
51{
52 const QMutexLocker locker( &mManager->mMutex );
53
54 if ( mManager->mShuttingDown )
55 {
56 // here we HAVE to build up a list of replies from the queue before do anything
57 // with them. Otherwise we can hit the situation where aborting the replies
58 // triggers immediately their removal from the queue, and we'll be modifying
59 // mQueue elsewhere while still trying to iterate over it here => crash
60 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
61 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
62 std::vector< QNetworkReply * > replies;
63 replies.reserve( mManager->mQueue.size() );
64 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
65 {
66 replies.emplace_back( it->networkReply );
67 }
68 // now abort all replies
69 for ( QNetworkReply *reply : replies )
70 {
71 reply->abort();
72 }
73
74 quitThread();
75 return;
76 }
77
78 if ( mIdleTimer.isActive() && !mManager->mQueue.empty() )
79 {
80 // if timer to kill thread is running: stop the timer, we have work to do
81 mIdleTimer.stop();
82 }
83
84 // There's a potential race here -- if a reply finishes while we're still in the middle of iterating over the queue,
85 // then the associated queue entry would get removed while we're iterating over the queue here.
86 // So instead defer the actual queue removal until we've finished iterating over the queue.
87 // WARNING: there may be event loops/processEvents in play here, because in some circumstances
88 // (authentication handling, ssl errors) QgsNetworkAccessManager will trigger these.
89 mManager->mStageQueueRemovals = true;
90 for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
91 {
92 if ( !it->networkReply )
93 {
94 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
95 // start entries which are not in progress
96
97 it->networkReply = QgsNetworkAccessManager::instance()->get( it->request );
98 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
99
100 ++mManager->mStats.networkRequestsStarted;
101 }
102 }
103 mManager->mStageQueueRemovals = false;
104 mManager->processStagedEntryRemovals();
105}
106
107void QgsTileDownloadManagerWorker::quitThread()
108{
109 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: stopping worker thread" ), 2 );
110
111 mManager->mWorker->deleteLater();
112 mManager->mWorker = nullptr;
113 // we signal to our worker thread it's time to go. Its finished() signal is connected
114 // to deleteLater() call, so it will get deleted automatically
115 mManager->mWorkerThread->quit();
116 mManager->mWorkerThread = nullptr;
117 mManager->mShuttingDown = false;
118}
119
120void QgsTileDownloadManagerWorker::idleTimerTimeout()
121{
122 const QMutexLocker locker( &mManager->mMutex );
123 Q_ASSERT( mManager->mQueue.empty() );
124 quitThread();
125}
126
127
129
130
131void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
132{
133 const QMutexLocker locker( &mManager->mMutex );
134
135 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
136
137 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
138 QByteArray data;
139
140 if ( reply->error() == QNetworkReply::NoError )
141 {
142 ++mManager->mStats.networkRequestsOk;
143 data = reply->readAll();
144 }
145 else
146 {
147 ++mManager->mStats.networkRequestsFailed;
148 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
149 if ( contentType.startsWith( QLatin1String( "text/plain" ) ) )
150 data = reply->readAll();
151 }
152
153 QMap<QNetworkRequest::Attribute, QVariant> attributes;
154 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
155 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
156 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
157 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
158
159 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
160 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
161
162 // Save loaded data to cache
163 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
164 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
165 {
166 mManager->mRangesCache->registerEntry( mRequest, data );
167 }
168
169 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
170
171 reply->deleteLater();
172
173 // kill the worker obj
174 deleteLater();
175
176 mManager->removeEntry( mRequest );
177
178 if ( mManager->mQueue.empty() )
179 {
180 // if this was the last thing in the queue, start a timer to kill thread after X seconds
181 mManager->mWorker->startIdleTimer();
182 }
183}
184
186
188
189
191{
192 mRangesCache.reset( new QgsRangeRequestCache );
193
194 const QgsSettings settings;
196 if ( cacheDirectory.isEmpty() )
197 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
198 if ( !cacheDirectory.endsWith( QDir::separator() ) )
199 {
200 cacheDirectory.push_back( QDir::separator() );
201 }
202 cacheDirectory += QLatin1String( "http-ranges" );
203 mRangesCache->setCacheDirectory( cacheDirectory );
205 mRangesCache->setCacheSize( cacheSize );
206}
207
209{
210 // make sure the worker thread is gone and any pending requests are canceled
211 shutdown();
212}
213
215{
216 const QMutexLocker locker( &mMutex );
217
218 if ( isCachedRangeRequest( request ) )
219 {
220 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
221 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
222 return reply;
223 }
224
225 if ( !mWorker )
226 {
227 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting worker thread" ), 2 );
228 mWorkerThread = new QThread;
229 mWorker = new QgsTileDownloadManagerWorker( this );
230 mWorker->moveToThread( mWorkerThread );
231 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
232 mWorkerThread->start();
233 }
234
235 QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
236
237 ++mStats.requestsTotal;
238
239 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
240 if ( !entry.isValid() )
241 {
242 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
243 // create a new entry and add it to queue
244 entry.request = request;
245 entry.objWorker = new QgsTileDownloadManagerReplyWorkerObject( this, request );
246 entry.objWorker->moveToThread( mWorkerThread );
247
248 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
249
250 addEntry( entry );
251 }
252 else
253 {
254 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
255
256 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
257
258 ++mStats.requestsMerged;
259 }
260
261 signalQueueModified();
262
263 return reply;
264}
265
267{
268 const QMutexLocker locker( &mMutex );
269
270 return !mQueue.empty();
271}
272
274{
275 QElapsedTimer t;
276 t.start();
277
278 while ( msec == -1 || t.elapsed() < msec )
279 {
280 {
281 const QMutexLocker locker( &mMutex );
282 if ( mQueue.empty() )
283 return true;
284 }
285 QThread::usleep( 1000 );
286 }
287
288 return false;
289}
290
292{
293 {
294 const QMutexLocker locker( &mMutex );
295 if ( !mWorkerThread )
296 return; // nothing to stop
297
298 // let's signal to the thread
299 mShuttingDown = true;
300 signalQueueModified();
301 }
302
303 // wait until the thread is gone
304 while ( 1 )
305 {
306 {
307 const QMutexLocker locker( &mMutex );
308 if ( !mWorkerThread )
309 return; // the thread has stopped
310 }
311
312 QThread::usleep( 1000 );
313 }
314}
315
317{
318 return mWorkerThread && mWorkerThread->isRunning();
319}
320
322{
323 const QMutexLocker locker( &mMutex );
325}
326
327QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest( const QNetworkRequest &request )
328{
329 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
330 {
331 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
332 return *it;
333 }
334 return QgsTileDownloadManager::QueueEntry();
335}
336
337void QgsTileDownloadManager::addEntry( const QgsTileDownloadManager::QueueEntry &entry )
338{
339 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
340 {
341 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader( "Range" ) != it->request.rawHeader( "Range" ) );
342 }
343
344 mQueue.emplace_back( entry );
345}
346
347void QgsTileDownloadManager::updateEntry( const QgsTileDownloadManager::QueueEntry &entry )
348{
349 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
350 {
351 if ( entry.request.url() == it->request.url() && entry.request.rawHeader( "Range" ) == it->request.rawHeader( "Range" ) )
352 {
353 *it = entry;
354 return;
355 }
356 }
357 Q_ASSERT( false );
358}
359
360void QgsTileDownloadManager::removeEntry( const QNetworkRequest &request )
361{
362 if ( mStageQueueRemovals )
363 {
364 mStagedQueueRemovals.emplace_back( request );
365 }
366 else
367 {
368 for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
369 {
370 if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
371 {
372 mQueue.erase( it );
373 return;
374 }
375 }
376 Q_ASSERT( false );
377 }
378}
379
380void QgsTileDownloadManager::processStagedEntryRemovals()
381{
382 Q_ASSERT( !mStageQueueRemovals );
383 for ( const QNetworkRequest &request : mStagedQueueRemovals )
384 {
385 removeEntry( request );
386 }
387 mStagedQueueRemovals.clear();
388}
389
390void QgsTileDownloadManager::signalQueueModified()
391{
392 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
393}
394
395bool QgsTileDownloadManager::isRangeRequest( const QNetworkRequest &request )
396{
397 if ( request.rawHeader( "Range" ).isEmpty() )
398 return false;
399 const thread_local QRegularExpression regex( "^bytes=\\d+-\\d+$" );
400 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader( "Range" ) ) );
401 return match.hasMatch();
402}
403
404bool QgsTileDownloadManager::isCachedRangeRequest( const QNetworkRequest &request )
405{
406 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
407 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
408 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
409}
410
412
413
414QgsTileDownloadManagerReply::QgsTileDownloadManagerReply( QgsTileDownloadManager *manager, const QNetworkRequest &request )
415 : mManager( manager )
416 , mRequest( request )
417{
418}
419
421{
422 const QMutexLocker locker( &mManager->mMutex );
423
424 if ( !mHasFinished )
425 {
426 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
427
428 ++mManager->mStats.requestsEarlyDeleted;
429 }
430}
431
432void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url, const QMap<QNetworkRequest::Attribute, QVariant> &attributes, const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers, const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error, const QString &errorString )
433{
434 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
435
436 mHasFinished = true;
437 mData = data;
438 mUrl = url;
439 mAttributes = attributes;
440 mHeaders = headers;
441 mRawHeaderPairs = rawHeaderPairs;
442 mError = error;
443 mErrorString = errorString;
444 emit finished();
445}
446
447void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
448{
449 QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
450 mHasFinished = true;
451 mData = mManager->mRangesCache->entry( mRequest );
452 mUrl = mRequest.url();
453 emit finished();
454}
455
456QVariant QgsTileDownloadManagerReply::attribute( QNetworkRequest::Attribute code )
457{
458 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
459}
460
461QVariant QgsTileDownloadManagerReply::header( QNetworkRequest::KnownHeaders header )
462{
463 return mHeaders.contains( header ) ? mHeaders.value( header ) : QVariant();
464}
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
T value(const QString &dynamicKeyPart=QString()) const
Returns settings value.
static const QgsSettingsEntryInteger64 * settingsNetworkCacheSize
Settings entry network cache directory.
static const QgsSettingsEntryString * settingsNetworkCacheDirectory
Settings entry network cache directory.
This class is a composition of two QSettings instances:
Definition qgssettings.h:64
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1) const
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)
Definition qgslogger.h:39