QGIS API Documentation  3.27.0-Master (aef1b1ec20)
qgstiledownloadmanager.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgstiledownloadmanager.cpp
3  --------------------------
4  begin : January 2021
5  copyright : (C) 2021 by Martin Dobias
6  email : wonder dot sk at gmail dot com
7  ***************************************************************************/
8 
9 /***************************************************************************
10  * *
11  * This program is free software; you can redistribute it and/or modify *
12  * it under the terms of the GNU General Public License as published by *
13  * the Free Software Foundation; either version 2 of the License, or *
14  * (at your option) any later version. *
15  * *
16  ***************************************************************************/
17 
18 #include "qgstiledownloadmanager.h"
19 
20 #include "qgslogger.h"
22 #include "qgsrangerequestcache.h"
23 
24 #include <QElapsedTimer>
25 #include <QNetworkReply>
26 #include <QStandardPaths>
27 #include <QRegularExpression>
28 
30 
31 QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker( QgsTileDownloadManager *manager, QObject *parent )
32  : QObject( parent )
33  , mManager( manager )
34  , mIdleTimer( this )
35 {
36  connect( &mIdleTimer, &QTimer::timeout, this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
37 }
38 
39 void QgsTileDownloadManagerWorker::startIdleTimer()
40 {
41  if ( !mIdleTimer.isActive() )
42  {
43  mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
44  }
45 }
46 
47 void QgsTileDownloadManagerWorker::queueUpdated()
48 {
49  const QMutexLocker locker( &mManager->mMutex );
50 
51  if ( mManager->mShuttingDown )
52  {
53  for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
54  {
55  it->networkReply->abort();
56  }
57 
58  quitThread();
59  return;
60  }
61 
62  if ( mIdleTimer.isActive() && !mManager->mQueue.isEmpty() )
63  {
64  // if timer to kill thread is running: stop the timer, we have work to do
65  mIdleTimer.stop();
66  }
67 
68  for ( auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
69  {
70  if ( !it->networkReply )
71  {
72  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
73  // start entries which are not in progress
74 
75  it->networkReply = QgsNetworkAccessManager::instance()->get( it->request );
76  connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
77 
78  ++mManager->mStats.networkRequestsStarted;
79  }
80  }
81 }
82 
83 void QgsTileDownloadManagerWorker::quitThread()
84 {
85  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: stopping worker thread" ), 2 );
86 
87  mManager->mWorker->deleteLater();
88  mManager->mWorker = nullptr;
89  // we signal to our worker thread it's time to go. Its finished() signal is connected
90  // to deleteLater() call, so it will get deleted automatically
91  mManager->mWorkerThread->quit();
92  mManager->mWorkerThread = nullptr;
93  mManager->mShuttingDown = false;
94 }
95 
96 void QgsTileDownloadManagerWorker::idleTimerTimeout()
97 {
98  const QMutexLocker locker( &mManager->mMutex );
99  Q_ASSERT( mManager->mQueue.isEmpty() );
100  quitThread();
101 }
102 
103 
105 
106 
107 void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
108 {
109  const QMutexLocker locker( &mManager->mMutex );
110 
111  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
112 
113  QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
114  QByteArray data;
115 
116  if ( reply->error() == QNetworkReply::NoError )
117  {
118  ++mManager->mStats.networkRequestsOk;
119  data = reply->readAll();
120  }
121  else
122  {
123  ++mManager->mStats.networkRequestsFailed;
124  const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
125  if ( contentType.startsWith( QLatin1String( "text/plain" ) ) )
126  data = reply->readAll();
127  }
128 
129  QMap<QNetworkRequest::Attribute, QVariant> attributes;
130  attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
131  attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
132  attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
133  attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
134 
135  QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
136  headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
137 
138  // Save loaded data to cache
139  int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
140  if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
141  {
142  mManager->mRangesCache->registerEntry( mRequest, data );
143  }
144 
145  emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
146 
147  reply->deleteLater();
148 
149  // kill the worker obj
150  deleteLater();
151 
152  mManager->removeEntry( mRequest );
153 
154  if ( mManager->mQueue.isEmpty() )
155  {
156  // if this was the last thing in the queue, start a timer to kill thread after X seconds
157  mManager->mWorker->startIdleTimer();
158  }
159 }
160 
162 
164 
165 
167 #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
168  : mMutex( QMutex::Recursive )
169 #endif
170 {
171  mRangesCache.reset( new QgsRangeRequestCache );
172 
173  const QgsSettings settings;
174  QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString();
175  if ( cacheDirectory.isEmpty() )
176  cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
177  if ( !cacheDirectory.endsWith( QDir::separator() ) )
178  {
179  cacheDirectory.push_back( QDir::separator() );
180  }
181  cacheDirectory += QLatin1String( "http-ranges" );
182  const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong();
183 
184  mRangesCache->setCacheDirectory( cacheDirectory );
185  mRangesCache->setCacheSize( cacheSize );
186 }
187 
189 {
190  // make sure the worker thread is gone and any pending requests are canceled
191  shutdown();
192 }
193 
195 {
196  const QMutexLocker locker( &mMutex );
197 
198  if ( isCachedRangeRequest( request ) )
199  {
200  QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
201  QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
202  return reply;
203  }
204 
205  if ( !mWorker )
206  {
207  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: starting worker thread" ), 2 );
208  mWorkerThread = new QThread;
209  mWorker = new QgsTileDownloadManagerWorker( this );
210  mWorker->moveToThread( mWorkerThread );
211  QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
212  mWorkerThread->start();
213  }
214 
215  QgsTileDownloadManagerReply *reply = new QgsTileDownloadManagerReply( this, request ); // lives in the same thread as the caller
216 
217  ++mStats.requestsTotal;
218 
219  QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
220  if ( !entry.isValid() )
221  {
222  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
223  // create a new entry and add it to queue
224  entry.request = request;
225  entry.objWorker = new QgsTileDownloadManagerReplyWorkerObject( this, request );
226  entry.objWorker->moveToThread( mWorkerThread );
227 
228  QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
229 
230  addEntry( entry );
231  }
232  else
233  {
234  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
235 
236  QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished ); // should be queued connection
237 
238  ++mStats.requestsMerged;
239  }
240 
241  signalQueueModified();
242 
243  return reply;
244 }
245 
247 {
248  const QMutexLocker locker( &mMutex );
249 
250  return !mQueue.isEmpty();
251 }
252 
254 {
255  QElapsedTimer t;
256  t.start();
257 
258  while ( msec == -1 || t.elapsed() < msec )
259  {
260  {
261  const QMutexLocker locker( &mMutex );
262  if ( mQueue.isEmpty() )
263  return true;
264  }
265  QThread::usleep( 1000 );
266  }
267 
268  return false;
269 }
270 
272 {
273  {
274  const QMutexLocker locker( &mMutex );
275  if ( !mWorkerThread )
276  return; // nothing to stop
277 
278  // let's signal to the thread
279  mShuttingDown = true;
280  signalQueueModified();
281  }
282 
283  // wait until the thread is gone
284  while ( 1 )
285  {
286  {
287  const QMutexLocker locker( &mMutex );
288  if ( !mWorkerThread )
289  return; // the thread has stopped
290  }
291 
292  QThread::usleep( 1000 );
293  }
294 }
295 
297 {
298  return mWorkerThread && mWorkerThread->isRunning();
299 }
300 
302 {
303  const QMutexLocker locker( &mMutex );
305 }
306 
307 QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest( const QNetworkRequest &request )
308 {
309  for ( auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it )
310  {
311  if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
312  return *it;
313  }
314  return QgsTileDownloadManager::QueueEntry();
315 }
316 
317 void QgsTileDownloadManager::addEntry( const QgsTileDownloadManager::QueueEntry &entry )
318 {
319  for ( auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it )
320  {
321  Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader( "Range" ) != it->request.rawHeader( "Range" ) );
322  }
323 
324  mQueue.append( entry );
325 }
326 
327 void QgsTileDownloadManager::updateEntry( const QgsTileDownloadManager::QueueEntry &entry )
328 {
329  for ( auto it = mQueue.begin(); it != mQueue.end(); ++it )
330  {
331  if ( entry.request.url() == it->request.url() && entry.request.rawHeader( "Range" ) == it->request.rawHeader( "Range" ) )
332  {
333  *it = entry;
334  return;
335  }
336  }
337  Q_ASSERT( false );
338 }
339 
340 void QgsTileDownloadManager::removeEntry( const QNetworkRequest &request )
341 {
342  int i = 0;
343  for ( auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it, ++i )
344  {
345  if ( it->request.url() == request.url() && it->request.rawHeader( "Range" ) == request.rawHeader( "Range" ) )
346  {
347  mQueue.removeAt( i );
348  return;
349  }
350  }
351  Q_ASSERT( false );
352 }
353 
354 void QgsTileDownloadManager::signalQueueModified()
355 {
356  QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
357 }
358 
359 bool QgsTileDownloadManager::isRangeRequest( const QNetworkRequest &request )
360 {
361  if ( request.rawHeader( "Range" ).isEmpty() )
362  return false;
363  QRegularExpression regex( "^bytes=\\d+-\\d+$" );
364  QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader( "Range" ) ) );
365  return match.hasMatch();
366 }
367 
368 bool QgsTileDownloadManager::isCachedRangeRequest( const QNetworkRequest &request )
369 {
370  QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
371  bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
372  return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
373 }
374 
376 
377 
378 QgsTileDownloadManagerReply::QgsTileDownloadManagerReply( QgsTileDownloadManager *manager, const QNetworkRequest &request )
379  : mManager( manager )
380  , mRequest( request )
381 {
382 }
383 
385 {
386  const QMutexLocker locker( &mManager->mMutex );
387 
388  if ( !mHasFinished )
389  {
390  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
391 
392  ++mManager->mStats.requestsEarlyDeleted;
393  }
394 }
395 
396 void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url, const QMap<QNetworkRequest::Attribute, QVariant> &attributes, const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers, const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error, const QString &errorString )
397 {
398  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
399 
400  mHasFinished = true;
401  mData = data;
402  mUrl = url;
403  mAttributes = attributes;
404  mHeaders = headers;
405  mRawHeaderPairs = rawHeaderPairs;
406  mError = error;
407  mErrorString = errorString;
408  emit finished();
409 }
410 
411 void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
412 {
413  QgsDebugMsgLevel( QStringLiteral( "Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
414  mHasFinished = true;
415  mData = mManager->mRangesCache->entry( mRequest );
416  mUrl = mRequest.url();
417  emit finished();
418 }
419 
420 QVariant QgsTileDownloadManagerReply::attribute( QNetworkRequest::Attribute code )
421 {
422  return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
423 }
424 
425 QVariant QgsTileDownloadManagerReply::header( QNetworkRequest::KnownHeaders header )
426 {
427  return mHeaders.contains( header ) ? mHeaders.value( header ) : QVariant();
428 }
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
This class is a composition of two QSettings instances:
Definition: qgssettings.h:62
QVariant value(const QString &key, const QVariant &defaultValue=QVariant(), Section section=NoSection) const
Returns the value for setting key.
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1)
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39