24 #include <QElapsedTimer>
25 #include <QNetworkReply>
26 #include <QStandardPaths>
27 #include <QRegularExpression>
31 QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker(
QgsTileDownloadManager *manager, QObject *parent )
36 connect( &mIdleTimer, &QTimer::timeout,
this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
39 void QgsTileDownloadManagerWorker::startIdleTimer()
41 if ( !mIdleTimer.isActive() )
43 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
47 void QgsTileDownloadManagerWorker::queueUpdated()
49 const QMutexLocker locker( &mManager->mMutex );
51 if ( mManager->mShuttingDown )
53 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
55 it->networkReply->abort();
62 if ( mIdleTimer.isActive() && !mManager->mQueue.isEmpty() )
68 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
70 if ( !it->networkReply )
72 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
76 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
78 ++mManager->mStats.networkRequestsStarted;
83 void QgsTileDownloadManagerWorker::quitThread()
85 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: stopping worker thread" ), 2 );
87 mManager->mWorker->deleteLater();
88 mManager->mWorker =
nullptr;
91 mManager->mWorkerThread->quit();
92 mManager->mWorkerThread =
nullptr;
93 mManager->mShuttingDown =
false;
96 void QgsTileDownloadManagerWorker::idleTimerTimeout()
98 const QMutexLocker locker( &mManager->mMutex );
99 Q_ASSERT( mManager->mQueue.isEmpty() );
107 void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
109 const QMutexLocker locker( &mManager->mMutex );
111 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
113 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
116 if ( reply->error() == QNetworkReply::NoError )
118 ++mManager->mStats.networkRequestsOk;
119 data = reply->readAll();
123 ++mManager->mStats.networkRequestsFailed;
124 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
125 if ( contentType.startsWith( QLatin1String(
"text/plain" ) ) )
126 data = reply->readAll();
129 QMap<QNetworkRequest::Attribute, QVariant> attributes;
130 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
131 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
132 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
133 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
135 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
136 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
139 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
140 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
142 mManager->mRangesCache->registerEntry( mRequest, data );
145 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
147 reply->deleteLater();
152 mManager->removeEntry( mRequest );
154 if ( mManager->mQueue.isEmpty() )
157 mManager->mWorker->startIdleTimer();
167 #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
168 : mMutex( QMutex::Recursive )
174 QString cacheDirectory = settings.
value( QStringLiteral(
"cache/directory" ) ).toString();
175 if ( cacheDirectory.isEmpty() )
176 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
177 if ( !cacheDirectory.endsWith( QDir::separator() ) )
179 cacheDirectory.push_back( QDir::separator() );
181 cacheDirectory += QLatin1String(
"http-ranges" );
182 const qint64 cacheSize = settings.
value( QStringLiteral(
"cache/size" ), 256 * 1024 * 1024 ).toLongLong();
184 mRangesCache->setCacheDirectory( cacheDirectory );
185 mRangesCache->setCacheSize( cacheSize );
196 const QMutexLocker locker( &mMutex );
198 if ( isCachedRangeRequest( request ) )
201 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
207 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting worker thread" ), 2 );
208 mWorkerThread =
new QThread;
210 mWorker->moveToThread( mWorkerThread );
211 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
212 mWorkerThread->start();
219 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
220 if ( !entry.isValid() )
222 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
224 entry.request = request;
226 entry.objWorker->moveToThread( mWorkerThread );
228 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
234 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
236 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
241 signalQueueModified();
248 const QMutexLocker locker( &mMutex );
250 return !mQueue.isEmpty();
258 while ( msec == -1 || t.elapsed() < msec )
261 const QMutexLocker locker( &mMutex );
262 if ( mQueue.isEmpty() )
265 QThread::usleep( 1000 );
274 const QMutexLocker locker( &mMutex );
275 if ( !mWorkerThread )
279 mShuttingDown =
true;
280 signalQueueModified();
287 const QMutexLocker locker( &mMutex );
288 if ( !mWorkerThread )
292 QThread::usleep( 1000 );
298 return mWorkerThread && mWorkerThread->isRunning();
303 const QMutexLocker locker( &mMutex );
307 QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest(
const QNetworkRequest &request )
309 for (
auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it )
311 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
314 return QgsTileDownloadManager::QueueEntry();
317 void QgsTileDownloadManager::addEntry(
const QgsTileDownloadManager::QueueEntry &entry )
319 for (
auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it )
321 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader(
"Range" ) != it->request.rawHeader(
"Range" ) );
324 mQueue.append( entry );
327 void QgsTileDownloadManager::updateEntry(
const QgsTileDownloadManager::QueueEntry &entry )
329 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
331 if ( entry.request.url() == it->request.url() && entry.request.rawHeader(
"Range" ) == it->request.rawHeader(
"Range" ) )
340 void QgsTileDownloadManager::removeEntry(
const QNetworkRequest &request )
343 for (
auto it = mQueue.constBegin(); it != mQueue.constEnd(); ++it, ++i )
345 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
347 mQueue.removeAt( i );
354 void QgsTileDownloadManager::signalQueueModified()
356 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
359 bool QgsTileDownloadManager::isRangeRequest(
const QNetworkRequest &request )
361 if ( request.rawHeader(
"Range" ).isEmpty() )
363 QRegularExpression regex(
"^bytes=\\d+-\\d+$" );
364 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader(
"Range" ) ) );
365 return match.hasMatch();
368 bool QgsTileDownloadManager::isCachedRangeRequest(
const QNetworkRequest &request )
370 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
371 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
372 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
378 QgsTileDownloadManagerReply::QgsTileDownloadManagerReply(
QgsTileDownloadManager *manager,
const QNetworkRequest &request )
379 : mManager( manager )
380 , mRequest( request )
386 const QMutexLocker locker( &mManager->mMutex );
390 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
396 void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url,
const QMap<QNetworkRequest::Attribute, QVariant> &attributes,
const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers,
const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error,
const QString &errorString )
398 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
403 mAttributes = attributes;
411 void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
413 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
415 mData = mManager->mRangesCache->entry( mRequest );
416 mUrl = mRequest.url();
422 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
427 return mHeaders.contains(
header ) ? mHeaders.value(
header ) : QVariant();
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
This class is a composition of two QSettings instances:
QVariant value(const QString &key, const QVariant &defaultValue=QVariant(), Section section=NoSection) const
Returns the value for setting key.
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
~QgsTileDownloadManagerReply()
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1)
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
friend class QgsTileDownloadManagerReply
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
friend class QgsTileDownloadManagerWorker
~QgsTileDownloadManager()
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)