19#include "moc_qgstiledownloadmanager.cpp"
28#include <QElapsedTimer>
29#include <QNetworkReply>
30#include <QStandardPaths>
31#include <QRegularExpression>
35QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker(
QgsTileDownloadManager *manager, QObject *parent )
40 connect( &mIdleTimer, &QTimer::timeout,
this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
43void QgsTileDownloadManagerWorker::startIdleTimer()
45 if ( !mIdleTimer.isActive() )
47 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
51void QgsTileDownloadManagerWorker::queueUpdated()
53 const QMutexLocker locker( &mManager->mMutex );
55 if ( mManager->mShuttingDown )
63 std::vector< QNetworkReply * > replies;
64 replies.reserve( mManager->mQueue.size() );
65 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
67 replies.emplace_back( it->networkReply );
70 for ( QNetworkReply *reply : replies )
79 if ( mIdleTimer.isActive() && !mManager->mQueue.empty() )
90 mManager->mStageQueueRemovals =
true;
91 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
93 if ( !it->networkReply )
95 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
99 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
101 ++mManager->mStats.networkRequestsStarted;
104 mManager->mStageQueueRemovals =
false;
105 mManager->processStagedEntryRemovals();
108void QgsTileDownloadManagerWorker::quitThread()
110 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: stopping worker thread" ), 2 );
112 mManager->mWorker->deleteLater();
113 mManager->mWorker =
nullptr;
116 mManager->mWorkerThread->quit();
117 mManager->mWorkerThread =
nullptr;
118 mManager->mShuttingDown =
false;
121void QgsTileDownloadManagerWorker::idleTimerTimeout()
123 const QMutexLocker locker( &mManager->mMutex );
124 Q_ASSERT( mManager->mQueue.empty() );
132void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
134 const QMutexLocker locker( &mManager->mMutex );
136 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
138 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
141 if ( reply->error() == QNetworkReply::NoError )
143 ++mManager->mStats.networkRequestsOk;
144 data = reply->readAll();
148 ++mManager->mStats.networkRequestsFailed;
149 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
150 if ( contentType.startsWith( QLatin1String(
"text/plain" ) ) )
151 data = reply->readAll();
154 QMap<QNetworkRequest::Attribute, QVariant> attributes;
155 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
156 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
157 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
158 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
160 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
161 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
164 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
165 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
167 mManager->mRangesCache->registerEntry( mRequest, data );
170 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
172 reply->deleteLater();
177 mManager->removeEntry( mRequest );
179 if ( mManager->mQueue.empty() )
182 mManager->mWorker->startIdleTimer();
197 if ( cacheDirectory.isEmpty() )
198 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
199 if ( !cacheDirectory.endsWith( QDir::separator() ) )
201 cacheDirectory.push_back( QDir::separator() );
203 cacheDirectory += QLatin1String(
"http-ranges" );
204 mRangesCache->setCacheDirectory( cacheDirectory );
206 mRangesCache->setCacheSize( cacheSize );
217 const QMutexLocker locker( &mMutex );
219 if ( isCachedRangeRequest( request ) )
222 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
228 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting worker thread" ), 2 );
229 mWorkerThread =
new QThread;
231 mWorker->moveToThread( mWorkerThread );
232 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
233 mWorkerThread->start();
240 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
241 if ( !entry.isValid() )
243 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
245 entry.request = request;
247 entry.objWorker->moveToThread( mWorkerThread );
249 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
255 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
257 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
262 signalQueueModified();
269 const QMutexLocker locker( &mMutex );
271 return !mQueue.empty();
279 while ( msec == -1 || t.elapsed() < msec )
282 const QMutexLocker locker( &mMutex );
283 if ( mQueue.empty() )
286 QThread::usleep( 1000 );
295 const QMutexLocker locker( &mMutex );
296 if ( !mWorkerThread )
300 mShuttingDown =
true;
301 signalQueueModified();
308 const QMutexLocker locker( &mMutex );
309 if ( !mWorkerThread )
313 QThread::usleep( 1000 );
319 return mWorkerThread && mWorkerThread->isRunning();
324 const QMutexLocker locker( &mMutex );
328QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest(
const QNetworkRequest &request )
330 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
332 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
335 return QgsTileDownloadManager::QueueEntry();
338void QgsTileDownloadManager::addEntry(
const QgsTileDownloadManager::QueueEntry &entry )
340 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
342 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader(
"Range" ) != it->request.rawHeader(
"Range" ) );
345 mQueue.emplace_back( entry );
348void QgsTileDownloadManager::updateEntry(
const QgsTileDownloadManager::QueueEntry &entry )
350 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
352 if ( entry.request.url() == it->request.url() && entry.request.rawHeader(
"Range" ) == it->request.rawHeader(
"Range" ) )
361void QgsTileDownloadManager::removeEntry(
const QNetworkRequest &request )
363 if ( mStageQueueRemovals )
365 mStagedQueueRemovals.emplace_back( request );
369 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
371 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
381void QgsTileDownloadManager::processStagedEntryRemovals()
383 Q_ASSERT( !mStageQueueRemovals );
384 for (
const QNetworkRequest &request : mStagedQueueRemovals )
386 removeEntry( request );
388 mStagedQueueRemovals.clear();
391void QgsTileDownloadManager::signalQueueModified()
393 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
396bool QgsTileDownloadManager::isRangeRequest(
const QNetworkRequest &request )
398 if ( request.rawHeader(
"Range" ).isEmpty() )
400 const thread_local QRegularExpression regex(
"^bytes=\\d+-\\d+$" );
401 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader(
"Range" ) ) );
402 return match.hasMatch();
405bool QgsTileDownloadManager::isCachedRangeRequest(
const QNetworkRequest &request )
407 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
408 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
409 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
415QgsTileDownloadManagerReply::QgsTileDownloadManagerReply(
QgsTileDownloadManager *manager,
const QNetworkRequest &request )
416 : mManager( manager )
417 , mRequest( request )
423 const QMutexLocker locker( &mManager->mMutex );
427 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
433void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url,
const QMap<QNetworkRequest::Attribute, QVariant> &attributes,
const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers,
const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error,
const QString &errorString )
435 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
440 mAttributes = attributes;
448void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
450 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
452 mData = mManager->mRangesCache->entry( mRequest );
453 mUrl = mRequest.url();
459 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
464 return mHeaders.contains(
header ) ? mHeaders.value(
header ) : QVariant();
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
T value(const QString &dynamicKeyPart=QString()) const
Returns settings value.
static const QgsSettingsEntryInteger64 * settingsNetworkCacheSize
Settings entry network cache directory.
static const QgsSettingsEntryString * settingsNetworkCacheDirectory
Settings entry network cache directory.
This class is a composition of two QSettings instances:
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
~QgsTileDownloadManagerReply()
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1) const
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
friend class QgsTileDownloadManagerReply
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
friend class QgsTileDownloadManagerWorker
~QgsTileDownloadManager()
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)