27#include <QElapsedTimer>
28#include <QNetworkReply>
29#include <QStandardPaths>
30#include <QRegularExpression>
34QgsTileDownloadManagerWorker::QgsTileDownloadManagerWorker(
QgsTileDownloadManager *manager, QObject *parent )
39 connect( &mIdleTimer, &QTimer::timeout,
this, &QgsTileDownloadManagerWorker::idleTimerTimeout );
42void QgsTileDownloadManagerWorker::startIdleTimer()
44 if ( !mIdleTimer.isActive() )
46 mIdleTimer.start( mManager->mIdleThreadTimeoutMs );
50void QgsTileDownloadManagerWorker::queueUpdated()
52 const QMutexLocker locker( &mManager->mMutex );
54 if ( mManager->mShuttingDown )
62 std::vector< QNetworkReply * > replies;
63 replies.reserve( mManager->mQueue.size() );
64 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
66 replies.emplace_back( it->networkReply );
69 for ( QNetworkReply *reply : replies )
78 if ( mIdleTimer.isActive() && !mManager->mQueue.empty() )
89 mManager->mStageQueueRemovals =
true;
90 for (
auto it = mManager->mQueue.begin(); it != mManager->mQueue.end(); ++it )
92 if ( !it->networkReply )
94 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting request: " ) + it->request.url().toString(), 2 );
98 connect( it->networkReply, &QNetworkReply::finished, it->objWorker, &QgsTileDownloadManagerReplyWorkerObject::replyFinished );
100 ++mManager->mStats.networkRequestsStarted;
103 mManager->mStageQueueRemovals =
false;
104 mManager->processStagedEntryRemovals();
107void QgsTileDownloadManagerWorker::quitThread()
109 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: stopping worker thread" ), 2 );
111 mManager->mWorker->deleteLater();
112 mManager->mWorker =
nullptr;
115 mManager->mWorkerThread->quit();
116 mManager->mWorkerThread =
nullptr;
117 mManager->mShuttingDown =
false;
120void QgsTileDownloadManagerWorker::idleTimerTimeout()
122 const QMutexLocker locker( &mManager->mMutex );
123 Q_ASSERT( mManager->mQueue.empty() );
131void QgsTileDownloadManagerReplyWorkerObject::replyFinished()
133 const QMutexLocker locker( &mManager->mMutex );
135 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal reply finished: " ) + mRequest.url().toString(), 2 );
137 QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
140 if ( reply->error() == QNetworkReply::NoError )
142 ++mManager->mStats.networkRequestsOk;
143 data = reply->readAll();
147 ++mManager->mStats.networkRequestsFailed;
148 const QString contentType = reply->header( QNetworkRequest::ContentTypeHeader ).toString();
149 if ( contentType.startsWith( QLatin1String(
"text/plain" ) ) )
150 data = reply->readAll();
153 QMap<QNetworkRequest::Attribute, QVariant> attributes;
154 attributes.insert( QNetworkRequest::SourceIsFromCacheAttribute, reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ) );
155 attributes.insert( QNetworkRequest::RedirectionTargetAttribute, reply->attribute( QNetworkRequest::RedirectionTargetAttribute ) );
156 attributes.insert( QNetworkRequest::HttpStatusCodeAttribute, reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ) );
157 attributes.insert( QNetworkRequest::HttpReasonPhraseAttribute, reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute ) );
159 QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
160 headers.insert( QNetworkRequest::ContentTypeHeader, reply->header( QNetworkRequest::ContentTypeHeader ) );
163 int httpStatusCode = reply->attribute( QNetworkRequest::Attribute::HttpStatusCodeAttribute ).toInt();
164 if ( httpStatusCode == 206 && mManager->isRangeRequest( mRequest ) )
166 mManager->mRangesCache->registerEntry( mRequest, data );
169 emit finished( data, reply->url(), attributes, headers, reply->rawHeaderPairs(), reply->error(), reply->errorString() );
171 reply->deleteLater();
176 mManager->removeEntry( mRequest );
178 if ( mManager->mQueue.empty() )
181 mManager->mWorker->startIdleTimer();
196 if ( cacheDirectory.isEmpty() )
197 cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
198 if ( !cacheDirectory.endsWith( QDir::separator() ) )
200 cacheDirectory.push_back( QDir::separator() );
202 cacheDirectory += QLatin1String(
"http-ranges" );
203 mRangesCache->setCacheDirectory( cacheDirectory );
205 mRangesCache->setCacheSize( cacheSize );
216 const QMutexLocker locker( &mMutex );
218 if ( isCachedRangeRequest( request ) )
221 QTimer::singleShot( 0, reply, &QgsTileDownloadManagerReply::cachedRangeRequestFinished );
227 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: starting worker thread" ), 2 );
228 mWorkerThread =
new QThread;
230 mWorker->moveToThread( mWorkerThread );
231 QObject::connect( mWorkerThread, &QThread::finished, mWorker, &QObject::deleteLater );
232 mWorkerThread->start();
239 QgsTileDownloadManager::QueueEntry entry = findEntryForRequest( request );
240 if ( !entry.isValid() )
242 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (new entry): " ) + request.url().toString(), 2 );
244 entry.request = request;
246 entry.objWorker->moveToThread( mWorkerThread );
248 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
254 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: get (existing entry): " ) + request.url().toString(), 2 );
256 QObject::connect( entry.objWorker, &QgsTileDownloadManagerReplyWorkerObject::finished, reply, &QgsTileDownloadManagerReply::requestFinished );
261 signalQueueModified();
268 const QMutexLocker locker( &mMutex );
270 return !mQueue.empty();
278 while ( msec == -1 || t.elapsed() < msec )
281 const QMutexLocker locker( &mMutex );
282 if ( mQueue.empty() )
285 QThread::usleep( 1000 );
294 const QMutexLocker locker( &mMutex );
295 if ( !mWorkerThread )
299 mShuttingDown =
true;
300 signalQueueModified();
307 const QMutexLocker locker( &mMutex );
308 if ( !mWorkerThread )
312 QThread::usleep( 1000 );
318 return mWorkerThread && mWorkerThread->isRunning();
323 const QMutexLocker locker( &mMutex );
327QgsTileDownloadManager::QueueEntry QgsTileDownloadManager::findEntryForRequest(
const QNetworkRequest &request )
329 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
331 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
334 return QgsTileDownloadManager::QueueEntry();
337void QgsTileDownloadManager::addEntry(
const QgsTileDownloadManager::QueueEntry &entry )
339 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
341 Q_ASSERT( entry.request.url() != it->request.url() || entry.request.rawHeader(
"Range" ) != it->request.rawHeader(
"Range" ) );
344 mQueue.emplace_back( entry );
347void QgsTileDownloadManager::updateEntry(
const QgsTileDownloadManager::QueueEntry &entry )
349 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
351 if ( entry.request.url() == it->request.url() && entry.request.rawHeader(
"Range" ) == it->request.rawHeader(
"Range" ) )
360void QgsTileDownloadManager::removeEntry(
const QNetworkRequest &request )
362 if ( mStageQueueRemovals )
364 mStagedQueueRemovals.emplace_back( request );
368 for (
auto it = mQueue.begin(); it != mQueue.end(); ++it )
370 if ( it->request.url() == request.url() && it->request.rawHeader(
"Range" ) == request.rawHeader(
"Range" ) )
380void QgsTileDownloadManager::processStagedEntryRemovals()
382 Q_ASSERT( !mStageQueueRemovals );
383 for (
const QNetworkRequest &request : mStagedQueueRemovals )
385 removeEntry( request );
387 mStagedQueueRemovals.clear();
390void QgsTileDownloadManager::signalQueueModified()
392 QMetaObject::invokeMethod( mWorker, &QgsTileDownloadManagerWorker::queueUpdated, Qt::QueuedConnection );
395bool QgsTileDownloadManager::isRangeRequest(
const QNetworkRequest &request )
397 if ( request.rawHeader(
"Range" ).isEmpty() )
399 const thread_local QRegularExpression regex(
"^bytes=\\d+-\\d+$" );
400 QRegularExpressionMatch match = regex.match( QString::fromUtf8( request.rawHeader(
"Range" ) ) );
401 return match.hasMatch();
404bool QgsTileDownloadManager::isCachedRangeRequest(
const QNetworkRequest &request )
406 QNetworkRequest::CacheLoadControl loadControl = ( QNetworkRequest::CacheLoadControl ) request.attribute( QNetworkRequest::CacheLoadControlAttribute ).toInt();
407 bool saveControl = request.attribute( QNetworkRequest::CacheSaveControlAttribute ).toBool();
408 return isRangeRequest( request ) && saveControl && loadControl != QNetworkRequest::AlwaysNetwork && mRangesCache->hasEntry( request );
414QgsTileDownloadManagerReply::QgsTileDownloadManagerReply(
QgsTileDownloadManager *manager,
const QNetworkRequest &request )
415 : mManager( manager )
416 , mRequest( request )
422 const QMutexLocker locker( &mManager->mMutex );
426 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply deleted before finished: " ) + mRequest.url().toString(), 2 );
432void QgsTileDownloadManagerReply::requestFinished( QByteArray data, QUrl url,
const QMap<QNetworkRequest::Attribute, QVariant> &attributes,
const QMap<QNetworkRequest::KnownHeaders, QVariant> &headers,
const QList<QNetworkReply::RawHeaderPair> rawHeaderPairs, QNetworkReply::NetworkError error,
const QString &errorString )
434 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: reply finished: " ) + mRequest.url().toString(), 2 );
439 mAttributes = attributes;
447void QgsTileDownloadManagerReply::cachedRangeRequestFinished()
449 QgsDebugMsgLevel( QStringLiteral(
"Tile download manager: internal range request reply loaded from cache: " ) + mRequest.url().toString(), 2 );
451 mData = mManager->mRangesCache->entry( mRequest );
452 mUrl = mRequest.url();
458 return mAttributes.contains( code ) ? mAttributes.value( code ) : QVariant();
463 return mHeaders.contains(
header ) ? mHeaders.value(
header ) : QVariant();
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
A custom cache for handling the storage and retrieval of HTTP range requests on disk.
T value(const QString &dynamicKeyPart=QString()) const
Returns settings value.
static const QgsSettingsEntryInteger64 * settingsNetworkCacheSize
Settings entry network cache directory.
static const QgsSettingsEntryString * settingsNetworkCacheDirectory
Settings entry network cache directory.
This class is a composition of two QSettings instances:
Reply object for tile download manager requests returned from calls to QgsTileDownloadManager::get().
QString errorString() const
Returns error string (only valid when already finished)
~QgsTileDownloadManagerReply()
const QList< QNetworkReply::RawHeaderPair > rawHeaderPairs() const
Returns a list of raw header pairs.
QByteArray data() const
Returns binary data returned in the reply (only valid when already finished)
QNetworkReply::NetworkError error() const
Returns error code (only valid when already finished)
QUrl url() const
Returns the reply URL.
QVariant header(QNetworkRequest::KnownHeaders header)
Returns the value of the known header header.
QVariant attribute(QNetworkRequest::Attribute code)
Returns the attribute associated with the code.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
Encapsulates any statistics we would like to keep about requests.
int requestsMerged
How many requests were same as some other pending request and got "merged".
int requestsEarlyDeleted
How many requests were deleted early by the client (i.e. lost interest)
int requestsTotal
How many requests were done through the download manager.
Tile download manager handles downloads of map tiles for the purpose of map rendering.
bool hasWorkerThreadRunning() const
Returns whether the worker thread is running currently (it may be stopped if there were no requests r...
friend class QgsTileDownloadManagerReplyWorkerObject
bool waitForPendingRequests(int msec=-1) const
Blocks the current thread until the queue is empty.
QgsTileDownloadManagerReply * get(const QNetworkRequest &request)
Starts a request.
friend class QgsTileDownloadManagerReply
bool hasPendingRequests() const
Returns whether there are any pending requests in the queue.
void resetStatistics()
Resets statistics of numbers of queries handled by this class.
friend class QgsTileDownloadManagerWorker
~QgsTileDownloadManager()
void shutdown()
Asks the worker thread to stop and blocks until it is not stopped.
#define QgsDebugMsgLevel(str, level)