QGIS API Documentation 3.40.0-Bratislava (b56115d8743)
Loading...
Searching...
No Matches
qgsblockingnetworkrequest.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgsblockingnetworkrequest.cpp
3 -----------------------------
4 begin : November 2018
5 copyright : (C) 2018 by Nyall Dawson
6 email : nyall dot dawson at gmail dot com
7 ***************************************************************************
8 * *
9 * This program is free software; you can redistribute it and/or modify *
10 * it under the terms of the GNU General Public License as published by *
11 * the Free Software Foundation; either version 2 of the License, or *
12 * (at your option) any later version. *
13 * *
14 ***************************************************************************/
15
17#include "qgslogger.h"
18#include "qgsapplication.h"
20#include "qgsauthmanager.h"
21#include "qgsmessagelog.h"
22#include "qgsfeedback.h"
23#include "qgsvariantutils.h"
24#include <QUrl>
25#include <QNetworkRequest>
26#include <QNetworkReply>
27#include <QMutex>
28#include <QWaitCondition>
29#include <QNetworkCacheMetaData>
30#include <QAuthenticator>
31#include <QBuffer>
32
34{
35 connect( QgsNetworkAccessManager::instance(), qOverload< QNetworkReply * >( &QgsNetworkAccessManager::requestTimedOut ), this, &QgsBlockingNetworkRequest::requestTimedOut );
36}
37
42
43void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply )
44{
45 if ( reply == mReply )
46 mTimedout = true;
47}
48
50{
51 return mAuthCfg;
52}
53
54void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg )
55{
56 mAuthCfg = authCfg;
57}
58
59QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback, RequestFlags requestFlags )
60{
61 return doRequest( Get, request, forceRefresh, feedback, requestFlags );
62}
63
64QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh, QgsFeedback *feedback )
65{
66 QByteArray ldata( data );
67 QBuffer buffer( &ldata );
68 buffer.open( QIODevice::ReadOnly );
69 return post( request, &buffer, forceRefresh, feedback );
70}
71
72QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, QIODevice *data, bool forceRefresh, QgsFeedback *feedback )
73{
74 mPayloadData = data;
75 return doRequest( Post, request, forceRefresh, feedback );
76}
77
78QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::head( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
79{
80 return doRequest( Head, request, forceRefresh, feedback );
81}
82
83QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback )
84{
85 QByteArray ldata( data );
86 QBuffer buffer( &ldata );
87 buffer.open( QIODevice::ReadOnly );
88 return put( request, &buffer, feedback );
89}
90
91QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback )
92{
93 mPayloadData = data;
94 return doRequest( Put, request, true, feedback );
95}
96
98{
99 return doRequest( Delete, request, true, feedback );
100}
101
102void QgsBlockingNetworkRequest::sendRequestToNetworkAccessManager( const QNetworkRequest &request )
103{
104 switch ( mMethod )
105 {
106 case Get:
107 mReply = QgsNetworkAccessManager::instance()->get( request );
108 break;
109
110 case Post:
111 mReply = QgsNetworkAccessManager::instance()->post( request, mPayloadData );
112 break;
113
114 case Head:
115 mReply = QgsNetworkAccessManager::instance()->head( request );
116 break;
117
118 case Put:
119 mReply = QgsNetworkAccessManager::instance()->put( request, mPayloadData );
120 break;
121
122 case Delete:
123 mReply = QgsNetworkAccessManager::instance()->deleteResource( request );
124 break;
125 };
126}
127
128QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback, RequestFlags requestFlags )
129{
130 mMethod = method;
131 mFeedback = feedback;
132
133 abort(); // cancel previous
134 mIsAborted = false;
135 mTimedout = false;
136 mGotNonEmptyResponse = false;
137 mRequestFlags = requestFlags;
138
139 mErrorMessage.clear();
140 mErrorCode = NoError;
141 mForceRefresh = forceRefresh;
142 mReplyContent.clear();
143
144 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
145 {
146 mErrorCode = NetworkError;
147 mErrorMessage = errorMessageFailedAuth();
148 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
149 return NetworkError;
150 }
151
152 QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 );
153
154 request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
155 request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
156
157 QWaitCondition authRequestBufferNotEmpty;
158 QMutex waitConditionMutex;
159
160 bool threadFinished = false;
161 bool success = false;
162
163 const bool requestMadeFromMainThread = QThread::currentThread() == QApplication::instance()->thread();
164
165 if ( mFeedback )
166 connect( mFeedback, &QgsFeedback::canceled, this, &QgsBlockingNetworkRequest::abort );
167
168 const std::function<void()> downloaderFunction = [ this, request, &waitConditionMutex, &authRequestBufferNotEmpty, &threadFinished, &success, requestMadeFromMainThread ]()
169 {
170 // this function will always be run in worker threads -- either the blocking call is being made in a worker thread,
171 // or the blocking call has been made from the main thread and we've fired up a new thread for this function
172 Q_ASSERT( QThread::currentThread() != QgsApplication::instance()->thread() );
173
174 QgsNetworkAccessManager::instance( Qt::DirectConnection );
175
176 success = true;
177
178 sendRequestToNetworkAccessManager( request );
179
180 if ( mFeedback )
181 connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
182
183 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
184 {
185 mErrorCode = NetworkError;
186 mErrorMessage = errorMessageFailedAuth();
187 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
188 if ( requestMadeFromMainThread )
189 authRequestBufferNotEmpty.wakeAll();
190 success = false;
191 }
192 else
193 {
194 // We are able to use direct connection here, because we
195 // * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway
196 // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread)
197 connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
198 connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
199 connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
200
201 if ( request.hasRawHeader( "Range" ) )
202 connect( mReply, &QNetworkReply::metaDataChanged, this, &QgsBlockingNetworkRequest::abortIfNotPartialContentReturned, Qt::DirectConnection );
203
204 auto resumeMainThread = [&waitConditionMutex, &authRequestBufferNotEmpty ]()
205 {
206 // when this method is called we have "produced" a single authentication request -- so the buffer is now full
207 // and it's time for the "consumer" (main thread) to do its part
208 waitConditionMutex.lock();
209 authRequestBufferNotEmpty.wakeAll();
210 waitConditionMutex.unlock();
211
212 // note that we don't need to handle waking this thread back up - that's done automatically by QgsNetworkAccessManager
213 };
214
215 if ( requestMadeFromMainThread )
216 {
217 connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authRequestOccurred, this, resumeMainThread, Qt::DirectConnection );
218 connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection );
219
220#ifndef QT_NO_SSL
221 connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrorsOccurred, this, resumeMainThread, Qt::DirectConnection );
222#endif
223 }
224 QEventLoop loop;
225 // connecting to aboutToQuit avoids an on-going request to remain stalled
226 // when QThreadPool::globalInstance()->waitForDone()
227 // is called at process termination
228 connect( qApp, &QCoreApplication::aboutToQuit, &loop, &QEventLoop::quit, Qt::DirectConnection );
229 connect( this, &QgsBlockingNetworkRequest::finished, &loop, &QEventLoop::quit, Qt::DirectConnection );
230 loop.exec();
231 }
232
233 if ( requestMadeFromMainThread )
234 {
235 waitConditionMutex.lock();
236 threadFinished = true;
237 authRequestBufferNotEmpty.wakeAll();
238 waitConditionMutex.unlock();
239 }
240 };
241
242 if ( requestMadeFromMainThread )
243 {
244 std::unique_ptr<DownloaderThread> downloaderThread = std::make_unique<DownloaderThread>( downloaderFunction );
245 downloaderThread->start();
246
247 while ( true )
248 {
249 waitConditionMutex.lock();
250 if ( threadFinished )
251 {
252 waitConditionMutex.unlock();
253 break;
254 }
255 authRequestBufferNotEmpty.wait( &waitConditionMutex );
256
257 // If the downloader thread wakes us (the main thread) up and is not yet finished
258 // then it has "produced" an authentication request which we need to now "consume".
259 // The processEvents() call gives the auth manager the chance to show a dialog and
260 // once done with that, we can wake the downloaderThread again and continue the download.
261 if ( !threadFinished )
262 {
263 waitConditionMutex.unlock();
264
265 QgsApplication::processEvents();
266 // we don't need to wake up the worker thread - it will automatically be woken when
267 // the auth request has been dealt with by QgsNetworkAccessManager
268 }
269 else
270 {
271 waitConditionMutex.unlock();
272 }
273 }
274 // wait for thread to gracefully exit
275 downloaderThread->wait();
276 }
277 else
278 {
279 downloaderFunction();
280 }
281 return mErrorCode;
282}
283
285{
286 mIsAborted = true;
287 if ( mReply )
288 {
289 mReply->deleteLater();
290 mReply = nullptr;
291 }
292}
293
294void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal )
295{
296 QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 );
297
298 if ( bytesReceived != 0 )
299 mGotNonEmptyResponse = true;
300
301 if ( !mIsAborted && mReply && ( !mFeedback || !mFeedback->isCanceled() ) )
302 {
303 if ( mReply->error() == QNetworkReply::NoError )
304 {
305 const QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
306 if ( !QgsVariantUtils::isNull( redirect ) )
307 {
308 // We don't want to emit downloadProgress() for a redirect
309 return;
310 }
311 }
312 }
313
314 if ( mMethod == Put || mMethod == Post )
315 emit uploadProgress( bytesReceived, bytesTotal );
316 else
317 emit downloadProgress( bytesReceived, bytesTotal );
318}
319
320void QgsBlockingNetworkRequest::replyFinished()
321{
322 if ( !mIsAborted && mReply )
323 {
324
325 if ( mReply->error() == QNetworkReply::NoError && ( !mFeedback || !mFeedback->isCanceled() ) )
326 {
327 QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 );
328 const QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
329 if ( !QgsVariantUtils::isNull( redirect ) )
330 {
331 QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 );
332
333 const QUrl &toUrl = redirect.toUrl();
334 mReply->request();
335 if ( toUrl == mReply->url() )
336 {
337 mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
338 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
339 mReplyContent.clear();
340 }
341 else
342 {
343 QNetworkRequest request( toUrl );
344
345 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
346 {
347 mReplyContent.clear();
348 mErrorMessage = errorMessageFailedAuth();
349 mErrorCode = NetworkError;
350 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
351 emit finished();
353 emit downloadFinished();
355 return;
356 }
357
358 request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
359 request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
360
361 // if that was a range request, use the same range for the redirected request
362 if ( mReply->request().hasRawHeader( "Range" ) )
363 request.setRawHeader( "Range", mReply->request().rawHeader( "Range" ) );
364
365 mReply->deleteLater();
366 mReply = nullptr;
367
368 QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 );
369
370 sendRequestToNetworkAccessManager( request );
371
372 if ( mFeedback )
373 connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
374
375 if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
376 {
377 mReplyContent.clear();
378 mErrorMessage = errorMessageFailedAuth();
379 mErrorCode = NetworkError;
380 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
381 emit finished();
383 emit downloadFinished();
385 return;
386 }
387
388 connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
389 connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
390 connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
391
392 if ( request.hasRawHeader( "Range" ) )
393 connect( mReply, &QNetworkReply::metaDataChanged, this, &QgsBlockingNetworkRequest::abortIfNotPartialContentReturned, Qt::DirectConnection );
394
395 return;
396 }
397 }
398 else
399 {
401
402 if ( nam->cache() )
403 {
404 QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
405
406 QNetworkCacheMetaData::RawHeaderList hl;
407 const auto constRawHeaders = cmd.rawHeaders();
408 for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
409 {
410 if ( h.first != "Cache-Control" )
411 hl.append( h );
412 }
413 cmd.setRawHeaders( hl );
414
415 QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 );
416 if ( cmd.expirationDate().isNull() )
417 {
418 cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) );
419 }
420
421 nam->cache()->updateMetaData( cmd );
422 }
423 else
424 {
425 QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 );
426 }
427
428#ifdef QGISDEBUG
429 const bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
430 QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 );
431#endif
432
433 mReplyContent = QgsNetworkReplyContent( mReply );
434 const QByteArray content = mReply->readAll();
435 if ( !( mRequestFlags & RequestFlag::EmptyResponseIsValid ) && content.isEmpty() && !mGotNonEmptyResponse && mMethod == Get )
436 {
437 mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
438 mErrorCode = ServerExceptionError;
439 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
440 }
441 mReplyContent.setContent( content );
442 }
443 }
444 else
445 {
446 if ( mReply->error() != QNetworkReply::OperationCanceledError )
447 {
448 mErrorMessage = mReply->errorString();
449 mErrorCode = ServerExceptionError;
450 QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
451 }
452 mReplyContent = QgsNetworkReplyContent( mReply );
453 mReplyContent.setContent( mReply->readAll() );
454 }
455 }
456 if ( mTimedout )
457 mErrorCode = TimeoutError;
458
459 if ( mReply )
460 {
461 mReply->deleteLater();
462 mReply = nullptr;
463 }
464
465 emit finished();
467 emit downloadFinished();
469}
470
471QString QgsBlockingNetworkRequest::errorMessageFailedAuth()
472{
473 return tr( "network request update failed for authentication config" );
474}
475
476void QgsBlockingNetworkRequest::abortIfNotPartialContentReturned()
477{
478 if ( mReply && mReply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt() == 200 )
479 {
480 // We're expecting a 206 - Partial Content but the server returned 200
481 // It seems it does not support range requests and is returning the whole file!
482 mReply->abort();
483 mErrorMessage = tr( "The server does not support range requests" );
484 mErrorCode = ServerExceptionError;
485 }
486}
static QgsApplication * instance()
Returns the singleton instance of the QgsApplication.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
bool updateNetworkRequest(QNetworkRequest &request, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkRequest with an authentication config.
bool updateNetworkReply(QNetworkReply *reply, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkReply with an authentication config (used to skip known SSL errors,...
QgsBlockingNetworkRequest()
Constructor for QgsBlockingNetworkRequest.
ErrorCode put(QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback=nullptr)
Performs a "put" operation on the specified request, using the given data.
void uploadProgress(qint64 bytesReceived, qint64 bytesTotal)
Emitted when when data are sent during a request.
ErrorCode head(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "head" operation on the specified request.
void abort()
Aborts the network request immediately.
Q_DECL_DEPRECATED void downloadFinished()
Emitted once a request has finished downloading.
ErrorCode post(QNetworkRequest &request, QIODevice *data, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "post" operation on the specified request, using the given data.
ErrorCode deleteResource(QNetworkRequest &request, QgsFeedback *feedback=nullptr)
Performs a "delete" operation on the specified request.
void finished()
Emitted once a request has finished.
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
QString authCfg() const
Returns the authentication config id which will be used during the request.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Emitted when when data arrives during a request.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ EmptyResponseIsValid
Do not generate an error if getting an empty response (e.g. HTTP 204)
@ NetworkError
A network error occurred.
@ ServerExceptionError
An exception was raised by the server.
@ NoError
No error was encountered.
@ TimeoutError
Timeout was reached before a reply was received.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
Base class for feedback objects to be used for cancellation of something running in a worker thread.
Definition qgsfeedback.h:44
void canceled()
Internal routines can connect to this signal if they use event loop.
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
network access manager for QGIS
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
void requestTimedOut(QgsNetworkRequestParameters request)
Emitted when a network request has timed out.
Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between...
void setContent(const QByteArray &content)
Sets the reply content.
void clear()
Clears the reply, resetting it back to a default, empty reply.
static bool isNull(const QVariant &variant, bool silenceNullWarnings=false)
Returns true if the specified variant should be considered a NULL value.
#define Q_NOWARN_DEPRECATED_POP
Definition qgis.h:6494
#define Q_NOWARN_DEPRECATED_PUSH
Definition qgis.h:6493
#define QgsDebugMsgLevel(str, level)
Definition qgslogger.h:39