QGIS API Documentation  3.27.0-Master (aef1b1ec20)
qgsblockingnetworkrequest.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsblockingnetworkrequest.cpp
3  -----------------------------
4  begin : November 2018
5  copyright : (C) 2018 by Nyall Dawson
6  email : nyall dot dawson at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
17 #include "qgslogger.h"
18 #include "qgsapplication.h"
20 #include "qgsauthmanager.h"
21 #include "qgsmessagelog.h"
22 #include "qgsfeedback.h"
23 #include <QUrl>
24 #include <QNetworkRequest>
25 #include <QNetworkReply>
26 #include <QMutex>
27 #include <QWaitCondition>
28 #include <QNetworkCacheMetaData>
29 #include <QAuthenticator>
30 #include <QBuffer>
31 
33 {
34  connect( QgsNetworkAccessManager::instance(), qOverload< QNetworkReply * >( &QgsNetworkAccessManager::requestTimedOut ), this, &QgsBlockingNetworkRequest::requestTimedOut );
35 }
36 
38 {
39  abort();
40 }
41 
42 void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply )
43 {
44  if ( reply == mReply )
45  mTimedout = true;
46 }
47 
49 {
50  return mAuthCfg;
51 }
52 
53 void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg )
54 {
55  mAuthCfg = authCfg;
56 }
57 
58 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
59 {
60  return doRequest( Get, request, forceRefresh, feedback );
61 }
62 
63 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh, QgsFeedback *feedback )
64 {
65  QByteArray ldata( data );
66  QBuffer buffer( &ldata );
67  buffer.open( QIODevice::ReadOnly );
68  return post( request, &buffer, forceRefresh, feedback );
69 }
70 
71 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, QIODevice *data, bool forceRefresh, QgsFeedback *feedback )
72 {
73  mPayloadData = data;
74  return doRequest( Post, request, forceRefresh, feedback );
75 }
76 
77 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::head( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
78 {
79  return doRequest( Head, request, forceRefresh, feedback );
80 }
81 
82 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback )
83 {
84  QByteArray ldata( data );
85  QBuffer buffer( &ldata );
86  buffer.open( QIODevice::ReadOnly );
87  return put( request, &buffer, feedback );
88 }
89 
90 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback )
91 {
92  mPayloadData = data;
93  return doRequest( Put, request, true, feedback );
94 }
95 
97 {
98  return doRequest( Delete, request, true, feedback );
99 }
100 
101 void QgsBlockingNetworkRequest::sendRequestToNetworkAccessManager( const QNetworkRequest &request )
102 {
103  switch ( mMethod )
104  {
105  case Get:
106  mReply = QgsNetworkAccessManager::instance()->get( request );
107  break;
108 
109  case Post:
110  mReply = QgsNetworkAccessManager::instance()->post( request, mPayloadData );
111  break;
112 
113  case Head:
114  mReply = QgsNetworkAccessManager::instance()->head( request );
115  break;
116 
117  case Put:
118  mReply = QgsNetworkAccessManager::instance()->put( request, mPayloadData );
119  break;
120 
121  case Delete:
122  mReply = QgsNetworkAccessManager::instance()->deleteResource( request );
123  break;
124  };
125 }
126 
127 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
128 {
129  mMethod = method;
130  mFeedback = feedback;
131 
132  abort(); // cancel previous
133  mIsAborted = false;
134  mTimedout = false;
135  mGotNonEmptyResponse = false;
136 
137  mErrorMessage.clear();
138  mErrorCode = NoError;
139  mForceRefresh = forceRefresh;
140  mReplyContent.clear();
141 
142  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
143  {
144  mErrorCode = NetworkError;
145  mErrorMessage = errorMessageFailedAuth();
146  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
147  return NetworkError;
148  }
149 
150  QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 );
151 
152  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
153  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
154 
155  QWaitCondition authRequestBufferNotEmpty;
156  QMutex waitConditionMutex;
157 
158  bool threadFinished = false;
159  bool success = false;
160 
161  const bool requestMadeFromMainThread = QThread::currentThread() == QApplication::instance()->thread();
162 
163  if ( mFeedback )
164  connect( mFeedback, &QgsFeedback::canceled, this, &QgsBlockingNetworkRequest::abort );
165 
166  const std::function<void()> downloaderFunction = [ this, request, &waitConditionMutex, &authRequestBufferNotEmpty, &threadFinished, &success, requestMadeFromMainThread ]()
167  {
168  // this function will always be run in worker threads -- either the blocking call is being made in a worker thread,
169  // or the blocking call has been made from the main thread and we've fired up a new thread for this function
170  Q_ASSERT( QThread::currentThread() != QgsApplication::instance()->thread() );
171 
172  QgsNetworkAccessManager::instance( Qt::DirectConnection );
173 
174  success = true;
175 
176  sendRequestToNetworkAccessManager( request );
177 
178  if ( mFeedback )
179  connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
180 
181  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
182  {
183  mErrorCode = NetworkError;
184  mErrorMessage = errorMessageFailedAuth();
185  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
186  if ( requestMadeFromMainThread )
187  authRequestBufferNotEmpty.wakeAll();
188  success = false;
189  }
190  else
191  {
192  // We are able to use direct connection here, because we
193  // * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway
194  // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread)
195  connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
196  connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
197  connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
198 
199  auto resumeMainThread = [&waitConditionMutex, &authRequestBufferNotEmpty ]()
200  {
201  // when this method is called we have "produced" a single authentication request -- so the buffer is now full
202  // and it's time for the "consumer" (main thread) to do its part
203  waitConditionMutex.lock();
204  authRequestBufferNotEmpty.wakeAll();
205  waitConditionMutex.unlock();
206 
207  // note that we don't need to handle waking this thread back up - that's done automatically by QgsNetworkAccessManager
208  };
209 
210  if ( requestMadeFromMainThread )
211  {
212  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authRequestOccurred, this, resumeMainThread, Qt::DirectConnection );
213  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection );
214 
215 #ifndef QT_NO_SSL
216  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrorsOccurred, this, resumeMainThread, Qt::DirectConnection );
217 #endif
218  }
219  QEventLoop loop;
220  // connecting to aboutToQuit avoids an on-going request to remain stalled
221  // when QThreadPool::globalInstance()->waitForDone()
222  // is called at process termination
223  connect( qApp, &QCoreApplication::aboutToQuit, &loop, &QEventLoop::quit, Qt::DirectConnection );
224  connect( this, &QgsBlockingNetworkRequest::finished, &loop, &QEventLoop::quit, Qt::DirectConnection );
225  loop.exec();
226  }
227 
228  if ( requestMadeFromMainThread )
229  {
230  waitConditionMutex.lock();
231  threadFinished = true;
232  authRequestBufferNotEmpty.wakeAll();
233  waitConditionMutex.unlock();
234  }
235  };
236 
237  if ( requestMadeFromMainThread )
238  {
239  std::unique_ptr<DownloaderThread> downloaderThread = std::make_unique<DownloaderThread>( downloaderFunction );
240  downloaderThread->start();
241 
242  while ( true )
243  {
244  waitConditionMutex.lock();
245  if ( threadFinished )
246  {
247  waitConditionMutex.unlock();
248  break;
249  }
250  authRequestBufferNotEmpty.wait( &waitConditionMutex );
251 
252  // If the downloader thread wakes us (the main thread) up and is not yet finished
253  // then it has "produced" an authentication request which we need to now "consume".
254  // The processEvents() call gives the auth manager the chance to show a dialog and
255  // once done with that, we can wake the downloaderThread again and continue the download.
256  if ( !threadFinished )
257  {
258  waitConditionMutex.unlock();
259 
260  QgsApplication::processEvents();
261  // we don't need to wake up the worker thread - it will automatically be woken when
262  // the auth request has been dealt with by QgsNetworkAccessManager
263  }
264  else
265  {
266  waitConditionMutex.unlock();
267  }
268  }
269  // wait for thread to gracefully exit
270  downloaderThread->wait();
271  }
272  else
273  {
274  downloaderFunction();
275  }
276  return mErrorCode;
277 }
278 
280 {
281  mIsAborted = true;
282  if ( mReply )
283  {
284  mReply->deleteLater();
285  mReply = nullptr;
286  }
287 }
288 
289 void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal )
290 {
291  QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 );
292 
293  if ( bytesReceived != 0 )
294  mGotNonEmptyResponse = true;
295 
296  if ( !mIsAborted && mReply && ( !mFeedback || !mFeedback->isCanceled() ) )
297  {
298  if ( mReply->error() == QNetworkReply::NoError )
299  {
300  const QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
301  if ( !redirect.isNull() )
302  {
303  // We don't want to emit downloadProgress() for a redirect
304  return;
305  }
306  }
307  }
308 
309  if ( mMethod == Put || mMethod == Post )
310  emit uploadProgress( bytesReceived, bytesTotal );
311  else
312  emit downloadProgress( bytesReceived, bytesTotal );
313 }
314 
315 void QgsBlockingNetworkRequest::replyFinished()
316 {
317  if ( !mIsAborted && mReply )
318  {
319 
320  if ( mReply->error() == QNetworkReply::NoError && ( !mFeedback || !mFeedback->isCanceled() ) )
321  {
322  QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 );
323  const QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
324  if ( !redirect.isNull() )
325  {
326  QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 );
327 
328  const QUrl &toUrl = redirect.toUrl();
329  mReply->request();
330  if ( toUrl == mReply->url() )
331  {
332  mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
333  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
334  mReplyContent.clear();
335  }
336  else
337  {
338  QNetworkRequest request( toUrl );
339 
340  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
341  {
342  mReplyContent.clear();
343  mErrorMessage = errorMessageFailedAuth();
344  mErrorCode = NetworkError;
345  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
346  emit finished();
348  emit downloadFinished();
350  return;
351  }
352 
353  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
354  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
355 
356  mReply->deleteLater();
357  mReply = nullptr;
358 
359  QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 );
360 
361  sendRequestToNetworkAccessManager( request );
362 
363  if ( mFeedback )
364  connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
365 
366  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
367  {
368  mReplyContent.clear();
369  mErrorMessage = errorMessageFailedAuth();
370  mErrorCode = NetworkError;
371  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
372  emit finished();
374  emit downloadFinished();
376  return;
377  }
378 
379  connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
380  connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
381  connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
382  return;
383  }
384  }
385  else
386  {
388 
389  if ( nam->cache() )
390  {
391  QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
392 
393  QNetworkCacheMetaData::RawHeaderList hl;
394  const auto constRawHeaders = cmd.rawHeaders();
395  for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
396  {
397  if ( h.first != "Cache-Control" )
398  hl.append( h );
399  }
400  cmd.setRawHeaders( hl );
401 
402  QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 );
403  if ( cmd.expirationDate().isNull() )
404  {
405  cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) );
406  }
407 
408  nam->cache()->updateMetaData( cmd );
409  }
410  else
411  {
412  QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 );
413  }
414 
415 #ifdef QGISDEBUG
416  const bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
417  QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 );
418 #endif
419 
420  mReplyContent = QgsNetworkReplyContent( mReply );
421  const QByteArray content = mReply->readAll();
422  if ( content.isEmpty() && !mGotNonEmptyResponse && mMethod == Get )
423  {
424  mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
425  mErrorCode = ServerExceptionError;
426  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
427  }
428  mReplyContent.setContent( content );
429  }
430  }
431  else
432  {
433  if ( mReply->error() != QNetworkReply::OperationCanceledError )
434  {
435  mErrorMessage = mReply->errorString();
436  mErrorCode = ServerExceptionError;
437  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
438  }
439  mReplyContent = QgsNetworkReplyContent( mReply );
440  mReplyContent.setContent( mReply->readAll() );
441  }
442  }
443  if ( mTimedout )
444  mErrorCode = TimeoutError;
445 
446  if ( mReply )
447  {
448  mReply->deleteLater();
449  mReply = nullptr;
450  }
451 
452  emit finished();
454  emit downloadFinished();
456 }
457 
458 QString QgsBlockingNetworkRequest::errorMessageFailedAuth()
459 {
460  return tr( "network request update failed for authentication config" );
461 }
static QgsApplication * instance()
Returns the singleton instance of the QgsApplication.
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
bool updateNetworkRequest(QNetworkRequest &request, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkRequest with an authentication config.
bool updateNetworkReply(QNetworkReply *reply, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkReply with an authentication config (used to skip known SSL errors,...
void downloadProgress(qint64, qint64)
Emitted when when data arrives during a request.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "get" operation on the specified request.
QgsBlockingNetworkRequest()
Constructor for QgsBlockingNetworkRequest.
ErrorCode put(QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback=nullptr)
Performs a "put" operation on the specified request, using the given data.
ErrorCode head(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "head" operation on the specified request.
void abort()
Aborts the network request immediately.
Q_DECL_DEPRECATED void downloadFinished()
Emitted once a request has finished downloading.
ErrorCode post(QNetworkRequest &request, QIODevice *data, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "post" operation on the specified request, using the given data.
ErrorCode deleteResource(QNetworkRequest &request, QgsFeedback *feedback=nullptr)
Performs a "delete" operation on the specified request.
void finished()
Emitted once a request has finished.
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
QString authCfg() const
Returns the authentication config id which will be used during the request.
void uploadProgress(qint64, qint64)
Emitted when when data are sent during a request.
@ NetworkError
A network error occurred.
@ ServerExceptionError
An exception was raised by the server.
@ NoError
No error was encountered.
@ TimeoutError
Timeout was reached before a reply was received.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
Base class for feedback objects to be used for cancellation of something running in a worker thread.
Definition: qgsfeedback.h:45
void canceled()
Internal routines can connect to this signal if they use event loop.
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
network access manager for QGIS
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
void requestTimedOut(QgsNetworkRequestParameters request)
Emitted when a network request has timed out.
Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between...
void setContent(const QByteArray &content)
Sets the reply content.
void clear()
Clears the reply, resetting it back to a default, empty reply.
#define Q_NOWARN_DEPRECATED_POP
Definition: qgis.h:2815
#define Q_NOWARN_DEPRECATED_PUSH
Definition: qgis.h:2814
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39