QGIS API Documentation  3.16.0-Hannover (43b64b13f3)
qgsblockingnetworkrequest.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsblockingnetworkrequest.cpp
3  -----------------------------
4  begin : November 2018
5  copyright : (C) 2018 by Nyall Dawson
6  email : nyall dot dawson at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
17 #include "qgslogger.h"
18 #include "qgsapplication.h"
20 #include "qgsauthmanager.h"
21 #include "qgsmessagelog.h"
22 #include "qgsfeedback.h"
23 #include <QUrl>
24 #include <QNetworkRequest>
25 #include <QNetworkReply>
26 #include <QMutex>
27 #include <QWaitCondition>
28 #include <QNetworkCacheMetaData>
29 #include <QAuthenticator>
30 
32 {
33  connect( QgsNetworkAccessManager::instance(), qgis::overload< QNetworkReply * >::of( &QgsNetworkAccessManager::requestTimedOut ), this, &QgsBlockingNetworkRequest::requestTimedOut );
34 }
35 
37 {
38  abort();
39 }
40 
41 void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply )
42 {
43  if ( reply == mReply )
44  mTimedout = true;
45 }
46 
48 {
49  return mAuthCfg;
50 }
51 
52 void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg )
53 {
54  mAuthCfg = authCfg;
55 }
56 
57 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
58 {
59  return doRequest( Get, request, forceRefresh, feedback );
60 }
61 
62 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh, QgsFeedback *feedback )
63 {
64  mPostData = data;
65  return doRequest( Post, request, forceRefresh, feedback );
66 }
67 
68 QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback )
69 {
70  mMethod = method;
71  mFeedback = feedback;
72 
73  abort(); // cancel previous
74  mIsAborted = false;
75  mTimedout = false;
76  mGotNonEmptyResponse = false;
77 
78  mErrorMessage.clear();
79  mErrorCode = NoError;
80  mForceRefresh = forceRefresh;
81  mReplyContent.clear();
82 
83  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
84  {
85  mErrorCode = NetworkError;
86  mErrorMessage = errorMessageFailedAuth();
87  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
88  return NetworkError;
89  }
90 
91  QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 );
92 
93  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
94  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
95 
96  QWaitCondition authRequestBufferNotEmpty;
97  QMutex waitConditionMutex;
98 
99  bool threadFinished = false;
100  bool success = false;
101 
102  const bool requestMadeFromMainThread = QThread::currentThread() == QApplication::instance()->thread();
103 
104  if ( mFeedback )
105  connect( mFeedback, &QgsFeedback::canceled, this, &QgsBlockingNetworkRequest::abort );
106 
107  std::function<void()> downloaderFunction = [ this, request, &waitConditionMutex, &authRequestBufferNotEmpty, &threadFinished, &success, requestMadeFromMainThread ]()
108  {
109  // this function will always be run in worker threads -- either the blocking call is being made in a worker thread,
110  // or the blocking call has been made from the main thread and we've fired up a new thread for this function
111  Q_ASSERT( QThread::currentThread() != QgsApplication::instance()->thread() );
112 
113  QgsNetworkAccessManager::instance( Qt::DirectConnection );
114 
115  success = true;
116 
117  switch ( mMethod )
118  {
119  case Get:
120  mReply = QgsNetworkAccessManager::instance()->get( request );
121  break;
122 
123  case Post:
124  mReply = QgsNetworkAccessManager::instance()->post( request, mPostData );
125  break;
126  };
127 
128  if ( mFeedback )
129  connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
130 
131  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
132  {
133  mErrorCode = NetworkError;
134  mErrorMessage = errorMessageFailedAuth();
135  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
136  if ( requestMadeFromMainThread )
137  authRequestBufferNotEmpty.wakeAll();
138  success = false;
139  }
140  else
141  {
142  // We are able to use direct connection here, because we
143  // * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway
144  // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread)
145  connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
146  connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
147 
148  auto resumeMainThread = [&waitConditionMutex, &authRequestBufferNotEmpty ]()
149  {
150  // when this method is called we have "produced" a single authentication request -- so the buffer is now full
151  // and it's time for the "consumer" (main thread) to do its part
152  waitConditionMutex.lock();
153  authRequestBufferNotEmpty.wakeAll();
154  waitConditionMutex.unlock();
155 
156  // note that we don't need to handle waking this thread back up - that's done automatically by QgsNetworkAccessManager
157  };
158 
159  if ( requestMadeFromMainThread )
160  {
161  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authRequestOccurred, this, resumeMainThread, Qt::DirectConnection );
162  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection );
163 
164 #ifndef QT_NO_SSL
165  connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrorsOccurred, this, resumeMainThread, Qt::DirectConnection );
166 #endif
167  }
168  QEventLoop loop;
169  connect( this, &QgsBlockingNetworkRequest::downloadFinished, &loop, &QEventLoop::quit, Qt::DirectConnection );
170  loop.exec();
171  }
172 
173  if ( requestMadeFromMainThread )
174  {
175  waitConditionMutex.lock();
176  threadFinished = true;
177  authRequestBufferNotEmpty.wakeAll();
178  waitConditionMutex.unlock();
179  }
180  };
181 
182  if ( requestMadeFromMainThread )
183  {
184  std::unique_ptr<DownloaderThread> downloaderThread = qgis::make_unique<DownloaderThread>( downloaderFunction );
185  downloaderThread->start();
186 
187  while ( true )
188  {
189  waitConditionMutex.lock();
190  if ( threadFinished )
191  {
192  waitConditionMutex.unlock();
193  break;
194  }
195  authRequestBufferNotEmpty.wait( &waitConditionMutex );
196 
197  // If the downloader thread wakes us (the main thread) up and is not yet finished
198  // then it has "produced" an authentication request which we need to now "consume".
199  // The processEvents() call gives the auth manager the chance to show a dialog and
200  // once done with that, we can wake the downloaderThread again and continue the download.
201  if ( !threadFinished )
202  {
203  waitConditionMutex.unlock();
204 
205  QgsApplication::instance()->processEvents();
206  // we don't need to wake up the worker thread - it will automatically be woken when
207  // the auth request has been dealt with by QgsNetworkAccessManager
208  }
209  else
210  {
211  waitConditionMutex.unlock();
212  }
213  }
214  // wait for thread to gracefully exit
215  downloaderThread->wait();
216  }
217  else
218  {
219  downloaderFunction();
220  }
221  return mErrorCode;
222 }
223 
225 {
226  mIsAborted = true;
227  if ( mReply )
228  {
229  mReply->deleteLater();
230  mReply = nullptr;
231  }
232 }
233 
234 void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal )
235 {
236  QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 );
237 
238  if ( bytesReceived != 0 )
239  mGotNonEmptyResponse = true;
240 
241  if ( !mIsAborted && mReply && ( !mFeedback || !mFeedback->isCanceled() ) )
242  {
243  if ( mReply->error() == QNetworkReply::NoError )
244  {
245  QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
246  if ( !redirect.isNull() )
247  {
248  // We don't want to emit downloadProgress() for a redirect
249  return;
250  }
251  }
252  }
253 
254  emit downloadProgress( bytesReceived, bytesTotal );
255 }
256 
257 void QgsBlockingNetworkRequest::replyFinished()
258 {
259  if ( !mIsAborted && mReply )
260  {
261  if ( mReply->error() == QNetworkReply::NoError && ( !mFeedback || !mFeedback->isCanceled() ) )
262  {
263  QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 );
264  QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
265  if ( !redirect.isNull() )
266  {
267  QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 );
268 
269  const QUrl &toUrl = redirect.toUrl();
270  mReply->request();
271  if ( toUrl == mReply->url() )
272  {
273  mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
274  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
275  mReplyContent.clear();
276  }
277  else
278  {
279  QNetworkRequest request( toUrl );
280 
281  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
282  {
283  mReplyContent.clear();
284  mErrorMessage = errorMessageFailedAuth();
285  mErrorCode = NetworkError;
286  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
287  emit downloadFinished();
288  return;
289  }
290 
291  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
292  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
293 
294  mReply->deleteLater();
295  mReply = nullptr;
296 
297  QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 );
298  switch ( mMethod )
299  {
300  case Get:
301  mReply = QgsNetworkAccessManager::instance()->get( request );
302  break;
303 
304  case Post:
305  mReply = QgsNetworkAccessManager::instance()->post( request, mPostData );
306  break;
307  };
308 
309  if ( mFeedback )
310  connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort );
311 
312  if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
313  {
314  mReplyContent.clear();
315  mErrorMessage = errorMessageFailedAuth();
316  mErrorCode = NetworkError;
317  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
318  emit downloadFinished();
319  return;
320  }
321 
322  connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
323  connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
324  return;
325  }
326  }
327  else
328  {
330 
331  if ( nam->cache() )
332  {
333  QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
334 
335  QNetworkCacheMetaData::RawHeaderList hl;
336  const auto constRawHeaders = cmd.rawHeaders();
337  for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
338  {
339  if ( h.first != "Cache-Control" )
340  hl.append( h );
341  }
342  cmd.setRawHeaders( hl );
343 
344  QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 );
345  if ( cmd.expirationDate().isNull() )
346  {
347  cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) );
348  }
349 
350  nam->cache()->updateMetaData( cmd );
351  }
352  else
353  {
354  QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 );
355  }
356 
357 #ifdef QGISDEBUG
358  bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
359  QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 );
360 #endif
361 
362  mReplyContent = QgsNetworkReplyContent( mReply );
363  const QByteArray content = mReply->readAll();
364  if ( content.isEmpty() && !mGotNonEmptyResponse )
365  {
366  mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
367  mErrorCode = ServerExceptionError;
368  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
369  }
370  mReplyContent.setContent( content );
371  }
372  }
373  else
374  {
375  if ( mReply->error() != QNetworkReply::OperationCanceledError )
376  {
377  mErrorMessage = mReply->errorString();
378  mErrorCode = ServerExceptionError;
379  QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
380  }
381  mReplyContent = QgsNetworkReplyContent( mReply );
382  }
383  }
384  if ( mTimedout )
385  mErrorCode = TimeoutError;
386 
387  if ( mReply )
388  {
389  mReply->deleteLater();
390  mReply = nullptr;
391  }
392 
393  emit downloadFinished();
394 }
395 
396 QString QgsBlockingNetworkRequest::errorMessageFailedAuth()
397 {
398  return tr( "network request update failed for authentication config" );
399 }
QgsNetworkReplyContent::clear
void clear()
Clears the reply, resetting it back to a default, empty reply.
Definition: qgsnetworkreply.cpp:41
QgsNetworkAccessManager::requestTimedOut
void requestTimedOut(QgsNetworkRequestParameters request)
Emitted when a network request has timed out.
QgsDebugMsgLevel
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:39
qgsauthmanager.h
qgsblockingnetworkrequest.h
QgsFeedback::canceled
void canceled()
Internal routines can connect to this signal if they use event loop.
QgsBlockingNetworkRequest::reply
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get() or post() request has been made.
Definition: qgsblockingnetworkrequest.h:117
QgsBlockingNetworkRequest::~QgsBlockingNetworkRequest
~QgsBlockingNetworkRequest() override
Definition: qgsblockingnetworkrequest.cpp:36
QgsAuthManager::updateNetworkReply
bool updateNetworkReply(QNetworkReply *reply, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkReply with an authentication config (used to skip known SSL errors,...
Definition: qgsauthmanager.cpp:1483
QgsBlockingNetworkRequest::QgsBlockingNetworkRequest
QgsBlockingNetworkRequest()
Constructor for QgsBlockingNetworkRequest.
Definition: qgsblockingnetworkrequest.cpp:31
QgsApplication::instance
static QgsApplication * instance()
Returns the singleton instance of the QgsApplication.
Definition: qgsapplication.cpp:411
QgsBlockingNetworkRequest::ServerExceptionError
@ ServerExceptionError
An exception was raised by the server.
Definition: qgsblockingnetworkrequest.h:57
QgsApplication::authManager
static QgsAuthManager * authManager()
Returns the application's authentication manager instance.
Definition: qgsapplication.cpp:1282
qgsapplication.h
QgsBlockingNetworkRequest::ErrorCode
ErrorCode
Error codes.
Definition: qgsblockingnetworkrequest.h:53
QgsFeedback
Base class for feedback objects to be used for cancellation of something running in a worker thread.
Definition: qgsfeedback.h:44
qgsnetworkaccessmanager.h
QgsBlockingNetworkRequest::setAuthCfg
void setAuthCfg(const QString &authCfg)
Sets the authentication config id which should be used during the request.
Definition: qgsblockingnetworkrequest.cpp:52
QgsBlockingNetworkRequest::NoError
@ NoError
No error was encountered.
Definition: qgsblockingnetworkrequest.h:54
QgsNetworkAccessManager
network access manager for QGIS
Definition: qgsnetworkaccessmanager.h:252
QgsNetworkAccessManager::instance
static QgsNetworkAccessManager * instance(Qt::ConnectionType connectionType=Qt::BlockingQueuedConnection)
Returns a pointer to the active QgsNetworkAccessManager for the current thread.
Definition: qgsnetworkaccessmanager.cpp:121
QgsBlockingNetworkRequest::abort
void abort()
Aborts the network request immediately.
Definition: qgsblockingnetworkrequest.cpp:224
QgsBlockingNetworkRequest::TimeoutError
@ TimeoutError
Timeout was reached before a reply was received.
Definition: qgsblockingnetworkrequest.h:56
QgsBlockingNetworkRequest::get
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "get" operation on the specified request.
Definition: qgsblockingnetworkrequest.cpp:57
QgsAuthManager::updateNetworkRequest
bool updateNetworkRequest(QNetworkRequest &request, const QString &authcfg, const QString &dataprovider=QString())
Provider call to update a QNetworkRequest with an authentication config.
Definition: qgsauthmanager.cpp:1458
QgsBlockingNetworkRequest::post
ErrorCode post(QNetworkRequest &request, const QByteArray &data, bool forceRefresh=false, QgsFeedback *feedback=nullptr)
Performs a "post" operation on the specified request, using the given data.
Definition: qgsblockingnetworkrequest.cpp:62
QgsMessageLog::logMessage
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
Definition: qgsmessagelog.cpp:27
QgsNetworkReplyContent
Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between...
Definition: qgsnetworkreply.h:29
QgsNetworkReplyContent::setContent
void setContent(const QByteArray &content)
Sets the reply content.
Definition: qgsnetworkreply.h:148
QgsBlockingNetworkRequest::downloadFinished
void downloadFinished()
Emitted once a request has finished downloading.
QgsBlockingNetworkRequest::NetworkError
@ NetworkError
A network error occurred.
Definition: qgsblockingnetworkrequest.h:55
qgslogger.h
QgsBlockingNetworkRequest::downloadProgress
void downloadProgress(qint64, qint64)
Emitted when when data arrives during a request.
qgsfeedback.h
QgsBlockingNetworkRequest::authCfg
QString authCfg() const
Returns the authentication config id which will be used during the request.
Definition: qgsblockingnetworkrequest.cpp:47
qgsmessagelog.h