QGIS API Documentation  3.22.4-Białowieża (ce8e65e95e)
qgsalgorithmfiledownloader.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsalgorithmfiledownloader.cpp
3  ---------------------
4  begin : October 2017
5  copyright : (C) 2017 by Etienne Trimaille
6  email : etienne at kartoza dot com
7  ***************************************************************************/
8 
9 /***************************************************************************
10  * *
11  * This program is free software; you can redistribute it and/or modify *
12  * it under the terms of the GNU General Public License as published by *
13  * the Free Software Foundation; either version 2 of the License, or *
14  * (at your option) any later version. *
15  * *
16  ***************************************************************************/
17 
20 #include "qgis.h"
21 #include "qgsfiledownloader.h"
22 #include "qgsfileutils.h"
23 
24 #include <QEventLoop>
25 #include <QFileInfo>
26 #include <QTimer>
27 #include <QUrl>
28 
30 
31 QString QgsFileDownloaderAlgorithm::name() const
32 {
33  return QStringLiteral( "filedownloader" );
34 }
35 
36 QString QgsFileDownloaderAlgorithm::displayName() const
37 {
38  return tr( "Download file" );
39 }
40 
41 QStringList QgsFileDownloaderAlgorithm::tags() const
42 {
43  return tr( "file,downloader,internet,url,fetch,get,https" ).split( ',' );
44 }
45 
46 QString QgsFileDownloaderAlgorithm::group() const
47 {
48  return tr( "File tools" );
49 }
50 
51 QString QgsFileDownloaderAlgorithm::groupId() const
52 {
53  return QStringLiteral( "filetools" );
54 }
55 
56 QString QgsFileDownloaderAlgorithm::shortHelpString() const
57 {
58  return tr( "This algorithm downloads a URL on the file system." );
59 }
60 
61 QgsFileDownloaderAlgorithm *QgsFileDownloaderAlgorithm::createInstance() const
62 {
63  return new QgsFileDownloaderAlgorithm();
64 }
65 
66 void QgsFileDownloaderAlgorithm::initAlgorithm( const QVariantMap & )
67 {
68  addParameter( new QgsProcessingParameterString( QStringLiteral( "URL" ), tr( "URL" ), QVariant(), false, false ) );
69 
70  std::unique_ptr< QgsProcessingParameterEnum > methodParam = std::make_unique < QgsProcessingParameterEnum > (
71  QStringLiteral( "METHOD" ),
72  QObject::tr( "Method" ),
73  QStringList()
74  << QObject::tr( "GET" )
75  << QObject::tr( "POST" ),
76  false,
77  0
78  );
79  methodParam->setHelp( QObject::tr( "The HTTP method to use for the request" ) );
80  methodParam->setFlags( methodParam->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
81  addParameter( methodParam.release() );
82 
83  std::unique_ptr< QgsProcessingParameterString > dataParam = std::make_unique < QgsProcessingParameterString >(
84  QStringLiteral( "DATA" ), tr( "Data" ), QVariant(), false, true );
85  dataParam->setHelp( QObject::tr( "The data to add in the body if the request is a POST" ) );
86  dataParam->setFlags( dataParam->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
87  addParameter( dataParam.release() );
88 
89  addParameter( new QgsProcessingParameterFileDestination( QStringLiteral( "OUTPUT" ),
90  tr( "File destination" ), QObject::tr( "All files (*.*)" ), QVariant(), true ) );
91 }
92 
93 QVariantMap QgsFileDownloaderAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
94 {
95  mFeedback = feedback;
96  QString url = parameterAsString( parameters, QStringLiteral( "URL" ), context );
97  if ( url.isEmpty() )
98  throw QgsProcessingException( tr( "No URL specified" ) );
99 
100  QString data = parameterAsString( parameters, QStringLiteral( "DATA" ), context );
101  QString outputFile = parameterAsFileOutput( parameters, QStringLiteral( "OUTPUT" ), context );
102 
103  QEventLoop loop;
104  QTimer timer;
105  QUrl downloadedUrl;
106  QStringList errors;
107 
108  Qgis::HttpMethod httpMethod = static_cast< Qgis::HttpMethod>( parameterAsEnum( parameters, QStringLiteral( "METHOD" ), context ) );
109 
110  if ( httpMethod == Qgis::HttpMethod::Get && ! data.isEmpty() )
111  {
112  feedback->pushWarning( tr( "DATA parameter is not used when it's a GET request." ) );
113  data = QString();
114  }
115 
116  QgsFileDownloader *downloader = new QgsFileDownloader( QUrl( url ), outputFile, QString(), true, httpMethod, data.toUtf8() );
117  connect( mFeedback, &QgsFeedback::canceled, downloader, &QgsFileDownloader::cancelDownload );
118  connect( downloader, &QgsFileDownloader::downloadError, this, [&errors, &loop]( const QStringList & e ) { errors = e; loop.exit(); } );
119  connect( downloader, &QgsFileDownloader::downloadProgress, this, &QgsFileDownloaderAlgorithm::receiveProgressFromDownloader );
120  connect( downloader, &QgsFileDownloader::downloadCompleted, this, [&downloadedUrl]( const QUrl url ) { downloadedUrl = url; } );
121  connect( downloader, &QgsFileDownloader::downloadExited, this, [&loop]() { loop.exit(); } );
122  connect( &timer, &QTimer::timeout, this, &QgsFileDownloaderAlgorithm::sendProgressFeedback );
123  downloader->startDownload();
124  timer.start( 1000 );
125 
126  loop.exec();
127 
128  timer.stop();
129  if ( errors.size() > 0 )
130  throw QgsProcessingException( errors.join( '\n' ) );
131 
132  const bool exists = QFileInfo::exists( outputFile );
133  if ( !feedback->isCanceled() && !exists )
134  throw QgsProcessingException( tr( "Output file doesn't exist." ) );
135 
136  url = downloadedUrl.toDisplayString();
137  feedback->pushInfo( QObject::tr( "Successfully downloaded %1" ).arg( url ) );
138 
139  if ( outputFile.startsWith( QgsProcessingUtils::tempFolder() ) )
140  {
141  // the output is temporary and its file name automatically generated, try to add a file extension
142  const int length = url.size();
143  const int lastDotIndex = url.lastIndexOf( "." );
144  const int lastSlashIndex = url.lastIndexOf( "/" );
145  if ( lastDotIndex > -1 && lastDotIndex > lastSlashIndex && length - lastDotIndex <= 6 )
146  {
147  QFile tmpFile( outputFile );
148  tmpFile.rename( tmpFile.fileName() + url.mid( lastDotIndex ) );
149  outputFile += url.mid( lastDotIndex );
150  }
151  }
152 
153  QVariantMap outputs;
154  outputs.insert( QStringLiteral( "OUTPUT" ), exists ? outputFile : QString() );
155  return outputs;
156 }
157 
158 void QgsFileDownloaderAlgorithm::sendProgressFeedback()
159 {
160  if ( !mReceived.isEmpty() && mLastReport != mReceived )
161  {
162  mLastReport = mReceived;
163  if ( mTotal.isEmpty() )
164  mFeedback->pushInfo( tr( "%1 downloaded" ).arg( mReceived ) );
165  else
166  mFeedback->pushInfo( tr( "%1 of %2 downloaded" ).arg( mReceived, mTotal ) );
167  }
168 }
169 
170 void QgsFileDownloaderAlgorithm::receiveProgressFromDownloader( qint64 bytesReceived, qint64 bytesTotal )
171 {
172  mReceived = QgsFileUtils::representFileSize( bytesReceived );
173  if ( bytesTotal > 0 )
174  {
175  if ( mTotal.isEmpty() )
176  mTotal = QgsFileUtils::representFileSize( bytesTotal );
177 
178  mFeedback->setProgress( ( bytesReceived * 100 ) / bytesTotal );
179  }
180 }
181 
HttpMethod
Different methods of HTTP requests.
Definition: qgis.h:335
@ Get
GET method.
void canceled()
Internal routines can connect to this signal if they use event loop.
bool isCanceled() const SIP_HOLDGIL
Tells whether the operation has been canceled already.
Definition: qgsfeedback.h:54
QgsFileDownloader is a utility class for downloading files.
void cancelDownload()
Call to abort the download and delete this object after the cancellation has been processed.
void downloadExited()
Emitted always when the downloader exits.
void downloadError(QStringList errorMessages)
Emitted when an error makes the download fail.
void startDownload()
Called to start the download.
void downloadCompleted(const QUrl &url)
Emitted when the download has completed successfully.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Emitted when data are ready to be processed.
static QString representFileSize(qint64 bytes)
Returns the human size from bytes.
Contains information about the context in which a processing algorithm is executed.
Custom exception class for processing related exceptions.
Definition: qgsexception.h:83
Base class for providing feedback from a processing algorithm.
virtual void pushInfo(const QString &info)
Pushes a general informational message from the algorithm.
virtual void pushWarning(const QString &warning)
Pushes a warning informational message from the algorithm.
@ FlagAdvanced
Parameter is an advanced parameter which should be hidden from users by default.
A generic file based destination parameter, for specifying the destination path for a file (non-map l...
A string parameter for processing algorithms.
static QString tempFolder()
Returns a session specific processing temporary folder for use in processing algorithms.