QGIS API Documentation 3.40.0-Bratislava (b56115d8743)
Loading...
Searching...
No Matches
qgsalgorithmfiledownloader.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgsalgorithmfiledownloader.cpp
3 ---------------------
4 begin : October 2017
5 copyright : (C) 2017 by Etienne Trimaille
6 email : etienne at kartoza dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
20#include "qgis.h"
21#include "qgsfiledownloader.h"
22#include "qgsfileutils.h"
23
24#include <QEventLoop>
25#include <QFileInfo>
26#include <QTimer>
27#include <QUrl>
28
30
31QString QgsFileDownloaderAlgorithm::name() const
32{
33 return QStringLiteral( "filedownloader" );
34}
35
36QString QgsFileDownloaderAlgorithm::displayName() const
37{
38 return tr( "Download file via HTTP(S)" );
39}
40
41QString QgsFileDownloaderAlgorithm::shortDescription() const
42{
43 return tr( "Downloads a URL to the file system with an HTTP(S) GET or POST request" );
44}
45
46QStringList QgsFileDownloaderAlgorithm::tags() const
47{
48 return tr( "file,downloader,internet,url,fetch,get,post,request,https" ).split( ',' );
49}
50
51QString QgsFileDownloaderAlgorithm::group() const
52{
53 return tr( "File tools" );
54}
55
56QString QgsFileDownloaderAlgorithm::groupId() const
57{
58 return QStringLiteral( "filetools" );
59}
60
61QString QgsFileDownloaderAlgorithm::shortHelpString() const
62{
63 return tr( "This algorithm downloads a URL to the file system with an HTTP(S) GET or POST request" );
64}
65
66QgsFileDownloaderAlgorithm *QgsFileDownloaderAlgorithm::createInstance() const
67{
68 return new QgsFileDownloaderAlgorithm();
69}
70
71void QgsFileDownloaderAlgorithm::initAlgorithm( const QVariantMap & )
72{
73 addParameter( new QgsProcessingParameterString( QStringLiteral( "URL" ), tr( "URL" ), QVariant(), false, false ) );
74
75 std::unique_ptr< QgsProcessingParameterEnum > methodParam = std::make_unique < QgsProcessingParameterEnum > (
76 QStringLiteral( "METHOD" ),
77 QObject::tr( "Method" ),
78 QStringList()
79 << QObject::tr( "GET" )
80 << QObject::tr( "POST" ),
81 false,
82 0
83 );
84 methodParam->setHelp( QObject::tr( "The HTTP method to use for the request" ) );
85 methodParam->setFlags( methodParam->flags() | Qgis::ProcessingParameterFlag::Advanced );
86 addParameter( methodParam.release() );
87
88 std::unique_ptr< QgsProcessingParameterString > dataParam = std::make_unique < QgsProcessingParameterString >(
89 QStringLiteral( "DATA" ), tr( "Data" ), QVariant(), false, true );
90 dataParam->setHelp( QObject::tr( "The data to add in the body if the request is a POST" ) );
91 dataParam->setFlags( dataParam->flags() | Qgis::ProcessingParameterFlag::Advanced );
92 addParameter( dataParam.release() );
93 addParameter( new QgsProcessingParameterFileDestination( QStringLiteral( "OUTPUT" ),
94 tr( "File destination" ), QObject::tr( "All files (*.*)" ), QVariant(), false ) );
95}
96
97QVariantMap QgsFileDownloaderAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
98{
99 mFeedback = feedback;
100 QString url = parameterAsString( parameters, QStringLiteral( "URL" ), context );
101 if ( url.isEmpty() )
102 throw QgsProcessingException( tr( "No URL specified" ) );
103
104 QString data = parameterAsString( parameters, QStringLiteral( "DATA" ), context );
105 QString outputFile = parameterAsFileOutput( parameters, QStringLiteral( "OUTPUT" ), context );
106
107 QEventLoop loop;
108 QTimer timer;
109 QUrl downloadedUrl;
110 QStringList errors;
111
112 Qgis::HttpMethod httpMethod = static_cast< Qgis::HttpMethod>( parameterAsEnum( parameters, QStringLiteral( "METHOD" ), context ) );
113
114 if ( httpMethod == Qgis::HttpMethod::Get && ! data.isEmpty() )
115 {
116 feedback->pushWarning( tr( "DATA parameter is not used when it's a GET request." ) );
117 data = QString();
118 }
119
120 QgsFileDownloader *downloader = new QgsFileDownloader( QUrl( url ), outputFile, QString(), true, httpMethod, data.toUtf8() );
121 connect( mFeedback, &QgsFeedback::canceled, downloader, &QgsFileDownloader::cancelDownload );
122 connect( downloader, &QgsFileDownloader::downloadError, this, [&errors, &loop]( const QStringList & e ) { errors = e; loop.exit(); } );
123 connect( downloader, &QgsFileDownloader::downloadProgress, this, &QgsFileDownloaderAlgorithm::receiveProgressFromDownloader );
124 connect( downloader, &QgsFileDownloader::downloadCompleted, this, [&downloadedUrl]( const QUrl url ) { downloadedUrl = url; } );
125 connect( downloader, &QgsFileDownloader::downloadExited, this, [&loop]() { loop.exit(); } );
126 connect( &timer, &QTimer::timeout, this, &QgsFileDownloaderAlgorithm::sendProgressFeedback );
127 downloader->startDownload();
128 timer.start( 1000 );
129
130 loop.exec();
131
132 timer.stop();
133 if ( errors.size() > 0 )
134 throw QgsProcessingException( errors.join( '\n' ) );
135
136 const bool exists = QFileInfo::exists( outputFile );
137 if ( !feedback->isCanceled() && !exists )
138 throw QgsProcessingException( tr( "Output file doesn't exist." ) );
139
140 url = downloadedUrl.toDisplayString();
141 feedback->pushInfo( QObject::tr( "Successfully downloaded %1" ).arg( url ) );
142
143 if ( outputFile.startsWith( QgsProcessingUtils::tempFolder( &context ) ) )
144 {
145 // the output is temporary and its file name automatically generated, try to add a file extension
146 const int length = url.size();
147 const int lastDotIndex = url.lastIndexOf( "." );
148 const int lastSlashIndex = url.lastIndexOf( "/" );
149 if ( lastDotIndex > -1 && lastDotIndex > lastSlashIndex && length - lastDotIndex <= 6 )
150 {
151 QFile tmpFile( outputFile );
152 tmpFile.rename( tmpFile.fileName() + url.mid( lastDotIndex ) );
153 outputFile += url.mid( lastDotIndex );
154 }
155 }
156
157 QVariantMap outputs;
158 outputs.insert( QStringLiteral( "OUTPUT" ), exists ? outputFile : QString() );
159 return outputs;
160}
161
162void QgsFileDownloaderAlgorithm::sendProgressFeedback()
163{
164 if ( !mReceived.isEmpty() && mLastReport != mReceived )
165 {
166 mLastReport = mReceived;
167 if ( mTotal.isEmpty() )
168 mFeedback->pushInfo( tr( "%1 downloaded" ).arg( mReceived ) );
169 else
170 mFeedback->pushInfo( tr( "%1 of %2 downloaded" ).arg( mReceived, mTotal ) );
171 }
172}
173
174void QgsFileDownloaderAlgorithm::receiveProgressFromDownloader( qint64 bytesReceived, qint64 bytesTotal )
175{
176 mReceived = QgsFileUtils::representFileSize( bytesReceived );
177 if ( bytesTotal > 0 )
178 {
179 if ( mTotal.isEmpty() )
180 mTotal = QgsFileUtils::representFileSize( bytesTotal );
181
182 mFeedback->setProgress( ( bytesReceived * 100 ) / bytesTotal );
183 }
184}
185
HttpMethod
Different methods of HTTP requests.
Definition qgis.h:971
@ Get
GET method.
@ Advanced
Parameter is an advanced parameter which should be hidden from users by default.
bool isCanceled() const
Tells whether the operation has been canceled already.
Definition qgsfeedback.h:53
void canceled()
Internal routines can connect to this signal if they use event loop.
QgsFileDownloader is a utility class for downloading files.
void cancelDownload()
Call to abort the download and delete this object after the cancellation has been processed.
void downloadExited()
Emitted always when the downloader exits.
void downloadError(QStringList errorMessages)
Emitted when an error makes the download fail.
void startDownload()
Called to start the download.
void downloadCompleted(const QUrl &url)
Emitted when the download has completed successfully.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Emitted when data are ready to be processed.
static QString representFileSize(qint64 bytes)
Returns the human size from bytes.
Contains information about the context in which a processing algorithm is executed.
Custom exception class for processing related exceptions.
Base class for providing feedback from a processing algorithm.
virtual void pushInfo(const QString &info)
Pushes a general informational message from the algorithm.
virtual void pushWarning(const QString &warning)
Pushes a warning informational message from the algorithm.
A generic file based destination parameter, for specifying the destination path for a file (non-map l...
A string parameter for processing algorithms.
static QString tempFolder(const QgsProcessingContext *context=nullptr)
Returns a session specific processing temporary folder for use in processing algorithms.