QGIS API Documentation 3.28.0-Firenze (ed3ad0430f)
qgsalgorithmfiledownloader.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgsalgorithmfiledownloader.cpp
3 ---------------------
4 begin : October 2017
5 copyright : (C) 2017 by Etienne Trimaille
6 email : etienne at kartoza dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
20#include "qgis.h"
21#include "qgsfiledownloader.h"
22#include "qgsfileutils.h"
23
24#include <QEventLoop>
25#include <QFileInfo>
26#include <QTimer>
27#include <QUrl>
28
30
31QString QgsFileDownloaderAlgorithm::name() const
32{
33 return QStringLiteral( "filedownloader" );
34}
35
36QString QgsFileDownloaderAlgorithm::displayName() const
37{
38 return tr( "Download file" );
39}
40
41QStringList QgsFileDownloaderAlgorithm::tags() const
42{
43 return tr( "file,downloader,internet,url,fetch,get,https" ).split( ',' );
44}
45
46QString QgsFileDownloaderAlgorithm::group() const
47{
48 return tr( "File tools" );
49}
50
51QString QgsFileDownloaderAlgorithm::groupId() const
52{
53 return QStringLiteral( "filetools" );
54}
55
56QString QgsFileDownloaderAlgorithm::shortHelpString() const
57{
58 return tr( "This algorithm downloads a URL on the file system." );
59}
60
61QgsFileDownloaderAlgorithm *QgsFileDownloaderAlgorithm::createInstance() const
62{
63 return new QgsFileDownloaderAlgorithm();
64}
65
66void QgsFileDownloaderAlgorithm::initAlgorithm( const QVariantMap & )
67{
68 addParameter( new QgsProcessingParameterString( QStringLiteral( "URL" ), tr( "URL" ), QVariant(), false, false ) );
69
70 std::unique_ptr< QgsProcessingParameterEnum > methodParam = std::make_unique < QgsProcessingParameterEnum > (
71 QStringLiteral( "METHOD" ),
72 QObject::tr( "Method" ),
73 QStringList()
74 << QObject::tr( "GET" )
75 << QObject::tr( "POST" ),
76 false,
77 0
78 );
79 methodParam->setHelp( QObject::tr( "The HTTP method to use for the request" ) );
80 methodParam->setFlags( methodParam->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
81 addParameter( methodParam.release() );
82
83 std::unique_ptr< QgsProcessingParameterString > dataParam = std::make_unique < QgsProcessingParameterString >(
84 QStringLiteral( "DATA" ), tr( "Data" ), QVariant(), false, true );
85 dataParam->setHelp( QObject::tr( "The data to add in the body if the request is a POST" ) );
86 dataParam->setFlags( dataParam->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
87 addParameter( dataParam.release() );
88
89 addParameter( new QgsProcessingParameterFileDestination( QStringLiteral( "OUTPUT" ),
90 tr( "File destination" ), QObject::tr( "All files (*.*)" ), QVariant(), true ) );
91}
92
93QVariantMap QgsFileDownloaderAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
94{
95 mFeedback = feedback;
96 QString url = parameterAsString( parameters, QStringLiteral( "URL" ), context );
97 if ( url.isEmpty() )
98 throw QgsProcessingException( tr( "No URL specified" ) );
99
100 QString data = parameterAsString( parameters, QStringLiteral( "DATA" ), context );
101 QString outputFile = parameterAsFileOutput( parameters, QStringLiteral( "OUTPUT" ), context );
102
103 QEventLoop loop;
104 QTimer timer;
105 QUrl downloadedUrl;
106 QStringList errors;
107
108 Qgis::HttpMethod httpMethod = static_cast< Qgis::HttpMethod>( parameterAsEnum( parameters, QStringLiteral( "METHOD" ), context ) );
109
110 if ( httpMethod == Qgis::HttpMethod::Get && ! data.isEmpty() )
111 {
112 feedback->pushWarning( tr( "DATA parameter is not used when it's a GET request." ) );
113 data = QString();
114 }
115
116 QgsFileDownloader *downloader = new QgsFileDownloader( QUrl( url ), outputFile, QString(), true, httpMethod, data.toUtf8() );
117 connect( mFeedback, &QgsFeedback::canceled, downloader, &QgsFileDownloader::cancelDownload );
118 connect( downloader, &QgsFileDownloader::downloadError, this, [&errors, &loop]( const QStringList & e ) { errors = e; loop.exit(); } );
119 connect( downloader, &QgsFileDownloader::downloadProgress, this, &QgsFileDownloaderAlgorithm::receiveProgressFromDownloader );
120 connect( downloader, &QgsFileDownloader::downloadCompleted, this, [&downloadedUrl]( const QUrl url ) { downloadedUrl = url; } );
121 connect( downloader, &QgsFileDownloader::downloadExited, this, [&loop]() { loop.exit(); } );
122 connect( &timer, &QTimer::timeout, this, &QgsFileDownloaderAlgorithm::sendProgressFeedback );
123 downloader->startDownload();
124 timer.start( 1000 );
125
126 loop.exec();
127
128 timer.stop();
129 if ( errors.size() > 0 )
130 throw QgsProcessingException( errors.join( '\n' ) );
131
132 const bool exists = QFileInfo::exists( outputFile );
133 if ( !feedback->isCanceled() && !exists )
134 throw QgsProcessingException( tr( "Output file doesn't exist." ) );
135
136 url = downloadedUrl.toDisplayString();
137 feedback->pushInfo( QObject::tr( "Successfully downloaded %1" ).arg( url ) );
138
139 if ( outputFile.startsWith( QgsProcessingUtils::tempFolder() ) )
140 {
141 // the output is temporary and its file name automatically generated, try to add a file extension
142 const int length = url.size();
143 const int lastDotIndex = url.lastIndexOf( "." );
144 const int lastSlashIndex = url.lastIndexOf( "/" );
145 if ( lastDotIndex > -1 && lastDotIndex > lastSlashIndex && length - lastDotIndex <= 6 )
146 {
147 QFile tmpFile( outputFile );
148 tmpFile.rename( tmpFile.fileName() + url.mid( lastDotIndex ) );
149 outputFile += url.mid( lastDotIndex );
150 }
151 }
152
153 QVariantMap outputs;
154 outputs.insert( QStringLiteral( "OUTPUT" ), exists ? outputFile : QString() );
155 return outputs;
156}
157
158void QgsFileDownloaderAlgorithm::sendProgressFeedback()
159{
160 if ( !mReceived.isEmpty() && mLastReport != mReceived )
161 {
162 mLastReport = mReceived;
163 if ( mTotal.isEmpty() )
164 mFeedback->pushInfo( tr( "%1 downloaded" ).arg( mReceived ) );
165 else
166 mFeedback->pushInfo( tr( "%1 of %2 downloaded" ).arg( mReceived, mTotal ) );
167 }
168}
169
170void QgsFileDownloaderAlgorithm::receiveProgressFromDownloader( qint64 bytesReceived, qint64 bytesTotal )
171{
172 mReceived = QgsFileUtils::representFileSize( bytesReceived );
173 if ( bytesTotal > 0 )
174 {
175 if ( mTotal.isEmpty() )
176 mTotal = QgsFileUtils::representFileSize( bytesTotal );
177
178 mFeedback->setProgress( ( bytesReceived * 100 ) / bytesTotal );
179 }
180}
181
HttpMethod
Different methods of HTTP requests.
Definition: qgis.h:438
@ Get
GET method.
void canceled()
Internal routines can connect to this signal if they use event loop.
bool isCanceled() const SIP_HOLDGIL
Tells whether the operation has been canceled already.
Definition: qgsfeedback.h:54
QgsFileDownloader is a utility class for downloading files.
void cancelDownload()
Call to abort the download and delete this object after the cancellation has been processed.
void downloadExited()
Emitted always when the downloader exits.
void downloadError(QStringList errorMessages)
Emitted when an error makes the download fail.
void startDownload()
Called to start the download.
void downloadCompleted(const QUrl &url)
Emitted when the download has completed successfully.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Emitted when data are ready to be processed.
static QString representFileSize(qint64 bytes)
Returns the human size from bytes.
Contains information about the context in which a processing algorithm is executed.
Custom exception class for processing related exceptions.
Definition: qgsexception.h:83
Base class for providing feedback from a processing algorithm.
virtual void pushInfo(const QString &info)
Pushes a general informational message from the algorithm.
virtual void pushWarning(const QString &warning)
Pushes a warning informational message from the algorithm.
@ FlagAdvanced
Parameter is an advanced parameter which should be hidden from users by default.
A generic file based destination parameter, for specifying the destination path for a file (non-map l...
A string parameter for processing algorithms.
static QString tempFolder()
Returns a session specific processing temporary folder for use in processing algorithms.