QGIS API Documentation 3.29.0-Master (006c3c0232)
qgsabstractcontentcache.h
Go to the documentation of this file.
1/***************************************************************************
2 qgsabstractcontentcache.h
3 ---------------
4 begin : December 2018
5 copyright : (C) 2018 by Nyall Dawson
6 email : nyall dot dawson at gmail dot com
7 ***************************************************************************/
8
9/***************************************************************************
10 * *
11 * This program is free software; you can redistribute it and/or modify *
12 * it under the terms of the GNU General Public License as published by *
13 * the Free Software Foundation; either version 2 of the License, or *
14 * (at your option) any later version. *
15 * *
16 ***************************************************************************/
17
18#ifndef QGSABSTRACTCONTENTCACHE_H
19#define QGSABSTRACTCONTENTCACHE_H
20
21#include "qgis_core.h"
22#include "qgis_sip.h"
23#include "qgslogger.h"
24#include "qgsmessagelog.h"
25#include "qgsapplication.h"
28#include "qgsvariantutils.h"
29
30#include <QObject>
31#include <QRecursiveMutex>
32#include <QCache>
33#include <QSet>
34#include <QDateTime>
35#include <QList>
36#include <QFile>
37#include <QNetworkReply>
38#include <QFileInfo>
39#include <QUrl>
40
52{
53 public:
54
58 QgsAbstractContentCacheEntry( const QString &path ) ;
59
60 virtual ~QgsAbstractContentCacheEntry() = default;
61
66
70 QString path;
71
73 QDateTime fileModified;
74
77
79 int mFileModifiedCheckTimeout = 30000;
80
85 QgsAbstractContentCacheEntry *nextEntry = nullptr;
86
91 QgsAbstractContentCacheEntry *previousEntry = nullptr;
92
93 bool operator==( const QgsAbstractContentCacheEntry &other ) const
94 {
95 return other.path == path;
96 }
97
101 virtual int dataSize() const = 0;
102
106 virtual void dump() const = 0;
107
108 protected:
109
115 virtual bool isEqual( const QgsAbstractContentCacheEntry *other ) const = 0;
116
117 private:
118#ifdef SIP_RUN
120#endif
121
122};
123
134class CORE_EXPORT QgsAbstractContentCacheBase: public QObject
135{
136 Q_OBJECT
137
138 public:
139
143 QgsAbstractContentCacheBase( QObject *parent );
144
145 signals:
146
150 void remoteContentFetched( const QString &url );
151
152 protected:
153
158 virtual bool checkReply( QNetworkReply *reply, const QString &path ) const
159 {
160 Q_UNUSED( reply )
161 Q_UNUSED( path )
162 return true;
163 }
164
165 protected slots:
166
173 virtual void onRemoteContentFetched( const QString &url, bool success );
174
175};
176
177#ifndef SIP_RUN
178
192template<class T>
194{
195
196 public:
197
209 QgsAbstractContentCache( QObject *parent SIP_TRANSFERTHIS = nullptr,
210 const QString &typeString = QString(),
211 long maxCacheSize = 20000000,
212 int fileModifiedCheckTimeout = 30000 )
214 , mMaxCacheSize( maxCacheSize )
215 , mFileModifiedCheckTimeout( fileModifiedCheckTimeout )
216 , mTypeString( typeString.isEmpty() ? QObject::tr( "Content" ) : typeString )
217 {
218 }
219
221 {
222 qDeleteAll( mEntryLookup );
223 }
224
225 protected:
226
231 {
232 //only one entry in cache
233 if ( mLeastRecentEntry == mMostRecentEntry )
234 {
235 return;
236 }
237 T *entry = mLeastRecentEntry;
238 while ( entry && ( mTotalSize > mMaxCacheSize ) )
239 {
240 T *bkEntry = entry;
241 entry = static_cast< T * >( entry->nextEntry );
242
243 takeEntryFromList( bkEntry );
244 mEntryLookup.remove( bkEntry->path, bkEntry );
245 mTotalSize -= bkEntry->dataSize();
246 delete bkEntry;
247 }
248 }
249
263 QByteArray getContent( const QString &path, const QByteArray &missingContent, const QByteArray &fetchingContent, bool blocking = false ) const
264 {
265 // is it a path to local file?
266 QFile file( path );
267 if ( file.exists() )
268 {
269 if ( file.open( QIODevice::ReadOnly ) )
270 {
271 return file.readAll();
272 }
273 else
274 {
275 return missingContent;
276 }
277 }
278
279 // maybe it's an embedded base64 string
280 if ( path.startsWith( QLatin1String( "base64:" ), Qt::CaseInsensitive ) )
281 {
282 const QByteArray base64 = path.mid( 7 ).toLocal8Bit(); // strip 'base64:' prefix
283 return QByteArray::fromBase64( base64, QByteArray::OmitTrailingEquals );
284 }
285
286 // maybe it's a url...
287 if ( !path.contains( QLatin1String( "://" ) ) ) // otherwise short, relative SVG paths might be considered URLs
288 {
289 return missingContent;
290 }
291
292 const QUrl url( path );
293 if ( !url.isValid() )
294 {
295 return missingContent;
296 }
297
298 // check whether it's a url pointing to a local file
299 if ( url.scheme().compare( QLatin1String( "file" ), Qt::CaseInsensitive ) == 0 )
300 {
301 file.setFileName( url.toLocalFile() );
302 if ( file.exists() )
303 {
304 if ( file.open( QIODevice::ReadOnly ) )
305 {
306 return file.readAll();
307 }
308 }
309
310 // not found...
311 return missingContent;
312 }
313
314 const QMutexLocker locker( &mMutex );
315
316 // already a request in progress for this url
317 if ( mPendingRemoteUrls.contains( path ) )
318 {
319 // it's a non blocking request so return fetching content
320 if ( !blocking )
321 {
322 return fetchingContent;
323 }
324
325 // it's a blocking request so try to find the task and wait for task finished
326 const auto constActiveTasks = QgsApplication::taskManager()->activeTasks();
327 for ( QgsTask *task : constActiveTasks )
328 {
329 // the network content fetcher task's description ends with the path
330 if ( !task->description().endsWith( path ) )
331 {
332 continue;
333 }
334
335 // cast task to network content fetcher task
336 QgsNetworkContentFetcherTask *ncfTask = qobject_cast<QgsNetworkContentFetcherTask *>( task );
337 if ( ncfTask )
338 {
339 // wait for task finished
340 if ( waitForTaskFinished( ncfTask ) )
341 {
342 if ( mRemoteContentCache.contains( path ) )
343 {
344 // We got the file!
345 return *mRemoteContentCache[ path ];
346 }
347 }
348 }
349 // task found, no needs to continue
350 break;
351 }
352 // if no content returns the content is probably in remote content cache
353 // or a new task will be created
354 }
355
356 if ( mRemoteContentCache.contains( path ) )
357 {
358 // already fetched this content - phew. Just return what we already got.
359 return *mRemoteContentCache[ path ];
360 }
361
362 mPendingRemoteUrls.insert( path );
363 //fire up task to fetch content in background
364 QNetworkRequest request( url );
365 QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsAbstractContentCache<%1>" ).arg( mTypeString ) );
366 request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
367 request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
368
370 connect( task, &QgsNetworkContentFetcherTask::fetched, this, [this, task, path, missingContent]
371 {
372 const QMutexLocker locker( &mMutex );
373
374 QNetworkReply *reply = task->reply();
375 if ( !reply )
376 {
377 // canceled
378 QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, false ) );
379 return;
380 }
381
382 if ( reply->error() != QNetworkReply::NoError )
383 {
384 QgsMessageLog::logMessage( tr( "%3 request failed [error: %1 - url: %2]" ).arg( reply->errorString(), path, mTypeString ), mTypeString );
385 return;
386 }
387
388 bool ok = true;
389
390 const QVariant status = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute );
391 if ( !QgsVariantUtils::isNull( status ) && status.toInt() >= 400 )
392 {
393 const QVariant phrase = reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute );
394 QgsMessageLog::logMessage( tr( "%4 request error [status: %1 - reason phrase: %2] for %3" ).arg( status.toInt() ).arg( phrase.toString(), path, mTypeString ), mTypeString );
395 mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
396 ok = false;
397 }
398
399 if ( !checkReply( reply, path ) )
400 {
401 mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
402 ok = false;
403 }
404
405 if ( ok )
406 {
407 // read the content data
408 const QByteArray ba = reply->readAll();
409
410 // because of the fragility listed below in waitForTaskFinished, this slot may get called twice. In that case
411 // the second time will have an empty reply (we've already read it all...)
412 if ( !ba.isEmpty() )
413 mRemoteContentCache.insert( path, new QByteArray( ba ) );
414 }
415 QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, true ) );
416 } );
417
419
420 // if blocking, wait for finished
421 if ( blocking )
422 {
423 if ( waitForTaskFinished( task ) )
424 {
425 if ( mRemoteContentCache.contains( path ) )
426 {
427 // We got the file!
428 return *mRemoteContentCache[ path ];
429 }
430 }
431 }
432 return fetchingContent;
433 }
434
435 void onRemoteContentFetched( const QString &url, bool success ) override
436 {
437 const QMutexLocker locker( &mMutex );
438 mPendingRemoteUrls.remove( url );
439
440 T *nextEntry = mLeastRecentEntry;
441 while ( T *entry = nextEntry )
442 {
443 nextEntry = static_cast< T * >( entry->nextEntry );
444 if ( entry->path == url )
445 {
446 takeEntryFromList( entry );
447 mEntryLookup.remove( entry->path, entry );
448 mTotalSize -= entry->dataSize();
449 delete entry;
450 }
451 }
452
453 if ( success )
454 emit remoteContentFetched( url );
455 }
456
468 {
469 // Wait up to timeout seconds for task finished
471 {
472 // The wait did not time out
473 // Third step, check status as complete
474 if ( task->status() == QgsTask::Complete )
475 {
476 // Fourth step, force the signal fetched to be sure reply has been checked
477
478 // ARGH this is BAD BAD BAD. The connection will get called twice as a result!!!
479 task->fetched();
480 return true;
481 }
482 }
483 return false;
484 }
485
495 T *findExistingEntry( T *entryTemplate )
496 {
497 //search entries in mEntryLookup
498 const QString path = entryTemplate->path;
499 T *currentEntry = nullptr;
500 const QList<T *> entries = mEntryLookup.values( path );
501 QDateTime modified;
502 for ( T *cacheEntry : entries )
503 {
504 if ( cacheEntry->isEqual( entryTemplate ) )
505 {
506 if ( mFileModifiedCheckTimeout <= 0 || cacheEntry->fileModifiedLastCheckTimer.hasExpired( mFileModifiedCheckTimeout ) )
507 {
508 if ( !modified.isValid() )
509 modified = QFileInfo( path ).lastModified();
510
511 if ( cacheEntry->fileModified != modified )
512 continue;
513 else
514 cacheEntry->fileModifiedLastCheckTimer.restart();
515 }
516 currentEntry = cacheEntry;
517 break;
518 }
519 }
520
521 //if not found: insert entryTemplate as a new entry
522 if ( !currentEntry )
523 {
524 currentEntry = insertCacheEntry( entryTemplate );
525 }
526 else
527 {
528 delete entryTemplate;
529 entryTemplate = nullptr;
530 ( void )entryTemplate;
531 takeEntryFromList( currentEntry );
532 if ( !mMostRecentEntry ) //list is empty
533 {
534 mMostRecentEntry = currentEntry;
535 mLeastRecentEntry = currentEntry;
536 }
537 else
538 {
539 mMostRecentEntry->nextEntry = currentEntry;
540 currentEntry->previousEntry = mMostRecentEntry;
541 currentEntry->nextEntry = nullptr;
542 mMostRecentEntry = currentEntry;
543 }
544 }
545
546 //debugging
547 //printEntryList();
548
549 return currentEntry;
550 }
551 mutable QRecursiveMutex mMutex;
552
554 long mTotalSize = 0;
555
557 long mMaxCacheSize = 20000000;
558
559 private:
560
566 T *insertCacheEntry( T *entry )
567 {
568 entry->mFileModifiedCheckTimeout = mFileModifiedCheckTimeout;
569
570 if ( !entry->path.startsWith( QLatin1String( "base64:" ) ) )
571 {
572 entry->fileModified = QFileInfo( entry->path ).lastModified();
573 entry->fileModifiedLastCheckTimer.start();
574 }
575
576 mEntryLookup.insert( entry->path, entry );
577
578 //insert to most recent place in entry list
579 if ( !mMostRecentEntry ) //inserting first entry
580 {
581 mLeastRecentEntry = entry;
582 mMostRecentEntry = entry;
583 entry->previousEntry = nullptr;
584 entry->nextEntry = nullptr;
585 }
586 else
587 {
588 entry->previousEntry = mMostRecentEntry;
589 entry->nextEntry = nullptr;
590 mMostRecentEntry->nextEntry = entry;
591 mMostRecentEntry = entry;
592 }
593
594 trimToMaximumSize();
595 return entry;
596 }
597
598
602 void takeEntryFromList( T *entry )
603 {
604 if ( !entry )
605 {
606 return;
607 }
608
609 if ( entry->previousEntry )
610 {
611 entry->previousEntry->nextEntry = entry->nextEntry;
612 }
613 else
614 {
615 mLeastRecentEntry = static_cast< T * >( entry->nextEntry );
616 }
617 if ( entry->nextEntry )
618 {
619 entry->nextEntry->previousEntry = entry->previousEntry;
620 }
621 else
622 {
623 mMostRecentEntry = static_cast< T * >( entry->previousEntry );
624 }
625 }
626
630 void printEntryList()
631 {
632 QgsDebugMsg( QStringLiteral( "****************cache entry list*************************" ) );
633 QgsDebugMsg( "Cache size: " + QString::number( mTotalSize ) );
634 T *entry = mLeastRecentEntry;
635 while ( entry )
636 {
637 QgsDebugMsg( QStringLiteral( "***Entry:" ) );
638 entry->dump();
639 entry = entry->nextEntry;
640 }
641 }
642
644 QMultiHash< QString, T * > mEntryLookup;
645
647 int mFileModifiedCheckTimeout = 30000;
648
649 //The content cache keeps the entries on a double connected list, moving the current entry to the front.
650 //That way, removing entries for more space can start with the least used objects.
651 T *mLeastRecentEntry = nullptr;
652 T *mMostRecentEntry = nullptr;
653
654 mutable QCache< QString, QByteArray > mRemoteContentCache;
655 mutable QSet< QString > mPendingRemoteUrls;
656
657 QString mTypeString;
658
659 friend class TestQgsSvgCache;
660 friend class TestQgsImageCache;
661};
662
663#endif
664
665#endif // QGSABSTRACTCONTENTCACHE_H
A QObject derived base class for QgsAbstractContentCache.
void remoteContentFetched(const QString &url)
Emitted when the cache has finished retrieving content from a remote url.
virtual bool checkReply(QNetworkReply *reply, const QString &path) const
Runs additional checks on a network reply to ensure that the reply content is consistent with that re...
Base class for entries in a QgsAbstractContentCache.
virtual int dataSize() const =0
Returns the memory usage in bytes for the entry.
virtual void dump() const =0
Dumps debugging strings containing the item's properties.
virtual ~QgsAbstractContentCacheEntry()=default
QElapsedTimer fileModifiedLastCheckTimer
Time since last check of file modified date.
QgsAbstractContentCacheEntry(const QgsAbstractContentCacheEntry &rh)=delete
QgsAbstractContentCacheEntry cannot be copied.
QgsAbstractContentCacheEntry & operator=(const QgsAbstractContentCacheEntry &rh)=delete
QgsAbstractContentCacheEntry cannot be copied.
QString path
Represents the absolute path to a file, a remote URL, or a base64 encoded string.
virtual bool isEqual(const QgsAbstractContentCacheEntry *other) const =0
Tests whether this entry matches another entry.
QDateTime fileModified
Timestamp when file was last modified.
bool operator==(const QgsAbstractContentCacheEntry &other) const
Abstract base class for file content caches, such as SVG or raster image caches.
QByteArray getContent(const QString &path, const QByteArray &missingContent, const QByteArray &fetchingContent, bool blocking=false) const
Gets the file content corresponding to the given path.
T * findExistingEntry(T *entryTemplate)
Returns the existing entry from the cache which matches entryTemplate (deleting entryTemplate when do...
void onRemoteContentFetched(const QString &url, bool success) override
Triggered after remote content (i.e.
QgsAbstractContentCache(QObject *parent=nullptr, const QString &typeString=QString(), long maxCacheSize=20000000, int fileModifiedCheckTimeout=30000)
Constructor for QgsAbstractContentCache, with the specified parent object.
void trimToMaximumSize()
Removes the least used cache entries until the maximum cache size is under the predefined size limit.
bool waitForTaskFinished(QgsNetworkContentFetcherTask *task) const
Blocks the current thread until the task finishes (or user's preset network timeout expires)
static QgsTaskManager * taskManager()
Returns the application's task manager, used for managing application wide background task handling.
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
static int timeout()
Returns the network timeout length, in milliseconds.
Handles HTTP network content fetching in a background task.
void fetched()
Emitted when the network content has been fetched, regardless of whether the fetch was successful or ...
QNetworkReply * reply()
Returns the network reply.
QList< QgsTask * > activeTasks() const
Returns a list of the active (queued or running) tasks.
long addTask(QgsTask *task, int priority=0)
Adds a task to the manager.
Abstract base class for long running background tasks.
TaskStatus status() const
Returns the current task status.
@ Complete
Task successfully completed.
bool waitForFinished(int timeout=30000)
Blocks the current thread until the task finishes or a maximum of timeout milliseconds.
static bool isNull(const QVariant &variant)
Returns true if the specified variant should be considered a NULL value.
#define SIP_TRANSFERTHIS
Definition: qgis_sip.h:53
#define QgsDebugMsg(str)
Definition: qgslogger.h:38
#define QgsSetRequestInitiatorClass(request, _class)