Merge pull request #1666 from deprov447/Limit_log_upload_size

[logUpload] Limit upload size

src/branding/default/branding.desc

@@ -220,13 +220,19 @@ slideshowAPI: 2
 
 
 # These options are to customize online uploading of logs to pastebins:
-#  - type : Defines the kind of pastebin service to be used. Currently
+#  - type      : Defines the kind of pastebin service to be used. Currently
-#           it accepts two values:
+#                it accepts two values:
-#           - none    :    disables the pastebin functionality
+#                - none    :    disables the pastebin functionality
-#           - fiche   :    use fiche pastebin server
+#                - fiche   :    use fiche pastebin server
-#  - url  : Defines the address of pastebin service to be used.
+#  - url       : Defines the address of pastebin service to be used.
-#           Takes string as input. Important bits are the host and port,
+#                Takes string as input. Important bits are the host and port,
-#           the scheme is not used.
+#                the scheme is not used.
+#  - sizeLimit : Defines maximum size limit (in KiB) of log file to be pasted.
+#                Takes integer as input. If < 0, no limit will be forced,
+#                else only last (approximately) 'n' KiB of log file will be pasted.
+#                Please note that upload size may be slightly over the limit (due
+#                to last minute logging), so provide a suitable value.
 uploadServer :
     type :    "fiche"
     url :     "http://termbin.com:9999"
+    sizeLimit : -1

src/libcalamaresui/Branding.cpp

@@ -18,6 +18,7 @@
 #include "utils/ImageRegistry.h"
 #include "utils/Logger.h"
 #include "utils/NamedEnum.h"
+#include "utils/Units.h"
 #include "utils/Yaml.h"
 
 #include <QDir>
@@ -153,15 +154,18 @@ uploadServerFromMap( const QVariantMap& map )
 
     QString typestring = map[ "type" ].toString();
     QString urlstring = map[ "url" ].toString();
+    qint64 sizeLimitKiB = map[ "sizeLimit" ].toLongLong();
 
     if ( typestring.isEmpty() || urlstring.isEmpty() )
     {
-        return Branding::UploadServerInfo( Branding::UploadServerType::None, QUrl() );
+        return Branding::UploadServerInfo( Branding::UploadServerType::None, QUrl(), 0 );
     }
 
     bool bogus = false;  // we don't care about type-name lookup success here
-    return Branding::UploadServerInfo( names.find( typestring, bogus ),
+    return Branding::UploadServerInfo(
-                                       QUrl( urlstring, QUrl::ParsingMode::StrictMode ) );
+        names.find( typestring, bogus ),
+        QUrl( urlstring, QUrl::ParsingMode::StrictMode ),
+        sizeLimitKiB >= 0 ? CalamaresUtils::KiBtoBytes( static_cast< unsigned long long >( sizeLimitKiB ) ) : -1 );
 }
 
 /** @brief Load the @p map with strings from @p config

src/libcalamaresui/Branding.h

@@ -223,10 +223,11 @@ public:
 
     /** @brief Upload server configuration
      *
-     * This is both the type (which may be none, in which case the URL
+     * This object has 3 items : the type (which may be none, in which case the URL
-     * is irrelevant and usually empty) and the URL for the upload.
+     * is irrelevant and usually empty), the URL for the upload and the size limit of upload
+     * in bytes (for configuration value < 0, it serves -1, which stands for having no limit).
      */
-    using UploadServerInfo = QPair< UploadServerType, QUrl >;
+    using UploadServerInfo = std::tuple< UploadServerType, QUrl, qint64 >;
     UploadServerInfo uploadServer() const { return m_uploadServer; }
 
     /**

src/libcalamaresui/ViewManager.cpp

@@ -143,8 +143,9 @@ ViewManager::insertViewStep( int before, ViewStep* step )
 void
 ViewManager::onInstallationFailed( const QString& message, const QString& details )
 {
-    bool shouldOfferWebPaste
+    bool shouldOfferWebPaste = std::get< 0 >( Calamares::Branding::instance()->uploadServer() )
-        = Calamares::Branding::instance()->uploadServer().first != Calamares::Branding::UploadServerType::None;
+            != Calamares::Branding::UploadServerType::None
+        and std::get< 2 >( Calamares::Branding::instance()->uploadServer() ) != 0;
 
     cError() << "Installation failed:" << message;
     cDebug() << Logger::SubEntry << "- message:" << message;

src/libcalamaresui/utils/Paste.cpp

@@ -30,8 +30,12 @@ using namespace CalamaresUtils::Units;
  * Returns an empty QByteArray() on any kind of error.
  */
 STATICTEST QByteArray
-logFileContents()
+logFileContents( const qint64 sizeLimitBytes )
 {
+    if ( sizeLimitBytes != -1 )
+    {
+        cDebug() << "Log upload size limit was limited to" << sizeLimitBytes << "bytes";
+    }
     const QString name = Logger::logFile();
     QFile pasteSourceFile( name );
     if ( !pasteSourceFile.open( QIODevice::ReadOnly | QIODevice::Text ) )
@@ -39,12 +43,18 @@ logFileContents()
         cWarning() << "Could not open log file" << name;
         return QByteArray();
     }
-    QFileInfo fi( pasteSourceFile );
+    if ( sizeLimitBytes == -1 )
-    if ( fi.size() > 16_KiB )
     {
-        pasteSourceFile.seek( fi.size() - 16_KiB );
+        return pasteSourceFile.readAll();
     }
-    return pasteSourceFile.read( 16_KiB );
+    QFileInfo fi( pasteSourceFile );
+    if ( fi.size() > sizeLimitBytes )
+    {
+        cDebug() << "Only last" << sizeLimitBytes << "bytes of log file (sized" << fi.size() << "bytes) uploaded";
+        fi.refresh();
+        pasteSourceFile.seek( fi.size() - sizeLimitBytes );
+    }
+    return pasteSourceFile.read( sizeLimitBytes );
 }
 
 
@@ -101,7 +111,7 @@ ficheLogUpload( const QByteArray& pasteData, const QUrl& serverUrl, QObject* par
 QString
 CalamaresUtils::Paste::doLogUpload( QObject* parent )
 {
-    auto [ type, serverUrl ] = Calamares::Branding::instance()->uploadServer();
+    auto [ type, serverUrl, sizeLimitBytes ] = Calamares::Branding::instance()->uploadServer();
     if ( !serverUrl.isValid() )
     {
         cWarning() << "Upload configure with invalid URL";
@@ -113,7 +123,7 @@ CalamaresUtils::Paste::doLogUpload( QObject* parent )
         return QString();
     }
 
-    QByteArray pasteData = logFileContents();
+    QByteArray pasteData = logFileContents( sizeLimitBytes );
     if ( pasteData.isEmpty() )
     {
         // An error has already been logged
@@ -165,6 +175,6 @@ CalamaresUtils::Paste::doLogUploadUI( QWidget* parent )
 bool
 CalamaresUtils::Paste::isEnabled()
 {
-    auto [ type, serverUrl ] = Calamares::Branding::instance()->uploadServer();
+    auto [ type, serverUrl, sizeLimitBytes ] = Calamares::Branding::instance()->uploadServer();
     return type != Calamares::Branding::UploadServerType::None;
 }

src/libcalamaresui/utils/TestPaste.cpp

@@ -10,13 +10,14 @@
  */
 
 #include "Paste.h"
+#include "network/Manager.h"
 
 #include "utils/Logger.h"
 
 #include <QDateTime>
 #include <QtTest/QtTest>
 
-extern QByteArray logFileContents();
+extern QByteArray logFileContents( qint64 sizeLimitBytes );
 extern QString ficheLogUpload( const QByteArray& pasteData, const QUrl& serverUrl, QObject* parent );
 
 class TestPaste : public QObject
@@ -30,6 +31,7 @@ public:
 private Q_SLOTS:
     void testGetLogFile();
     void testFichePaste();
+    void testUploadSize();
 };
 
 void
@@ -37,14 +39,18 @@ TestPaste::testGetLogFile()
 {
     QFile::remove( Logger::logFile() );
     // This test assumes nothing **else** has set up logging yet
-    QByteArray contentsOfLogfileBefore = logFileContents();
+    QByteArray logLimitedBefore = logFileContents( 16 );
-    QVERIFY( contentsOfLogfileBefore.isEmpty() );
+    QVERIFY( logLimitedBefore.isEmpty() );
+    QByteArray logUnlimitedBefore = logFileContents( -1 );
+    QVERIFY( logUnlimitedBefore.isEmpty() );
 
     Logger::setupLogLevel( Logger::LOGDEBUG );
     Logger::setupLogfile();
 
-    QByteArray contentsOfLogfileAfterSetup = logFileContents();
+    QByteArray logLimitedAfter = logFileContents( 16 );
-    QVERIFY( !contentsOfLogfileAfterSetup.isEmpty() );
+    QVERIFY( !logLimitedAfter.isEmpty() );
+    QByteArray logUnlimitedAfter = logFileContents( -1 );
+    QVERIFY( !logUnlimitedAfter.isEmpty() );
 }
 
 void
@@ -60,7 +66,19 @@ TestPaste::testFichePaste()
     QVERIFY( !s.isEmpty() );
 }
 
+void
+TestPaste::testUploadSize()
+{
+    QByteArray logContent = logFileContents( 100 );
+    QString s = ficheLogUpload( logContent, QUrl( "http://termbin.com:9999" ), nullptr );
 
+    QVERIFY( !s.isEmpty() );
+
+    QUrl url( s );
+    QByteArray returnedData = CalamaresUtils::Network::Manager::instance().synchronousGet( url );
+
+    QCOMPARE( returnedData.size(), 100 );
+}
 QTEST_GUILESS_MAIN( TestPaste )
 
 #include "utils/moc-warnings.h"