Merge branch 'issue-1579' into calamares

FIXES #1579

CHANGES

@@ -21,6 +21,11 @@ This release contains contributions from (alphabetically by first name):
    coded setup (which Calamares has had for a long time with @home
    and similar) and introduce a custom btrfs configuration through the
    `mount.conf` file.
+ - *netinstall* now supports fallbacks for the groups data.
+   Instead of a single URL, multiple URLs may be specified in
+   a list and Calamares goes through them until one is successfully
+   retrieved. Older configurations with a single string are
+   treated like a one-item list. #1579
  - The *usersq* module now connects to the internal configuration
    object and may be usable for regular installations.
 

src/libcalamares/CMakeLists.txt

@@ -59,6 +59,9 @@ set( libSources
     # Network service
     network/Manager.cpp
 
+    # Packages service
+    packages/Globals.cpp
+
     # Partition service
     partition/Mount.cpp
     partition/PartitionSize.cpp
@@ -228,6 +231,12 @@ calamares_add_test(
         network/Tests.cpp
 )
 
+calamares_add_test(
+    libcalamarespackagestest
+    SOURCES
+        packages/Tests.cpp
+)
+
 calamares_add_test(
     libcalamarespartitiontest
     SOURCES

src/libcalamares/packages/Globals.cpp

@@ -0,0 +1,68 @@
+/* === This file is part of Calamares - <https://calamares.io> ===
+ *
+ *   SPDX-FileCopyrightText: 2021 Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#include "Globals.h"
+
+#include "GlobalStorage.h"
+#include "utils/Logger.h"
+
+bool
+CalamaresUtils::Packages::setGSPackageAdditions( Calamares::GlobalStorage* gs,
+                                                 const Calamares::ModuleSystem::InstanceKey& module,
+                                                 const QVariantList& installPackages,
+                                                 const QVariantList& tryInstallPackages )
+{
+    static const char PACKAGEOP[] = "packageOperations";
+
+    // Check if there's already a PACAKGEOP entry in GS, and if so we'll
+    // extend that one (overwriting the value in GS at the end of this method)
+    QVariantList packageOperations = gs->contains( PACKAGEOP ) ? gs->value( PACKAGEOP ).toList() : QVariantList();
+    cDebug() << "Existing package operations length" << packageOperations.length();
+
+    const QString key = module.toString();
+
+    // Clear out existing operations for this module, going backwards:
+    // Sometimes we remove an item, and we don't want the index to
+    // fall off the end of the list.
+    bool somethingRemoved = false;
+    for ( int index = packageOperations.length() - 1; 0 <= index; index-- )
+    {
+        const QVariantMap op = packageOperations.at( index ).toMap();
+        if ( op.contains( "source" ) && op.value( "source" ).toString() == key )
+        {
+            cDebug() << Logger::SubEntry << "Removing existing operations for" << key;
+            packageOperations.removeAt( index );
+            somethingRemoved = true;
+        }
+    }
+
+    if ( !installPackages.empty() )
+    {
+        QVariantMap op;
+        op.insert( "install", QVariant( installPackages ) );
+        op.insert( "source", key );
+        packageOperations.append( op );
+        cDebug() << Logger::SubEntry << installPackages.length() << "critical packages.";
+    }
+    if ( !tryInstallPackages.empty() )
+    {
+        QVariantMap op;
+        op.insert( "try_install", QVariant( tryInstallPackages ) );
+        op.insert( "source", key );
+        packageOperations.append( op );
+        cDebug() << Logger::SubEntry << tryInstallPackages.length() << "non-critical packages.";
+    }
+
+    if ( somethingRemoved || !packageOperations.isEmpty() )
+    {
+        gs->insert( PACKAGEOP, packageOperations );
+        return true;
+    }
+    return false;
+}

src/libcalamares/packages/Globals.h

@@ -0,0 +1,36 @@
+/* === This file is part of Calamares - <https://calamares.io> ===
+ *
+ *   SPDX-FileCopyrightText: 2021 Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#ifndef LIBCALAMARES_PACKAGES_GLOBALS_H
+#define LIBCALAMARES_PACKAGES_GLOBALS_H
+
+#include "GlobalStorage.h"
+#include "modulesystem/InstanceKey.h"
+
+namespace CalamaresUtils
+{
+namespace Packages
+{
+/** @brief Sets the install-packages GS keys for the given module
+ *
+ * This replaces previously-set install-packages lists for the
+ * given module by the two new lists.
+ *
+ * Returns @c true if anything was changed, @c false otherwise.
+ */
+bool setGSPackageAdditions( Calamares::GlobalStorage* gs,
+                            const Calamares::ModuleSystem::InstanceKey& module,
+                            const QVariantList& installPackages,
+                            const QVariantList& tryInstallPackages );
+// void setGSPackageRemovals( const Calamares::ModuleSystem::InstanceKey& key, const QVariantList& removePackages );
+}  // namespace Packages
+}  // namespace CalamaresUtils
+
+
+#endif

src/libcalamares/packages/Tests.cpp

@@ -0,0 +1,88 @@
+/* === This file is part of Calamares - <https://calamares.io> ===
+ *
+ *   SPDX-FileCopyrightText: 2021 Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#include "Globals.h"
+
+#include "GlobalStorage.h"
+#include "utils/Logger.h"
+
+#include <QtTest/QtTest>
+
+class PackagesTests : public QObject
+{
+    Q_OBJECT
+public:
+    PackagesTests() {}
+    ~PackagesTests() override {}
+private Q_SLOTS:
+    void initTestCase();
+
+    void testEmpty();
+    void testAdd();
+};
+
+void
+PackagesTests::initTestCase()
+{
+    Logger::setupLogLevel( Logger::LOGDEBUG );
+}
+
+void
+PackagesTests::testEmpty()
+{
+    Calamares::GlobalStorage gs;
+    const QString topKey( "packageOperations" );
+    Calamares::ModuleSystem::InstanceKey k( "this", "that" );
+
+    QVERIFY( !gs.contains( topKey ) );
+    QCOMPARE( k.toString(), "this@that" );
+
+    // Adding nothing at all does nothing
+    QVERIFY( !CalamaresUtils::Packages::setGSPackageAdditions( &gs, k, QVariantList(), QVariantList() ) );
+    QVERIFY( !gs.contains( topKey ) );
+}
+
+void
+PackagesTests::testAdd()
+{
+    Calamares::GlobalStorage gs;
+    const QString topKey( "packageOperations" );
+    Calamares::ModuleSystem::InstanceKey k( "this", "that" );
+
+    QVERIFY( !gs.contains( topKey ) );
+    QVERIFY(
+        CalamaresUtils::Packages::setGSPackageAdditions( &gs, k, QVariantList { QString( "vim" ) }, QVariantList() ) );
+    QVERIFY( gs.contains( topKey ) );
+    auto actionList = gs.value( topKey ).toList();
+    QCOMPARE( actionList.length(), 1 );
+    auto action = actionList[ 0 ].toMap();
+    QVERIFY( action.contains( "install" ) );
+    auto op = action[ "install" ].toList();
+    QCOMPARE( op.length(), 1 );
+    cDebug() << op;
+
+    QVERIFY( CalamaresUtils::Packages::setGSPackageAdditions(
+        &gs, k, QVariantList { QString( "vim" ), QString( "emacs" ) }, QVariantList() ) );
+    QVERIFY( gs.contains( topKey ) );
+    actionList = gs.value( topKey ).toList();
+    QCOMPARE( actionList.length(), 1 );
+    action = actionList[ 0 ].toMap();
+    QVERIFY( action.contains( "install" ) );
+    op = action[ "install" ].toList();
+    QCOMPARE( op.length(), 2 );
+    QCOMPARE( action[ "source" ].toString(), k.toString() );
+    cDebug() << op;
+}
+
+
+QTEST_GUILESS_MAIN( PackagesTests )
+
+#include "utils/moc-warnings.h"
+
+#include "Tests.moc"

src/modules/netinstall/CMakeLists.txt

@@ -8,6 +8,7 @@ calamares_add_plugin( netinstall
     EXPORT_MACRO PLUGINDLLEXPORT_PRO
     SOURCES
         Config.cpp
+        LoaderQueue.cpp
         NetInstallViewStep.cpp
         NetInstallPage.cpp
         PackageTreeItem.cpp

src/modules/netinstall/Config.cpp

@@ -12,10 +12,15 @@
 
 #include "Config.h"
 
+#include "LoaderQueue.h"
+
+#include "GlobalStorage.h"
+#include "JobQueue.h"
 #include "network/Manager.h"
+#include "packages/Globals.h"
 #include "utils/Logger.h"
-#include "utils/RAII.h"
+#include "utils/Retranslator.h"
-#include "utils/Yaml.h"
+#include "utils/Variant.h"
 
 #include <QNetworkReply>
 
@@ -23,10 +28,20 @@ Config::Config( QObject* parent )
     : QObject( parent )
     , m_model( new PackageModel( this ) )
 {
+    CALAMARES_RETRANSLATE_SLOT( &Config::retranslate );
 }
 
 Config::~Config() {}
 
+void
+Config::retranslate()
+{
+    emit statusChanged( status() );
+    emit sidebarLabelChanged( sidebarLabel() );
+    emit titleLabelChanged( titleLabel() );
+}
+
+
 QString
 Config::status() const
 {
@@ -39,9 +54,11 @@ Config::status() const
     case Status::FailedBadData:
         return tr( "Network Installation. (Disabled: Received invalid groups data)" );
     case Status::FailedInternalError:
-        return tr( "Network Installation. (Disabled: internal error)" );
+        return tr( "Network Installation. (Disabled: Internal error)" );
     case Status::FailedNetworkError:
         return tr( "Network Installation. (Disabled: Unable to fetch package lists, check your network connection)" );
+    case Status::FailedNoData:
+        return tr( "Network Installation. (Disabled: No package list)" );
     }
     __builtin_unreachable();
 }
@@ -54,92 +71,114 @@ Config::setStatus( Status s )
     emit statusChanged( status() );
 }
 
+QString
+Config::sidebarLabel() const
+{
+    return m_sidebarLabel ? m_sidebarLabel->get() : tr( "Package selection" );
+}
+
+QString
+Config::titleLabel() const
+{
+    return m_titleLabel ? m_titleLabel->get() : QString();
+}
+
+
 void
 Config::loadGroupList( const QVariantList& groupData )
 {
     m_model->setupModelData( groupData );
+    if ( m_model->rowCount() < 1 )
+    {
+        cWarning() << "NetInstall groups data was empty.";
+        setStatus( Status::FailedNoData );
+    }
+    else
+    {
+        setStatus( Status::Ok );
+    }
     emit statusReady();
 }
 
 void
-Config::loadGroupList( const QUrl& url )
+Config::loadingDone()
 {
-    if ( !url.isValid() )
+    if ( m_queue )
     {
-        setStatus( Status::FailedBadConfiguration );
+        m_queue->deleteLater();
+        m_queue = nullptr;
+    }
+}
+
+
+void
+Config::setConfigurationMap( const QVariantMap& configurationMap )
+{
+    setRequired( CalamaresUtils::getBool( configurationMap, "required", false ) );
+
+    // Get the translations, if any
+    bool bogus = false;
+    auto label = CalamaresUtils::getSubMap( configurationMap, "label", bogus );
+    // Use a different class name for translation lookup because the
+    // .. table of strings lives in NetInstallViewStep.cpp and moving them
+    // .. around is annoying for translators.
+    static const char className[] = "NetInstallViewStep";
+
+    if ( label.contains( "sidebar" ) )
+    {
+        m_sidebarLabel = new CalamaresUtils::Locale::TranslatedString( label, "sidebar", className );
+    }
+    if ( label.contains( "title" ) )
+    {
+        m_titleLabel = new CalamaresUtils::Locale::TranslatedString( label, "title", className );
     }
 
-    using namespace CalamaresUtils::Network;
+    // Lastly, load the groups data
-
+    const QString key = QStringLiteral( "groupsUrl" );
-    cDebug() << "NetInstall loading groups from" << url;
+    const auto& groupsUrlVariant = configurationMap.value( key );
-    QNetworkReply* reply = Manager::instance().asynchronousGet(
+    if ( groupsUrlVariant.type() == QVariant::String )
-        url,
-        RequestOptions( RequestOptions::FakeUserAgent | RequestOptions::FollowRedirect, std::chrono::seconds( 30 ) ) );
-
-    if ( !reply )
     {
-        cDebug() << Logger::Continuation << "request failed immediately.";
+        m_queue = new LoaderQueue( this );
-        setStatus( Status::FailedBadConfiguration );
+        m_queue->append( SourceItem::makeSourceItem( groupsUrlVariant.toString(), configurationMap ) );
     }
-    else
+    else if ( groupsUrlVariant.type() == QVariant::List )
     {
-        m_reply = reply;
+        m_queue = new LoaderQueue( this );
-        connect( reply, &QNetworkReply::finished, this, &Config::receivedGroupData );
+        for ( const auto& s : groupsUrlVariant.toStringList() )
+        {
+            m_queue->append( SourceItem::makeSourceItem( s, configurationMap ) );
+        }
+    }
+    if ( m_queue && m_queue->count() > 0 )
+    {
+        cDebug() << "Loading netinstall from" << m_queue->count() << "alternate sources.";
+        connect( m_queue, &LoaderQueue::done, this, &Config::loadingDone );
+        m_queue->load();
     }
 }
 
 void
-Config::receivedGroupData()
+Config::finalizeGlobalStorage( const Calamares::ModuleSystem::InstanceKey& key )
 {
-    if ( !m_reply || !m_reply->isFinished() )
+    auto packages = model()->getPackages();
+
+    // This netinstall module may add two sub-steps to the packageOperations,
+    // one for installing and one for try-installing.
+    QVariantList installPackages;
+    QVariantList tryInstallPackages;
+
+    for ( const auto& package : packages )
     {
-        cWarning() << "NetInstall data called too early.";
+        if ( package->isCritical() )
-        setStatus( Status::FailedInternalError );
-        return;
-    }
-
-    cDebug() << "NetInstall group data received" << m_reply->size() << "bytes from" << m_reply->url();
-
-    cqDeleter< QNetworkReply > d { m_reply };
-
-    // If m_required is *false* then we still say we're ready
-    // even if the reply is corrupt or missing.
-    if ( m_reply->error() != QNetworkReply::NoError )
-    {
-        cWarning() << "unable to fetch netinstall package lists.";
-        cDebug() << Logger::SubEntry << "Netinstall reply error: " << m_reply->error();
-        cDebug() << Logger::SubEntry << "Request for url: " << m_reply->url().toString()
-                 << " failed with: " << m_reply->errorString();
-        setStatus( Status::FailedNetworkError );
-        return;
-    }
-
-    QByteArray yamlData = m_reply->readAll();
-    try
-    {
-        YAML::Node groups = YAML::Load( yamlData.constData() );
-
-        if ( groups.IsSequence() )
         {
-            loadGroupList( CalamaresUtils::yamlSequenceToVariant( groups ) );
+            installPackages.append( package->toOperation() );
-        }
-        else if ( groups.IsMap() )
-        {
-            auto map = CalamaresUtils::yamlMapToVariant( groups );
-            loadGroupList( map.value( "groups" ).toList() );
         }
         else
         {
-            cWarning() << "NetInstall groups data does not form a sequence.";
+            tryInstallPackages.append( package->toOperation() );
-        }
-        if ( m_model->rowCount() < 1 )
-        {
-            cWarning() << "NetInstall groups data was empty.";
         }
     }
-    catch ( YAML::Exception& e )
+
-    {
+    CalamaresUtils::Packages::setGSPackageAdditions(
-        CalamaresUtils::explainYamlException( e, yamlData, "netinstall groups data" );
+        Calamares::JobQueue::instance()->globalStorage(), key, installPackages, tryInstallPackages );
-        setStatus( Status::FailedBadData );
-    }
 }

src/modules/netinstall/Config.h

@@ -14,10 +14,15 @@
 
 #include "PackageModel.h"
 
-#include <QObject>
+#include "locale/TranslatableConfiguration.h"
-#include <QUrl>
+#include "modulesystem/InstanceKey.h"
 
-class QNetworkReply;
+#include <QObject>
+#include <QVariantMap>
+
+#include <memory>
+
+class LoaderQueue;
 
 class Config : public QObject
 {
@@ -26,17 +31,25 @@ class Config : public QObject
     Q_PROPERTY( PackageModel* packageModel MEMBER m_model FINAL )
     Q_PROPERTY( QString status READ status NOTIFY statusChanged FINAL )
 
+    // Translations, of the module name (for sidebar) and above the list
+    Q_PROPERTY( QString sidebarLabel READ sidebarLabel NOTIFY sidebarLabelChanged FINAL )
+    Q_PROPERTY( QString titleLabel READ titleLabel NOTIFY titleLabelChanged FINAL )
+
 public:
     Config( QObject* parent = nullptr );
     ~Config() override;
 
+    void setConfigurationMap( const QVariantMap& configurationMap );
+
     enum class Status
     {
         Ok,
         FailedBadConfiguration,
         FailedInternalError,
         FailedNetworkError,
-        FailedBadData
+        FailedBadData,
+        FailedNoData
+
     };
 
     QString status() const;
@@ -45,12 +58,10 @@ public:
     bool required() const { return m_required; }
     void setRequired( bool r ) { m_required = r; }
 
-    /** @brief Retrieves the groups, with name, description and packages
+    PackageModel* model() const { return m_model; }
-     *
+
-     * Loads data from the given URL. Once done, the data is parsed
+    QString sidebarLabel() const;
-     * and passed on to the other loadGroupList() method.
+    QString titleLabel() const;
-     */
-    void loadGroupList( const QUrl& url );
 
     /** @brief Fill model from parsed data.
      *
@@ -59,18 +70,28 @@ public:
      */
     void loadGroupList( const QVariantList& groupData );
 
-    PackageModel* model() const { return m_model; }
+    /** @brief Write the selected package lists to global storage
+     *
+     * Since the config doesn't know what module it is for,
+     * pass in an instance key.
+     */
+    void finalizeGlobalStorage( const Calamares::ModuleSystem::InstanceKey& key );
 
-signals:
+Q_SIGNALS:
     void statusChanged( QString status );  ///< Something changed
+    void sidebarLabelChanged( QString label );
+    void titleLabelChanged( QString label );
     void statusReady();  ///< Loading groups is complete
 
-private slots:
+private Q_SLOTS:
-    void receivedGroupData();  ///< From async-loading group data
+    void retranslate();
+    void loadingDone();
 
 private:
+    CalamaresUtils::Locale::TranslatedString* m_sidebarLabel = nullptr;  // As it appears in the sidebar
+    CalamaresUtils::Locale::TranslatedString* m_titleLabel = nullptr;
     PackageModel* m_model = nullptr;
-    QNetworkReply* m_reply = nullptr;  // For fetching data
+    LoaderQueue* m_queue = nullptr;
     Status m_status = Status::Ok;
     bool m_required = false;
 };

src/modules/netinstall/LoaderQueue.cpp

@@ -0,0 +1,194 @@
+/*
+ *   SPDX-FileCopyrightText: 2016 Luca Giambonini <almack@chakraos.org>
+ *   SPDX-FileCopyrightText: 2016 Lisa Vitolo     <shainer@chakraos.org>
+ *   SPDX-FileCopyrightText: 2017 Kyle Robbertze  <krobbertze@gmail.com>
+ *   SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#include "LoaderQueue.h"
+
+#include "Config.h"
+#include "network/Manager.h"
+#include "utils/Logger.h"
+#include "utils/RAII.h"
+#include "utils/Yaml.h"
+
+#include <QNetworkReply>
+#include <QTimer>
+
+/** @brief Call fetchNext() on the queue if it can
+ *
+ * On destruction, a new call to fetchNext() is queued, so that
+ * the queue continues loading. Calling release() before the
+ * destructor skips the fetchNext(), ending the queue-loading.
+ */
+class FetchNextUnless
+{
+public:
+    FetchNextUnless( LoaderQueue* q )
+        : m_q( q )
+    {
+    }
+    ~FetchNextUnless()
+    {
+        if ( m_q )
+        {
+            QMetaObject::invokeMethod( m_q, "fetchNext", Qt::QueuedConnection );
+        }
+    }
+    void release() { m_q = nullptr; }
+
+private:
+    LoaderQueue* m_q = nullptr;
+};
+
+SourceItem
+SourceItem::makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap )
+{
+    if ( groupsUrl == QStringLiteral( "local" ) )
+    {
+        return SourceItem { QUrl(), configurationMap.value( "groups" ).toList() };
+    }
+    else
+    {
+        return SourceItem { QUrl { groupsUrl }, QVariantList() };
+    }
+}
+
+LoaderQueue::LoaderQueue( Config* parent )
+    : QObject( parent )
+    , m_config( parent )
+{
+}
+
+void
+LoaderQueue::append( SourceItem&& i )
+{
+    m_queue.append( std::move( i ) );
+}
+
+void
+LoaderQueue::load()
+{
+    QMetaObject::invokeMethod( this, "fetchNext", Qt::QueuedConnection );
+}
+
+
+void
+LoaderQueue::fetchNext()
+{
+    if ( m_queue.isEmpty() )
+    {
+        m_config->setStatus( Config::Status::FailedBadData );
+        emit done();
+        return;
+    }
+
+    auto source = m_queue.takeFirst();
+    if ( source.isLocal() )
+    {
+        m_config->loadGroupList( source.data );
+        emit done();
+    }
+    else
+    {
+        fetch( source.url );
+    }
+}
+
+void
+LoaderQueue::fetch( const QUrl& url )
+{
+    FetchNextUnless next( this );
+
+    if ( !url.isValid() )
+    {
+        m_config->setStatus( Config::Status::FailedBadConfiguration );
+        cDebug() << "Invalid URL" << url;
+        return;
+    }
+
+    using namespace CalamaresUtils::Network;
+
+    cDebug() << "NetInstall loading groups from" << url;
+    QNetworkReply* reply = Manager::instance().asynchronousGet(
+        url,
+        RequestOptions( RequestOptions::FakeUserAgent | RequestOptions::FollowRedirect, std::chrono::seconds( 30 ) ) );
+
+    if ( !reply )
+    {
+        cDebug() << Logger::SubEntry << "Request failed immediately.";
+        // If nobody sets a different status, this will remain
+        m_config->setStatus( Config::Status::FailedBadConfiguration );
+    }
+    else
+    {
+        // When the network request is done, **then** we might
+        // do the next item from the queue, so don't call fetchNext() now.
+        next.release();
+        m_reply = reply;
+        connect( reply, &QNetworkReply::finished, this, &LoaderQueue::dataArrived );
+    }
+}
+
+void
+LoaderQueue::dataArrived()
+{
+    FetchNextUnless finished( this );
+
+    if ( !m_reply || !m_reply->isFinished() )
+    {
+        cWarning() << "NetInstall data called too early.";
+        m_config->setStatus( Config::Status::FailedInternalError );
+        return;
+    }
+
+    cDebug() << "NetInstall group data received" << m_reply->size() << "bytes from" << m_reply->url();
+
+    cqDeleter< QNetworkReply > d { m_reply };
+
+    // If m_required is *false* then we still say we're ready
+    // even if the reply is corrupt or missing.
+    if ( m_reply->error() != QNetworkReply::NoError )
+    {
+        cWarning() << "unable to fetch netinstall package lists.";
+        cDebug() << Logger::SubEntry << "Netinstall reply error: " << m_reply->error();
+        cDebug() << Logger::SubEntry << "Request for url: " << m_reply->url().toString()
+                 << " failed with: " << m_reply->errorString();
+        m_config->setStatus( Config::Status::FailedNetworkError );
+        return;
+    }
+
+    QByteArray yamlData = m_reply->readAll();
+    try
+    {
+        YAML::Node groups = YAML::Load( yamlData.constData() );
+
+        if ( groups.IsSequence() )
+        {
+            finished.release();
+            m_config->loadGroupList( CalamaresUtils::yamlSequenceToVariant( groups ) );
+            emit done();
+        }
+        else if ( groups.IsMap() )
+        {
+            finished.release();
+            auto map = CalamaresUtils::yamlMapToVariant( groups );
+            m_config->loadGroupList( map.value( "groups" ).toList() );
+            emit done();
+        }
+        else
+        {
+            cWarning() << "NetInstall groups data does not form a sequence.";
+        }
+    }
+    catch ( YAML::Exception& e )
+    {
+        CalamaresUtils::explainYamlException( e, yamlData, "netinstall groups data" );
+        m_config->setStatus( Config::Status::FailedBadData );
+    }
+}

src/modules/netinstall/LoaderQueue.h

@@ -0,0 +1,77 @@
+/*
+ *   SPDX-FileCopyrightText: 2016 Luca Giambonini <almack@chakraos.org>
+ *   SPDX-FileCopyrightText: 2016 Lisa Vitolo     <shainer@chakraos.org>
+ *   SPDX-FileCopyrightText: 2017 Kyle Robbertze  <krobbertze@gmail.com>
+ *   SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#ifndef NETINSTALL_LOADERQUEUE_H
+#define NETINSTALL_LOADERQUEUE_H
+
+#include <QQueue>
+#include <QUrl>
+#include <QVariantList>
+
+class Config;
+class QNetworkReply;
+
+/** @brief Data about an entry in *groupsUrl*
+ *
+ * This can be a specific URL, or "local" which uses data stored
+ * in the configuration file itself.
+ */
+struct SourceItem
+{
+    QUrl url;
+    QVariantList data;
+
+    bool isUrl() const { return url.isValid(); }
+    bool isLocal() const { return !data.isEmpty(); }
+    bool isValid() const { return isUrl() || isLocal(); }
+    /** @brief Create a SourceItem
+     *
+     * If the @p groupsUrl is @c "local" then the *groups* key in
+     * the @p configurationMap is used as the source; otherwise the
+     * string is used as an actual URL.
+     */
+    static SourceItem makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap );
+};
+
+/** @brief Queue of source items to load
+ *
+ * Queue things up by calling append() and then kick things off
+ * by calling load(). This will try to load the items, in order;
+ * the first one that succeeds will end the loading process.
+ *
+ * Signal done() is emitted when done (also when all of the items fail).
+ */
+class LoaderQueue : public QObject
+{
+    Q_OBJECT
+public:
+    LoaderQueue( Config* parent );
+
+    void append( SourceItem&& i );
+    int count() const { return m_queue.count(); }
+
+public Q_SLOTS:
+    void load();
+
+    void fetchNext();
+    void fetch( const QUrl& url );
+    void dataArrived();
+
+Q_SIGNALS:
+    void done();
+
+private:
+    QQueue< SourceItem > m_queue;
+    Config* m_config = nullptr;
+    QNetworkReply* m_reply = nullptr;
+};
+
+#endif

src/modules/netinstall/NetInstallPage.cpp

@@ -33,40 +33,16 @@ NetInstallPage::NetInstallPage( Config* c, QWidget* parent )
     ui->setupUi( this );
     ui->groupswidget->header()->setSectionResizeMode( QHeaderView::ResizeToContents );
     ui->groupswidget->setModel( c->model() );
-    connect( c, &Config::statusChanged, this, &NetInstallPage::setStatus );
+    connect( c, &Config::statusChanged, ui->netinst_status, &QLabel::setText );
+    connect( c, &Config::titleLabelChanged, [ui = this->ui]( const QString title ) {
+        ui->label->setVisible( !title.isEmpty() );
+        ui->label->setText( title );
+    } );
     connect( c, &Config::statusReady, this, &NetInstallPage::expandGroups );
-
-    setPageTitle( nullptr );
-    CALAMARES_RETRANSLATE_SLOT( &NetInstallPage::retranslate );
 }
 
 NetInstallPage::~NetInstallPage() {}
 
-void
-NetInstallPage::setPageTitle( CalamaresUtils::Locale::TranslatedString* t )
-{
-    m_title.reset( t );
-    if ( !m_title )
-    {
-        ui->label->hide();
-    }
-    else
-    {
-        ui->label->show();
-    }
-    retranslate();
-}
-
-void
-NetInstallPage::retranslate()
-{
-    if ( m_title )
-    {
-        ui->label->setText( m_title->get() );  // That's get() on the TranslatedString
-    }
-    ui->netinst_status->setText( m_config->status() );
-}
-
 void
 NetInstallPage::expandGroups()
 {
@@ -82,12 +58,6 @@ NetInstallPage::expandGroups()
     }
 }
 
-void
-NetInstallPage::setStatus( QString s )
-{
-    ui->netinst_status->setText( s );
-}
-
 void
 NetInstallPage::onActivate()
 {

src/modules/netinstall/NetInstallPage.h

@@ -37,23 +37,8 @@ public:
     NetInstallPage( Config* config, QWidget* parent = nullptr );
     ~NetInstallPage() override;
 
-    /** @brief Sets the page title
-     *
-     * In situations where there is more than one netinstall page,
-     * or you want some explanatory title above the treeview,
-     * set the page title. This page takes ownership of the
-     * TranslatedString object.
-     *
-     * Set to nullptr to remove the title.
-     */
-    void setPageTitle( CalamaresUtils::Locale::TranslatedString* );
-
     void onActivate();
 
-public slots:
-    void retranslate();
-    void setStatus( QString s );
-
     /** @brief Expand entries that should be pre-expanded.
      *
      * Follows the *expanded* key / the startExpanded field in the
@@ -64,8 +49,6 @@ public slots:
 private:
     Config* m_config;
     Ui::Page_NetInst* ui;
-
-    std::unique_ptr< CalamaresUtils::Locale::TranslatedString > m_title;  // Above the treeview
 };
 
 #endif  // NETINSTALLPAGE_H

src/modules/netinstall/NetInstallViewStep.cpp

@@ -11,12 +11,6 @@
 
 #include "NetInstallViewStep.h"
 
-#include "GlobalStorage.h"
-#include "JobQueue.h"
-
-#include "utils/Logger.h"
-#include "utils/Variant.h"
-
 #include "NetInstallPage.h"
 
 CALAMARES_PLUGIN_FACTORY_DEFINITION( NetInstallViewStepFactory, registerPlugin< NetInstallViewStep >(); )
@@ -24,7 +18,6 @@ CALAMARES_PLUGIN_FACTORY_DEFINITION( NetInstallViewStepFactory, registerPlugin<
 NetInstallViewStep::NetInstallViewStep( QObject* parent )
     : Calamares::ViewStep( parent )
     , m_widget( new NetInstallPage( &m_config ) )
-    , m_sidebarLabel( nullptr )
     , m_nextEnabled( false )
 {
     connect( &m_config, &Config::statusReady, this, &NetInstallViewStep::nextIsReady );
@@ -37,20 +30,22 @@ NetInstallViewStep::~NetInstallViewStep()
     {
         m_widget->deleteLater();
     }
-    delete m_sidebarLabel;
 }
 
 
 QString
 NetInstallViewStep::prettyName() const
 {
-    return m_sidebarLabel ? m_sidebarLabel->get() : tr( "Package selection" );
+    return m_config.sidebarLabel();
 
 #if defined( TABLE_OF_TRANSLATIONS )
     __builtin_unreachable();
     // This is a table of "standard" labels for this module. If you use them
     // in the label: sidebar: section of the config file, the existing
     // translations can be used.
+    //
+    // These translations still live here, even though the lookup
+    // code is in the Config class.
     tr( "Package selection" );
     tr( "Office software" );
     tr( "Office package" );
@@ -125,70 +120,7 @@ NetInstallViewStep::onActivate()
 void
 NetInstallViewStep::onLeave()
 {
-    auto packages = m_config.model()->getPackages();
+    m_config.finalizeGlobalStorage( moduleInstanceKey() );
-    cDebug() << "Netinstall: Processing" << packages.length() << "packages.";
-
-    static const char PACKAGEOP[] = "packageOperations";
-
-    // Check if there's already a PACAKGEOP entry in GS, and if so we'll
-    // extend that one (overwriting the value in GS at the end of this method)
-    Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
-    QVariantList packageOperations = gs->contains( PACKAGEOP ) ? gs->value( PACKAGEOP ).toList() : QVariantList();
-    cDebug() << Logger::SubEntry << "Existing package operations length" << packageOperations.length();
-
-    // Clear out existing operations for this module, going backwards:
-    // Sometimes we remove an item, and we don't want the index to
-    // fall off the end of the list.
-    bool somethingRemoved = false;
-    for ( int index = packageOperations.length() - 1; 0 <= index; index-- )
-    {
-        const QVariantMap op = packageOperations.at( index ).toMap();
-        if ( op.contains( "source" ) && op.value( "source" ).toString() == moduleInstanceKey().toString() )
-        {
-            cDebug() << Logger::SubEntry << "Removing existing operations for" << moduleInstanceKey();
-            packageOperations.removeAt( index );
-            somethingRemoved = true;
-        }
-    }
-
-    // This netinstall module may add two sub-steps to the packageOperations,
-    // one for installing and one for try-installing.
-    QVariantList installPackages;
-    QVariantList tryInstallPackages;
-
-    for ( const auto& package : packages )
-    {
-        if ( package->isCritical() )
-        {
-            installPackages.append( package->toOperation() );
-        }
-        else
-        {
-            tryInstallPackages.append( package->toOperation() );
-        }
-    }
-
-    if ( !installPackages.empty() )
-    {
-        QVariantMap op;
-        op.insert( "install", QVariant( installPackages ) );
-        op.insert( "source", moduleInstanceKey().toString() );
-        packageOperations.append( op );
-        cDebug() << Logger::SubEntry << installPackages.length() << "critical packages.";
-    }
-    if ( !tryInstallPackages.empty() )
-    {
-        QVariantMap op;
-        op.insert( "try_install", QVariant( tryInstallPackages ) );
-        op.insert( "source", moduleInstanceKey().toString() );
-        packageOperations.append( op );
-        cDebug() << Logger::SubEntry << tryInstallPackages.length() << "non-critical packages.";
-    }
-
-    if ( somethingRemoved || !packageOperations.isEmpty() )
-    {
-        gs->insert( PACKAGEOP, packageOperations );
-    }
 }
 
 void
@@ -201,35 +133,5 @@ NetInstallViewStep::nextIsReady()
 void
 NetInstallViewStep::setConfigurationMap( const QVariantMap& configurationMap )
 {
-    m_config.setRequired( CalamaresUtils::getBool( configurationMap, "required", false ) );
+    m_config.setConfigurationMap( configurationMap );
-
-    QString groupsUrl = CalamaresUtils::getString( configurationMap, "groupsUrl" );
-    if ( !groupsUrl.isEmpty() )
-    {
-        // Keep putting groupsUrl into the global storage,
-        // even though it's no longer used for in-module data-passing.
-        Calamares::JobQueue::instance()->globalStorage()->insert( "groupsUrl", groupsUrl );
-        if ( groupsUrl == QStringLiteral( "local" ) )
-        {
-            QVariantList l = configurationMap.value( "groups" ).toList();
-            m_config.loadGroupList( l );
-        }
-        else
-        {
-            m_config.loadGroupList( groupsUrl );
-        }
-    }
-
-    bool bogus = false;
-    auto label = CalamaresUtils::getSubMap( configurationMap, "label", bogus );
-
-    if ( label.contains( "sidebar" ) )
-    {
-        m_sidebarLabel = new CalamaresUtils::Locale::TranslatedString( label, "sidebar", metaObject()->className() );
-    }
-    if ( label.contains( "title" ) )
-    {
-        m_widget->setPageTitle(
-            new CalamaresUtils::Locale::TranslatedString( label, "title", metaObject()->className() ) );
-    }
 }

src/modules/netinstall/NetInstallViewStep.h

@@ -14,7 +14,6 @@
 #include "Config.h"
 
 #include "DllMacro.h"
-#include "locale/TranslatableConfiguration.h"
 #include "utils/PluginFactory.h"
 #include "viewpages/ViewStep.h"
 
@@ -56,7 +55,6 @@ private:
     Config m_config;
 
     NetInstallPage* m_widget;
-    CalamaresUtils::Locale::TranslatedString* m_sidebarLabel;  // As it appears in the sidebar
     bool m_nextEnabled = false;
 };
 

src/modules/netinstall/netinstall.conf

@@ -32,20 +32,21 @@
 # This module supports multiple instances through the *label* key,
 # which allows you to distinguish them in the UI.
 ---
-# This is the URL that is retrieved to get the netinstall groups-and-packages
+# The *groupsUrl* determines where the data for the netinstall groups-and-
-# data (which should be in the format described in netinstall.yaml), e.g.:
+# packages comes from. The value of the key may be:
-#    ```
-#    groupsUrl: http://example.org/netinstall.php
-#    ```
-# or it can be a locally installed file:
-#    ```
-#    groupsUrl: file:///usr/share/calamares/netinstall.yaml
-#    ```
-# or it can be the special-case literal string "local":
-#    ```
-#    groupsUrl: local
-#    ```
 #
+# - a single string (this is treated as a list with just that string in it)
+# - a list of strings
+#
+# Each string is treated as a URL (see below for special cases. The
+# list is examined **in order** and each URL is tried in turn. The
+# first URL to load successfully -- even if it yields 0 packages --
+# ends the process. This allows using a network URL and a (fallback)
+# local URL for package lists, or for using multiple mirrors of
+# netinstall data.
+#
+# The URL must point to a YAML file that follows the format described
+# below at the key *groups* -- except for the special case URL "local".
 # Note that the contents of the groups file is the **important**
 # part of the configuration of this module. It specifies what
 # groups and packages the user may select (and so what commands are to
@@ -59,12 +60,27 @@
 # must have a list-of-groups as value; if the file does not have
 # a top-level key *groups*, then the file must contain only a list of groups.
 #
-# As a special case, setting *groupsUrl* to the literal string
+# Each item in the list *groupsUrl* may be:
-# `local` means that the data is obtained from **this** config
+# - A remote URL like `http://example.org/netinstall.php`
-# file, under the key *groups*.
+# - A local file URL like `file:///usr/share/calamares/netinstall.yaml`
+# - The special-case literal string `local`
+#
+# Non-special case URLs are loaded as YAML; if the load succeeds, then
+# they are interpreted like the *groups* key below. The special case
+# `local` loads the data directly from **this** file.
 #
 groupsUrl: local
 
+# Alternate form:
+# groupsUrl: [ local ]
+
+# Net-based package list, with fallback to local file
+# groupsUrl:
+#   - http://example.com/calamares/netinstall.yaml
+#   - file:///etc/calamares/modules/netinstall.yaml
+
+
+
 # If the installation can proceed without netinstall (e.g. the Live CD
 # can create a working installed system, but netinstall is preferred
 # to bring it up-to-date or extend functionality) leave this set to