diff --git a/CHANGES b/CHANGES index 7f4f18ab7..10647218d 100644 --- a/CHANGES +++ b/CHANGES @@ -21,6 +21,11 @@ This release contains contributions from (alphabetically by first name): coded setup (which Calamares has had for a long time with @home and similar) and introduce a custom btrfs configuration through the `mount.conf` file. + - *netinstall* now supports fallbacks for the groups data. + Instead of a single URL, multiple URLs may be specified in + a list and Calamares goes through them until one is successfully + retrieved. Older configurations with a single string are + treated like a one-item list. #1579 - The *usersq* module now connects to the internal configuration object and may be usable for regular installations. diff --git a/src/libcalamares/CMakeLists.txt b/src/libcalamares/CMakeLists.txt index 826f0bc41..95dad0530 100644 --- a/src/libcalamares/CMakeLists.txt +++ b/src/libcalamares/CMakeLists.txt @@ -59,6 +59,9 @@ set( libSources # Network service network/Manager.cpp + # Packages service + packages/Globals.cpp + # Partition service partition/Mount.cpp partition/PartitionSize.cpp @@ -228,6 +231,12 @@ calamares_add_test( network/Tests.cpp ) +calamares_add_test( + libcalamarespackagestest + SOURCES + packages/Tests.cpp +) + calamares_add_test( libcalamarespartitiontest SOURCES diff --git a/src/libcalamares/packages/Globals.cpp b/src/libcalamares/packages/Globals.cpp new file mode 100644 index 000000000..c5e882436 --- /dev/null +++ b/src/libcalamares/packages/Globals.cpp @@ -0,0 +1,68 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "Globals.h" + +#include "GlobalStorage.h" +#include "utils/Logger.h" + +bool +CalamaresUtils::Packages::setGSPackageAdditions( Calamares::GlobalStorage* gs, + const Calamares::ModuleSystem::InstanceKey& module, + const QVariantList& installPackages, + const QVariantList& tryInstallPackages ) +{ + static const char PACKAGEOP[] = "packageOperations"; + + // Check if there's already a PACAKGEOP entry in GS, and if so we'll + // extend that one (overwriting the value in GS at the end of this method) + QVariantList packageOperations = gs->contains( PACKAGEOP ) ? gs->value( PACKAGEOP ).toList() : QVariantList(); + cDebug() << "Existing package operations length" << packageOperations.length(); + + const QString key = module.toString(); + + // Clear out existing operations for this module, going backwards: + // Sometimes we remove an item, and we don't want the index to + // fall off the end of the list. + bool somethingRemoved = false; + for ( int index = packageOperations.length() - 1; 0 <= index; index-- ) + { + const QVariantMap op = packageOperations.at( index ).toMap(); + if ( op.contains( "source" ) && op.value( "source" ).toString() == key ) + { + cDebug() << Logger::SubEntry << "Removing existing operations for" << key; + packageOperations.removeAt( index ); + somethingRemoved = true; + } + } + + if ( !installPackages.empty() ) + { + QVariantMap op; + op.insert( "install", QVariant( installPackages ) ); + op.insert( "source", key ); + packageOperations.append( op ); + cDebug() << Logger::SubEntry << installPackages.length() << "critical packages."; + } + if ( !tryInstallPackages.empty() ) + { + QVariantMap op; + op.insert( "try_install", QVariant( tryInstallPackages ) ); + op.insert( "source", key ); + packageOperations.append( op ); + cDebug() << Logger::SubEntry << tryInstallPackages.length() << "non-critical packages."; + } + + if ( somethingRemoved || !packageOperations.isEmpty() ) + { + gs->insert( PACKAGEOP, packageOperations ); + return true; + } + return false; +} diff --git a/src/libcalamares/packages/Globals.h b/src/libcalamares/packages/Globals.h new file mode 100644 index 000000000..a47cf5ae1 --- /dev/null +++ b/src/libcalamares/packages/Globals.h @@ -0,0 +1,36 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef LIBCALAMARES_PACKAGES_GLOBALS_H +#define LIBCALAMARES_PACKAGES_GLOBALS_H + +#include "GlobalStorage.h" +#include "modulesystem/InstanceKey.h" + +namespace CalamaresUtils +{ +namespace Packages +{ +/** @brief Sets the install-packages GS keys for the given module + * + * This replaces previously-set install-packages lists for the + * given module by the two new lists. + * + * Returns @c true if anything was changed, @c false otherwise. + */ +bool setGSPackageAdditions( Calamares::GlobalStorage* gs, + const Calamares::ModuleSystem::InstanceKey& module, + const QVariantList& installPackages, + const QVariantList& tryInstallPackages ); +// void setGSPackageRemovals( const Calamares::ModuleSystem::InstanceKey& key, const QVariantList& removePackages ); +} // namespace Packages +} // namespace CalamaresUtils + + +#endif diff --git a/src/libcalamares/packages/Tests.cpp b/src/libcalamares/packages/Tests.cpp new file mode 100644 index 000000000..0a9be3a20 --- /dev/null +++ b/src/libcalamares/packages/Tests.cpp @@ -0,0 +1,88 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "Globals.h" + +#include "GlobalStorage.h" +#include "utils/Logger.h" + +#include + +class PackagesTests : public QObject +{ + Q_OBJECT +public: + PackagesTests() {} + ~PackagesTests() override {} +private Q_SLOTS: + void initTestCase(); + + void testEmpty(); + void testAdd(); +}; + +void +PackagesTests::initTestCase() +{ + Logger::setupLogLevel( Logger::LOGDEBUG ); +} + +void +PackagesTests::testEmpty() +{ + Calamares::GlobalStorage gs; + const QString topKey( "packageOperations" ); + Calamares::ModuleSystem::InstanceKey k( "this", "that" ); + + QVERIFY( !gs.contains( topKey ) ); + QCOMPARE( k.toString(), "this@that" ); + + // Adding nothing at all does nothing + QVERIFY( !CalamaresUtils::Packages::setGSPackageAdditions( &gs, k, QVariantList(), QVariantList() ) ); + QVERIFY( !gs.contains( topKey ) ); +} + +void +PackagesTests::testAdd() +{ + Calamares::GlobalStorage gs; + const QString topKey( "packageOperations" ); + Calamares::ModuleSystem::InstanceKey k( "this", "that" ); + + QVERIFY( !gs.contains( topKey ) ); + QVERIFY( + CalamaresUtils::Packages::setGSPackageAdditions( &gs, k, QVariantList { QString( "vim" ) }, QVariantList() ) ); + QVERIFY( gs.contains( topKey ) ); + auto actionList = gs.value( topKey ).toList(); + QCOMPARE( actionList.length(), 1 ); + auto action = actionList[ 0 ].toMap(); + QVERIFY( action.contains( "install" ) ); + auto op = action[ "install" ].toList(); + QCOMPARE( op.length(), 1 ); + cDebug() << op; + + QVERIFY( CalamaresUtils::Packages::setGSPackageAdditions( + &gs, k, QVariantList { QString( "vim" ), QString( "emacs" ) }, QVariantList() ) ); + QVERIFY( gs.contains( topKey ) ); + actionList = gs.value( topKey ).toList(); + QCOMPARE( actionList.length(), 1 ); + action = actionList[ 0 ].toMap(); + QVERIFY( action.contains( "install" ) ); + op = action[ "install" ].toList(); + QCOMPARE( op.length(), 2 ); + QCOMPARE( action[ "source" ].toString(), k.toString() ); + cDebug() << op; +} + + +QTEST_GUILESS_MAIN( PackagesTests ) + +#include "utils/moc-warnings.h" + +#include "Tests.moc" diff --git a/src/modules/netinstall/CMakeLists.txt b/src/modules/netinstall/CMakeLists.txt index 4500a314f..ec926c9d3 100644 --- a/src/modules/netinstall/CMakeLists.txt +++ b/src/modules/netinstall/CMakeLists.txt @@ -8,6 +8,7 @@ calamares_add_plugin( netinstall EXPORT_MACRO PLUGINDLLEXPORT_PRO SOURCES Config.cpp + LoaderQueue.cpp NetInstallViewStep.cpp NetInstallPage.cpp PackageTreeItem.cpp diff --git a/src/modules/netinstall/Config.cpp b/src/modules/netinstall/Config.cpp index e69b3c2a4..2d663829c 100644 --- a/src/modules/netinstall/Config.cpp +++ b/src/modules/netinstall/Config.cpp @@ -12,10 +12,15 @@ #include "Config.h" +#include "LoaderQueue.h" + +#include "GlobalStorage.h" +#include "JobQueue.h" #include "network/Manager.h" +#include "packages/Globals.h" #include "utils/Logger.h" -#include "utils/RAII.h" -#include "utils/Yaml.h" +#include "utils/Retranslator.h" +#include "utils/Variant.h" #include @@ -23,10 +28,20 @@ Config::Config( QObject* parent ) : QObject( parent ) , m_model( new PackageModel( this ) ) { + CALAMARES_RETRANSLATE_SLOT( &Config::retranslate ); } Config::~Config() {} +void +Config::retranslate() +{ + emit statusChanged( status() ); + emit sidebarLabelChanged( sidebarLabel() ); + emit titleLabelChanged( titleLabel() ); +} + + QString Config::status() const { @@ -39,9 +54,11 @@ Config::status() const case Status::FailedBadData: return tr( "Network Installation. (Disabled: Received invalid groups data)" ); case Status::FailedInternalError: - return tr( "Network Installation. (Disabled: internal error)" ); + return tr( "Network Installation. (Disabled: Internal error)" ); case Status::FailedNetworkError: return tr( "Network Installation. (Disabled: Unable to fetch package lists, check your network connection)" ); + case Status::FailedNoData: + return tr( "Network Installation. (Disabled: No package list)" ); } __builtin_unreachable(); } @@ -54,92 +71,114 @@ Config::setStatus( Status s ) emit statusChanged( status() ); } +QString +Config::sidebarLabel() const +{ + return m_sidebarLabel ? m_sidebarLabel->get() : tr( "Package selection" ); +} + +QString +Config::titleLabel() const +{ + return m_titleLabel ? m_titleLabel->get() : QString(); +} + + void Config::loadGroupList( const QVariantList& groupData ) { m_model->setupModelData( groupData ); + if ( m_model->rowCount() < 1 ) + { + cWarning() << "NetInstall groups data was empty."; + setStatus( Status::FailedNoData ); + } + else + { + setStatus( Status::Ok ); + } emit statusReady(); } void -Config::loadGroupList( const QUrl& url ) +Config::loadingDone() { - if ( !url.isValid() ) + if ( m_queue ) { - setStatus( Status::FailedBadConfiguration ); + m_queue->deleteLater(); + m_queue = nullptr; + } +} + + +void +Config::setConfigurationMap( const QVariantMap& configurationMap ) +{ + setRequired( CalamaresUtils::getBool( configurationMap, "required", false ) ); + + // Get the translations, if any + bool bogus = false; + auto label = CalamaresUtils::getSubMap( configurationMap, "label", bogus ); + // Use a different class name for translation lookup because the + // .. table of strings lives in NetInstallViewStep.cpp and moving them + // .. around is annoying for translators. + static const char className[] = "NetInstallViewStep"; + + if ( label.contains( "sidebar" ) ) + { + m_sidebarLabel = new CalamaresUtils::Locale::TranslatedString( label, "sidebar", className ); + } + if ( label.contains( "title" ) ) + { + m_titleLabel = new CalamaresUtils::Locale::TranslatedString( label, "title", className ); } - using namespace CalamaresUtils::Network; - - cDebug() << "NetInstall loading groups from" << url; - QNetworkReply* reply = Manager::instance().asynchronousGet( - url, - RequestOptions( RequestOptions::FakeUserAgent | RequestOptions::FollowRedirect, std::chrono::seconds( 30 ) ) ); - - if ( !reply ) + // Lastly, load the groups data + const QString key = QStringLiteral( "groupsUrl" ); + const auto& groupsUrlVariant = configurationMap.value( key ); + if ( groupsUrlVariant.type() == QVariant::String ) { - cDebug() << Logger::Continuation << "request failed immediately."; - setStatus( Status::FailedBadConfiguration ); + m_queue = new LoaderQueue( this ); + m_queue->append( SourceItem::makeSourceItem( groupsUrlVariant.toString(), configurationMap ) ); } - else + else if ( groupsUrlVariant.type() == QVariant::List ) { - m_reply = reply; - connect( reply, &QNetworkReply::finished, this, &Config::receivedGroupData ); + m_queue = new LoaderQueue( this ); + for ( const auto& s : groupsUrlVariant.toStringList() ) + { + m_queue->append( SourceItem::makeSourceItem( s, configurationMap ) ); + } + } + if ( m_queue && m_queue->count() > 0 ) + { + cDebug() << "Loading netinstall from" << m_queue->count() << "alternate sources."; + connect( m_queue, &LoaderQueue::done, this, &Config::loadingDone ); + m_queue->load(); } } void -Config::receivedGroupData() +Config::finalizeGlobalStorage( const Calamares::ModuleSystem::InstanceKey& key ) { - if ( !m_reply || !m_reply->isFinished() ) + auto packages = model()->getPackages(); + + // This netinstall module may add two sub-steps to the packageOperations, + // one for installing and one for try-installing. + QVariantList installPackages; + QVariantList tryInstallPackages; + + for ( const auto& package : packages ) { - cWarning() << "NetInstall data called too early."; - setStatus( Status::FailedInternalError ); - return; - } - - cDebug() << "NetInstall group data received" << m_reply->size() << "bytes from" << m_reply->url(); - - cqDeleter< QNetworkReply > d { m_reply }; - - // If m_required is *false* then we still say we're ready - // even if the reply is corrupt or missing. - if ( m_reply->error() != QNetworkReply::NoError ) - { - cWarning() << "unable to fetch netinstall package lists."; - cDebug() << Logger::SubEntry << "Netinstall reply error: " << m_reply->error(); - cDebug() << Logger::SubEntry << "Request for url: " << m_reply->url().toString() - << " failed with: " << m_reply->errorString(); - setStatus( Status::FailedNetworkError ); - return; - } - - QByteArray yamlData = m_reply->readAll(); - try - { - YAML::Node groups = YAML::Load( yamlData.constData() ); - - if ( groups.IsSequence() ) + if ( package->isCritical() ) { - loadGroupList( CalamaresUtils::yamlSequenceToVariant( groups ) ); - } - else if ( groups.IsMap() ) - { - auto map = CalamaresUtils::yamlMapToVariant( groups ); - loadGroupList( map.value( "groups" ).toList() ); + installPackages.append( package->toOperation() ); } else { - cWarning() << "NetInstall groups data does not form a sequence."; - } - if ( m_model->rowCount() < 1 ) - { - cWarning() << "NetInstall groups data was empty."; + tryInstallPackages.append( package->toOperation() ); } } - catch ( YAML::Exception& e ) - { - CalamaresUtils::explainYamlException( e, yamlData, "netinstall groups data" ); - setStatus( Status::FailedBadData ); - } + + CalamaresUtils::Packages::setGSPackageAdditions( + Calamares::JobQueue::instance()->globalStorage(), key, installPackages, tryInstallPackages ); } diff --git a/src/modules/netinstall/Config.h b/src/modules/netinstall/Config.h index 13eb098c6..b676a7d39 100644 --- a/src/modules/netinstall/Config.h +++ b/src/modules/netinstall/Config.h @@ -14,10 +14,15 @@ #include "PackageModel.h" -#include -#include +#include "locale/TranslatableConfiguration.h" +#include "modulesystem/InstanceKey.h" -class QNetworkReply; +#include +#include + +#include + +class LoaderQueue; class Config : public QObject { @@ -26,17 +31,25 @@ class Config : public QObject Q_PROPERTY( PackageModel* packageModel MEMBER m_model FINAL ) Q_PROPERTY( QString status READ status NOTIFY statusChanged FINAL ) + // Translations, of the module name (for sidebar) and above the list + Q_PROPERTY( QString sidebarLabel READ sidebarLabel NOTIFY sidebarLabelChanged FINAL ) + Q_PROPERTY( QString titleLabel READ titleLabel NOTIFY titleLabelChanged FINAL ) + public: Config( QObject* parent = nullptr ); ~Config() override; + void setConfigurationMap( const QVariantMap& configurationMap ); + enum class Status { Ok, FailedBadConfiguration, FailedInternalError, FailedNetworkError, - FailedBadData + FailedBadData, + FailedNoData + }; QString status() const; @@ -45,12 +58,10 @@ public: bool required() const { return m_required; } void setRequired( bool r ) { m_required = r; } - /** @brief Retrieves the groups, with name, description and packages - * - * Loads data from the given URL. Once done, the data is parsed - * and passed on to the other loadGroupList() method. - */ - void loadGroupList( const QUrl& url ); + PackageModel* model() const { return m_model; } + + QString sidebarLabel() const; + QString titleLabel() const; /** @brief Fill model from parsed data. * @@ -59,18 +70,28 @@ public: */ void loadGroupList( const QVariantList& groupData ); - PackageModel* model() const { return m_model; } + /** @brief Write the selected package lists to global storage + * + * Since the config doesn't know what module it is for, + * pass in an instance key. + */ + void finalizeGlobalStorage( const Calamares::ModuleSystem::InstanceKey& key ); -signals: +Q_SIGNALS: void statusChanged( QString status ); ///< Something changed + void sidebarLabelChanged( QString label ); + void titleLabelChanged( QString label ); void statusReady(); ///< Loading groups is complete -private slots: - void receivedGroupData(); ///< From async-loading group data +private Q_SLOTS: + void retranslate(); + void loadingDone(); private: + CalamaresUtils::Locale::TranslatedString* m_sidebarLabel = nullptr; // As it appears in the sidebar + CalamaresUtils::Locale::TranslatedString* m_titleLabel = nullptr; PackageModel* m_model = nullptr; - QNetworkReply* m_reply = nullptr; // For fetching data + LoaderQueue* m_queue = nullptr; Status m_status = Status::Ok; bool m_required = false; }; diff --git a/src/modules/netinstall/LoaderQueue.cpp b/src/modules/netinstall/LoaderQueue.cpp new file mode 100644 index 000000000..f8ba17cff --- /dev/null +++ b/src/modules/netinstall/LoaderQueue.cpp @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: 2016 Luca Giambonini + * SPDX-FileCopyrightText: 2016 Lisa Vitolo + * SPDX-FileCopyrightText: 2017 Kyle Robbertze + * SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "LoaderQueue.h" + +#include "Config.h" +#include "network/Manager.h" +#include "utils/Logger.h" +#include "utils/RAII.h" +#include "utils/Yaml.h" + +#include +#include + +/** @brief Call fetchNext() on the queue if it can + * + * On destruction, a new call to fetchNext() is queued, so that + * the queue continues loading. Calling release() before the + * destructor skips the fetchNext(), ending the queue-loading. + */ +class FetchNextUnless +{ +public: + FetchNextUnless( LoaderQueue* q ) + : m_q( q ) + { + } + ~FetchNextUnless() + { + if ( m_q ) + { + QMetaObject::invokeMethod( m_q, "fetchNext", Qt::QueuedConnection ); + } + } + void release() { m_q = nullptr; } + +private: + LoaderQueue* m_q = nullptr; +}; + +SourceItem +SourceItem::makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap ) +{ + if ( groupsUrl == QStringLiteral( "local" ) ) + { + return SourceItem { QUrl(), configurationMap.value( "groups" ).toList() }; + } + else + { + return SourceItem { QUrl { groupsUrl }, QVariantList() }; + } +} + +LoaderQueue::LoaderQueue( Config* parent ) + : QObject( parent ) + , m_config( parent ) +{ +} + +void +LoaderQueue::append( SourceItem&& i ) +{ + m_queue.append( std::move( i ) ); +} + +void +LoaderQueue::load() +{ + QMetaObject::invokeMethod( this, "fetchNext", Qt::QueuedConnection ); +} + + +void +LoaderQueue::fetchNext() +{ + if ( m_queue.isEmpty() ) + { + m_config->setStatus( Config::Status::FailedBadData ); + emit done(); + return; + } + + auto source = m_queue.takeFirst(); + if ( source.isLocal() ) + { + m_config->loadGroupList( source.data ); + emit done(); + } + else + { + fetch( source.url ); + } +} + +void +LoaderQueue::fetch( const QUrl& url ) +{ + FetchNextUnless next( this ); + + if ( !url.isValid() ) + { + m_config->setStatus( Config::Status::FailedBadConfiguration ); + cDebug() << "Invalid URL" << url; + return; + } + + using namespace CalamaresUtils::Network; + + cDebug() << "NetInstall loading groups from" << url; + QNetworkReply* reply = Manager::instance().asynchronousGet( + url, + RequestOptions( RequestOptions::FakeUserAgent | RequestOptions::FollowRedirect, std::chrono::seconds( 30 ) ) ); + + if ( !reply ) + { + cDebug() << Logger::SubEntry << "Request failed immediately."; + // If nobody sets a different status, this will remain + m_config->setStatus( Config::Status::FailedBadConfiguration ); + } + else + { + // When the network request is done, **then** we might + // do the next item from the queue, so don't call fetchNext() now. + next.release(); + m_reply = reply; + connect( reply, &QNetworkReply::finished, this, &LoaderQueue::dataArrived ); + } +} + +void +LoaderQueue::dataArrived() +{ + FetchNextUnless finished( this ); + + if ( !m_reply || !m_reply->isFinished() ) + { + cWarning() << "NetInstall data called too early."; + m_config->setStatus( Config::Status::FailedInternalError ); + return; + } + + cDebug() << "NetInstall group data received" << m_reply->size() << "bytes from" << m_reply->url(); + + cqDeleter< QNetworkReply > d { m_reply }; + + // If m_required is *false* then we still say we're ready + // even if the reply is corrupt or missing. + if ( m_reply->error() != QNetworkReply::NoError ) + { + cWarning() << "unable to fetch netinstall package lists."; + cDebug() << Logger::SubEntry << "Netinstall reply error: " << m_reply->error(); + cDebug() << Logger::SubEntry << "Request for url: " << m_reply->url().toString() + << " failed with: " << m_reply->errorString(); + m_config->setStatus( Config::Status::FailedNetworkError ); + return; + } + + QByteArray yamlData = m_reply->readAll(); + try + { + YAML::Node groups = YAML::Load( yamlData.constData() ); + + if ( groups.IsSequence() ) + { + finished.release(); + m_config->loadGroupList( CalamaresUtils::yamlSequenceToVariant( groups ) ); + emit done(); + } + else if ( groups.IsMap() ) + { + finished.release(); + auto map = CalamaresUtils::yamlMapToVariant( groups ); + m_config->loadGroupList( map.value( "groups" ).toList() ); + emit done(); + } + else + { + cWarning() << "NetInstall groups data does not form a sequence."; + } + } + catch ( YAML::Exception& e ) + { + CalamaresUtils::explainYamlException( e, yamlData, "netinstall groups data" ); + m_config->setStatus( Config::Status::FailedBadData ); + } +} diff --git a/src/modules/netinstall/LoaderQueue.h b/src/modules/netinstall/LoaderQueue.h new file mode 100644 index 000000000..d7baf58d4 --- /dev/null +++ b/src/modules/netinstall/LoaderQueue.h @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2016 Luca Giambonini + * SPDX-FileCopyrightText: 2016 Lisa Vitolo + * SPDX-FileCopyrightText: 2017 Kyle Robbertze + * SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef NETINSTALL_LOADERQUEUE_H +#define NETINSTALL_LOADERQUEUE_H + +#include +#include +#include + +class Config; +class QNetworkReply; + +/** @brief Data about an entry in *groupsUrl* + * + * This can be a specific URL, or "local" which uses data stored + * in the configuration file itself. + */ +struct SourceItem +{ + QUrl url; + QVariantList data; + + bool isUrl() const { return url.isValid(); } + bool isLocal() const { return !data.isEmpty(); } + bool isValid() const { return isUrl() || isLocal(); } + /** @brief Create a SourceItem + * + * If the @p groupsUrl is @c "local" then the *groups* key in + * the @p configurationMap is used as the source; otherwise the + * string is used as an actual URL. + */ + static SourceItem makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap ); +}; + +/** @brief Queue of source items to load + * + * Queue things up by calling append() and then kick things off + * by calling load(). This will try to load the items, in order; + * the first one that succeeds will end the loading process. + * + * Signal done() is emitted when done (also when all of the items fail). + */ +class LoaderQueue : public QObject +{ + Q_OBJECT +public: + LoaderQueue( Config* parent ); + + void append( SourceItem&& i ); + int count() const { return m_queue.count(); } + +public Q_SLOTS: + void load(); + + void fetchNext(); + void fetch( const QUrl& url ); + void dataArrived(); + +Q_SIGNALS: + void done(); + +private: + QQueue< SourceItem > m_queue; + Config* m_config = nullptr; + QNetworkReply* m_reply = nullptr; +}; + +#endif diff --git a/src/modules/netinstall/NetInstallPage.cpp b/src/modules/netinstall/NetInstallPage.cpp index 3addf86b3..a1a86294e 100644 --- a/src/modules/netinstall/NetInstallPage.cpp +++ b/src/modules/netinstall/NetInstallPage.cpp @@ -33,40 +33,16 @@ NetInstallPage::NetInstallPage( Config* c, QWidget* parent ) ui->setupUi( this ); ui->groupswidget->header()->setSectionResizeMode( QHeaderView::ResizeToContents ); ui->groupswidget->setModel( c->model() ); - connect( c, &Config::statusChanged, this, &NetInstallPage::setStatus ); + connect( c, &Config::statusChanged, ui->netinst_status, &QLabel::setText ); + connect( c, &Config::titleLabelChanged, [ui = this->ui]( const QString title ) { + ui->label->setVisible( !title.isEmpty() ); + ui->label->setText( title ); + } ); connect( c, &Config::statusReady, this, &NetInstallPage::expandGroups ); - - setPageTitle( nullptr ); - CALAMARES_RETRANSLATE_SLOT( &NetInstallPage::retranslate ); } NetInstallPage::~NetInstallPage() {} -void -NetInstallPage::setPageTitle( CalamaresUtils::Locale::TranslatedString* t ) -{ - m_title.reset( t ); - if ( !m_title ) - { - ui->label->hide(); - } - else - { - ui->label->show(); - } - retranslate(); -} - -void -NetInstallPage::retranslate() -{ - if ( m_title ) - { - ui->label->setText( m_title->get() ); // That's get() on the TranslatedString - } - ui->netinst_status->setText( m_config->status() ); -} - void NetInstallPage::expandGroups() { @@ -82,12 +58,6 @@ NetInstallPage::expandGroups() } } -void -NetInstallPage::setStatus( QString s ) -{ - ui->netinst_status->setText( s ); -} - void NetInstallPage::onActivate() { diff --git a/src/modules/netinstall/NetInstallPage.h b/src/modules/netinstall/NetInstallPage.h index 1c97423da..72375d0f0 100644 --- a/src/modules/netinstall/NetInstallPage.h +++ b/src/modules/netinstall/NetInstallPage.h @@ -37,23 +37,8 @@ public: NetInstallPage( Config* config, QWidget* parent = nullptr ); ~NetInstallPage() override; - /** @brief Sets the page title - * - * In situations where there is more than one netinstall page, - * or you want some explanatory title above the treeview, - * set the page title. This page takes ownership of the - * TranslatedString object. - * - * Set to nullptr to remove the title. - */ - void setPageTitle( CalamaresUtils::Locale::TranslatedString* ); - void onActivate(); -public slots: - void retranslate(); - void setStatus( QString s ); - /** @brief Expand entries that should be pre-expanded. * * Follows the *expanded* key / the startExpanded field in the @@ -64,8 +49,6 @@ public slots: private: Config* m_config; Ui::Page_NetInst* ui; - - std::unique_ptr< CalamaresUtils::Locale::TranslatedString > m_title; // Above the treeview }; #endif // NETINSTALLPAGE_H diff --git a/src/modules/netinstall/NetInstallViewStep.cpp b/src/modules/netinstall/NetInstallViewStep.cpp index d92058e51..2ac0e73c9 100644 --- a/src/modules/netinstall/NetInstallViewStep.cpp +++ b/src/modules/netinstall/NetInstallViewStep.cpp @@ -11,12 +11,6 @@ #include "NetInstallViewStep.h" -#include "GlobalStorage.h" -#include "JobQueue.h" - -#include "utils/Logger.h" -#include "utils/Variant.h" - #include "NetInstallPage.h" CALAMARES_PLUGIN_FACTORY_DEFINITION( NetInstallViewStepFactory, registerPlugin< NetInstallViewStep >(); ) @@ -24,7 +18,6 @@ CALAMARES_PLUGIN_FACTORY_DEFINITION( NetInstallViewStepFactory, registerPlugin< NetInstallViewStep::NetInstallViewStep( QObject* parent ) : Calamares::ViewStep( parent ) , m_widget( new NetInstallPage( &m_config ) ) - , m_sidebarLabel( nullptr ) , m_nextEnabled( false ) { connect( &m_config, &Config::statusReady, this, &NetInstallViewStep::nextIsReady ); @@ -37,20 +30,22 @@ NetInstallViewStep::~NetInstallViewStep() { m_widget->deleteLater(); } - delete m_sidebarLabel; } QString NetInstallViewStep::prettyName() const { - return m_sidebarLabel ? m_sidebarLabel->get() : tr( "Package selection" ); + return m_config.sidebarLabel(); #if defined( TABLE_OF_TRANSLATIONS ) __builtin_unreachable(); // This is a table of "standard" labels for this module. If you use them // in the label: sidebar: section of the config file, the existing // translations can be used. + // + // These translations still live here, even though the lookup + // code is in the Config class. tr( "Package selection" ); tr( "Office software" ); tr( "Office package" ); @@ -125,70 +120,7 @@ NetInstallViewStep::onActivate() void NetInstallViewStep::onLeave() { - auto packages = m_config.model()->getPackages(); - cDebug() << "Netinstall: Processing" << packages.length() << "packages."; - - static const char PACKAGEOP[] = "packageOperations"; - - // Check if there's already a PACAKGEOP entry in GS, and if so we'll - // extend that one (overwriting the value in GS at the end of this method) - Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage(); - QVariantList packageOperations = gs->contains( PACKAGEOP ) ? gs->value( PACKAGEOP ).toList() : QVariantList(); - cDebug() << Logger::SubEntry << "Existing package operations length" << packageOperations.length(); - - // Clear out existing operations for this module, going backwards: - // Sometimes we remove an item, and we don't want the index to - // fall off the end of the list. - bool somethingRemoved = false; - for ( int index = packageOperations.length() - 1; 0 <= index; index-- ) - { - const QVariantMap op = packageOperations.at( index ).toMap(); - if ( op.contains( "source" ) && op.value( "source" ).toString() == moduleInstanceKey().toString() ) - { - cDebug() << Logger::SubEntry << "Removing existing operations for" << moduleInstanceKey(); - packageOperations.removeAt( index ); - somethingRemoved = true; - } - } - - // This netinstall module may add two sub-steps to the packageOperations, - // one for installing and one for try-installing. - QVariantList installPackages; - QVariantList tryInstallPackages; - - for ( const auto& package : packages ) - { - if ( package->isCritical() ) - { - installPackages.append( package->toOperation() ); - } - else - { - tryInstallPackages.append( package->toOperation() ); - } - } - - if ( !installPackages.empty() ) - { - QVariantMap op; - op.insert( "install", QVariant( installPackages ) ); - op.insert( "source", moduleInstanceKey().toString() ); - packageOperations.append( op ); - cDebug() << Logger::SubEntry << installPackages.length() << "critical packages."; - } - if ( !tryInstallPackages.empty() ) - { - QVariantMap op; - op.insert( "try_install", QVariant( tryInstallPackages ) ); - op.insert( "source", moduleInstanceKey().toString() ); - packageOperations.append( op ); - cDebug() << Logger::SubEntry << tryInstallPackages.length() << "non-critical packages."; - } - - if ( somethingRemoved || !packageOperations.isEmpty() ) - { - gs->insert( PACKAGEOP, packageOperations ); - } + m_config.finalizeGlobalStorage( moduleInstanceKey() ); } void @@ -201,35 +133,5 @@ NetInstallViewStep::nextIsReady() void NetInstallViewStep::setConfigurationMap( const QVariantMap& configurationMap ) { - m_config.setRequired( CalamaresUtils::getBool( configurationMap, "required", false ) ); - - QString groupsUrl = CalamaresUtils::getString( configurationMap, "groupsUrl" ); - if ( !groupsUrl.isEmpty() ) - { - // Keep putting groupsUrl into the global storage, - // even though it's no longer used for in-module data-passing. - Calamares::JobQueue::instance()->globalStorage()->insert( "groupsUrl", groupsUrl ); - if ( groupsUrl == QStringLiteral( "local" ) ) - { - QVariantList l = configurationMap.value( "groups" ).toList(); - m_config.loadGroupList( l ); - } - else - { - m_config.loadGroupList( groupsUrl ); - } - } - - bool bogus = false; - auto label = CalamaresUtils::getSubMap( configurationMap, "label", bogus ); - - if ( label.contains( "sidebar" ) ) - { - m_sidebarLabel = new CalamaresUtils::Locale::TranslatedString( label, "sidebar", metaObject()->className() ); - } - if ( label.contains( "title" ) ) - { - m_widget->setPageTitle( - new CalamaresUtils::Locale::TranslatedString( label, "title", metaObject()->className() ) ); - } + m_config.setConfigurationMap( configurationMap ); } diff --git a/src/modules/netinstall/NetInstallViewStep.h b/src/modules/netinstall/NetInstallViewStep.h index c500cbcd9..8949632c1 100644 --- a/src/modules/netinstall/NetInstallViewStep.h +++ b/src/modules/netinstall/NetInstallViewStep.h @@ -14,7 +14,6 @@ #include "Config.h" #include "DllMacro.h" -#include "locale/TranslatableConfiguration.h" #include "utils/PluginFactory.h" #include "viewpages/ViewStep.h" @@ -56,7 +55,6 @@ private: Config m_config; NetInstallPage* m_widget; - CalamaresUtils::Locale::TranslatedString* m_sidebarLabel; // As it appears in the sidebar bool m_nextEnabled = false; }; diff --git a/src/modules/netinstall/netinstall.conf b/src/modules/netinstall/netinstall.conf index c377b526e..d38c0b040 100644 --- a/src/modules/netinstall/netinstall.conf +++ b/src/modules/netinstall/netinstall.conf @@ -32,20 +32,21 @@ # This module supports multiple instances through the *label* key, # which allows you to distinguish them in the UI. --- -# This is the URL that is retrieved to get the netinstall groups-and-packages -# data (which should be in the format described in netinstall.yaml), e.g.: -# ``` -# groupsUrl: http://example.org/netinstall.php -# ``` -# or it can be a locally installed file: -# ``` -# groupsUrl: file:///usr/share/calamares/netinstall.yaml -# ``` -# or it can be the special-case literal string "local": -# ``` -# groupsUrl: local -# ``` +# The *groupsUrl* determines where the data for the netinstall groups-and- +# packages comes from. The value of the key may be: # +# - a single string (this is treated as a list with just that string in it) +# - a list of strings +# +# Each string is treated as a URL (see below for special cases. The +# list is examined **in order** and each URL is tried in turn. The +# first URL to load successfully -- even if it yields 0 packages -- +# ends the process. This allows using a network URL and a (fallback) +# local URL for package lists, or for using multiple mirrors of +# netinstall data. +# +# The URL must point to a YAML file that follows the format described +# below at the key *groups* -- except for the special case URL "local". # Note that the contents of the groups file is the **important** # part of the configuration of this module. It specifies what # groups and packages the user may select (and so what commands are to @@ -59,12 +60,27 @@ # must have a list-of-groups as value; if the file does not have # a top-level key *groups*, then the file must contain only a list of groups. # -# As a special case, setting *groupsUrl* to the literal string -# `local` means that the data is obtained from **this** config -# file, under the key *groups*. +# Each item in the list *groupsUrl* may be: +# - A remote URL like `http://example.org/netinstall.php` +# - A local file URL like `file:///usr/share/calamares/netinstall.yaml` +# - The special-case literal string `local` +# +# Non-special case URLs are loaded as YAML; if the load succeeds, then +# they are interpreted like the *groups* key below. The special case +# `local` loads the data directly from **this** file. # groupsUrl: local +# Alternate form: +# groupsUrl: [ local ] + +# Net-based package list, with fallback to local file +# groupsUrl: +# - http://example.com/calamares/netinstall.yaml +# - file:///etc/calamares/modules/netinstall.yaml + + + # If the installation can proceed without netinstall (e.g. the Live CD # can create a working installed system, but netinstall is preferred # to bring it up-to-date or extend functionality) leave this set to