diff --git a/CHANGES-3.3 b/CHANGES-3.3 index f719bad28..bf02374d0 100644 --- a/CHANGES-3.3 +++ b/CHANGES-3.3 @@ -7,16 +7,23 @@ contributors are listed. Note that Calamares does not have a historical changelog -- this log starts with version 3.3.0. See CHANGES-3.2 for the history of the 3.2 series (2018-05 - 2022-08). -# 3.3.11 (unreleased) +# 3.3.11 (2024-11-05) This release contains contributions from (alphabetically by given name): - - Nobody yet + - Adriaan de Groot + - Jakob Petsovits + - Simon Quigley ## Core ## - Nothing yet ## Modules ## - - Nothing yet + - *unpackfs* now supports a `condition` configuration option for + conditional installation / unsquash. (thanks Simon) + - *unpackfsc* module imported from Calamares-extensions, and extended + with the same `condition` configuration. + - *partition* crash fixed when swap was using the wrong end-sector + in some GPT configurations. (thanks Jakob, #2367) # 3.3.10 (2024-10-21) diff --git a/CMakeLists.txt b/CMakeLists.txt index c969e20fb..ad1148676 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,7 +49,7 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) set(CALAMARES_VERSION 3.3.11) -set(CALAMARES_RELEASE_MODE OFF) # Set to ON during a release +set(CALAMARES_RELEASE_MODE ON) # Set to ON during a release if(CMAKE_SCRIPT_MODE_FILE) include(${CMAKE_CURRENT_LIST_DIR}/CMakeModules/ExtendedVersion.cmake) diff --git a/src/libcalamares/GlobalStorage.cpp b/src/libcalamares/GlobalStorage.cpp index 0c78201d3..e78a1b04a 100644 --- a/src/libcalamares/GlobalStorage.cpp +++ b/src/libcalamares/GlobalStorage.cpp @@ -190,4 +190,55 @@ GlobalStorage::loadYaml( const QString& filename ) return false; } +///@brief Implementation for recursively looking up dotted selector parts. +static QVariant +lookup( const QStringList& nestedKey, int index, const QVariant& v, bool& ok ) +{ + if ( !v.canConvert< QVariantMap >() ) + { + // Mismatch: we're still looking for keys, but v is not a submap + ok = false; + return {}; + } + if ( index >= nestedKey.length() ) + { + cError() << "Recursion error looking at index" << index << "of" << nestedKey; + ok = false; + return {}; + } + + const QVariantMap map = v.toMap(); + const QString& key = nestedKey.at( index ); + if ( index == nestedKey.length() - 1 ) + { + ok = map.contains( key ); + return ok ? map.value( key ) : QVariant(); + } + else + { + return lookup( nestedKey, index + 1, map.value( key ), ok ); + } +} + +QVariant +lookup( const GlobalStorage* storage, const QString& nestedKey, bool& ok ) +{ + ok = false; + if ( !storage ) + { + return {}; + } + + if ( nestedKey.contains( '.' ) ) + { + QStringList steps = nestedKey.split( '.' ); + return lookup( steps, 1, storage->value( steps.first() ), ok ); + } + else + { + ok = storage->contains( nestedKey ); + return ok ? storage->value( nestedKey ) : QVariant(); + } +} + } // namespace Calamares diff --git a/src/libcalamares/GlobalStorage.h b/src/libcalamares/GlobalStorage.h index 5bb6d4e97..37ea332d2 100644 --- a/src/libcalamares/GlobalStorage.h +++ b/src/libcalamares/GlobalStorage.h @@ -167,6 +167,26 @@ private: mutable QMutex m_mutex; }; + +/** @brief Gets a value from the store + * + * When @p nestedKey contains no '.' characters, equivalent + * to `gs->value(nestedKey)`. Otherwise recursively looks up + * the '.'-separated parts of @p nestedKey in successive sub-maps + * of the store, returning the value in the innermost one. + * + * Example: `lookup(gs, "branding.name")` finds the value of the + * 'name' key in the 'branding' submap of the store. + * + * Sets @p ok to @c true if a value was found. Returns the value + * as a variant. If no value is found (e.g. the key is missing + * or some prefix submap is missing) sets @p ok to @c false + * and returns an invalid QVariant. + * + * @see GlobalStorage::value + */ +DLLEXPORT QVariant lookup( const GlobalStorage* gs, const QString& nestedKey, bool& ok ); + } // namespace Calamares #endif // CALAMARES_GLOBALSTORAGE_H diff --git a/src/libcalamares/Tests.cpp b/src/libcalamares/Tests.cpp index 42bdbbe38..0b718d14e 100644 --- a/src/libcalamares/Tests.cpp +++ b/src/libcalamares/Tests.cpp @@ -32,6 +32,7 @@ private Q_SLOTS: void testGSLoadSave(); void testGSLoadSave2(); void testGSLoadSaveYAMLStringList(); + void testGSNestedLookup(); void testInstanceKey(); void testInstanceDescription(); @@ -126,17 +127,14 @@ TestLibCalamares::testGSLoadSave2() { Logger::setupLogLevel( Logger::LOGDEBUG ); - const QString filename( "../src/libcalamares/testdata/yaml-list.conf" ); - if ( !QFile::exists( filename ) ) - { - return; - } + const QString filename( BUILD_AS_TEST "/testdata/yaml-list.conf" ); + QVERIFY2( QFile::exists( filename ), qPrintable( filename ) ); Calamares::GlobalStorage gs1; const QString key( "dwarfs" ); QVERIFY( gs1.loadYaml( filename ) ); - QCOMPARE( gs1.count(), 3 ); // From examining the file + QCOMPARE( gs1.count(), 4 ); // From examining the file QVERIFY( gs1.contains( key ) ); cDebug() << Calamares::typeOf( gs1.value( key ) ) << gs1.value( key ); QCOMPARE( Calamares::typeOf( gs1.value( key ) ), Calamares::ListVariantType ); @@ -177,6 +175,38 @@ TestLibCalamares::testGSLoadSaveYAMLStringList() QCOMPARE( gs2.value( "dwarfs" ).toString(), QStringLiteral( "" ) ); // .. they're gone } +void +TestLibCalamares::testGSNestedLookup() +{ + Logger::setupLogLevel( Logger::LOGDEBUG ); + + const QString filename( BUILD_AS_TEST "/testdata/yaml-list.conf" ); + QVERIFY2( QFile::exists( filename ), qPrintable( filename ) ); + + Calamares::GlobalStorage gs2; + QVERIFY( gs2.loadYaml( filename ) ); + + bool ok = false; + const auto v0 = Calamares::lookup( &gs2, "horse.colors.neck", ok ); + QVERIFY( ok ); + QVERIFY( v0.canConvert< QString >() ); + QCOMPARE( v0.toString(), QStringLiteral( "roan" ) ); + const auto v1 = Calamares::lookup( &gs2, "horse.colors.nose", ok ); + QVERIFY( !ok ); + QVERIFY( !v1.isValid() ); + const auto v2 = Calamares::lookup( &gs2, "cow.colors.nose", ok ); + QVERIFY( !ok ); + QVERIFY( !v2.isValid() ); + const auto v3 = Calamares::lookup( &gs2, "dwarfs", ok ); + QVERIFY( ok ); + QVERIFY( v3.canConvert< QVariantList >() ); // because it's a list-valued thing + const auto v4 = Calamares::lookup( &gs2, "dwarfs.sleepy", ok ); + QVERIFY( !ok ); // Sleepy is a value in the list of dwarfs, not a key + const auto v5 = Calamares::lookup( &gs2, "derp", ok ); + QVERIFY( ok ); + QCOMPARE( v5.toInt(), 17 ); +} + void TestLibCalamares::testInstanceKey() { diff --git a/src/libcalamares/testdata/yaml-list.conf b/src/libcalamares/testdata/yaml-list.conf index d8d2178d1..fca1c4c2d 100644 --- a/src/libcalamares/testdata/yaml-list.conf +++ b/src/libcalamares/testdata/yaml-list.conf @@ -9,3 +9,9 @@ - "sleepy" - "sneezy" - "doc" +horse: + hoofs: 4 + colors: + mane: black + neck: roan + tail: white diff --git a/src/libcalamares/utils/Runner.h b/src/libcalamares/utils/Runner.h index d6adf3bdd..aa8087628 100644 --- a/src/libcalamares/utils/Runner.h +++ b/src/libcalamares/utils/Runner.h @@ -44,7 +44,7 @@ using ProcessResult = Calamares::ProcessResult; * * Processes are always run with LC_ALL and LANG set to "C". */ -class Runner : public QObject +class DLLEXPORT Runner : public QObject { Q_OBJECT diff --git a/src/modules/contextualprocess/ContextualProcessJob.cpp b/src/modules/contextualprocess/ContextualProcessJob.cpp index 30d57947e..9b34db42f 100644 --- a/src/modules/contextualprocess/ContextualProcessJob.cpp +++ b/src/modules/contextualprocess/ContextualProcessJob.cpp @@ -58,45 +58,18 @@ ContextualProcessBinding::run( const QString& value ) const return Calamares::JobResult::ok(); } -///@brief Implementation of fetch() for recursively looking up dotted selector parts. -static bool -fetch( QString& value, QStringList& selector, int index, const QVariant& v ) -{ - if ( !v.canConvert< QVariantMap >() ) - { - return false; - } - const QVariantMap map = v.toMap(); - const QString& key = selector.at( index ); - if ( index == selector.length() - 1 ) - { - value = map.value( key ).toString(); - return map.contains( key ); - } - else - { - return fetch( value, selector, index + 1, map.value( key ) ); - } -} - bool ContextualProcessBinding::fetch( Calamares::GlobalStorage* storage, QString& value ) const { value.clear(); - if ( !storage ) + bool ok = false; + const auto v = Calamares::lookup( storage, m_variable, ok ); + if ( !ok ) { return false; } - if ( m_variable.contains( '.' ) ) - { - QStringList steps = m_variable.split( '.' ); - return ::fetch( value, steps, 1, storage->value( steps.first() ) ); - } - else - { - value = storage->value( m_variable ).toString(); - return storage->contains( m_variable ); - } + value = v.toString(); + return true; } ContextualProcessJob::ContextualProcessJob( QObject* parent ) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 814556f8b..a6a84c3d4 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -48,7 +48,7 @@ class UnpackEntry: :param destination: """ __slots__ = ('source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile', - 'mountPoint', 'weight') + 'mountPoint', 'weight', 'condition') def __init__(self, source, sourcefs, destination): """ @@ -71,6 +71,7 @@ class UnpackEntry: self.total = 0 self.mountPoint = None self.weight = 1 + self.condition = True def is_file(self): return self.sourcefs == "file" @@ -419,6 +420,18 @@ def extract_weight(entry): return 1 +def fetch_from_globalstorage(keys_list): + value = libcalamares.globalstorage.value(keys_list[0]) + if value is None: + return None + for key in keys_list[1:]: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value + + def run(): """ Unsquash filesystem. @@ -474,6 +487,28 @@ def run(): sourcefs = entry["sourcefs"] destination = os.path.abspath(root_mount_point + entry["destination"]) + condition = entry.get("condition", True) + if isinstance(condition, bool): + pass # 'condition' is already True or False + elif isinstance(condition, str): + keys = condition.split(".") + gs_value = fetch_from_globalstorage(keys) + if gs_value is None: + libcalamares.utils.warning("Condition key '{}' not found in global storage, assuming False".format(condition)) + condition = False + elif isinstance(gs_value, bool): + condition = gs_value + else: + libcalamares.utils.warning("Condition key '{}' is not a boolean, assuming True".format(condition)) + condition = True + else: + libcalamares.utils.warning("Invalid 'condition' value '{}', assuming True".format(condition)) + condition = True + + if not condition: + libcalamares.utils.debug("Skipping unpack of {} due to 'condition' being False".format(source)) + continue + if not os.path.isdir(destination) and sourcefs != "file": libcalamares.utils.warning(("The destination \"{}\" in the target system is not a directory").format(destination)) if is_first: diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index d12110b60..42f3a943d 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -86,6 +86,22 @@ # of trailing slashes apply. In order to *rename* a file as it is # copied, specify one single file (e.g. CHANGES) and a full pathname # for its destination name, as in the example below. +# +# It is also possible to dynamically (conditionally) unpack a source by passing a boolean +# value for *condition*. This may be true or false (constant) or name a globalstorage +# value. Use '.' to separate parts of a globalstorage name if it is nested. +# +# This is used in e.g. stacked squashfses, where the user can select a specific +# install type. The default value of *condition* is true. +# +# - source: ./example.minimal.sqfs +# sourcefs: squashfs +# destination: "" +# condition: false +# - source: ./example.standard.sqfs +# sourcefs: squashfs +# destination: "" +# condition: exampleGlobalStorageVariable.subkey unpack: - source: ../CHANGES diff --git a/src/modules/unpackfs/unpackfs.schema.yaml b/src/modules/unpackfs/unpackfs.schema.yaml index 0d96fe9cb..03faa9440 100644 --- a/src/modules/unpackfs/unpackfs.schema.yaml +++ b/src/modules/unpackfs/unpackfs.schema.yaml @@ -18,4 +18,8 @@ properties: excludeFile: { type: string } exclude: { type: array, items: { type: string } } weight: { type: integer, exclusiveMinimum: 0 } + condition: + anyOf: + - type: boolean + - type: string required: [ source , sourcefs, destination ] diff --git a/src/modules/unpackfsc/CMakeLists.txt b/src/modules/unpackfsc/CMakeLists.txt new file mode 100644 index 000000000..a045253ff --- /dev/null +++ b/src/modules/unpackfsc/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2021 Adriaan de Groot +# SPDX-License-Identifier: GPL-3.0-or-later + +calamares_add_plugin( unpackfsc + TYPE job + EXPORT_MACRO PLUGINDLLEXPORT_PRO + SOURCES + UnpackFSCJob.cpp + # The workers for differently-packed filesystems + Runners.cpp + FSArchiverRunner.cpp + TarballRunner.cpp + UnsquashRunner.cpp + SHARED_LIB +) diff --git a/src/modules/unpackfsc/FSArchiverRunner.cpp b/src/modules/unpackfsc/FSArchiverRunner.cpp new file mode 100644 index 000000000..41b6b7122 --- /dev/null +++ b/src/modules/unpackfsc/FSArchiverRunner.cpp @@ -0,0 +1,117 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "FSArchiverRunner.h" + +#include "utils/Logger.h" +#include "utils/Runner.h" + +#include + +static constexpr const int chunk_size = 137; +static const QString& +toolName() +{ + static const QString name = QStringLiteral( "fsarchiver" ); + return name; +} + +void +FSArchiverRunner::fsarchiverProgress( QString line ) +{ + m_since++; + // Typical line of output is this: + // -[00][ 99%][REGFILEM] /boot/thing + // 5 9 ^21 + if ( m_since >= chunk_size && line.length() > 21 && line[ 5 ] == '[' && line[ 9 ] == '%' ) + { + m_since = 0; + double p = double( line.mid( 6, 3 ).toInt() ) / 100.0; + const QString filename = line.mid( 22 ); + Q_EMIT progress( p, filename ); + } +} + +Calamares::JobResult +FSArchiverRunner::checkPrerequisites( QString& fsarchiverExecutable ) const +{ + if ( !checkToolExists( toolName(), fsarchiverExecutable ) ) + { + return Calamares::JobResult::internalError( + tr( "Missing tools" ), + tr( "The %1 tool is not installed on the system." ).arg( toolName() ), + Calamares::JobResult::MissingRequirements ); + } + + if ( !checkSourceExists() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid fsarchiver configuration" ), + tr( "The source archive %1 does not exist." ).arg( m_source ), + Calamares::JobResult::InvalidConfiguration ); + } + return Calamares::JobResult::ok(); +} + +Calamares::JobResult +FSArchiverRunner::checkDestination( QString& destinationPath ) const +{ + destinationPath = Calamares::System::instance()->targetPath( m_destination ); + if ( destinationPath.isEmpty() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid fsarchiver configuration" ), + tr( "No destination could be found for %1." ).arg( m_destination ), + Calamares::JobResult::InvalidConfiguration ); + } + + return Calamares::JobResult::ok(); +} + +Calamares::JobResult +FSArchiverDirRunner::run() +{ + QString fsarchiverExecutable; + if ( auto res = checkPrerequisites( fsarchiverExecutable ); !res ) + { + return res; + } + QString destinationPath; + if ( auto res = checkDestination( destinationPath ); !res ) + { + return res; + } + + Calamares::Utils::Runner r( + { fsarchiverExecutable, QStringLiteral( "-v" ), QStringLiteral( "restdir" ), m_source, destinationPath } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + connect( &r, &decltype( r )::output, this, &FSArchiverDirRunner::fsarchiverProgress ); + return r.run().explainProcess( toolName(), std::chrono::seconds( 0 ) ); +} + +Calamares::JobResult +FSArchiverFSRunner::run() +{ + QString fsarchiverExecutable; + if ( auto res = checkPrerequisites( fsarchiverExecutable ); !res ) + { + return res; + } + QString destinationPath; + if ( auto res = checkDestination( destinationPath ); !res ) + { + return res; + } + + Calamares::Utils::Runner r( + { fsarchiverExecutable, QStringLiteral( "-v" ), QStringLiteral( "restfs" ), m_source, destinationPath } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + connect( &r, &decltype( r )::output, this, &FSArchiverFSRunner::fsarchiverProgress ); + return r.run().explainProcess( toolName(), std::chrono::seconds( 0 ) ); +} diff --git a/src/modules/unpackfsc/FSArchiverRunner.h b/src/modules/unpackfsc/FSArchiverRunner.h new file mode 100644 index 000000000..39be6233f --- /dev/null +++ b/src/modules/unpackfsc/FSArchiverRunner.h @@ -0,0 +1,59 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef UNPACKFSC_FSARCHIVERRUNNER_H +#define UNPACKFSC_FSARCHIVERRUNNER_H + +#include "Runners.h" + +/** @brief Base class for runners of FSArchiver + * + */ +class FSArchiverRunner : public Runner +{ + Q_OBJECT +public: + using Runner::Runner; + +protected Q_SLOTS: + void fsarchiverProgress( QString line ); + +protected: + /** @brief Checks prerequisites, sets full path of fsarchiver in @p executable + */ + Calamares::JobResult checkPrerequisites( QString& executable ) const; + Calamares::JobResult checkDestination( QString& destinationPath ) const; + + int m_since = 0; +}; + +/** @brief Running FSArchiver in **dir** mode + * + */ +class FSArchiverDirRunner : public FSArchiverRunner +{ +public: + using FSArchiverRunner::FSArchiverRunner; + + Calamares::JobResult run() override; +}; + +/** @brief Running FSArchiver in **dir** mode + * + */ +class FSArchiverFSRunner : public FSArchiverRunner +{ +public: + using FSArchiverRunner::FSArchiverRunner; + + Calamares::JobResult run() override; +}; + + +#endif diff --git a/src/modules/unpackfsc/Runners.cpp b/src/modules/unpackfsc/Runners.cpp new file mode 100644 index 000000000..3b3862260 --- /dev/null +++ b/src/modules/unpackfsc/Runners.cpp @@ -0,0 +1,38 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "Runners.h" + +#include +#include + +#include +#include + +Runner::Runner( const QString& source, const QString& destination ) + : m_source( source ) + , m_destination( destination ) +{ +} + +Runner::~Runner() { } + +bool +Runner::checkSourceExists() const +{ + QFileInfo fi( m_source ); + return fi.exists() && fi.isReadable(); +} + +bool +Runner::checkToolExists( const QString& toolName, QString& fullPath ) +{ + fullPath = QStandardPaths::findExecutable( toolName ); + return !fullPath.isEmpty(); +} diff --git a/src/modules/unpackfsc/Runners.h b/src/modules/unpackfsc/Runners.h new file mode 100644 index 000000000..e8f385ca3 --- /dev/null +++ b/src/modules/unpackfsc/Runners.h @@ -0,0 +1,48 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef UNPACKFSC_RUNNERS_H +#define UNPACKFSC_RUNNERS_H + +#include + +class Runner : public QObject +{ + Q_OBJECT + +public: + Runner( const QString& source, const QString& destination ); + ~Runner() override; + + virtual Calamares::JobResult run() = 0; + + /** @brief Check that the (configured) source file exists. + * + * Returns @c true if it's a file and readable. + */ + bool checkSourceExists() const; + + /** @brief Check that a named tool (executable) exists in the search path. + * + * Returns @c true if the tool is found and sets @p fullPath + * to the full path of that tool; returns @c false and clears + * @p fullPath otherwise. + */ + static bool checkToolExists( const QString& toolName, QString& fullPath ); + +Q_SIGNALS: + // See Calamares Job::progress + void progress( qreal percent, const QString& message ); + +protected: + QString m_source; + QString m_destination; +}; + +#endif diff --git a/src/modules/unpackfsc/TarballRunner.cpp b/src/modules/unpackfsc/TarballRunner.cpp new file mode 100644 index 000000000..79dbec9aa --- /dev/null +++ b/src/modules/unpackfsc/TarballRunner.cpp @@ -0,0 +1,86 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2022 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "TarballRunner.h" + +#include +#include +#include + +#include + +static constexpr const int chunk_size = 107; + +Calamares::JobResult +TarballRunner::run() +{ + if ( !checkSourceExists() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid tarball configuration" ), + tr( "The source archive %1 does not exist." ).arg( m_source ), + Calamares::JobResult::InvalidConfiguration ); + } + + const QString toolName = QStringLiteral( "tar" ); + QString tarExecutable; + if ( !checkToolExists( toolName, tarExecutable ) ) + { + return Calamares::JobResult::internalError( + tr( "Missing tools" ), + tr( "The %1 tool is not installed on the system." ).arg( toolName ), + Calamares::JobResult::MissingRequirements ); + } + + const QString destinationPath = Calamares::System::instance()->targetPath( m_destination ); + if ( destinationPath.isEmpty() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid tarball configuration" ), + tr( "No destination could be found for %1." ).arg( m_destination ), + Calamares::JobResult::InvalidConfiguration ); + } + + // Get the stats (number of inodes) from the FS + { + m_total = 0; + Calamares::Utils::Runner r( { tarExecutable, QStringLiteral( "-tf" ), m_source } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + QObject::connect( &r, &decltype( r )::output, [ & ]( QString line ) { m_total++; } ); + /* ignored */ r.run(); + } + if ( m_total <= 0 ) + { + cWarning() << "No stats could be obtained from" << tarExecutable << "-tf" << m_source; + } + + // Now do the actual unpack + { + m_processed = 0; + m_since = 0; + Calamares::Utils::Runner r( + { tarExecutable, QStringLiteral( "-xpvf" ), m_source, QStringLiteral( "-C" ), destinationPath } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + connect( &r, &decltype( r )::output, this, &TarballRunner::tarballProgress ); + return r.run().explainProcess( toolName, std::chrono::seconds( 0 ) ); + } +} + +void +TarballRunner::tarballProgress( QString line ) +{ + m_processed++; + m_since++; + if ( m_since > chunk_size ) + { + m_since = 0; + double p = m_total > 0 ? ( double( m_processed ) / double( m_total ) ) : 0.5; + Q_EMIT progress( p, tr( "Tarball extract file %1" ).arg( line ) ); + } +} diff --git a/src/modules/unpackfsc/TarballRunner.h b/src/modules/unpackfsc/TarballRunner.h new file mode 100644 index 000000000..8573cc3a7 --- /dev/null +++ b/src/modules/unpackfsc/TarballRunner.h @@ -0,0 +1,35 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2022 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef UNPACKFSC_TARBALLRUNNER_H +#define UNPACKFSC_TARBALLRUNNER_H + +#include "Runners.h" + +/** @brief Use (GNU) tar for extracting a filesystem + * + */ +class TarballRunner : public Runner +{ +public: + using Runner::Runner; + + Calamares::JobResult run() override; + +protected Q_SLOTS: + void tarballProgress( QString line ); + +private: + // Progress reporting + int m_total = 0; + int m_processed = 0; + int m_since = 0; +}; + +#endif diff --git a/src/modules/unpackfsc/UnpackFSCJob.cpp b/src/modules/unpackfsc/UnpackFSCJob.cpp new file mode 100644 index 000000000..48b1e8456 --- /dev/null +++ b/src/modules/unpackfsc/UnpackFSCJob.cpp @@ -0,0 +1,194 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "UnpackFSCJob.h" + +#include "FSArchiverRunner.h" +#include "TarballRunner.h" +#include "UnsquashRunner.h" + +#include "GlobalStorage.h" +#include "JobQueue.h" +#include "compat/Variant.h" +#include "utils/Logger.h" +#include "utils/NamedEnum.h" +#include "utils/RAII.h" +#include "utils/Variant.h" + +#include + +static const NamedEnumTable< UnpackFSCJob::Type > +typeNames() +{ + using T = UnpackFSCJob::Type; + // clang-format off + static const NamedEnumTable< T > names + { + { "none", T::None }, + { "fsarchiver", T::FSArchive }, + { "fsarchive", T::FSArchive }, + { "fsa", T::FSArchive }, + { "fsa-dir", T::FSArchive }, + { "fsa-block", T::FSArchiveFS }, + { "fsa-fs", T::FSArchiveFS }, + { "squashfs", T::Squashfs }, + { "squash", T::Squashfs }, + { "unsquash", T::Squashfs }, + { "tar", T::Tarball }, + { "tarball", T::Tarball }, + { "tgz", T::Tarball }, + }; + // clang-format on + return names; +} + +UnpackFSCJob::UnpackFSCJob( QObject* parent ) + : Calamares::CppJob( parent ) +{ +} + +UnpackFSCJob::~UnpackFSCJob() {} + +QString +UnpackFSCJob::prettyName() const +{ + return tr( "Unpack filesystems" ); +} + +QString +UnpackFSCJob::prettyStatusMessage() const +{ + return m_progressMessage; +} + +static bool +checkCondition( const QString& condition ) +{ + if ( condition.isEmpty() ) + { + return true; + } + + Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage(); + + bool ok = false; + const auto v = Calamares::lookup( gs, condition, ok ); + if ( !ok ) + { + cWarning() << "Item has condition '" << condition << "' which is not set at all (assuming 'true')."; + return true; + } + + if ( !v.canConvert< bool >() ) + { + cWarning() << "Item has condition '" << condition << "' with value" << v << "(assuming 'true')."; + return true; + } + + return v.toBool(); +} + +Calamares::JobResult +UnpackFSCJob::exec() +{ + if ( !checkCondition( m_condition ) ) + { + cDebug() << "Skipping item with condition '" << m_condition << "' which is set to false."; + return Calamares::JobResult::ok(); + } + + cScopedAssignment messageClearer( &m_progressMessage, QString() ); + std::unique_ptr< Runner > r; + switch ( m_type ) + { + case Type::FSArchive: + r = std::make_unique< FSArchiverDirRunner >( m_source, m_destination ); + break; + case Type::FSArchiveFS: + r = std::make_unique< FSArchiverFSRunner >( m_source, m_destination ); + break; + case Type::Squashfs: + r = std::make_unique< UnsquashRunner >( m_source, m_destination ); + break; + case Type::Tarball: + r = std::make_unique< TarballRunner >( m_source, m_destination ); + break; + case Type::None: + default: + cDebug() << "Nothing to do."; + return Calamares::JobResult::ok(); + } + + connect( r.get(), + &Runner::progress, + [ = ]( qreal percent, const QString& message ) + { + m_progressMessage = message; + Q_EMIT progress( percent ); + } ); + return r->run(); +} + +void +UnpackFSCJob::setConfigurationMap( const QVariantMap& map ) +{ + m_type = Type::None; + + const QString source = Calamares::getString( map, "source" ); + const QString sourceTypeName = Calamares::getString( map, "sourcefs" ); + if ( source.isEmpty() || sourceTypeName.isEmpty() ) + { + cWarning() << "Skipping item with bad source data:" << map; + return; + } + bool bogus = false; + Type sourceType = typeNames().find( sourceTypeName, bogus ); + if ( sourceType == Type::None ) + { + cWarning() << "Skipping item with source type None"; + return; + } + const QString destination = Calamares::getString( map, "destination" ); + if ( destination.isEmpty() ) + { + cWarning() << "Skipping item with empty destination"; + return; + } + const auto conditionKey = QStringLiteral( "condition" ); + if ( map.contains( conditionKey ) ) + { + const auto value = map[ conditionKey ]; + if ( Calamares::typeOf( value ) == Calamares::BoolVariantType ) + { + if ( !value.toBool() ) + { + cDebug() << "Skipping item with condition set to false."; + // Leave type set to None, which will be skipped later + return; + } + // Else the condition is true, and we're fine leaving the string empty because that defaults to true + } + else + { + const auto variable = value.toString(); + if ( variable.isEmpty() ) + { + cDebug() << "Skipping item with condition '" << value << "' that is empty (use 'true' instead)."; + return; + } + m_condition = variable; + } + } + + m_source = source; + m_destination = destination; + m_type = sourceType; +} + +CALAMARES_PLUGIN_FACTORY_DEFINITION( UnpackFSCFactory, registerPlugin< UnpackFSCJob >(); ) diff --git a/src/modules/unpackfsc/UnpackFSCJob.h b/src/modules/unpackfsc/UnpackFSCJob.h new file mode 100644 index 000000000..416efe1f9 --- /dev/null +++ b/src/modules/unpackfsc/UnpackFSCJob.h @@ -0,0 +1,51 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef UNPACKFSC_UNPACKFSCJOB_H +#define UNPACKFSC_UNPACKFSCJOB_H + +#include +#include +#include + +class PLUGINDLLEXPORT UnpackFSCJob : public Calamares::CppJob +{ + Q_OBJECT + +public: + enum class Type + { + None, /// << Invalid + FSArchive, + FSArchiveFS, + Squashfs, + Tarball, + }; + + explicit UnpackFSCJob( QObject* parent = nullptr ); + ~UnpackFSCJob() override; + + QString prettyName() const override; + QString prettyStatusMessage() const override; + + Calamares::JobResult exec() override; + + void setConfigurationMap( const QVariantMap& configurationMap ) override; + +private: + QString m_source; + QString m_destination; + Type m_type = Type::None; + QString m_progressMessage; + QString m_condition; ///< May be empty to express condition "true" +}; + +CALAMARES_PLUGIN_FACTORY_DECLARATION( UnpackFSCFactory ) + +#endif diff --git a/src/modules/unpackfsc/UnsquashRunner.cpp b/src/modules/unpackfsc/UnsquashRunner.cpp new file mode 100644 index 000000000..b3712b997 --- /dev/null +++ b/src/modules/unpackfsc/UnsquashRunner.cpp @@ -0,0 +1,101 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#include "UnsquashRunner.h" + +#include +#include +#include + +#include + +static constexpr const int chunk_size = 107; + +Calamares::JobResult +UnsquashRunner::run() +{ + if ( !checkSourceExists() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid unsquash configuration" ), + tr( "The source archive %1 does not exist." ).arg( m_source ), + Calamares::JobResult::InvalidConfiguration ); + } + + const QString toolName = QStringLiteral( "unsquashfs" ); + QString unsquashExecutable; + if ( !checkToolExists( toolName, unsquashExecutable ) ) + { + return Calamares::JobResult::internalError( + tr( "Missing tools" ), + tr( "The %1 tool is not installed on the system." ).arg( toolName ), + Calamares::JobResult::MissingRequirements ); + } + + const QString destinationPath = Calamares::System::instance()->targetPath( m_destination ); + if ( destinationPath.isEmpty() ) + { + return Calamares::JobResult::internalError( + tr( "Invalid unsquash configuration" ), + tr( "No destination could be found for %1." ).arg( m_destination ), + Calamares::JobResult::InvalidConfiguration ); + } + + // Get the stats (number of inodes) from the FS + { + m_inodes = -1; + Calamares::Utils::Runner r( { unsquashExecutable, QStringLiteral( "-s" ), m_source } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + QObject::connect( &r, + &decltype( r )::output, + [ & ]( QString line ) + { + if ( line.startsWith( "Number of inodes " ) ) + { + m_inodes = line.split( ' ', SplitSkipEmptyParts ).last().toInt(); + } + } ); + /* ignored */ r.run(); + } + if ( m_inodes <= 0 ) + { + cWarning() << "No stats could be obtained from" << unsquashExecutable << "-s"; + } + + // Now do the actual unpack + { + m_processed = 0; + Calamares::Utils::Runner r( { unsquashExecutable, + QStringLiteral( "-i" ), // List files + QStringLiteral( "-f" ), // Force-overwrite + QStringLiteral( "-d" ), + destinationPath, + m_source } ); + r.setLocation( Calamares::Utils::RunLocation::RunInHost ).enableOutputProcessing(); + connect( &r, &decltype( r )::output, this, &UnsquashRunner::unsquashProgress ); + return r.run().explainProcess( toolName, std::chrono::seconds( 0 ) ); + } +} + +void +UnsquashRunner::unsquashProgress( QString line ) +{ + m_processed++; + m_since++; + if ( m_since > chunk_size && line.contains( '/' ) ) + { + const QString filename = line.split( '/', SplitSkipEmptyParts ).last().trimmed(); + if ( !filename.isEmpty() ) + { + m_since = 0; + double p = m_inodes > 0 ? ( double( m_processed ) / double( m_inodes ) ) : 0.5; + Q_EMIT progress( p, tr( "Unsquash file %1" ).arg( filename ) ); + } + } +} diff --git a/src/modules/unpackfsc/UnsquashRunner.h b/src/modules/unpackfsc/UnsquashRunner.h new file mode 100644 index 000000000..c9b57c0e6 --- /dev/null +++ b/src/modules/unpackfsc/UnsquashRunner.h @@ -0,0 +1,36 @@ +/* === This file is part of Calamares - === + * + * SPDX-FileCopyrightText: 2021 Adriaan de Groot + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Calamares is Free Software: see the License-Identifier above. + * + */ + +#ifndef UNPACKFSC_UNSQUASHRUNNER_H +#define UNPACKFSC_UNSQUASHRUNNER_H + +#include "Runners.h" + +/** @brief Use Unsquash for extracting a filesystem + * + */ +class UnsquashRunner : public Runner +{ +public: + using Runner::Runner; + + Calamares::JobResult run() override; + +protected Q_SLOTS: + void unsquashProgress( QString line ); + +private: + int m_inodes = 0; // Total in the FS + + // Progress reporting + int m_processed = 0; + int m_since = 0; +}; + +#endif diff --git a/src/modules/unpackfsc/tests/1.global b/src/modules/unpackfsc/tests/1.global new file mode 100644 index 000000000..064ac2a8b --- /dev/null +++ b/src/modules/unpackfsc/tests/1.global @@ -0,0 +1,2 @@ +--- +rootMountPoint: /tmp/fstest diff --git a/src/modules/unpackfsc/tests/1.job b/src/modules/unpackfsc/tests/1.job new file mode 100644 index 000000000..9c6e46dff --- /dev/null +++ b/src/modules/unpackfsc/tests/1.job @@ -0,0 +1,4 @@ +--- +source: /tmp/src.fsa +sourcefs: fsarchive +destination: "/calasrc" diff --git a/src/modules/unpackfsc/unpackfsc.conf b/src/modules/unpackfsc/unpackfsc.conf new file mode 100644 index 000000000..7cb4c3234 --- /dev/null +++ b/src/modules/unpackfsc/unpackfsc.conf @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +# +# Unpack a filesystem. Supported ways to "pack" the filesystem are: +# - fsarchiver in *savedir/restdir* mode (directories, not block devices) +# - squashfs +# +# Configuration: +# +# from globalstorage: rootMountPoint +# from job configuration: the item to unpack +# + +--- +# This module is configured a lot like the items in the *unpackfs* +# module, but with only **one** item. Use multiple instances for +# unpacking more than one filesystem. +# +# There are the following **mandatory** keys: +# - *source* path relative to the live / intstalling system to the image +# - *sourcefs* the type of the source files; valid entries are +# - `none` (this entry is ignored; kind of useless) +# - `fsarchiver` +# Aliases of this are `fsarchive`, `fsa` and `fsa-dir`. Uses +# fsarchiver in "restdir" mode. +# - `fsarchiver-block` +# Aliases of this are `fsa-block` and `fsa-fs`. Uses fsarchiver +# in "restfs" mode. +# - `squashfs` +# Aliases of this are `squash` and `unsquash`. +# - `tar` +# - *destination* path relative to rootMountPoint (so in the target +# system) where this filesystem is unpacked. It may be an +# empty string, which effectively is / (the root) of the target +# system. +# +# +# There are the following **optional** keys: +# - *condition* sets a dynamic condition on unpacking the item in +# this job. This may be true or false (constant) or name a globalstorage +# value. Use '.' to separate parts of a globalstorage name if it is nested. +# Remember to quote names. +# +# A condition is used in e.g. stacked squashfses, where the user can select +# a specific install type. The default value of *condition* is true. + +source: /data/rootfs.fsa +sourcefs: fsarchiver +destination: "/" +# condition: true diff --git a/src/modules/unpackfsc/unpackfsc.schema.yaml b/src/modules/unpackfsc/unpackfsc.schema.yaml new file mode 100644 index 000000000..14493b687 --- /dev/null +++ b/src/modules/unpackfsc/unpackfsc.schema.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2020 Adriaan de Groot +# SPDX-License-Identifier: GPL-3.0-or-later +--- +$schema: https://json-schema.org/schema# +$id: https://calamares.io/schemas/unpackfsc +additionalProperties: false +type: object +properties: + unpack: + type: array + items: + type: object + additionalProperties: false + properties: + source: { type: string } + sourcefs: { type: string } + destination: { type: string } + condition: + anyOf: + - type: boolean + - type: string + required: [ source , sourcefs, destination ]