diff --git a/CHANGES b/CHANGES index e6a16fac8..896a9fee2 100644 --- a/CHANGES +++ b/CHANGES @@ -10,15 +10,27 @@ This release contains contributions from (alphabetically by first name): - Caio Carvalho - Kevin Kofler - Philip Mueller + - Scott Harvey ## Core ## +There are no core changes in this version. + ## Modules ## * The *partition* module supports RAID devices, but only when Calamares is compiled with the newest KPMCore release. + * The calculation of required space -- including swap -- has been simplified, + and Calamares no longer reserves 2GiB of space in calculations for internal + use (this means that it no longer mysteriously drops swap when the disk + size is close to the required installation size). * The *keyboard* module now handles the (bogus) Austrian keymap for the system console properly. + * The *preservefiles* module now has a mechanism for setting the permissions + (and ownership) of preserved files. + * New module *fsresizer* can be used to resize filesystems. It is intended + for use in OEM installs where an image of fixed size is created, + and then sized to the actual SD card the user has used. # 3.2.2 (2018-09-04) # diff --git a/ci/travis-continuous.sh b/ci/travis-continuous.sh index eccb6743e..1b3841e54 100755 --- a/ci/travis-continuous.sh +++ b/ci/travis-continuous.sh @@ -51,4 +51,4 @@ df -h echo "# Install results" install_debugging "$DESTDIR" -$result # Result of make install, above +$result || { echo "! Install failed" ; exit 1 ; } # Result of make install, above diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp index c22342f9a..a8b363209 100644 --- a/src/calamares/testmain.cpp +++ b/src/calamares/testmain.cpp @@ -59,6 +59,7 @@ handle_args( QCoreApplication& a ) parser.addOption( debugLevelOption ); parser.addPositionalArgument( "module", "Path or name of module to run." ); + parser.addPositionalArgument( "config", "Path of job-config file to use.", "[config]"); parser.process( a ); @@ -140,6 +141,8 @@ load_module( const ModuleConfig& moduleConfig ) ? moduleDirectory + '/' + name + ".conf" : moduleConfig.configFile() ); + cDebug() << "Module" << moduleName << "job-configuration:" << configFile; + Calamares::Module* module = Calamares::Module::fromDescriptor( descriptor, name, configFile, moduleDirectory ); @@ -158,7 +161,7 @@ main( int argc, char* argv[] ) std::unique_ptr< Calamares::Settings > settings_p( new Calamares::Settings( QString(), true ) ); std::unique_ptr< Calamares::JobQueue > jobqueue_p( new Calamares::JobQueue( nullptr ) ); - cDebug() << "Calamares test module-loader" << module.moduleName(); + cDebug() << "Calamares module-loader testing" << module.moduleName(); Calamares::Module* m = load_module( module ); if ( !m ) { @@ -175,16 +178,27 @@ main( int argc, char* argv[] ) return 1; } - cDebug() << "Module" << m->name() << m->typeString() << m->interfaceString(); + using TR = Logger::DebugRow; + cDebug() << "Module metadata" + << TR( "name", m->name() ) + << TR( "type", m->typeString() ) + << TR( "interface", m->interfaceString() ); + + cDebug() << "Job outputs:"; Calamares::JobList jobList = m->jobs(); unsigned int count = 1; for ( const auto& p : jobList ) { - cDebug() << count << p->prettyName(); + cDebug() << "Job #" << count << "name" << p->prettyName(); Calamares::JobResult r = p->exec(); if ( !r ) - cDebug() << count << ".. failed" << r; + { + using TR = Logger::DebugRow; + cDebug() << count << ".. failed" + << TR( "summary", r.message() ) + << TR( "details", r.details() ); + } ++count; } diff --git a/src/libcalamaresui/modulesystem/Module.cpp b/src/libcalamaresui/modulesystem/Module.cpp index 8d92c37ad..a1349c280 100644 --- a/src/libcalamaresui/modulesystem/Module.cpp +++ b/src/libcalamaresui/modulesystem/Module.cpp @@ -145,8 +145,15 @@ moduleConfigurationCandidates( bool assumeBuildDir, const QString& moduleName, c paths << CalamaresUtils::appDataDir().absoluteFilePath( QString( "modules/%1" ).arg( configFileName ) ); else { + // If an absolute path is given, in debug mode, look for it + // first. The case contains('/'), below, will add the absolute + // path a second time, though. + if ( assumeBuildDir && configFileName.startsWith( '/' ) ) + paths << configFileName; if ( assumeBuildDir ) paths << QDir().absoluteFilePath(QString( "src/modules/%1/%2" ).arg( moduleName ).arg( configFileName ) ); + if ( assumeBuildDir && configFileName.contains( '/' ) ) + paths << QDir().absoluteFilePath( configFileName ); paths << QString( "/etc/calamares/modules/%1" ).arg( configFileName ); paths << CalamaresUtils::appDataDir().absoluteFilePath( QString( "modules/%1" ).arg( configFileName ) ); @@ -168,6 +175,7 @@ Module::loadConfigurationFile( const QString& configFileName ) //throws YAML::Ex YAML::Node doc = YAML::Load( ba.constData() ); if ( doc.IsNull() ) { + cDebug() << "Found empty module configuration" << path; // Special case: empty config files are valid, // but aren't a map. return; @@ -178,14 +186,13 @@ Module::loadConfigurationFile( const QString& configFileName ) //throws YAML::Ex return; } + cDebug() << "Loaded module configuration" << path; m_configurationMap = CalamaresUtils::yamlMapToVariant( doc ).toMap(); m_emergency = m_maybe_emergency && m_configurationMap.contains( EMERGENCY ) && m_configurationMap[ EMERGENCY ].toBool(); return; } - else - continue; } } diff --git a/src/modules/fsresizer/CMakeLists.txt b/src/modules/fsresizer/CMakeLists.txt new file mode 100644 index 000000000..5de329cad --- /dev/null +++ b/src/modules/fsresizer/CMakeLists.txt @@ -0,0 +1,44 @@ +find_package( KPMcore 3.3 ) +find_package( Qt5 REQUIRED DBus ) # Needed for KPMCore +find_package( KF5 REQUIRED I18n WidgetsAddons ) # Needed for KPMCore + +if ( KPMcore_FOUND ) + include_directories( ${KPMCORE_INCLUDE_DIR} ) + include_directories( ${PROJECT_BINARY_DIR}/src/libcalamares ) + + # The PartitionIterator is a small class, and it's easiest -- but also a + # gross hack -- to just compile it again from the partition module tree. + calamares_add_plugin( fsresizer + TYPE job + EXPORT_MACRO PLUGINDLLEXPORT_PRO + SOURCES + ResizeFSJob.cpp + ${PROJECT_SOURCE_DIR}/src/modules/partition/core/PartitionIterator.cpp + LINK_PRIVATE_LIBRARIES + kpmcore + calamares + SHARED_LIB + ) + + if( ECM_FOUND ) + find_package( Qt5 COMPONENTS Test REQUIRED ) + include( ECMAddTests ) + + ecm_add_test( + Tests.cpp + TEST_NAME + fsresizertest + LINK_LIBRARIES + ${CALAMARES_LIBRARIES} + calamares + calamares_job_fsresizer # From above + ${YAMLCPP_LIBRARY} + Qt5::Core + Qt5::Test + ) + set_target_properties( fsresizertest PROPERTIES AUTOMOC TRUE ) + target_include_directories(fsresizertest PRIVATE /usr/local/include ) + endif() +else() + calamares_skip_module( "fsresizer (missing suitable KPMcore)" ) +endif() diff --git a/src/modules/fsresizer/ResizeFSJob.cpp b/src/modules/fsresizer/ResizeFSJob.cpp new file mode 100644 index 000000000..4320baba1 --- /dev/null +++ b/src/modules/fsresizer/ResizeFSJob.cpp @@ -0,0 +1,347 @@ +/* === This file is part of Calamares - === + * + * Copyright 2018, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#include "ResizeFSJob.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "CalamaresVersion.h" +#include "JobQueue.h" +#include "GlobalStorage.h" + +#include "utils/CalamaresUtils.h" +#include "utils/Logger.h" +#include "utils/Units.h" + +#include "modules/partition/core/PartitionIterator.h" + +ResizeFSJob::RelativeSize::RelativeSize() + : m_value( 0 ) + , m_unit( None ) +{ +} + + +template +void matchUnitSuffix( + const QString& s, + const char ( &suffix )[N], + ResizeFSJob::RelativeSize::Unit matchedUnit, + int& value, + ResizeFSJob::RelativeSize::Unit& unit +) +{ + if ( s.endsWith( suffix ) ) + { + value = s.left( s.length() - N + 1 ).toInt(); + unit = matchedUnit; + } +} + + +ResizeFSJob::RelativeSize::RelativeSize( const QString& s ) + : m_value( 0 ) + , m_unit( None ) +{ + matchUnitSuffix( s, "%", Percent, m_value, m_unit ); + matchUnitSuffix( s, "MiB", Absolute, m_value, m_unit ); + + if ( ( unit() == Percent ) && ( value() > 100 ) ) + { + cDebug() << "Percent value" << value() << "is not valid."; + m_value = 0; + m_unit = None; + } + + if ( !m_value ) + m_unit = None; +} + +qint64 +ResizeFSJob::RelativeSize::apply( qint64 totalSectors, qint64 sectorSize ) +{ + if ( !isValid() ) + return -1; + if ( sectorSize < 1 ) + return -1; + + switch ( m_unit ) + { + case None: + return -1; + case Absolute: + return CalamaresUtils::MiBtoBytes( value() ) / sectorSize; + case Percent: + if ( value() == 100 ) + return totalSectors; // Common-case, avoid futzing around + else + return totalSectors * value() / 100; + } + + // notreached + return -1; +} + +qint64 +ResizeFSJob::RelativeSize::apply( Device* d ) +{ + return apply( d->totalLogical(), d->logicalSize() ); +} + +ResizeFSJob::ResizeFSJob( QObject* parent ) + : Calamares::CppJob( parent ) + , m_required( false ) +{ +} + + +ResizeFSJob::~ResizeFSJob() +{ +} + + +QString +ResizeFSJob::prettyName() const +{ + return tr( "Resize Filesystem Job" ); +} + +ResizeFSJob::PartitionMatch +ResizeFSJob::findPartition( CoreBackend* backend ) +{ + using DeviceList = QList< Device* >; + DeviceList devices = backend->scanDevices( false ); + cDebug() << "ResizeFSJob found" << devices.count() << "devices."; + for ( DeviceList::iterator dev_it = devices.begin(); dev_it != devices.end(); ++dev_it ) + { + if ( ! ( *dev_it ) ) + continue; + cDebug() << "ResizeFSJob found" << ( *dev_it )->deviceNode(); + for ( auto part_it = PartitionIterator::begin( *dev_it ); part_it != PartitionIterator::end( *dev_it ); ++part_it ) + { + cDebug() << ".." << ( *part_it )->mountPoint() << "on" << ( *part_it )->deviceNode(); + if ( ( !m_fsname.isEmpty() && ( *part_it )->mountPoint() == m_fsname ) || + ( !m_devicename.isEmpty() && ( *part_it )->deviceNode() == m_devicename ) ) + { + cDebug() << ".. matched configuration dev=" << m_devicename << "fs=" << m_fsname; + return PartitionMatch( *dev_it, *part_it ); + } + } + } + + cDebug() << "No match for configuration dev=" << m_devicename << "fs=" << m_fsname; + return PartitionMatch( nullptr, nullptr ); +} + +/** @brief Returns the last sector the matched partition should occupy. + * + * Returns a sector number. Returns -1 if something is wrong (e.g. + * can't resize at all, or missing data). Returns 0 if the resize + * won't fit because it doesn't satisfy the settings for atleast + * and size (or won't grow at all because the partition is blocked + * by occupied space after it). + */ +qint64 +ResizeFSJob::findGrownEnd( ResizeFSJob::PartitionMatch m ) +{ + if ( !m.first || !m.second ) + return -1; // Missing device data + if ( !ResizeOperation::canGrow( m.second ) ) + return -1; // Operation is doomed + if ( !m_size.isValid() ) + return -1; // Must have a grow-size + + cDebug() << "Containing device size" << m.first->totalLogical(); + qint64 last_available = m.first->totalLogical() - 1; // Numbered from 0 + qint64 last_currently = m.second->lastSector(); + cDebug() << "Growing partition" << m.second->firstSector() << '-' << last_currently; + + for ( auto part_it = PartitionIterator::begin( m.first ); part_it != PartitionIterator::end( m.first ); ++part_it ) + { + qint64 next_start = ( *part_it )->firstSector(); + qint64 next_end = ( *part_it )->lastSector(); + if ( next_start > next_end ) + { + cWarning() << "Corrupt partition has end" << next_end << " < start" << next_start; + std::swap( next_start, next_end ); + } + if ( ( *part_it )->roles().has( PartitionRole::Unallocated ) ) + { + cDebug() << ".. ignoring unallocated" << next_start << '-' << next_end; + continue; + } + cDebug() << ".. comparing" << next_start << '-' << next_end; + if ( ( next_start > last_currently ) && ( next_start < last_available ) ) + { + cDebug() << " .. shrunk last available to" << next_start; + last_available = next_start - 1; // Before that one starts + } + } + + if ( !( last_available > last_currently ) ) + { + cDebug() << "Partition cannot grow larger."; + return 0; + } + + qint64 expand = last_available - last_currently; // number of sectors + if ( m_atleast.isValid() ) + { + qint64 required = m_atleast.apply( m.first ); + if ( expand < required ) + { + cDebug() << ".. need to expand by" << required << "but only" << expand << "is available."; + return 0; + } + } + + qint64 wanted = m_size.apply( expand, m.first->logicalSize() ); + if ( wanted < expand ) + { + cDebug() << ".. only growing by" << wanted << "instead of full" << expand; + last_available -= ( expand - wanted ); + } + + return last_available; +} + + +Calamares::JobResult +ResizeFSJob::exec() +{ + if ( !isValid() ) + return Calamares::JobResult::error( + tr( "Invalid configuration" ), + tr( "The file-system resize job has an invalid configuration and will not run." ) ); + + // Get KPMCore + auto backend_p = CoreBackendManager::self()->backend(); + if ( backend_p ) + cDebug() << "KPMCore backend @" << ( void* )backend_p << backend_p->id() << backend_p->version(); + else + { + cDebug() << "No KPMCore backend loaded yet"; + QByteArray backendName = qgetenv( "KPMCORE_BACKEND" ); + if ( !CoreBackendManager::self()->load( backendName.isEmpty() ? CoreBackendManager::defaultBackendName() : backendName ) ) + { + cWarning() << "Could not load KPMCore backend."; + return Calamares::JobResult::error( + tr( "KPMCore not Available" ), + tr( "Calamares cannot start KPMCore for the file-system resize job." ) ); + } + + backend_p = CoreBackendManager::self()->backend(); + } + if ( !backend_p ) + { + cWarning() << "Could not load KPMCore backend (2)."; + return Calamares::JobResult::error( + tr( "KPMCore not Available" ), + tr( "Calamares cannot start KPMCore for the file-system resize job." ) ); + } + backend_p->initFSSupport(); // Might not be enough, see below + + // Now get the partition and FS we want to work on + PartitionMatch m = findPartition( backend_p ); + if ( !m.first || !m.second ) + return Calamares::JobResult::error( + tr( "Resize Failed" ), + !m_fsname.isEmpty() ? tr( "The filesystem %1 could not be found in this system, and cannot be resized." ).arg( m_fsname ) + : tr( "The device %1 could not be found in this system, and cannot be resized." ).arg( m_devicename ) ); + + m.second->fileSystem().init(); // Initialize support for specific FS + if ( !ResizeOperation::canGrow( m.second ) ) + { + cDebug() << "canGrow() returned false."; + return Calamares::JobResult::error( + tr( "Resize Failed" ), + !m_fsname.isEmpty() ? tr( "The filesystem %1 cannot be resized." ).arg( m_fsname ) + : tr( "The device %1 cannot be resized." ).arg( m_devicename ) ); + } + + qint64 new_end = findGrownEnd( m ); + cDebug() << "Resize from" + << m.second->firstSector() << '-' << m.second->lastSector() + << '(' << m.second->length() << ')' + << "to -" << new_end; + + if ( new_end < 0 ) + return Calamares::JobResult::error( + tr( "Resize Failed" ), + !m_fsname.isEmpty() ? tr( "The filesystem %1 cannot be resized." ).arg( m_fsname ) + : tr( "The device %1 cannot be resized." ).arg( m_devicename ) ); + if ( new_end == 0 ) + { + cWarning() << "Resize operation on" << m_fsname << m_devicename + << "skipped as not-useful."; + if ( m_required ) + return Calamares::JobResult::error( + tr( "Resize Failed" ), + !m_fsname.isEmpty() ? tr( "The filesystem %1 must be resized, but cannot." ).arg( m_fsname ) + : tr( "The device %11 must be resized, but cannot" ).arg( m_fsname ) ); + + return Calamares::JobResult::ok(); + } + + if ( ( new_end > 0 ) && ( new_end > m.second->lastSector() ) ) + { + ResizeOperation op( *m.first, *m.second, m.second->firstSector(), new_end ); + Report op_report( nullptr ); + if ( op.execute( op_report ) ) + cDebug() << "Resize operation OK."; + else + { + cDebug() << "Resize failed." << op_report.output(); + return Calamares::JobResult::error( + tr( "Resize Failed" ), + op_report.toText() ); + } + } + + return Calamares::JobResult::ok(); +} + + +void +ResizeFSJob::setConfigurationMap( const QVariantMap& configurationMap ) +{ + m_fsname = configurationMap["fs"].toString(); + m_devicename = configurationMap["dev"].toString(); + + if ( m_fsname.isEmpty() && m_devicename.isEmpty() ) + { + cWarning() << "No fs or dev configured for resize."; + return; + } + + m_size = RelativeSize( configurationMap["size"].toString() ); + m_atleast = RelativeSize( configurationMap["atleast"].toString() ); + + m_required = CalamaresUtils::getBool( configurationMap, "required", false ); +} + +CALAMARES_PLUGIN_FACTORY_DEFINITION( ResizeFSJobFactory, registerPlugin(); ) diff --git a/src/modules/fsresizer/ResizeFSJob.h b/src/modules/fsresizer/ResizeFSJob.h new file mode 100644 index 000000000..c34ccb865 --- /dev/null +++ b/src/modules/fsresizer/ResizeFSJob.h @@ -0,0 +1,122 @@ +/* === This file is part of Calamares - === + * + * Copyright 2018, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#ifndef RESIZEFSJOB_H +#define RESIZEFSJOB_H + +#include +#include + +#include + +#include + +#include + +class CoreBackend; // From KPMCore +class Device; // From KPMCore +class Partition; + +class PLUGINDLLEXPORT ResizeFSJob : public Calamares::CppJob +{ + Q_OBJECT + +public: + /** @brief Size expressions + * + * Sizes can be specified in MiB or percent (of the device they + * are on). This class handles parsing of such strings from the + * config file. + */ + class RelativeSize + { + public: + RelativeSize(); + RelativeSize( const QString& ); + + enum Unit + { + None, + Percent, + Absolute + }; + + int value() const { return m_value; } + Unit unit() const { return m_unit; } + + bool isValid() const + { + return ( unit() != None ) && ( value() > 0 ); + } + + /** @brief Apply this size to the number of sectors @p totalSectors . + * + * Each sector has size @p sectorSize , for converting absolute + * sizes in MiB to sector counts. + * + * For invalid sizes, returns -1. + * For absolute sizes, returns the number of sectors needed. + * For percent sizes, returns that percent of the number of sectors. + */ + qint64 apply( qint64 totalSectors, qint64 sectorSize ); + + /** @brief Apply this size to the given device. + * + * Equivalent to apply( d->totalLogical(), d->logicalSize() ) + */ + qint64 apply( Device* d ); + + private: + int m_value; + Unit m_unit; + } ; + + explicit ResizeFSJob( QObject* parent = nullptr ); + virtual ~ResizeFSJob() override; + + QString prettyName() const override; + + Calamares::JobResult exec() override; + + void setConfigurationMap( const QVariantMap& configurationMap ) override; + + /** @brief Is the configuration of this job valid? */ + bool isValid() const + { + return ( !m_fsname.isEmpty() || !m_devicename.isEmpty() ) && + m_size.isValid(); + } + +private: + RelativeSize m_size; + RelativeSize m_atleast; + QString m_fsname; // Either this, or devicename, is set, not both + QString m_devicename; + bool m_required; + + using PartitionMatch = QPair; + /** @brief Find the configured FS using KPMCore @p backend */ + PartitionMatch findPartition( CoreBackend* backend ); + + /** @brief Return a new end-sector for the given dev-part pair. */ + qint64 findGrownEnd( PartitionMatch ); +}; + +CALAMARES_PLUGIN_FACTORY_DECLARATION( ResizeFSJobFactory ) + +#endif // RESIZEFSJOB_H diff --git a/src/modules/fsresizer/Tests.cpp b/src/modules/fsresizer/Tests.cpp new file mode 100644 index 000000000..255153fa2 --- /dev/null +++ b/src/modules/fsresizer/Tests.cpp @@ -0,0 +1,126 @@ +/* === This file is part of Calamares - === + * + * Copyright 2017-2018, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#include "Tests.h" + +#include "GlobalStorage.h" +#include "JobQueue.h" +#include "Settings.h" + +#include "utils/Logger.h" +#include "utils/YamlUtils.h" + +#include + +#include + +#include +#include + +#define private public +#include "ResizeFSJob.h" +#undef private + +QTEST_GUILESS_MAIN( FSResizerTests ) + +FSResizerTests::FSResizerTests() +{ +} + +FSResizerTests::~FSResizerTests() +{ +} + +void +FSResizerTests::initTestCase() +{ +} + +void FSResizerTests::testConfigurationRobust() +{ + ResizeFSJob j; + + // Empty config + j.setConfigurationMap( QVariantMap() ); + QVERIFY( j.m_fsname.isEmpty() ); + QVERIFY( j.m_devicename.isEmpty() ); + QCOMPARE( j.m_size.unit(), ResizeFSJob::RelativeSize::None ); + QCOMPARE( j.m_atleast.unit(), ResizeFSJob::RelativeSize::None ); + + // Config is missing fs and dev, so it isn't valid + YAML::Node doc0 = YAML::Load( R"(--- +size: 100% +atleast: 600MiB +)" ); + j.setConfigurationMap( CalamaresUtils::yamlMapToVariant( doc0 ).toMap() ); + QVERIFY( j.m_fsname.isEmpty() ); + QVERIFY( j.m_devicename.isEmpty() ); + QCOMPARE( j.m_size.unit(), ResizeFSJob::RelativeSize::None ); + QCOMPARE( j.m_atleast.unit(), ResizeFSJob::RelativeSize::None ); + QCOMPARE( j.m_size.value(), 0 ); + QCOMPARE( j.m_atleast.value(), 0 ); +} + +void FSResizerTests::testConfigurationValues() +{ + ResizeFSJob j; + + // Check both + YAML::Node doc0 = YAML::Load( R"(--- +fs: / +size: 100% +atleast: 600MiB +)" ); + j.setConfigurationMap( CalamaresUtils::yamlMapToVariant( doc0 ).toMap() ); + QVERIFY( !j.m_fsname.isEmpty() ); + QVERIFY( j.m_devicename.isEmpty() ); + QCOMPARE( j.m_size.unit(), ResizeFSJob::RelativeSize::Percent ); + QCOMPARE( j.m_atleast.unit(), ResizeFSJob::RelativeSize::Absolute ); + QCOMPARE( j.m_size.value(), 100 ); + QCOMPARE( j.m_atleast.value(), 600 ); + + // Silly config + doc0 = YAML::Load( R"(--- +fs: / +dev: /dev/m00 +size: 72 MiB +atleast: 127 % +)" ); + j.setConfigurationMap( CalamaresUtils::yamlMapToVariant( doc0 ).toMap() ); + QVERIFY( !j.m_fsname.isEmpty() ); + QVERIFY( !j.m_devicename.isEmpty() ); + QCOMPARE( j.m_size.unit(), ResizeFSJob::RelativeSize::Absolute ); + QCOMPARE( j.m_atleast.unit(), ResizeFSJob::RelativeSize::Percent ); + QCOMPARE( j.m_size.value(), 72 ); + QCOMPARE( j.m_atleast.value(), 127 ); + + // Silly config + doc0 = YAML::Load( R"(--- +fs: / +# dev: /dev/m00 +size: 71MiB +# atleast: 127% +)" ); + j.setConfigurationMap( CalamaresUtils::yamlMapToVariant( doc0 ).toMap() ); + QVERIFY( !j.m_fsname.isEmpty() ); + QVERIFY( j.m_devicename.isEmpty() ); + QCOMPARE( j.m_size.unit(), ResizeFSJob::RelativeSize::Absolute ); + QCOMPARE( j.m_atleast.unit(), ResizeFSJob::RelativeSize::None ); + QCOMPARE( j.m_size.value(), 71 ); + QCOMPARE( j.m_atleast.value(), 0 ); +} diff --git a/src/modules/fsresizer/Tests.h b/src/modules/fsresizer/Tests.h new file mode 100644 index 000000000..958c0e655 --- /dev/null +++ b/src/modules/fsresizer/Tests.h @@ -0,0 +1,39 @@ +/* === This file is part of Calamares - === + * + * Copyright 2018, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#ifndef TESTS_H +#define TESTS_H + +#include + +class FSResizerTests : public QObject +{ + Q_OBJECT +public: + FSResizerTests(); + ~FSResizerTests() override; + +private Q_SLOTS: + void initTestCase(); + // Can handle missing values + void testConfigurationRobust(); + // Can parse % and MiB values + void testConfigurationValues(); +}; + +#endif diff --git a/src/modules/fsresizer/fsresizer.conf b/src/modules/fsresizer/fsresizer.conf new file mode 100644 index 000000000..33329248d --- /dev/null +++ b/src/modules/fsresizer/fsresizer.conf @@ -0,0 +1,49 @@ +# Module that resizes a single FS to fill the entire (rest) of +# a device. This is used in OEM situations where an image is +# flashed onto an SD card (or similar) and used to boot a device, +# after which the FS should expand to fill the SD card. +# +# Example: a distro produces a 6GiB large image that is +# written to an 8GiB SD card; the FS should expand to take +# advantage of the unused 2GiB. The FS should expand much +# more if the same image is written to a 16GiB card. +--- + +# Which FS needs to be grown? Choose one way to identify it: +# - *fs* names a mount point which should already be mounted +# in the system. +# - *dev* names a device +fs: / +# dev: /dev/mmcblk0p1 + +# How much of the total remaining space should the FS use? +# The only sensible amount is "all of it". The value is +# in percent, so set it to 100. Perhaps a fixed size is +# needed (that would be weird though, since you don't know +# how big the card is), use MiB as suffix in that case. +# If missing, then it's assumed to be 0, and no resizing +# will happen. +# +# Percentages apply to **available space**. +size: 100% + +# Resizing might not be worth it, though. Set the minimum +# that it must grow; if it cannot grow that much, the +# resizing is skipped. Can be in percentage or absolute +# size, as above. If missing, then it's assumed to be 0, +# which means resizing is always worthwhile. +# +# If *atleast* is not zero, then the setting *required*, +# below, becomes relevant. +# +# Percentages apply to **total device size**. +#atleast: 1000MiB + +# When *atleast* is not zero, then the resize may be +# recommended (the default) or **required**. If the +# resize is required and cannot be carried out (because +# there's not enough space), then that is a fatal +# error for the installer. By default, resize is only +# recommended and it is not an error for no resize to be +# carried out. +required: false diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index 62a00b738..713b1e321 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -134,7 +134,7 @@ def run(): if os.path.exists(target_locale_gen_bak): shutil.copy2(target_locale_gen_bak, target_locale_gen) libcalamares.utils.debug("Restored backup {!s} -> {!s}" - .format(target_locale_gen_bak).format(target_locale_gen)) + .format(target_locale_gen_bak, target_locale_gen)) # run locale-gen if detected; this *will* cause an exception # if the live system has locale.gen, but the target does not: diff --git a/src/modules/partition/core/DeviceList.cpp b/src/modules/partition/core/DeviceList.cpp index 4a353efb4..f51eec047 100644 --- a/src/modules/partition/core/DeviceList.cpp +++ b/src/modules/partition/core/DeviceList.cpp @@ -113,28 +113,32 @@ QList< Device* > getDevices( DeviceType which, qint64 minimumSize ) // Remove the device which contains / from the list for ( DeviceList::iterator it = devices.begin(); it != devices.end(); ) - if ( ! ( *it ) || - ( *it )->deviceNode().startsWith( "/dev/zram" ) + if ( !( *it ) ) + { + cDebug() << " .. Skipping nullptr device"; + it = erase( devices, it); + } + else if ( ( *it )->deviceNode().startsWith( "/dev/zram" ) ) { cDebug() << " .. Removing zram" << it; - it = erase(devices, it ); + it = erase( devices, it ); } else if ( writableOnly && hasRootPartition( *it ) ) { cDebug() << " .. Removing device with root filesystem (/) on it" << it; - it = erase(devices, it ); + it = erase( devices, it ); } else if ( writableOnly && isIso9660( *it ) ) { cDebug() << " .. Removing device with iso9660 filesystem (probably a CD) on it" << it; - it = erase(devices, it ); + it = erase( devices, it ); } else if ( (minimumSize >= 0) && !( (*it)->capacity() > minimumSize ) ) { cDebug() << " .. Removing too-small" << it; - it = erase(devices, it ); + it = erase( devices, it ); } else ++it; diff --git a/src/modules/partition/core/PartitionIterator.cpp b/src/modules/partition/core/PartitionIterator.cpp index 5ed48fd91..8301835c6 100644 --- a/src/modules/partition/core/PartitionIterator.cpp +++ b/src/modules/partition/core/PartitionIterator.cpp @@ -18,7 +18,7 @@ * along with Calamares. If not, see . */ -#include +#include "PartitionIterator.h" // KPMcore #include diff --git a/src/modules/partition/gui/ChoicePage.cpp b/src/modules/partition/gui/ChoicePage.cpp index abe7795c0..5cbfb69f7 100644 --- a/src/modules/partition/gui/ChoicePage.cpp +++ b/src/modules/partition/gui/ChoicePage.cpp @@ -1174,6 +1174,9 @@ ChoicePage::setupActions() OsproberEntryList osproberEntriesForCurrentDevice = getOsproberEntriesForDevice( currentDevice ); + cDebug() << "Setting up actions for" << currentDevice->deviceNode() + << "with" << osproberEntriesForCurrentDevice.count() << "entries."; + if ( currentDevice->partitionTable() ) m_deviceInfoWidget->setPartitionTableType( currentDevice->partitionTable()->type() ); else @@ -1190,18 +1193,30 @@ ChoicePage::setupActions() #ifdef WITH_KPMCOREGT33 if ( currentDevice->type() == Device::Type::SoftwareRAID_Device && static_cast< SoftwareRAID* >(currentDevice)->status() == SoftwareRAID::Status::Inactive ) + { + cDebug() << ".. part of an inactive RAID device"; isInactiveRAID = true; + } #endif for ( auto it = PartitionIterator::begin( currentDevice ); it != PartitionIterator::end( currentDevice ); ++it ) { if ( PartUtils::canBeResized( *it ) ) + { + cDebug() << ".. contains resizable" << it; atLeastOneCanBeResized = true; + } if ( PartUtils::canBeReplaced( *it ) ) + { + cDebug() << ".. contains replacable" << it; atLeastOneCanBeReplaced = true; + } if ( (*it)->isMounted() ) + { + cDebug() << ".. contains mounted" << it; atLeastOneIsMounted = true; + } } if ( osproberEntriesForCurrentDevice.count() == 0 ) @@ -1318,7 +1333,12 @@ ChoicePage::setupActions() if ( !atLeastOneIsMounted && !isInactiveRAID ) m_eraseButton->show(); // None mounted else + { + cDebug() << "Erase button suppressed" + << "mount?" << atLeastOneIsMounted + << "raid?" << isInactiveRAID; force_uncheck( m_grp, m_eraseButton ); + } bool isEfi = PartUtils::isEfiSystem(); bool efiSystemPartitionFound = !m_core->efiSystemPartitions().isEmpty(); diff --git a/src/modules/preservefiles/CMakeLists.txt b/src/modules/preservefiles/CMakeLists.txt index 1ac979d1b..c1021eeda 100644 --- a/src/modules/preservefiles/CMakeLists.txt +++ b/src/modules/preservefiles/CMakeLists.txt @@ -4,6 +4,7 @@ calamares_add_plugin( preservefiles TYPE job EXPORT_MACRO PLUGINDLLEXPORT_PRO SOURCES + permissions.cpp PreserveFiles.cpp LINK_PRIVATE_LIBRARIES calamares diff --git a/src/modules/preservefiles/PreserveFiles.cpp b/src/modules/preservefiles/PreserveFiles.cpp index 0fe1d278b..2c1b85103 100644 --- a/src/modules/preservefiles/PreserveFiles.cpp +++ b/src/modules/preservefiles/PreserveFiles.cpp @@ -18,6 +18,8 @@ #include "PreserveFiles.h" +#include "permissions.h" + #include "CalamaresVersion.h" #include "JobQueue.h" #include "GlobalStorage.h" @@ -83,6 +85,38 @@ PreserveFiles::prettyName() const return tr( "Saving files for later ..." ); } +static bool +copy_file( const QString& source, const QString& dest ) +{ + QFile sourcef( source ); + if ( !sourcef.open( QFile::ReadOnly ) ) + { + cWarning() << "Could not read" << source; + return false; + } + + QFile destf( dest ); + if ( !destf.open( QFile::WriteOnly ) ) + { + sourcef.close(); + cWarning() << "Could not open" << destf.fileName() << "for writing; could not copy" << source; + return false; + } + + QByteArray b; + do + { + b = sourcef.read( 1_MiB ); + destf.write( b ); + } + while ( b.count() > 0 ); + + sourcef.close(); + destf.close(); + + return true; +} + Calamares::JobResult PreserveFiles::exec() { if ( m_items.isEmpty() ) @@ -96,7 +130,8 @@ Calamares::JobResult PreserveFiles::exec() for ( const auto& it : m_items ) { QString source = it.source; - QString dest = prefix + atReplacements( it.dest ); + QString bare_dest = atReplacements( it.dest ); + QString dest = prefix + bare_dest; if ( it.type == ItemType::Log ) source = Logger::logFile(); @@ -111,32 +146,29 @@ Calamares::JobResult PreserveFiles::exec() cWarning() << "Skipping unnamed source file for" << dest; else { - QFile sourcef( source ); - if ( !sourcef.open( QFile::ReadOnly ) ) + if ( copy_file( source, dest ) ) { - cWarning() << "Could not read" << source; - continue; + if ( it.perm.isValid() ) + { + auto s_p = CalamaresUtils::System::instance(); + + int r; + + r = s_p->targetEnvCall( QStringList{ "chown", it.perm.username(), bare_dest } ); + if ( r ) + cWarning() << "Could not chown target" << bare_dest; + + r = s_p->targetEnvCall( QStringList{ "chgrp", it.perm.group(), bare_dest } ); + if ( r ) + cWarning() << "Could not chgrp target" << bare_dest; + + r = s_p->targetEnvCall( QStringList{ "chmod", it.perm.octal(), bare_dest } ); + if ( r ) + cWarning() << "Could not chmod target" << bare_dest; + } + + ++count; } - - QFile destf( dest ); - if ( !destf.open( QFile::WriteOnly ) ) - { - sourcef.close(); - cWarning() << "Could not open" << destf.fileName() << "for writing; could not copy" << source; - continue; - } - - QByteArray b; - do - { - b = sourcef.read( 1_MiB ); - destf.write( b ); - } - while ( b.count() > 0 ); - - sourcef.close(); - destf.close(); - ++count; } } @@ -160,6 +192,10 @@ void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap) return; } + QString defaultPermissions = configurationMap[ "perm" ].toString(); + if ( defaultPermissions.isEmpty() ) + defaultPermissions = QStringLiteral( "root:root:0400" ); + QVariantList l = files.toList(); unsigned int c = 0; for ( const auto& li : l ) @@ -168,7 +204,7 @@ void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap) { QString filename = li.toString(); if ( !filename.isEmpty() ) - m_items.append( Item{ filename, filename, ItemType::Path } ); + m_items.append( Item{ filename, filename, Permissions( defaultPermissions ), ItemType::Path } ); else cDebug() << "Empty filename for preservefiles, item" << c; } @@ -181,6 +217,9 @@ void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap) ( from == "log" ) ? ItemType::Log : ( from == "config" ) ? ItemType::Config : ItemType::None; + QString perm = map[ "perm" ].toString(); + if ( perm.isEmpty() ) + perm = defaultPermissions; if ( dest.isEmpty() ) { @@ -192,7 +231,7 @@ void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap) } else { - m_items.append( Item{ QString(), dest, t } ); + m_items.append( Item{ QString(), dest, Permissions( perm ), t } ); } } else diff --git a/src/modules/preservefiles/PreserveFiles.h b/src/modules/preservefiles/PreserveFiles.h index 0c9216336..ed2fe889c 100644 --- a/src/modules/preservefiles/PreserveFiles.h +++ b/src/modules/preservefiles/PreserveFiles.h @@ -24,11 +24,11 @@ #include #include "CppJob.h" +#include "PluginDllMacro.h" #include "utils/PluginFactory.h" -#include "PluginDllMacro.h" - +#include "permissions.h" class PLUGINDLLEXPORT PreserveFiles : public Calamares::CppJob { @@ -46,6 +46,7 @@ class PLUGINDLLEXPORT PreserveFiles : public Calamares::CppJob { QString source; QString dest; + Permissions perm; ItemType type; } ; diff --git a/src/modules/preservefiles/permissions.cpp b/src/modules/preservefiles/permissions.cpp new file mode 100644 index 000000000..a3f8ac136 --- /dev/null +++ b/src/modules/preservefiles/permissions.cpp @@ -0,0 +1,75 @@ +/* === This file is part of Calamares - === + * + * Copyright (C) 2018 Scott Harvey + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#include +#include +#include "permissions.h" + +Permissions::Permissions() : + m_username(), + m_group(), + m_valid(false), + m_value(0) +{ +} + + +Permissions::Permissions(QString p) : Permissions() +{ + parsePermissions(p); +} + +void Permissions::parsePermissions(const QString& p) { + + QStringList segments = p.split(":"); + + if (segments.length() != 3) { + m_valid = false; + return; + } + + if (segments[0].isEmpty() || segments[1].isEmpty()) { + m_valid = false; + return; + } + + bool ok; + int octal = segments[2].toInt(&ok, 8); + if (!ok || octal == 0) { + m_valid = false; + return; + } else { + m_value = octal; + } + + // We have exactly three segments and the third is valid octal, + // so we can declare the string valid and set the user and group names + m_valid = true; + m_username = segments[0]; + m_group = segments[1]; + + return; + +} + + + + + + diff --git a/src/modules/preservefiles/permissions.h b/src/modules/preservefiles/permissions.h new file mode 100644 index 000000000..4cb70a2c2 --- /dev/null +++ b/src/modules/preservefiles/permissions.h @@ -0,0 +1,62 @@ +/* === This file is part of Calamares - === + * + * Copyright (C) 2018 Scott Harvey + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#ifndef PERMISSIONS_H +#define PERMISSIONS_H + +#include + +/** + * @brief The Permissions class takes a QString @p in the form of + * ::, checks it for validity, and makes the three + * components available indivdually. + */ +class Permissions +{ + +public: + + /** @brief Constructor + * + * Splits the string @p at the colon (":") into separate elements for + * , , and (permissions), where is returned as + * an **octal** integer. + */ + Permissions(QString p); + + /** @brief Default constructor of an invalid Permissions. */ + Permissions(); + + bool isValid() const { return m_valid; } + QString username() const { return m_username; } + QString group() const { return m_group; } + int value() const { return m_value; } + QString octal() const { return QString::number( m_value, 8 ); } + +private: + void parsePermissions(QString const &p); + + QString m_username; + QString m_group; + bool m_valid; + int m_value; + +}; + +#endif // PERMISSIONS_H diff --git a/src/modules/preservefiles/preservefiles.conf b/src/modules/preservefiles/preservefiles.conf index ab9114d20..671a308cc 100644 --- a/src/modules/preservefiles/preservefiles.conf +++ b/src/modules/preservefiles/preservefiles.conf @@ -9,13 +9,18 @@ # as the source). # - a map with a *dest* key. The *dest* value is a path interpreted in the # target system (if dontChroot is true, in the host system). Relative paths -# are not recommended. There are two possible other keys in the map: +# are not recommended. There are three possible other keys in the map: # - *from*, which must have one of the values, below; it is used to # preserve files whose pathname is known to Calamares internally. # - *src*, to refer to a path interpreted in the host system. Relative # paths are not recommended, and are interpreted relative to where # Calamares is being run. -# Only one of the two other keys (either *from* or *src*) may be set. +# - *perm*, is a colon-separated tuple of :: +# where is in octal (e.g. 4777 for wide-open, 0400 for read-only +# by owner). If set, the file's ownership and permissions are set to +# those values within the target system; if not set, no permissions +# are changed. +# Only one of the two source keys (either *from* or *src*) may be set. # # The target filename is modified as follows: # - `@@ROOT@@` is replaced by the path to the target root (may be /) @@ -32,5 +37,13 @@ files: - /etc/oem-information - from: log dest: /root/install.log + perm: root:wheel:644 - from: config dest: /root/install.cfg + perm: root:wheel:400 + +# The *perm* key contains a default value to apply to all files listed +# above that do not have a *perm* key of their own. If not set, +# root:root:0400 (highly restrictive) is used. +# +# perm: "root:root:0400" diff --git a/src/modules/test_conf.cpp b/src/modules/test_conf.cpp index b5362d25a..ca6b72cc7 100644 --- a/src/modules/test_conf.cpp +++ b/src/modules/test_conf.cpp @@ -87,7 +87,7 @@ int main(int argc, char** argv) if ( !doc.IsMap() ) { cerr << "WARNING:" << filename << '\n'; - cerr << "WARNING: not-a-YAML-map\n"; + cerr << "WARNING: not-a-YAML-map (type=" << doc.Type() << ")\n"; return 1; }