Merge branch 'filepreserver'

src/libcalamares/GlobalStorage.cpp

@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014-2015, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -22,6 +22,9 @@
 
 #include "utils/Logger.h"
 
+#include <QFile>
+#include <QJsonDocument>
+
 #ifdef WITH_PYTHON
 #include "PythonHelper.h"
 
@@ -94,6 +97,19 @@ GlobalStorage::debugDump() const
     }
 }
 
+bool
+GlobalStorage::save(const QString& filename)
+{
+    QFile f( filename );
+    if ( !f.open( QFile::WriteOnly ) )
+        return false;
+
+    f.write( QJsonDocument::fromVariant( m ).toJson() ) ;
+    f.close();
+    return true;
+}
+
+
 } // namespace Calamares
 
 #ifdef WITH_PYTHON

src/libcalamares/GlobalStorage.h

@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014-2015, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -58,7 +58,16 @@ public:
     int remove( const QString& key );
     QVariant value( const QString& key ) const;
 
+    /// @brief dump keys and values to the debug log
     void debugDump() const;
+    /** @brief write as JSON to the given filename
+     *
+     * No tidying, sanitization, or censoring is done -- for instance,
+     * the user module sets a slightly-obscured password in global storage,
+     * and this JSON file will contain that password in-the-only-slightly-
+     * obscured form.
+     */
+    bool save( const QString& filename );
 
 signals:
     void changed();

src/libcalamares/utils/CommandList.cpp

@@ -98,9 +98,19 @@ CommandList::~CommandList()
 {
 }
 
+static inline bool
+findInCommands( const CommandList& l, const QString& needle )
+{
+    for ( CommandList::const_iterator i = l.cbegin(); i != l.cend(); ++i )
+        if ( i->command().contains( needle ) )
+            return true;
+    return false;
+}
+
 Calamares::JobResult CommandList::run()
 {
     QLatin1Literal rootMagic( "@@ROOT@@" );
+    QLatin1Literal userMagic( "@@USER@@" );
 
     System::RunLocation location = m_doChroot ? System::RunLocation::RunInTarget : System::RunLocation::RunInHost;
 
@@ -108,14 +118,7 @@ Calamares::JobResult CommandList::run()
     QString root = QStringLiteral( "/" );
     Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
 
-    bool needsRootSubstitution = false;
+    bool needsRootSubstitution = findInCommands( *this, rootMagic );
-    for ( CommandList::const_iterator i = cbegin(); i != cend(); ++i )
-        if ( i->command().contains( rootMagic ) )
-        {
-            needsRootSubstitution = true;
-            break;
-        }
-
     if ( needsRootSubstitution && ( location == System::RunLocation::RunInHost ) )
     {
         if ( !gs || !gs->contains( "rootMountPoint" ) )
@@ -127,10 +130,20 @@ Calamares::JobResult CommandList::run()
         root = gs->value( "rootMountPoint" ).toString();
     }
 
+    bool needsUserSubstitution = findInCommands( *this, userMagic );
+    if ( needsUserSubstitution && ( !gs || !gs->contains( "username" ) ) )
+    {
+        cError() << "No username defined.";
+        return Calamares::JobResult::error(
+            QCoreApplication::translate( "CommandList", "Could not run command." ),
+            QCoreApplication::translate( "CommandList", "The command needs to know the user's name, but no username is defined." ) );
+    }
+    QString user = gs->value( "username" ).toString();  // may be blank if unset
+
     for ( CommandList::const_iterator i = cbegin(); i != cend(); ++i )
     {
         QString processed_cmd = i->command();
-        processed_cmd.replace( rootMagic, root );
+        processed_cmd.replace( rootMagic, root ).replace( userMagic, user );
         bool suppress_result = false;
         if ( processed_cmd.startsWith( '-' ) )
         {

src/libcalamares/utils/CommandList.h

@@ -74,6 +74,9 @@ using CommandList_t = QList< CommandLine >;
  * A list of commands; the list may have its own default timeout
  * for commands (which is then applied to each individual command
  * that doesn't have one of its own).
+ *
+ * Documentation for the format of commands can be found in
+ * `shellprocess.conf`.
  */
 class CommandList : protected CommandList_t
 {

src/libcalamares/utils/Logger.cpp

@@ -116,7 +116,7 @@ CalamaresLogHandler( QtMsgType type, const QMessageLogContext& context, const QS
 }
 
 
-static QString
+QString
 logFile()
 {
     return CalamaresUtils::appLogDir().filePath( "session.log" );

src/libcalamares/utils/Logger.h

@@ -64,6 +64,11 @@ namespace Logger
         virtual ~CDebug();
     };
 
+    /**
+     * @brief The full path of the log file.
+     */
+    DLLEXPORT QString logFile();
+
     /**
      * @brief Start logging to the log file.
      *

src/modules/partition/core/DeviceList.cpp

@@ -1,6 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2015-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -52,22 +53,6 @@ hasRootPartition( Device* device )
     return false;
 }
 
-/* Unused */
-static bool
-hasMountedPartitions( Device* device )
-{
-    cDebug() << "Checking for mounted partitions in" << device->deviceNode();
-    for ( auto it = PartitionIterator::begin( device ); it != PartitionIterator::end( device ); ++it )
-    {
-        if ( ! ( *it )->isMounted() )
-        {
-            cDebug() << "  .." << ( *it )->partitionPath() << "is mounted on" << ( *it )->mountPoint();
-            return true;
-        }
-    }
-    return false;
-}
-
 static bool
 isIso9660( const Device* device )
 {

src/modules/partition/core/PartUtils.cpp

@@ -1,6 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2015-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -139,7 +140,6 @@ canBeResized( PartitionCoreModule* core, const QString& partitionPath )
     if ( partitionWithOs.startsWith( "/dev/" ) )
     {
         cDebug() << partitionWithOs << "seems like a good path";
-        bool canResize = false;
         DeviceModel* dm = core->deviceModel();
         for ( int i = 0; i < dm->rowCount(); ++i )
         {

src/modules/preservefiles/CMakeLists.txt

@@ -0,0 +1,11 @@
+include_directories( ${PROJECT_BINARY_DIR}/src/libcalamaresui )
+
+calamares_add_plugin( preservefiles
+    TYPE job
+    EXPORT_MACRO PLUGINDLLEXPORT_PRO
+    SOURCES
+        PreserveFiles.cpp
+    LINK_PRIVATE_LIBRARIES
+        calamares
+    SHARED_LIB
+)

src/modules/preservefiles/PreserveFiles.cpp

@@ -0,0 +1,201 @@
+/* === This file is part of Calamares - <https://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "PreserveFiles.h"
+
+#include "CalamaresVersion.h"
+#include "JobQueue.h"
+#include "GlobalStorage.h"
+
+#include "utils/CalamaresUtils.h"
+#include "utils/CalamaresUtilsSystem.h"
+#include "utils/CommandList.h"
+#include "utils/Logger.h"
+#include "utils/Units.h"
+
+#include <QFile>
+
+using CalamaresUtils::operator""_MiB;
+
+QString targetPrefix()
+{
+    if ( CalamaresUtils::System::instance()->doChroot() )
+    {
+        Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+        if ( gs && gs->contains( "rootMountPoint" ) )
+        {
+            QString r = gs->value( "rootMountPoint" ).toString();
+            if ( !r.isEmpty() )
+                return r;
+            else
+                cDebug() << "RootMountPoint is empty";
+        }
+        else
+        {
+            cDebug() << "No rootMountPoint defined, preserving files to '/'";
+        }
+    }
+
+    return QLatin1Literal( "/" );
+}
+
+QString atReplacements( QString s )
+{
+    Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+    QString root( "/" );
+    QString user;
+
+    if ( gs && gs->contains( "rootMountPoint" ) )
+        root = gs->value( "rootMountPoint" ).toString();
+    if ( gs && gs->contains( "username" ) )
+        user = gs->value( "username" ).toString();
+
+    return s.replace( "@@ROOT@@", root ).replace( "@@USER@@", user );
+}
+
+PreserveFiles::PreserveFiles( QObject* parent )
+    : Calamares::CppJob( parent )
+{
+}
+
+PreserveFiles::~PreserveFiles()
+{
+}
+
+QString
+PreserveFiles::prettyName() const
+{
+    return tr( "Saving files for later ..." );
+}
+
+Calamares::JobResult PreserveFiles::exec()
+{
+    if ( m_items.isEmpty() )
+        return Calamares::JobResult::error( tr( "No files configured to save for later." ) );
+
+    QString prefix = targetPrefix();
+    if ( !prefix.endsWith( '/' ) )
+        prefix.append( '/' );
+
+    int count = 0;
+    for ( const auto it : m_items )
+    {
+        QString source = it.source;
+        QString dest = prefix + atReplacements( it.dest );
+
+        if ( it.type == ItemType::Log )
+            source = Logger::logFile();
+        if ( it.type == ItemType::Config )
+        {
+            if ( Calamares::JobQueue::instance()->globalStorage()->save( dest ) )
+                cWarning() << "Could not write config for" << dest;
+            else
+                ++count;
+        }
+        else if ( source.isEmpty() )
+            cWarning() << "Skipping unnamed source file for" << dest;
+        else
+        {
+            QFile sourcef( source );
+            if ( !sourcef.open( QFile::ReadOnly ) )
+            {
+                cWarning() << "Could not read" << source;
+                continue;
+            }
+
+            QFile destf( dest );
+            if ( !destf.open( QFile::WriteOnly ) )
+            {
+                sourcef.close();
+                cWarning() << "Could not open" << destf.fileName() << "for writing; could not copy" << source;
+                continue;
+            }
+
+            QByteArray b;
+            do
+            {
+                b = sourcef.read( 1_MiB );
+                destf.write( b );
+            }
+            while ( b.count() > 0 );
+
+            sourcef.close();
+            destf.close();
+            ++count;
+        }
+    }
+
+    return count == m_items.count() ?
+        Calamares::JobResult::ok() :
+        Calamares::JobResult::error( tr( "Not all of the configured files could be preserved." ) );
+}
+
+void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap)
+{
+    auto files = configurationMap[ "files" ];
+
+    if ( ! ( files.isValid() && ( files.type() == QVariant::List ) ) )
+    {
+        cDebug() << "No files: configuration key, or not a list.";
+        return;
+    }
+
+    QVariantList l = files.toList();
+    unsigned int c = 0;
+    for ( const auto li : l )
+    {
+        if ( li.type() == QVariant::String )
+        {
+            QString filename = li.toString();
+            if ( !filename.isEmpty() )
+                m_items.append( Item{ filename, filename, ItemType::Path } );
+            else
+                cDebug() << "Empty filename for preservefiles, item" << c;
+        }
+        else if ( li.type() == QVariant::Map )
+        {
+            const auto map = li.toMap();
+            QString dest = map[ "dest" ].toString();
+            QString from = map[ "from" ].toString();
+            ItemType t =
+                ( from == "log" ) ? ItemType::Log :
+                ( from == "config" ) ? ItemType::Config :
+                ItemType::None;
+
+            if ( dest.isEmpty() )
+            {
+                cDebug() << "Empty dest for preservefiles, item" << c;
+            }
+            else if ( t == ItemType::None )
+            {
+                cDebug() << "Invalid type for preservefiles, item" << c;
+            }
+            else
+            {
+                m_items.append( Item{ QString(), dest, t } );
+            }
+        }
+        else
+            cDebug() << "Invalid type for preservefiles, item" << c;
+
+        ++c;
+    }
+}
+
+CALAMARES_PLUGIN_FACTORY_DEFINITION( PreserveFilesFactory, registerPlugin<PreserveFiles>(); )
+

src/modules/preservefiles/PreserveFiles.h

@@ -0,0 +1,70 @@
+/* === This file is part of Calamares - <https://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef PRESERVEFILES_H
+#define PRESERVEFILES_H
+
+#include <QList>
+#include <QObject>
+#include <QVariantMap>
+
+#include "CppJob.h"
+
+#include "utils/PluginFactory.h"
+
+#include "PluginDllMacro.h"
+
+
+class PLUGINDLLEXPORT PreserveFiles : public Calamares::CppJob
+{
+    Q_OBJECT
+
+    enum class ItemType
+    {
+        None,
+        Path,
+        Log,
+        Config
+    } ;
+
+    struct Item
+    {
+        QString source;
+        QString dest;
+        ItemType type;
+    } ;
+
+    using ItemList = QList< Item >;
+
+public:
+    explicit PreserveFiles( QObject* parent = nullptr );
+    virtual ~PreserveFiles() override;
+
+    QString prettyName() const override;
+
+    Calamares::JobResult exec() override;
+
+    void setConfigurationMap( const QVariantMap& configurationMap ) override;
+
+private:
+    ItemList m_items;
+};
+
+CALAMARES_PLUGIN_FACTORY_DECLARATION( PreserveFilesFactory )
+
+#endif // PRESERVEFILES_H

src/modules/preservefiles/module.desc

@@ -0,0 +1,5 @@
+---
+type:      "job"
+name:      "preservefiles"
+interface: "qtplugin"
+load:      "libcalamares_job_preservefiles.so"

src/modules/preservefiles/preservefiles.conf

@@ -0,0 +1,36 @@
+# Configuration for the preserve-files job
+#
+# The *files* key contains a list of files to preserve. Each element of
+# the list should have one of these forms:
+#
+# - an absolute path (probably within the host system). This will be preserved
+#   as the same path within the target system (chroot). If, globally, dontChroot
+#   is true, then these items are ignored (since the destination is the same
+#   as the source).
+# - a map with a *dest* key. The *dest* value is a path interpreted in the
+#   target system (if dontChroot is true, in the host system). Relative paths
+#   are not recommended. There are two possible other keys in the map:
+#   - *from*, which must have one of the values, below; it is used to
+#     preserve files whose pathname is known to Calamares internally.
+#   - *src*, to refer to a path interpreted in the host system. Relative
+#     paths are not recommended, and are interpreted relative to where
+#     Calamares is being run.
+#   Only one of the two other keys (either *from* or *src*) may be set.
+#
+# The target filename is modified as follows:
+#   - `@@ROOT@@` is replaced by the path to the target root (may be /)
+#   - `@@USER@@` is replaced by the username entered by on the user
+#       page (may be empty, for instance if no user page is enabled)
+#
+# Special values for the key *from* are:
+# - *log*, for the complete log file (up to the moment the preservefiles
+#   module is run),
+# - *config*, for the Calamares configuration file
+# - *globals*, for a JSON dump of the contents of global storage
+---
+files:
+  - /etc/oem-information
+  - from: log
+    dest: /root/install.log
+  - from: config
+    dest: /root/install.cfg

src/modules/shellprocess/Tests.cpp

@@ -18,7 +18,12 @@
 
 #include "Tests.h"
 
+#include "GlobalStorage.h"
+#include "JobQueue.h"
+#include "Settings.h"
+
 #include "utils/CommandList.h"
+#include "utils/Logger.h"
 #include "utils/YamlUtils.h"
 
 #include <yaml-cpp/yaml.h>
@@ -149,3 +154,60 @@ script:
     QCOMPARE( cl.at(0).command(), QStringLiteral( "ls /tmp" ) );
     QCOMPARE( cl.at(1).timeout(), -1 );  // not set
 }
+
+void ShellProcessTests::testRootSubstitution()
+{
+    YAML::Node doc = YAML::Load( R"(---
+script:
+    - "ls /tmp"
+)" );
+    QVariant plainScript = CalamaresUtils::yamlMapToVariant( doc ).toMap().value( "script" );
+    QVariant rootScript = CalamaresUtils::yamlMapToVariant(
+        YAML::Load( R"(---
+script:
+    - "ls @@ROOT@@"
+)" ) ).toMap().value( "script" );
+    QVariant userScript = CalamaresUtils::yamlMapToVariant(
+        YAML::Load( R"(---
+script:
+    - mktemp -d @@ROOT@@/calatestXXXXXXXX
+    - "chown @@USER@@ @@ROOT@@/calatest*"
+    - rm -rf @@ROOT@@/calatest*
+)" ) ).toMap().value( "script" );
+
+    if ( !Calamares::JobQueue::instance() )
+        (void *)new Calamares::JobQueue( nullptr );
+    if ( !Calamares::Settings::instance() )
+        (void *)new Calamares::Settings( QString(), true );
+
+    Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+    QVERIFY( gs != nullptr );
+
+    qDebug() << "Expect WARNING, ERROR, WARNING";
+    // Doesn't use @@ROOT@@, so no failures
+    QVERIFY( bool(CommandList(plainScript, false, 10 ).run()) );
+
+    // Doesn't use @@ROOT@@, but does chroot, so fails
+    QVERIFY( !bool(CommandList(plainScript, true, 10 ).run()) );
+
+    // Does use @@ROOT@@, which is not set, so fails
+    QVERIFY( !bool(CommandList(rootScript, false, 10 ).run()) );
+    // .. fails for two reasons
+    QVERIFY( !bool(CommandList(rootScript, true, 10 ).run()) );
+
+    gs->insert( "rootMountPoint", "/tmp" );
+    // Now that the root is set, two variants work .. still can't
+    // chroot, unless the rootMountPoint contains a full system,
+    // *and* we're allowed to chroot (ie. running tests as root).
+    qDebug() << "Expect no output.";
+    QVERIFY( bool(CommandList(plainScript, false, 10 ).run()) );
+    QVERIFY( bool(CommandList(rootScript, false, 10 ).run()) );
+
+    qDebug() << "Expect ERROR";
+    // But no user set yet
+    QVERIFY( !bool(CommandList(userScript, false, 10 ).run()) );
+
+    // Now play dangerous games with shell expansion
+    gs->insert( "username", "`id -u`" );
+    QVERIFY( bool(CommandList(userScript, false, 10 ).run()) );
+}

src/modules/shellprocess/Tests.h

@@ -40,6 +40,8 @@ private Q_SLOTS:
     void testProcessFromObject();
     // Create from a complex YAML list
     void testProcessListFromObject();
+    // Check @@ROOT@@ substitution
+    void testRootSubstitution();
 };
 
 #endif

src/modules/shellprocess/shellprocess.conf

@@ -4,9 +4,11 @@
 # If the top-level key *dontChroot* is true, then the commands
 # are executed in the context of the live system, otherwise
 # in the context of the target system. In all of the commands,
-# `@@ROOT@@` is replaced by the root mount point of the **target**
+# the following substitutions will take place:
-# system from the point of view of the command (for chrooted
+#  - `@@ROOT@@` is replaced by the root mount point of the **target**
-# commands, that will be */*).
+#    system from the point of view of the command (for chrooted
+#    commands, that will be */*).
+#  - `@@USER@@` is replaced by the username, set on the user page.
 #
 # The (global) timeout for the command list can be set with
 # the *timeout* key. The value is a time in seconds, default