From 5bcd6eaef84b703d65aa036fa1cd227380473838 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Sat, 19 May 2018 09:08:11 -0400
Subject: [PATCH 01/21] [partition] Reduce warnings by removing unused code

---
 src/modules/partition/core/DeviceList.cpp | 17 +----------------
 src/modules/partition/core/PartUtils.cpp  |  2 +-
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/modules/partition/core/DeviceList.cpp b/src/modules/partition/core/DeviceList.cpp
index d7e6bab29..ebc40a03d 100644
--- a/src/modules/partition/core/DeviceList.cpp
+++ b/src/modules/partition/core/DeviceList.cpp
@@ -1,6 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2015-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -52,22 +53,6 @@ hasRootPartition( Device* device )
     return false;
 }
 
-/* Unused */
-static bool
-hasMountedPartitions( Device* device )
-{
-    cDebug() << "Checking for mounted partitions in" << device->deviceNode();
-    for ( auto it = PartitionIterator::begin( device ); it != PartitionIterator::end( device ); ++it )
-    {
-        if ( ! ( *it )->isMounted() )
-        {
-            cDebug() << "  .." << ( *it )->partitionPath() << "is mounted on" << ( *it )->mountPoint();
-            return true;
-        }
-    }
-    return false;
-}
-
 static bool
 isIso9660( const Device* device )
 {
diff --git a/src/modules/partition/core/PartUtils.cpp b/src/modules/partition/core/PartUtils.cpp
index 2c2944997..a39cb5395 100644
--- a/src/modules/partition/core/PartUtils.cpp
+++ b/src/modules/partition/core/PartUtils.cpp
@@ -1,6 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2015-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -139,7 +140,6 @@ canBeResized( PartitionCoreModule* core, const QString& partitionPath )
     if ( partitionWithOs.startsWith( "/dev/" ) )
     {
         cDebug() << partitionWithOs << "seems like a good path";
-        bool canResize = false;
         DeviceModel* dm = core->deviceModel();
         for ( int i = 0; i < dm->rowCount(); ++i )
         {

From 6c87747a5dc7df416452f4b9561e87b10d6cae7e Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Sun, 20 May 2018 10:42:26 -0400
Subject: [PATCH 02/21] [libcalamares] Make it possible to get the logfile name

---
 src/libcalamares/utils/Logger.cpp | 2 +-
 src/libcalamares/utils/Logger.h   | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libcalamares/utils/Logger.cpp b/src/libcalamares/utils/Logger.cpp
index 0a13881d3..735414b85 100644
--- a/src/libcalamares/utils/Logger.cpp
+++ b/src/libcalamares/utils/Logger.cpp
@@ -116,7 +116,7 @@ CalamaresLogHandler( QtMsgType type, const QMessageLogContext& context, const QS
 }
 
 
-static QString
+QString
 logFile()
 {
     return CalamaresUtils::appLogDir().filePath( "session.log" );
diff --git a/src/libcalamares/utils/Logger.h b/src/libcalamares/utils/Logger.h
index f7488b553..dba386eae 100644
--- a/src/libcalamares/utils/Logger.h
+++ b/src/libcalamares/utils/Logger.h
@@ -64,6 +64,11 @@ namespace Logger
         virtual ~CDebug();
     };
 
+    /**
+     * @brief The full path of the log file.
+     */
+    DLLEXPORT QString logFile();
+
     /**
      * @brief Start logging to the log file.
      *

From 6779a449913c687fe5261573225613c2419c2b39 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Sat, 19 May 2018 09:04:14 -0400
Subject: [PATCH 03/21] [preservefiles] New module preservefiles

 - more flexible way to keep (all kinds of) files from the host
   system, into the target system.
 - WIP: substitutions like in shellprocess (@@ROOT@@, @@HOME@@ probably)
 - WIP: creating a JSON file from global settings
---
 src/modules/preservefiles/CMakeLists.txt     |  11 ++
 src/modules/preservefiles/PreserveFiles.cpp  | 181 +++++++++++++++++++
 src/modules/preservefiles/PreserveFiles.h    |  70 +++++++
 src/modules/preservefiles/module.desc        |   5 +
 src/modules/preservefiles/preservefiles.conf |  31 ++++
 5 files changed, 298 insertions(+)
 create mode 100644 src/modules/preservefiles/CMakeLists.txt
 create mode 100644 src/modules/preservefiles/PreserveFiles.cpp
 create mode 100644 src/modules/preservefiles/PreserveFiles.h
 create mode 100644 src/modules/preservefiles/module.desc
 create mode 100644 src/modules/preservefiles/preservefiles.conf

diff --git a/src/modules/preservefiles/CMakeLists.txt b/src/modules/preservefiles/CMakeLists.txt
new file mode 100644
index 000000000..43602024c
--- /dev/null
+++ b/src/modules/preservefiles/CMakeLists.txt
@@ -0,0 +1,11 @@
+include_directories( ${PROJECT_BINARY_DIR}/src/libcalamaresui )
+
+calamares_add_plugin( preservefiles
+    TYPE job
+    EXPORT_MACRO PLUGINDLLEXPORT_PRO
+    SOURCES
+        PreserveFiles.cpp
+    LINK_PRIVATE_LIBRARIES
+        calamares
+    SHARED_LIB
+)
diff --git a/src/modules/preservefiles/PreserveFiles.cpp b/src/modules/preservefiles/PreserveFiles.cpp
new file mode 100644
index 000000000..9ef473f27
--- /dev/null
+++ b/src/modules/preservefiles/PreserveFiles.cpp
@@ -0,0 +1,181 @@
+/* === This file is part of Calamares - <https://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "PreserveFiles.h"
+
+#include "CalamaresVersion.h"
+#include "JobQueue.h"
+#include "GlobalStorage.h"
+
+#include "utils/CalamaresUtils.h"
+#include "utils/CalamaresUtilsSystem.h"
+#include "utils/CommandList.h"
+#include "utils/Logger.h"
+#include "utils/Units.h"
+
+#include <QFile>
+
+using CalamaresUtils::operator""_MiB;
+
+QString targetPrefix()
+{
+    if ( CalamaresUtils::System::instance()->doChroot() )
+    {
+        Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+        if ( gs && gs->contains( "rootMountPoint" ) )
+        {
+            QString r = gs->value( "rootMountPoint" ).toString();
+            if ( !r.isEmpty() )
+                return r;
+            else
+                cDebug() << "RootMountPoint is empty";
+        }
+        else
+        {
+            cDebug() << "No rootMountPoint defined, preserving files to '/'";
+        }
+    }
+
+    return QLatin1Literal( "/" );
+}
+
+PreserveFiles::PreserveFiles( QObject* parent )
+    : Calamares::CppJob( parent )
+{
+}
+
+PreserveFiles::~PreserveFiles()
+{
+}
+
+QString
+PreserveFiles::prettyName() const
+{
+    return tr( "Saving files for later ..." );
+}
+
+Calamares::JobResult PreserveFiles::exec()
+{
+    if ( m_items.isEmpty() )
+        return Calamares::JobResult::error( tr( "No files configured to save for later." ) );
+
+    QString prefix = targetPrefix();
+    if ( !prefix.endsWith( '/' ) )
+        prefix.append( '/' );
+
+    int count = 0;
+    for ( const auto it : m_items )
+    {
+        QString source = it.source;
+        if ( it.type == ItemType::Log )
+            source = Logger::logFile();
+        if ( it.type == ItemType::Config )
+            cDebug() << "Config-preserving is not implemented yet.";
+
+        if ( source.isEmpty() )
+            cWarning() << "Skipping unnamed source file for" << it.dest;
+        else
+        {
+            QFile sourcef( source );
+            if ( !sourcef.open( QFile::ReadOnly ) )
+            {
+                cWarning() << "Could not read" << source;
+                continue;
+            }
+
+            QFile destf( prefix + it.dest );
+            if ( !destf.open( QFile::WriteOnly ) )
+            {
+                sourcef.close();
+                cWarning() << "Could not open" << destf.fileName() << "for writing; could not copy" << source;
+                continue;
+            }
+
+            QByteArray b;
+            do
+            {
+                b = sourcef.read( 1_MiB );
+                destf.write( b );
+            }
+            while ( b.count() > 0 );
+
+            sourcef.close();
+            destf.close();
+            ++count;
+        }
+    }
+
+    return count == m_items.count() ?
+        Calamares::JobResult::ok() :
+        Calamares::JobResult::error( tr( "Not all of the configured files could be preserved." ) );
+}
+
+void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap)
+{
+    auto files = configurationMap[ "files" ];
+
+    if ( ! ( files.isValid() && ( files.type() == QVariant::List ) ) )
+    {
+        cDebug() << "No files: configuration key, or not a list.";
+        return;
+    }
+
+    QVariantList l = files.toList();
+    unsigned int c = 0;
+    for ( const auto li : l )
+    {
+        if ( li.type() == QVariant::String )
+        {
+            QString filename = li.toString();
+            if ( !filename.isEmpty() )
+                m_items.append( Item{ filename, filename, ItemType::Path } );
+            else
+                cDebug() << "Empty filename for preservefiles, item" << c;
+        }
+        else if ( li.type() == QVariant::Map )
+        {
+            const auto map = li.toMap();
+            QString dest = map[ "dest" ].toString();
+            QString from = map[ "from" ].toString();
+            ItemType t =
+                ( from == "log" ) ? ItemType::Log :
+                ( from == "config" ) ? ItemType::Config :
+                ItemType::None;
+
+            if ( dest.isEmpty() )
+            {
+                cDebug() << "Empty dest for preservefiles, item" << c;
+            }
+            else if ( t == ItemType::None )
+            {
+                cDebug() << "Invalid type for preservefiles, item" << c;
+            }
+            else
+            {
+                m_items.append( Item{ QString(), dest, t } );
+            }
+        }
+        else
+            cDebug() << "Invalid type for preservefiles, item" << c;
+
+        ++c;
+    }
+}
+
+CALAMARES_PLUGIN_FACTORY_DEFINITION( PreserveFilesFactory, registerPlugin<PreserveFiles>(); )
+
diff --git a/src/modules/preservefiles/PreserveFiles.h b/src/modules/preservefiles/PreserveFiles.h
new file mode 100644
index 000000000..0c9216336
--- /dev/null
+++ b/src/modules/preservefiles/PreserveFiles.h
@@ -0,0 +1,70 @@
+/* === This file is part of Calamares - <https://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef PRESERVEFILES_H
+#define PRESERVEFILES_H
+
+#include <QList>
+#include <QObject>
+#include <QVariantMap>
+
+#include "CppJob.h"
+
+#include "utils/PluginFactory.h"
+
+#include "PluginDllMacro.h"
+
+
+class PLUGINDLLEXPORT PreserveFiles : public Calamares::CppJob
+{
+    Q_OBJECT
+
+    enum class ItemType
+    {
+        None,
+        Path,
+        Log,
+        Config
+    } ;
+
+    struct Item
+    {
+        QString source;
+        QString dest;
+        ItemType type;
+    } ;
+
+    using ItemList = QList< Item >;
+
+public:
+    explicit PreserveFiles( QObject* parent = nullptr );
+    virtual ~PreserveFiles() override;
+
+    QString prettyName() const override;
+
+    Calamares::JobResult exec() override;
+
+    void setConfigurationMap( const QVariantMap& configurationMap ) override;
+
+private:
+    ItemList m_items;
+};
+
+CALAMARES_PLUGIN_FACTORY_DECLARATION( PreserveFilesFactory )
+
+#endif // PRESERVEFILES_H
diff --git a/src/modules/preservefiles/module.desc b/src/modules/preservefiles/module.desc
new file mode 100644
index 000000000..953d8c81b
--- /dev/null
+++ b/src/modules/preservefiles/module.desc
@@ -0,0 +1,5 @@
+---
+type:      "job"
+name:      "preservefiles"
+interface: "qtplugin"
+load:      "libcalamares_job_preservefiles.so"
diff --git a/src/modules/preservefiles/preservefiles.conf b/src/modules/preservefiles/preservefiles.conf
new file mode 100644
index 000000000..4c33d93d9
--- /dev/null
+++ b/src/modules/preservefiles/preservefiles.conf
@@ -0,0 +1,31 @@
+# Configuration for the preserve-files job
+#
+# The *files* key contains a list of files to preserve. Each element of
+# the list should have one of these forms:
+#
+# - an absolute path (probably within the host system). This will be preserved
+#   as the same path within the target system (chroot). If, globally, dontChroot
+#   is true, then these items are ignored (since the destination is the same
+#   as the source).
+# - a map with a *dest* key. The *dest* value is a path interpreted in the
+#   target system (if dontChroot is true, in the host system). Relative paths
+#   are not recommended. There are two possible other keys in the map:
+#   - *from*, which must have one of the values, below; it is used to
+#     preserve files whose pathname is known to Calamares internally.
+#   - *src*, to refer to a path interpreted in the host system. Relative
+#     paths are not recommended, and are interpreted relative to where
+#     Calamares is being run.
+#   Only one of the two other keys may be set.
+#
+# Special values for the key *from* are:
+# - *log*, for the complete log file (up to the moment the preservefiles
+#   module is run),
+# - *config*, for the Calamares configuration file
+# - *globals*, for a JSON dump of the contents of global storage
+---
+files:
+  - /etc/oem-information
+  - from: log
+    dest: /root/install.log
+  - from: config
+    dest: /root/install.cfg

From dae84d3bb14658ade2c03e6374d3a0ce597e3ae8 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 08:11:49 -0400
Subject: [PATCH 04/21] [users] Improve explanation of other steps of adding a
 user

---
 src/modules/users/CreateUserJob.cpp | 38 ++++++++++++-----------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/modules/users/CreateUserJob.cpp b/src/modules/users/CreateUserJob.cpp
index 119028059..052af87c6 100644
--- a/src/modules/users/CreateUserJob.cpp
+++ b/src/modules/users/CreateUserJob.cpp
@@ -161,29 +161,23 @@ CreateUserJob::exec()
         return pres.explainProcess( useradd, 10 /* bogus timeout */ );
     }
 
-    int ec = CalamaresUtils::System::instance()->
-             targetEnvCall( { "usermod",
-                              "-aG",
-                              defaultGroups,
-                              m_userName } );
-    if ( ec )
-        return Calamares::JobResult::error( tr( "Cannot add user %1 to groups: %2." )
-                                                .arg( m_userName )
-                                                .arg( defaultGroups ),
-                                            tr( "usermod terminated with error code %1." )
-                                                .arg( ec ) );
+    pres = CalamaresUtils::System::instance()->targetEnvCommand(
+        { "usermod", "-aG", defaultGroups, m_userName } );
+    if ( pres.getExitCode() )
+    {
+        cError() << "usermod failed" << pres.getExitCode();
+        return pres.explainProcess( "usermod", 10 );
+    }
 
-    ec = CalamaresUtils::System::instance()->
-                      targetEnvCall( { "chown",
-                                       "-R",
-                                       QString( "%1:%2" ).arg( m_userName )
-                                                         .arg( m_userName ),
-                                       QString( "/home/%1" ).arg( m_userName ) } );
-    if ( ec )
-        return Calamares::JobResult::error( tr( "Cannot set home directory ownership for user %1." )
-                                                .arg( m_userName ),
-                                            tr( "chown terminated with error code %1." )
-                                                .arg( ec ) );
+    QString userGroup = QString( "%1:%2" ).arg( m_userName ).arg( m_userName );
+    QString homeDir = QString( "/home/%1" ).arg( m_userName );
+    pres = CalamaresUtils::System::instance()->targetEnvCommand(
+        { "chown", "-R", userGroup, homeDir } );
+    if ( pres.getExitCode() )
+    {
+        cError() << "chown failed" << pres.getExitCode();
+        return pres.explainProcess( "chown", 10 );
+    }
 
     return Calamares::JobResult::ok();
 }

From ac287a0ac5b1f6cdbf2706663082aa3b3f66661f Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 08:53:11 -0400
Subject: [PATCH 05/21] [libcalamares] Add a save() method to global storage

 - This is a quick way to dump GS to JSON, which is useful
   for the preservefiles module #928
 - Also useful for, e.g., #466
---
 src/libcalamares/GlobalStorage.cpp | 18 +++++++++++++++++-
 src/libcalamares/GlobalStorage.h   | 11 ++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/libcalamares/GlobalStorage.cpp b/src/libcalamares/GlobalStorage.cpp
index fd72697cf..4f98ea2eb 100644
--- a/src/libcalamares/GlobalStorage.cpp
+++ b/src/libcalamares/GlobalStorage.cpp
@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014-2015, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -22,6 +22,9 @@
 
 #include "utils/Logger.h"
 
+#include <QFile>
+#include <QJsonDocument>
+
 #ifdef WITH_PYTHON
 #include "PythonHelper.h"
 
@@ -94,6 +97,19 @@ GlobalStorage::debugDump() const
     }
 }
 
+bool
+GlobalStorage::save(const QString& filename)
+{
+    QFile f( filename );
+    if ( !f.open( QFile::WriteOnly ) )
+        return false;
+
+    f.write( QJsonDocument::fromVariant( m ).toJson() ) ;
+    f.close();
+    return true;
+}
+
+
 } // namespace Calamares
 
 #ifdef WITH_PYTHON
diff --git a/src/libcalamares/GlobalStorage.h b/src/libcalamares/GlobalStorage.h
index dc79c55e8..72524ba4f 100644
--- a/src/libcalamares/GlobalStorage.h
+++ b/src/libcalamares/GlobalStorage.h
@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014-2015, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -58,7 +58,16 @@ public:
     int remove( const QString& key );
     QVariant value( const QString& key ) const;
 
+    /// @brief dump keys and values to the debug log
     void debugDump() const;
+    /** @brief write as JSON to the given filename
+     *
+     * No tidying, sanitization, or censoring is done -- for instance,
+     * the user module sets a slightly-obscured password in global storage,
+     * and this JSON file will contain that password in-the-only-slightly-
+     * obscured form.
+     */
+    bool save( const QString& filename );
 
 signals:
     void changed();

From b7890d865f33f59e630c2e715509d2bd9482e481 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 09:09:03 -0400
Subject: [PATCH 06/21] [preservefiles] Save GS, munge destination

 - using `from: config` now writes a JSON file
 - using @@ROOT@@ and @@USER@@ in dest does a sensible
   substitution.
---
 src/modules/preservefiles/PreserveFiles.cpp  | 30 ++++++++++++++++----
 src/modules/preservefiles/preservefiles.conf |  7 ++++-
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/modules/preservefiles/PreserveFiles.cpp b/src/modules/preservefiles/PreserveFiles.cpp
index 9ef473f27..6490f8303 100644
--- a/src/modules/preservefiles/PreserveFiles.cpp
+++ b/src/modules/preservefiles/PreserveFiles.cpp
@@ -54,6 +54,20 @@ QString targetPrefix()
     return QLatin1Literal( "/" );
 }
 
+QString atReplacements( QString s )
+{
+    Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+    QString root( "/" );
+    QString user;
+
+    if ( gs && gs->contains( "rootMountPoint" ) )
+        root = gs->value( "rootMountPoint" ).toString();
+    if ( gs && gs->contains( "username" ) )
+        user = gs->value( "username" ).toString();
+
+    return s.replace( "@@ROOT@@", root ).replace( "@@USER@@", user );
+}
+
 PreserveFiles::PreserveFiles( QObject* parent )
     : Calamares::CppJob( parent )
 {
@@ -82,13 +96,19 @@ Calamares::JobResult PreserveFiles::exec()
     for ( const auto it : m_items )
     {
         QString source = it.source;
+        QString dest = prefix + atReplacements( it.dest );
+
         if ( it.type == ItemType::Log )
             source = Logger::logFile();
         if ( it.type == ItemType::Config )
-            cDebug() << "Config-preserving is not implemented yet.";
-
-        if ( source.isEmpty() )
-            cWarning() << "Skipping unnamed source file for" << it.dest;
+        {
+            if ( Calamares::JobQueue::instance()->globalStorage()->save( dest ) )
+                cWarning() << "Could not write config for" << dest;
+            else
+                ++count;
+        }
+        else if ( source.isEmpty() )
+            cWarning() << "Skipping unnamed source file for" << dest;
         else
         {
             QFile sourcef( source );
@@ -98,7 +118,7 @@ Calamares::JobResult PreserveFiles::exec()
                 continue;
             }
 
-            QFile destf( prefix + it.dest );
+            QFile destf( dest );
             if ( !destf.open( QFile::WriteOnly ) )
             {
                 sourcef.close();
diff --git a/src/modules/preservefiles/preservefiles.conf b/src/modules/preservefiles/preservefiles.conf
index 4c33d93d9..ab9114d20 100644
--- a/src/modules/preservefiles/preservefiles.conf
+++ b/src/modules/preservefiles/preservefiles.conf
@@ -15,7 +15,12 @@
 #   - *src*, to refer to a path interpreted in the host system. Relative
 #     paths are not recommended, and are interpreted relative to where
 #     Calamares is being run.
-#   Only one of the two other keys may be set.
+#   Only one of the two other keys (either *from* or *src*) may be set.
+#
+# The target filename is modified as follows:
+#   - `@@ROOT@@` is replaced by the path to the target root (may be /)
+#   - `@@USER@@` is replaced by the username entered by on the user
+#       page (may be empty, for instance if no user page is enabled)
 #
 # Special values for the key *from* are:
 # - *log*, for the complete log file (up to the moment the preservefiles

From b5d900c1c6da2b25c185ad16f98d56861c4921d5 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 09:25:57 -0400
Subject: [PATCH 07/21] [libcalamares] Allow a @@USER@@ replacement in commands

 - Following example in preservefiles module, allow @@USER@@
   in commands (e.g. to do something specific in the home-dir
   of the new user).
---
 src/libcalamares/utils/CommandList.cpp     | 31 +++++++++++++++-------
 src/libcalamares/utils/CommandList.h       |  3 +++
 src/modules/shellprocess/shellprocess.conf |  8 +++---
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/src/libcalamares/utils/CommandList.cpp b/src/libcalamares/utils/CommandList.cpp
index 3b2935c55..8e332a066 100644
--- a/src/libcalamares/utils/CommandList.cpp
+++ b/src/libcalamares/utils/CommandList.cpp
@@ -98,9 +98,19 @@ CommandList::~CommandList()
 {
 }
 
+static inline bool
+findInCommands( const CommandList& l, const QString& needle )
+{
+    for ( CommandList::const_iterator i = l.cbegin(); i != l.cend(); ++i )
+        if ( i->command().contains( needle ) )
+            return true;
+    return false;
+}
+
 Calamares::JobResult CommandList::run()
 {
     QLatin1Literal rootMagic( "@@ROOT@@" );
+    QLatin1Literal userMagic( "@@USER@@" );
 
     System::RunLocation location = m_doChroot ? System::RunLocation::RunInTarget : System::RunLocation::RunInHost;
 
@@ -108,14 +118,7 @@ Calamares::JobResult CommandList::run()
     QString root = QStringLiteral( "/" );
     Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
 
-    bool needsRootSubstitution = false;
-    for ( CommandList::const_iterator i = cbegin(); i != cend(); ++i )
-        if ( i->command().contains( rootMagic ) )
-        {
-            needsRootSubstitution = true;
-            break;
-        }
-
+    bool needsRootSubstitution = findInCommands( *this, rootMagic );
     if ( needsRootSubstitution && ( location == System::RunLocation::RunInHost ) )
     {
         if ( !gs || !gs->contains( "rootMountPoint" ) )
@@ -127,10 +130,20 @@ Calamares::JobResult CommandList::run()
         root = gs->value( "rootMountPoint" ).toString();
     }
 
+    bool needsUserSubstitution = findInCommands( *this, userMagic );
+    if ( needsUserSubstitution && ( !gs || !gs->contains( "username" ) ) )
+    {
+        cError() << "No username defined.";
+        return Calamares::JobResult::error(
+            QCoreApplication::translate( "CommandList", "Could not run command." ),
+            QCoreApplication::translate( "CommandList", "The command needs to know the user's name, but no username is defined." ) );
+    }
+    QString user = gs->value( "username" ).toString();  // may be blank if unset
+
     for ( CommandList::const_iterator i = cbegin(); i != cend(); ++i )
     {
         QString processed_cmd = i->command();
-        processed_cmd.replace( rootMagic, root );
+        processed_cmd.replace( rootMagic, root ).replace( userMagic, user );
         bool suppress_result = false;
         if ( processed_cmd.startsWith( '-' ) )
         {
diff --git a/src/libcalamares/utils/CommandList.h b/src/libcalamares/utils/CommandList.h
index b766259c0..9faf705f2 100644
--- a/src/libcalamares/utils/CommandList.h
+++ b/src/libcalamares/utils/CommandList.h
@@ -74,6 +74,9 @@ using CommandList_t = QList< CommandLine >;
  * A list of commands; the list may have its own default timeout
  * for commands (which is then applied to each individual command
  * that doesn't have one of its own).
+ *
+ * Documentation for the format of commands can be found in
+ * `shellprocess.conf`.
  */
 class CommandList : protected CommandList_t
 {
diff --git a/src/modules/shellprocess/shellprocess.conf b/src/modules/shellprocess/shellprocess.conf
index ff53dc228..4734aaadd 100644
--- a/src/modules/shellprocess/shellprocess.conf
+++ b/src/modules/shellprocess/shellprocess.conf
@@ -4,9 +4,11 @@
 # If the top-level key *dontChroot* is true, then the commands
 # are executed in the context of the live system, otherwise
 # in the context of the target system. In all of the commands,
-# `@@ROOT@@` is replaced by the root mount point of the **target**
-# system from the point of view of the command (for chrooted
-# commands, that will be */*).
+# the following substitutions will take place:
+#  - `@@ROOT@@` is replaced by the root mount point of the **target**
+#    system from the point of view of the command (for chrooted
+#    commands, that will be */*).
+#  - `@@USER@@` is replaced by the username, set on the user page.
 #
 # The (global) timeout for the command list can be set with
 # the *timeout* key. The value is a time in seconds, default

From 8b00a03423b2c5fb6290e4aa7fefb5088d14d3e7 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 16 May 2018 11:36:59 -0400
Subject: [PATCH 08/21] [shellprocess] Test command-substitution

---
 src/modules/shellprocess/Tests.cpp | 13 +++++++++++++
 src/modules/shellprocess/Tests.h   |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/src/modules/shellprocess/Tests.cpp b/src/modules/shellprocess/Tests.cpp
index c6643325f..ebfb8046b 100644
--- a/src/modules/shellprocess/Tests.cpp
+++ b/src/modules/shellprocess/Tests.cpp
@@ -149,3 +149,16 @@ script:
     QCOMPARE( cl.at(0).command(), QStringLiteral( "ls /tmp" ) );
     QCOMPARE( cl.at(1).timeout(), -1 );  // not set
 }
+
+void ShellProcessTests::testRootSubstitution()
+{
+    YAML::Node doc = YAML::Load( R"(---
+script:
+    - "ls /tmp"
+)" );
+    QVariant script = CalamaresUtils::yamlMapToVariant( doc ).toMap().value( "script" );
+
+    // Doesn't use @@ROOT@@, so no failures
+    QVERIFY( bool(CommandList(script, true, 10 ).run()) );
+    QVERIFY( bool(CommandList(script, false, 10 ).run()) );
+}
diff --git a/src/modules/shellprocess/Tests.h b/src/modules/shellprocess/Tests.h
index af1f78487..5b4ebebbb 100644
--- a/src/modules/shellprocess/Tests.h
+++ b/src/modules/shellprocess/Tests.h
@@ -40,6 +40,8 @@ private Q_SLOTS:
     void testProcessFromObject();
     // Create from a complex YAML list
     void testProcessListFromObject();
+    // Check @@ROOT@@ substitution
+    void testRootSubstitution();
 };
 
 #endif

From 87b9c421588889b12defbae7da0c8cac597d9a82 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 09:53:03 -0400
Subject: [PATCH 09/21] [shellprocess] Don't crash test

 - May need to create a JobQueue before doing anything internal
 - May need to create global settings
 - Chroot always needs rootMountPath internally
---
 src/modules/shellprocess/Tests.cpp | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/modules/shellprocess/Tests.cpp b/src/modules/shellprocess/Tests.cpp
index ebfb8046b..55c5a9560 100644
--- a/src/modules/shellprocess/Tests.cpp
+++ b/src/modules/shellprocess/Tests.cpp
@@ -18,7 +18,11 @@
 
 #include "Tests.h"
 
+#include "JobQueue.h"
+#include "Settings.h"
+
 #include "utils/CommandList.h"
+#include "utils/Logger.h"
 #include "utils/YamlUtils.h"
 
 #include <yaml-cpp/yaml.h>
@@ -158,7 +162,17 @@ script:
 )" );
     QVariant script = CalamaresUtils::yamlMapToVariant( doc ).toMap().value( "script" );
 
+    if ( !Calamares::JobQueue::instance() )
+        (void *)new Calamares::JobQueue( nullptr );
+    if ( !Calamares::Settings::instance() )
+        (void *)new Calamares::Settings( QString(), true );
+
+    Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
+    QVERIFY( gs != nullptr );
+
     // Doesn't use @@ROOT@@, so no failures
-    QVERIFY( bool(CommandList(script, true, 10 ).run()) );
     QVERIFY( bool(CommandList(script, false, 10 ).run()) );
+
+    // Doesn't use @@ROOT@@, but does chroot, so fails
+    QVERIFY( !bool(CommandList(script, true, 10 ).run()) );
 }

From d3e57e9c9f3eb0464575365fc3cd6b9696a72662 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Wed, 23 May 2018 13:49:23 -0400
Subject: [PATCH 10/21] [shellprocess] Expand tests

These tests run (shell) commands as part of the test; this may be
a security problem, although I've tried to do things safely.
---
 src/modules/shellprocess/Tests.cpp | 41 +++++++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/modules/shellprocess/Tests.cpp b/src/modules/shellprocess/Tests.cpp
index 55c5a9560..7ebb8e624 100644
--- a/src/modules/shellprocess/Tests.cpp
+++ b/src/modules/shellprocess/Tests.cpp
@@ -18,6 +18,7 @@
 
 #include "Tests.h"
 
+#include "GlobalStorage.h"
 #include "JobQueue.h"
 #include "Settings.h"
 
@@ -160,7 +161,19 @@ void ShellProcessTests::testRootSubstitution()
 script:
     - "ls /tmp"
 )" );
-    QVariant script = CalamaresUtils::yamlMapToVariant( doc ).toMap().value( "script" );
+    QVariant plainScript = CalamaresUtils::yamlMapToVariant( doc ).toMap().value( "script" );
+    QVariant rootScript = CalamaresUtils::yamlMapToVariant(
+        YAML::Load( R"(---
+script:
+    - "ls @@ROOT@@"
+)" ) ).toMap().value( "script" );
+    QVariant userScript = CalamaresUtils::yamlMapToVariant(
+        YAML::Load( R"(---
+script:
+    - mktemp -d @@ROOT@@/calatestXXXXXXXX
+    - "chown @@USER@@ @@ROOT@@/calatest*"
+    - rm -rf @@ROOT@@/calatest*
+)" ) ).toMap().value( "script" );
 
     if ( !Calamares::JobQueue::instance() )
         (void *)new Calamares::JobQueue( nullptr );
@@ -170,9 +183,31 @@ script:
     Calamares::GlobalStorage* gs = Calamares::JobQueue::instance()->globalStorage();
     QVERIFY( gs != nullptr );
 
+    qDebug() << "Expect WARNING, ERROR, WARNING";
     // Doesn't use @@ROOT@@, so no failures
-    QVERIFY( bool(CommandList(script, false, 10 ).run()) );
+    QVERIFY( bool(CommandList(plainScript, false, 10 ).run()) );
 
     // Doesn't use @@ROOT@@, but does chroot, so fails
-    QVERIFY( !bool(CommandList(script, true, 10 ).run()) );
+    QVERIFY( !bool(CommandList(plainScript, true, 10 ).run()) );
+
+    // Does use @@ROOT@@, which is not set, so fails
+    QVERIFY( !bool(CommandList(rootScript, false, 10 ).run()) );
+    // .. fails for two reasons
+    QVERIFY( !bool(CommandList(rootScript, true, 10 ).run()) );
+
+    gs->insert( "rootMountPoint", "/tmp" );
+    // Now that the root is set, two variants work .. still can't
+    // chroot, unless the rootMountPoint contains a full system,
+    // *and* we're allowed to chroot (ie. running tests as root).
+    qDebug() << "Expect no output.";
+    QVERIFY( bool(CommandList(plainScript, false, 10 ).run()) );
+    QVERIFY( bool(CommandList(rootScript, false, 10 ).run()) );
+
+    qDebug() << "Expect ERROR";
+    // But no user set yet
+    QVERIFY( !bool(CommandList(userScript, false, 10 ).run()) );
+
+    // Now play dangerous games with shell expansion
+    gs->insert( "username", "`id -u`" );
+    QVERIFY( bool(CommandList(userScript, false, 10 ).run()) );
 }

From 45b95e1b65c403d37a4f03113d2de31c6b38decf Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 13:54:08 +0100
Subject: [PATCH 11/21] PythonQt: default to enabled

 - This just causes it to be enabled and used when present
   by default, rather than disabled by default (even when present).
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 113a491a1..0a9b26aee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -141,7 +141,7 @@ endif()
 
 option( INSTALL_CONFIG "Install configuration files" ON )
 option( WITH_PYTHON "Enable Python modules API (requires Boost.Python)." ON )
-option( WITH_PYTHONQT "Enable next generation Python modules API (experimental, requires PythonQt)." OFF )
+option( WITH_PYTHONQT "Enable next generation Python modules API (experimental, requires PythonQt)." ON )
 option( WITH_KF5Crash "Enable crash reporting with KCrash." ON )
 option( BUILD_TESTING "Build the testing tree." ON )
 

From f26ac63c078f1f18eff486fd2a96c7252961541b Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 14:22:24 +0100
Subject: [PATCH 12/21] [libcalamaresui] Make Python code const

 - This is always loaded into the Python context, so it may as well
   be done only once.
---
 src/libcalamaresui/modulesystem/PythonQtViewModule.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libcalamaresui/modulesystem/PythonQtViewModule.cpp b/src/libcalamaresui/modulesystem/PythonQtViewModule.cpp
index e2b497f2e..486d9a8e2 100644
--- a/src/libcalamaresui/modulesystem/PythonQtViewModule.cpp
+++ b/src/libcalamaresui/modulesystem/PythonQtViewModule.cpp
@@ -143,12 +143,12 @@ PythonQtViewModule::loadSelf()
             return;
         }
 
-        QString calamares_module_annotation =
+        static const QLatin1Literal calamares_module_annotation(
                 "_calamares_module_typename = ''\n"
                 "def calamares_module(viewmodule_type):\n"
                 "    global _calamares_module_typename\n"
                 "    _calamares_module_typename = viewmodule_type.__name__\n"
-                "    return viewmodule_type\n";
+                "    return viewmodule_type\n");
 
         // Load in the decorator
         PythonQt::self()->evalScript( cxt, calamares_module_annotation );

From 261c545476a4fa4a1b18524929dc0ec22a355bf1 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 15:04:20 +0100
Subject: [PATCH 13/21] [libcalamaresui] Refactor loading of YAML to
 QVariantMap

---
 src/libcalamares/utils/YamlUtils.cpp | 46 +++++++++++++++++++++++++++-
 src/libcalamares/utils/YamlUtils.h   | 12 +++++++-
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/src/libcalamares/utils/YamlUtils.cpp b/src/libcalamares/utils/YamlUtils.cpp
index 962cbd1da..3bc9c36d2 100644
--- a/src/libcalamares/utils/YamlUtils.cpp
+++ b/src/libcalamares/utils/YamlUtils.cpp
@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -23,6 +23,8 @@
 #include <yaml-cpp/yaml.h>
 
 #include <QByteArray>
+#include <QFile>
+#include <QFileInfo>
 #include <QRegExp>
 
 void
@@ -146,4 +148,46 @@ explainYamlException( const YAML::Exception& e, const QByteArray& yamlData, cons
     }
 }
 
+QVariantMap
+loadYaml(const QFileInfo& fi, bool* ok)
+{
+    return loadYaml( fi.absoluteFilePath(), ok );
+}
+
+QVariantMap
+loadYaml(const QString& filename, bool* ok)
+{
+    if ( ok )
+        *ok = false;
+
+    QFile descriptorFile( filename );
+    QVariant moduleDescriptor;
+    if ( descriptorFile.exists() && descriptorFile.open( QFile::ReadOnly | QFile::Text ) )
+    {
+        QByteArray ba = descriptorFile.readAll();
+        try
+        {
+            YAML::Node doc = YAML::Load( ba.constData() );
+            moduleDescriptor = CalamaresUtils::yamlToVariant( doc );
+        }
+        catch ( YAML::Exception& e )
+        {
+            explainYamlException( e, ba, filename.toLatin1().constData() );
+            return QVariantMap();
+        }
+    }
+
+
+    if ( moduleDescriptor.isValid() &&
+         !moduleDescriptor.isNull() &&
+         moduleDescriptor.type() == QVariant::Map )
+    {
+        if ( ok )
+            *ok = true;
+        return moduleDescriptor.toMap();
+    }
+
+    return QVariantMap();
+}
+
 }  // namespace
diff --git a/src/libcalamares/utils/YamlUtils.h b/src/libcalamares/utils/YamlUtils.h
index 33ee8dca0..ad8de7d57 100644
--- a/src/libcalamares/utils/YamlUtils.h
+++ b/src/libcalamares/utils/YamlUtils.h
@@ -1,7 +1,7 @@
 /* === This file is part of Calamares - <https://github.com/calamares> ===
  *
  *   Copyright 2014, Teo Mrnjavac <teo@kde.org>
- *   Copyright 2017, Adriaan de Groot <groot@kde.org>
+ *   Copyright 2017-2018, Adriaan de Groot <groot@kde.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include <QVariant>
 
 class QByteArray;
+class QFileInfo;
 
 namespace YAML
 {
@@ -35,6 +36,15 @@ void operator>>( const YAML::Node& node, QStringList& v );
 
 namespace CalamaresUtils
 {
+/**
+ * Loads a given @p filename and returns the YAML data
+ * as a QVariantMap. If filename doesn't exist, or is
+ * malformed in some way, returns an empty map and sets
+ * @p *ok to false. Otherwise sets @p *ok to true.
+ */
+QVariantMap loadYaml( const QString& filename, bool* ok = nullptr );
+/** Convenience overload. */
+QVariantMap loadYaml( const QFileInfo&, bool* ok = nullptr );
 
 QVariant yamlToVariant( const YAML::Node& node );
 QVariant yamlScalarToVariant( const YAML::Node& scalarNode );

From e5ca8e091f8ce36ac0b6d1753fc9fac753480258 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 15:06:13 +0100
Subject: [PATCH 14/21] [libcalamaresui] Use refactored loadYaml

---
 .../modulesystem/ModuleManager.cpp            | 41 ++++---------------
 1 file changed, 9 insertions(+), 32 deletions(-)

diff --git a/src/libcalamaresui/modulesystem/ModuleManager.cpp b/src/libcalamaresui/modulesystem/ModuleManager.cpp
index cbee61129..83273e924 100644
--- a/src/libcalamaresui/modulesystem/ModuleManager.cpp
+++ b/src/libcalamaresui/modulesystem/ModuleManager.cpp
@@ -103,39 +103,16 @@ ModuleManager::doInit()
                         continue;
                     }
 
-                    QFile descriptorFile( descriptorFileInfo.absoluteFilePath() );
-                    QVariant moduleDescriptor;
-                    if ( descriptorFile.exists() && descriptorFile.open( QFile::ReadOnly | QFile::Text ) )
+                    bool ok = false;
+                    QVariantMap moduleDescriptorMap = CalamaresUtils::loadYaml( descriptorFileInfo, &ok );
+                    QString moduleName = ok ? moduleDescriptorMap.value( "name" ).toString() : QString();
+
+                    if ( ok && ( moduleName == currentDir.dirName() ) &&
+                            !m_availableDescriptorsByModuleName.contains( moduleName ) )
                     {
-                        QByteArray ba = descriptorFile.readAll();
-                        try
-                        {
-                            YAML::Node doc = YAML::Load( ba.constData() );
-
-                            moduleDescriptor = CalamaresUtils::yamlToVariant( doc );
-                        }
-                        catch ( YAML::Exception& e )
-                        {
-                            cWarning() << "YAML parser error " << e.what();
-                            continue;
-                        }
-                    }
-
-
-                    if ( moduleDescriptor.isValid() &&
-                         !moduleDescriptor.isNull() &&
-                         moduleDescriptor.type() == QVariant::Map )
-                    {
-                        QVariantMap moduleDescriptorMap = moduleDescriptor.toMap();
-
-                        if ( moduleDescriptorMap.value( "name" ) == currentDir.dirName() &&
-                             !m_availableDescriptorsByModuleName.contains( moduleDescriptorMap.value( "name" ).toString() ) )
-                        {
-                            m_availableDescriptorsByModuleName.insert( moduleDescriptorMap.value( "name" ).toString(),
-                                                                       moduleDescriptorMap );
-                            m_moduleDirectoriesByModuleName.insert( moduleDescriptorMap.value( "name" ).toString(),
-                                                                    descriptorFileInfo.absoluteDir().absolutePath() );
-                        }
+                        m_availableDescriptorsByModuleName.insert( moduleName, moduleDescriptorMap );
+                        m_moduleDirectoriesByModuleName.insert( moduleName,
+                                                                descriptorFileInfo.absoluteDir().absolutePath() );
                     }
                 }
                 else

From fdda0e14aa04451f30a0f85ff5bc1b5642987ef2 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 20:07:06 +0100
Subject: [PATCH 15/21] [libcalamaresui] Improve explainYamlException

 - overloads for common kinds of label
 - improve error reporting when reading settings and branding files
---
 src/libcalamares/Settings.cpp        |  4 ++--
 src/libcalamares/utils/YamlUtils.cpp | 15 ++++++++++++++-
 src/libcalamares/utils/YamlUtils.h   |  2 ++
 src/libcalamaresui/Branding.cpp      |  4 ++--
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/libcalamares/Settings.cpp b/src/libcalamares/Settings.cpp
index 06178c621..732afa8d8 100644
--- a/src/libcalamares/Settings.cpp
+++ b/src/libcalamares/Settings.cpp
@@ -156,12 +156,12 @@ Settings::Settings( const QString& settingsFilePath,
         }
         catch ( YAML::Exception& e )
         {
-            cWarning() << "YAML parser error " << e.what() << "in" << file.fileName();
+            CalamaresUtils::explainYamlException( e, ba, file.fileName() );
         }
     }
     else
     {
-        cWarning() << "Cannot read " << file.fileName();
+        cWarning() << "Cannot read settings file" << file.fileName();
     }
 
     s_instance = this;
diff --git a/src/libcalamares/utils/YamlUtils.cpp b/src/libcalamares/utils/YamlUtils.cpp
index 3bc9c36d2..474ced2a1 100644
--- a/src/libcalamares/utils/YamlUtils.cpp
+++ b/src/libcalamares/utils/YamlUtils.cpp
@@ -114,6 +114,19 @@ void
 explainYamlException( const YAML::Exception& e, const QByteArray& yamlData, const char *label )
 {
     cWarning() << "YAML error " << e.what() << "in" << label << '.';
+    explainYamlException( e, yamlData );
+}
+
+void
+explainYamlException( const YAML::Exception& e, const QByteArray& yamlData, const QString& label )
+{
+    cWarning() << "YAML error " << e.what() << "in" << label << '.';
+    explainYamlException( e, yamlData );
+}
+
+void
+explainYamlException( const YAML::Exception& e, const QByteArray& yamlData )
+{
     if ( ( e.mark.line >= 0 ) && ( e.mark.column >= 0 ) )
     {
         // Try to show the line where it happened.
@@ -172,7 +185,7 @@ loadYaml(const QString& filename, bool* ok)
         }
         catch ( YAML::Exception& e )
         {
-            explainYamlException( e, ba, filename.toLatin1().constData() );
+            explainYamlException( e, ba, filename );
             return QVariantMap();
         }
     }
diff --git a/src/libcalamares/utils/YamlUtils.h b/src/libcalamares/utils/YamlUtils.h
index ad8de7d57..268c0bdc7 100644
--- a/src/libcalamares/utils/YamlUtils.h
+++ b/src/libcalamares/utils/YamlUtils.h
@@ -57,6 +57,8 @@ QVariant yamlMapToVariant( const YAML::Node& mapNode );
  * Uses @p label when labeling the data source (e.g. "netinstall data")
  */
 void explainYamlException( const YAML::Exception& e, const QByteArray& data, const char *label );
+void explainYamlException( const YAML::Exception& e, const QByteArray& data, const QString& label );
+void explainYamlException( const YAML::Exception& e, const QByteArray& data );
 
 } //ns
 
diff --git a/src/libcalamaresui/Branding.cpp b/src/libcalamaresui/Branding.cpp
index 584d85c0b..c71b1daa5 100644
--- a/src/libcalamaresui/Branding.cpp
+++ b/src/libcalamaresui/Branding.cpp
@@ -179,7 +179,7 @@ Branding::Branding( const QString& brandingFilePath,
         }
         catch ( YAML::Exception& e )
         {
-            cWarning() << "YAML parser error " << e.what() << "in" << file.fileName();
+            CalamaresUtils::explainYamlException( e, ba, file.fileName() );
         }
 
         QDir translationsDir( componentDir.filePath( "lang" ) );
@@ -192,7 +192,7 @@ Branding::Branding( const QString& brandingFilePath,
     }
     else
     {
-        cWarning() << "Cannot read " << file.fileName();
+        cWarning() << "Cannot read branding file" << file.fileName();
     }
 
     s_instance = this;

From 308f508c7ef2c4cdc3849f614c942b485c97eb17 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Mon, 26 Feb 2018 14:29:26 +0100
Subject: [PATCH 16/21] [calamares] Add a test-application.

This test-application should load a single module and execute it --
that can be used to quickly test configurations, loading, etc. This
is preparation for loading all sorts of Python modules by PythonQt.

The loader does some internals initialization and gets the module,
but doesn't actually run it yet.
---
 src/calamares/CMakeLists.txt |   6 ++
 src/calamares/testmain.cpp   | 153 +++++++++++++++++++++++++++++++++++
 2 files changed, 159 insertions(+)
 create mode 100644 src/calamares/testmain.cpp

diff --git a/src/calamares/CMakeLists.txt b/src/calamares/CMakeLists.txt
index 270abbb88..e1f8e4236 100644
--- a/src/calamares/CMakeLists.txt
+++ b/src/calamares/CMakeLists.txt
@@ -68,3 +68,9 @@ install( FILES ${CMAKE_SOURCE_DIR}/data/images/squid.svg
     RENAME calamares.svg
     DESTINATION ${CMAKE_INSTALL_DATADIR}/icons/hicolor/scalable/apps
 )
+
+if( BUILD_TESTING )
+    add_executable( loadmodule testmain.cpp )
+    target_link_libraries( loadmodule ${CALAMARES_LIBRARIES} Qt5::Core Qt5::Widgets calamaresui )
+    # Don't install, it's just for enable_testing
+endif()
diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp
new file mode 100644
index 000000000..dfd6ed7eb
--- /dev/null
+++ b/src/calamares/testmain.cpp
@@ -0,0 +1,153 @@
+/* === This file is part of Calamares - <https://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This executable loads and runs a Calamares Python module
+ * within a C++ application, in order to test the different
+ * bindings.
+ */
+
+#include "utils/Logger.h"
+#include "utils/YamlUtils.h"
+#include "modulesystem/Module.h"
+
+#include "Settings.h"
+
+#include <QCommandLineOption>
+#include <QCommandLineParser>
+#include <QCoreApplication>
+#include <QFileInfo>
+
+#include <memory>
+
+static QString
+handle_args( QCoreApplication& a )
+{
+    QCommandLineOption debugLevelOption( QStringLiteral("D"),
+                                          "Verbose output for debugging purposes (0-8).", "level" );
+
+    QCommandLineParser parser;
+    parser.setApplicationDescription( "Calamares module tester" );
+    parser.addHelpOption();
+    parser.addVersionOption();
+
+    parser.addOption( debugLevelOption );
+    parser.addPositionalArgument( "module", "Path or name of module to run." );
+
+    parser.process( a );
+
+    if ( parser.isSet( debugLevelOption ) )
+    {
+        bool ok = true;
+        int l = parser.value( debugLevelOption ).toInt( &ok );
+        unsigned int dlevel = 0;
+        if ( !ok || ( l < 0 ) )
+            dlevel = Logger::LOGVERBOSE;
+        else
+            dlevel = l;
+        Logger::setupLogLevel( dlevel );
+    }
+
+    const QStringList args = parser.positionalArguments();
+    if ( args.isEmpty() )
+    {
+        cError() << "Missing <module> path.\n";
+        parser.showHelp();
+        return QString();  // NOTREACHED
+    }
+    if ( args.size() > 1 )
+    {
+        cError() << "More than one <module> path.\n";
+        parser.showHelp();
+        return QString();  // NOTREACHED
+    }
+
+    return args.first();
+}
+
+
+static Calamares::Module*
+load_module( const QString& moduleName )
+{
+    QFileInfo fi;
+
+    bool ok = false;
+    QVariantMap descriptor;
+
+    for ( const QString& prefix : QStringList{ "./", "src/modules/", "modules/" } )
+    {
+        // Could be a complete path, eg. src/modules/dummycpp/module.desc
+        fi = QFileInfo( prefix + moduleName );
+        if ( fi.exists() && fi.isFile() )
+            descriptor = CalamaresUtils::loadYaml( fi, &ok );
+        if ( ok )
+            break;
+
+        // Could be a path without module.desc
+        fi = QFileInfo( prefix + moduleName );
+        if ( fi.exists() && fi.isDir() )
+        {
+            fi = QFileInfo( prefix + moduleName + "/module.desc" );
+            if ( fi.exists() && fi.isFile() )
+                descriptor = CalamaresUtils::loadYaml( fi, &ok );
+            if ( ok ) break;
+        }
+    }
+
+    if ( !ok )
+    {
+        cWarning() << "No suitable module descriptor found.";
+        return nullptr;
+    }
+
+    QString name = descriptor.value( "name" ).toString();
+    if ( name.isEmpty() )
+    {
+        cWarning() << "No name found in module descriptor" << fi.absoluteFilePath();
+        return nullptr;
+    }
+
+    QString moduleDirectory = fi.absolutePath();
+    QString configFile = moduleDirectory + '/' + name + ".conf";
+
+    Calamares::Module* module = Calamares::Module::fromDescriptor(
+        descriptor, name, configFile, moduleDirectory );
+
+    return module;
+}
+
+int
+main( int argc, char* argv[] )
+{
+    QCoreApplication a( argc, argv );
+
+    QString module = handle_args( a );
+    if ( module.isEmpty() )
+        return 1;
+
+    std::unique_ptr< Calamares::Settings > settings_p( new Calamares::Settings( QString(), true ) );
+    cDebug() << "Calamares test module-loader" << module;
+    Calamares::Module* m = load_module( module );
+    if ( !m )
+    {
+        cError() << "No module.desc data found in" << module;
+        return 1;
+    }
+
+    return 0;
+}

From 48771f968a68b0a54270bca6e08f1ffb2450397f Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Tue, 27 Feb 2018 01:04:35 +0100
Subject: [PATCH 17/21] [calamares] Load and execute the modules

This runs dummyprocess, at least, but the other three dummies coredump.
---
 src/calamares/testmain.cpp | 58 ++++++++++++++++++++++++++++++--------
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp
index dfd6ed7eb..1b5cb41f4 100644
--- a/src/calamares/testmain.cpp
+++ b/src/calamares/testmain.cpp
@@ -27,6 +27,7 @@
 #include "modulesystem/Module.h"
 
 #include "Settings.h"
+#include "Job.h"
 
 #include <QCommandLineOption>
 #include <QCommandLineParser>
@@ -35,7 +36,16 @@
 
 #include <memory>
 
-static QString
+struct ModuleConfig : public QPair< QString, QString >
+{
+    ModuleConfig( const QString& a, const QString& b ) : QPair< QString, QString >(a, b) { }
+    ModuleConfig() : QPair< QString, QString >( QString(), QString() ) { }
+
+    QString moduleName() const { return first; }
+    QString configFile() const { return second; }
+} ;
+
+static ModuleConfig
 handle_args( QCoreApplication& a )
 {
     QCommandLineOption debugLevelOption( QStringLiteral("D"),
@@ -68,22 +78,23 @@ handle_args( QCoreApplication& a )
     {
         cError() << "Missing <module> path.\n";
         parser.showHelp();
-        return QString();  // NOTREACHED
+        return ModuleConfig();  // NOTREACHED
     }
-    if ( args.size() > 1 )
+    if ( args.size() > 2 )
     {
         cError() << "More than one <module> path.\n";
         parser.showHelp();
-        return QString();  // NOTREACHED
+        return ModuleConfig();  // NOTREACHED
     }
 
-    return args.first();
+    return ModuleConfig( args.first(), args.size() == 2 ? args.at(1) : QString() );
 }
 
 
 static Calamares::Module*
-load_module( const QString& moduleName )
+load_module( const ModuleConfig& moduleConfig )
 {
+    QString moduleName = moduleConfig.moduleName();
     QFileInfo fi;
 
     bool ok = false;
@@ -123,7 +134,10 @@ load_module( const QString& moduleName )
     }
 
     QString moduleDirectory = fi.absolutePath();
-    QString configFile = moduleDirectory + '/' + name + ".conf";
+    QString configFile(
+        moduleConfig.configFile().isEmpty()
+        ? moduleDirectory + '/' + name + ".conf"
+        : moduleConfig.configFile() );
 
     Calamares::Module* module = Calamares::Module::fromDescriptor(
         descriptor, name, configFile, moduleDirectory );
@@ -136,18 +150,40 @@ main( int argc, char* argv[] )
 {
     QCoreApplication a( argc, argv );
 
-    QString module = handle_args( a );
-    if ( module.isEmpty() )
+    ModuleConfig module = handle_args( a );
+    if ( module.moduleName().isEmpty() )
         return 1;
 
     std::unique_ptr< Calamares::Settings > settings_p( new Calamares::Settings( QString(), true ) );
-    cDebug() << "Calamares test module-loader" << module;
+    cDebug() << "Calamares test module-loader" << module.moduleName();
     Calamares::Module* m = load_module( module );
     if ( !m )
     {
-        cError() << "No module.desc data found in" << module;
+        cError() << "No module.desc data found in" << module.moduleName();
         return 1;
     }
 
+    if ( !m->isLoaded() )
+        m->loadSelf();
+
+    if ( !m->isLoaded() )
+    {
+        cError() << "Module" << module.moduleName() << "could not be loaded.";
+        return 1;
+    }
+
+    cDebug() << "Module" << m->name() << m->typeString() << m->interfaceString();
+
+    Calamares::JobList jobList = m->jobs();
+    unsigned int count = 1;
+    for ( const auto& p : jobList )
+    {
+        cDebug() << count << p->prettyName();
+        Calamares::JobResult r = p->exec();
+        if ( !r )
+            cDebug() << count << ".. failed" << r;
+        ++count;
+    }
+
     return 0;
 }

From 182458ad5a0ae1819acfcf9fbc0ba06bda8c1dcd Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Tue, 27 Feb 2018 01:34:32 +0100
Subject: [PATCH 18/21] [calamares] Need a JobQueue and global storage before
 running any job.

The 'singleton' instances don't do a very good job of being singletons
or ensuring their own creation.
---
 src/calamares/testmain.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp
index 1b5cb41f4..4dcbfea96 100644
--- a/src/calamares/testmain.cpp
+++ b/src/calamares/testmain.cpp
@@ -28,6 +28,7 @@
 
 #include "Settings.h"
 #include "Job.h"
+#include "JobQueue.h"
 
 #include <QCommandLineOption>
 #include <QCommandLineParser>
@@ -155,6 +156,8 @@ main( int argc, char* argv[] )
         return 1;
 
     std::unique_ptr< Calamares::Settings > settings_p( new Calamares::Settings( QString(), true ) );
+    std::unique_ptr< Calamares::JobQueue > jobqueue_p( new Calamares::JobQueue( nullptr ) );
+
     cDebug() << "Calamares test module-loader" << module.moduleName();
     Calamares::Module* m = load_module( module );
     if ( !m )

From cdadc2f0036b40fc07188a3005b986102b51c44a Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Thu, 1 Mar 2018 12:24:00 +0100
Subject: [PATCH 19/21] [libcalamares] Improve error logging during module
 loading

---
 src/calamares/testmain.cpp                 | 2 +-
 src/libcalamaresui/modulesystem/Module.cpp | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp
index 4dcbfea96..c22342f9a 100644
--- a/src/calamares/testmain.cpp
+++ b/src/calamares/testmain.cpp
@@ -162,7 +162,7 @@ main( int argc, char* argv[] )
     Calamares::Module* m = load_module( module );
     if ( !m )
     {
-        cError() << "No module.desc data found in" << module.moduleName();
+        cError() << "Could not load module" << module.moduleName();
         return 1;
     }
 
diff --git a/src/libcalamaresui/modulesystem/Module.cpp b/src/libcalamaresui/modulesystem/Module.cpp
index f80f4d48b..05394e69f 100644
--- a/src/libcalamaresui/modulesystem/Module.cpp
+++ b/src/libcalamaresui/modulesystem/Module.cpp
@@ -86,7 +86,7 @@ Module::fromDescriptor( const QVariantMap& moduleDescriptor,
 #ifdef WITH_PYTHONQT
             m = new PythonQtViewModule();
 #else
-            cError() << "PythonQt modules are not supported in this version of Calamares.";
+            cError() << "PythonQt view modules are not supported in this version of Calamares.";
 #endif
         }
         else
@@ -118,7 +118,7 @@ Module::fromDescriptor( const QVariantMap& moduleDescriptor,
 
     if ( !m )
     {
-        cDebug() << "Bad module type (" << typeString
+        cError() << "Bad module type (" << typeString
             << ") or interface string (" << intfString
             << ") for module " << instanceId;
         return nullptr;
@@ -129,8 +129,7 @@ Module::fromDescriptor( const QVariantMap& moduleDescriptor,
         m->m_directory = moduleDir.absolutePath();
     else
     {
-        cError() << "Bad module directory" << moduleDirectory
-               << "for" << instanceId;
+        cError() << "Bad module directory" << moduleDirectory << "for" << instanceId;
         delete m;
         return nullptr;
     }
@@ -144,7 +143,7 @@ Module::fromDescriptor( const QVariantMap& moduleDescriptor,
     }
     catch ( YAML::Exception& e )
     {
-        cWarning() << "YAML parser error " << e.what();
+        cError() << "YAML parser error " << e.what();
         delete m;
         return nullptr;
     }

From ce3e09318a7fa343949dc232f7a8e05ae3024a88 Mon Sep 17 00:00:00 2001
From: Adriaan de Groot <groot@kde.org>
Date: Thu, 24 May 2018 07:03:19 -0400
Subject: [PATCH 20/21] [preservefiles] Improve failure messages

---
 src/modules/preservefiles/PreserveFiles.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/modules/preservefiles/PreserveFiles.cpp b/src/modules/preservefiles/PreserveFiles.cpp
index 6490f8303..29564fcd1 100644
--- a/src/modules/preservefiles/PreserveFiles.cpp
+++ b/src/modules/preservefiles/PreserveFiles.cpp
@@ -148,10 +148,15 @@ Calamares::JobResult PreserveFiles::exec()
 void PreserveFiles::setConfigurationMap(const QVariantMap& configurationMap)
 {
     auto files = configurationMap[ "files" ];
-
-    if ( ! ( files.isValid() && ( files.type() == QVariant::List ) ) )
+    if ( !files.isValid() )
     {
-        cDebug() << "No files: configuration key, or not a list.";
+        cDebug() << "No 'files' key for preservefiles.";
+        return;
+    }
+
+    if ( files.type() != QVariant::List )
+    {
+        cDebug() << "Configuration key 'files' is not a list for preservefiles.";
         return;
     }
 

From fc8d961049be3de32a0391672eba8af535b24c0a Mon Sep 17 00:00:00 2001
From: Philip <philm@manjaro.org>
Date: Sat, 26 May 2018 09:49:27 +0200
Subject: [PATCH 21/21] [ci] fix link in HACKING.md

---
 ci/HACKING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/HACKING.md b/ci/HACKING.md
index dfc768be9..ca3901f07 100644
--- a/ci/HACKING.md
+++ b/ci/HACKING.md
@@ -44,7 +44,7 @@ it's just a typo-fix which might not be copyrightable in all jurisdictions).
 Formatting C++
 --------------
 This formatting guide applies to C++ code only; for Python modules, we use
-[pycodestyle][https://github.com/PyCQA/pycodestyle] to apply a check of
+[pycodestyle](https://github.com/PyCQA/pycodestyle) to apply a check of
 some PEP8 guidelines.
 
 * Spaces, not tabs.