Merge branch 'improve-testing'

src/calamares/testmain.cpp

@@ -26,9 +26,10 @@
 #include "utils/YamlUtils.h"
 #include "modulesystem/Module.h"
 
-#include "Settings.h"
+#include "GlobalStorage.h"
 #include "Job.h"
 #include "JobQueue.h"
+#include "Settings.h"
 
 #include <QCommandLineOption>
 #include <QCommandLineParser>
@@ -37,13 +38,17 @@
 
 #include <memory>
 
-struct ModuleConfig : public QPair< QString, QString >
+struct ModuleConfig
 {
-    ModuleConfig( const QString& a, const QString& b ) : QPair< QString, QString >(a, b) { }
+    QString moduleName() const { return m_module; }
-    ModuleConfig() : QPair< QString, QString >( QString(), QString() ) { }
+    QString configFile() const { return m_jobConfig; }
+    QString language() const { return m_language; }
+    QString globalConfigFile() const { return m_globalConfig; }
 
-    QString moduleName() const { return first; }
+    QString m_module;
-    QString configFile() const { return second; }
+    QString m_jobConfig;
+    QString m_globalConfig;
+    QString m_language;
 } ;
 
 static ModuleConfig
@@ -51,6 +56,12 @@ handle_args( QCoreApplication& a )
 {
     QCommandLineOption debugLevelOption( QStringLiteral("D"),
                                           "Verbose output for debugging purposes (0-8).", "level" );
+    QCommandLineOption globalOption( QStringList() << QStringLiteral( "g" ) << QStringLiteral( "global "),
+                                     QStringLiteral( "Global settings document" ), "global.yaml" );
+    QCommandLineOption jobOption( QStringList() << QStringLiteral( "j" ) << QStringLiteral( "job"),
+                                  QStringLiteral( "Job settings document" ), "job.yaml" );
+    QCommandLineOption langOption( QStringList() << QStringLiteral( "l" ) << QStringLiteral( "language" ),
+                                   QStringLiteral( "Language (global)" ), "languagecode" );
 
     QCommandLineParser parser;
     parser.setApplicationDescription( "Calamares module tester" );
@@ -58,8 +69,11 @@ handle_args( QCoreApplication& a )
     parser.addVersionOption();
 
     parser.addOption( debugLevelOption );
+    parser.addOption( globalOption );
+    parser.addOption( jobOption );
+    parser.addOption( langOption );
     parser.addPositionalArgument( "module", "Path or name of module to run." );
-    parser.addPositionalArgument( "config", "Path of job-config file to use.", "[config]");
+    parser.addPositionalArgument( "job.yaml", "Path of job settings document to use.", "[job.yaml]");
 
     parser.process( a );
 
@@ -89,7 +103,11 @@ handle_args( QCoreApplication& a )
         return ModuleConfig();  // NOTREACHED
     }
 
-    return ModuleConfig( args.first(), args.size() == 2 ? args.at(1) : QString() );
+    QString jobSettings( parser.value( jobOption ) );
+    if ( jobSettings.isEmpty() && ( args.size() == 2 ) )
+        jobSettings = args.at(1);
+
+    return ModuleConfig{ args.first(), jobSettings, parser.value( globalOption ), parser.value( langOption ) };
 }
 
 
@@ -161,6 +179,17 @@ main( int argc, char* argv[] )
     std::unique_ptr< Calamares::Settings > settings_p( new Calamares::Settings( QString(), true ) );
     std::unique_ptr< Calamares::JobQueue > jobqueue_p( new Calamares::JobQueue( nullptr ) );
 
+    auto gs = jobqueue_p->globalStorage();
+    if ( !module.globalConfigFile().isEmpty() )
+        gs->loadYaml( module.globalConfigFile() );
+    if ( !module.language().isEmpty() )
+    {
+        QVariantMap vm;
+        vm.insert( "LANG", module.language() );
+        gs->insert( "localeConf", vm );
+    }
+
+
     cDebug() << "Calamares module-loader testing" << module.moduleName();
     Calamares::Module* m = load_module( module );
     if ( !m )

src/libcalamares/GlobalStorage.cpp

@@ -21,6 +21,7 @@
 #include "JobQueue.h"
 
 #include "utils/Logger.h"
+#include "utils/YamlUtils.h"
 
 #include <QFile>
 #include <QJsonDocument>
@@ -110,6 +111,23 @@ GlobalStorage::save(const QString& filename)
 }
 
 
+bool
+GlobalStorage::saveYaml( const QString& filename )
+{
+    return CalamaresUtils::saveYaml( filename, m );
+}
+
+bool
+GlobalStorage::loadYaml( const QString& filename )
+{
+    bool ok = false;
+    auto gs = CalamaresUtils::loadYaml( filename, &ok );
+    if ( ok )
+        m = gs;
+    return ok;
+}
+
+
 } // namespace Calamares
 
 #ifdef WITH_PYTHON

src/libcalamares/GlobalStorage.h

@@ -60,6 +60,7 @@ public:
 
     /// @brief dump keys and values to the debug log
     void debugDump() const;
+
     /** @brief write as JSON to the given filename
      *
      * No tidying, sanitization, or censoring is done -- for instance,
@@ -69,6 +70,15 @@ public:
      */
     bool save( const QString& filename );
 
+    /** @brief write as YAML to the given filename
+     *
+     * See also save(), above.
+     */
+    bool saveYaml( const QString& filename );
+
+    /// @brief reads settings from the given filename
+    bool loadYaml( const QString& filename );
+
 signals:
     void changed();
 

src/libcalamares/Tests.cpp

@@ -19,6 +19,7 @@
 #include "Tests.h"
 
 #include "utils/Logger.h"
+#include "utils/YamlUtils.h"
 
 #include <QtTest/QtTest>
 
@@ -57,3 +58,54 @@ LibCalamaresTests::testDebugLevels()
     }
 }
 
+void
+LibCalamaresTests::testLoadSaveYaml()
+{
+    QFile f( "settings.conf" );
+    QVERIFY( f.exists() );
+
+    auto map = CalamaresUtils::loadYaml( "settings.conf" );
+    CalamaresUtils::saveYaml( "out.yaml", map );
+
+    auto other_map = CalamaresUtils::loadYaml( "out.yaml" );
+    CalamaresUtils::saveYaml(" out2.yaml", other_map );
+    QCOMPARE( map, other_map );
+
+    QFile::remove( "out.yaml" );
+    QFile::remove( "out2.yaml" );
+}
+
+static QStringList
+findConf( const QDir& d )
+{
+    QStringList mine;
+    if ( d.exists() )
+    {
+        QString path = d.absolutePath();
+        path.append( d.separator() );
+        for ( const auto& confname : d.entryList( { "*.conf" } ) )
+            mine.append( path + confname );
+        for ( const auto& subdirname : d.entryList( QDir::AllDirs | QDir::NoDotAndDotDot ) )
+        {
+            QDir subdir( d );
+            subdir.cd( subdirname );
+            mine.append( findConf( subdir ) );
+        }
+    }
+    return mine;
+}
+
+
+void
+LibCalamaresTests::testLoadSaveYamlExtended()
+{
+    for ( const auto& confname : findConf( QDir( "../src" ) ) )
+    {
+        cDebug() << "Testing" << confname;
+        auto map = CalamaresUtils::loadYaml( confname );
+        QVERIFY( CalamaresUtils::saveYaml( "out.yaml", map ) );
+        auto othermap = CalamaresUtils::loadYaml( "out.yaml" );
+        QCOMPARE( map, othermap );
+    }
+    QFile::remove( "out.yaml" );
+}

src/libcalamares/Tests.h

@@ -31,6 +31,9 @@ public:
 private Q_SLOTS:
     void initTestCase();
     void testDebugLevels();
+
+    void testLoadSaveYaml();  // Just settings.conf
+    void testLoadSaveYamlExtended();  // Do a find() in the src dir
 };
 
 #endif

src/libcalamares/utils/YamlUtils.cpp

@@ -173,15 +173,15 @@ loadYaml(const QString& filename, bool* ok)
     if ( ok )
         *ok = false;
 
-    QFile descriptorFile( filename );
+    QFile yamlFile( filename );
-    QVariant moduleDescriptor;
+    QVariant yamlContents;
-    if ( descriptorFile.exists() && descriptorFile.open( QFile::ReadOnly | QFile::Text ) )
+    if ( yamlFile.exists() && yamlFile.open( QFile::ReadOnly | QFile::Text ) )
     {
-        QByteArray ba = descriptorFile.readAll();
+        QByteArray ba = yamlFile.readAll();
         try
         {
             YAML::Node doc = YAML::Load( ba.constData() );
-            moduleDescriptor = CalamaresUtils::yamlToVariant( doc );
+            yamlContents = CalamaresUtils::yamlToVariant( doc );
         }
         catch ( YAML::Exception& e )
         {
@@ -191,16 +191,108 @@ loadYaml(const QString& filename, bool* ok)
     }
 
 
-    if ( moduleDescriptor.isValid() &&
+    if ( yamlContents.isValid() &&
-         !moduleDescriptor.isNull() &&
+         !yamlContents.isNull() &&
-         moduleDescriptor.type() == QVariant::Map )
+            yamlContents.type() == QVariant::Map )
     {
         if ( ok )
             *ok = true;
-        return moduleDescriptor.toMap();
+        return yamlContents.toMap();
     }
 
     return QVariantMap();
 }
 
+/// @brief Convenience function writes @p indent times four spaces
+static void
+writeIndent( QFile& f, int indent )
+{
+    while ( indent-- > 0 )
+        f.write( "  " );
+}
+
+// forward declaration
+static bool dumpYaml( QFile& f, const QVariantMap& map, int indent );
+
+// It's a quote
+static const char quote[] = "\"";
+static const char newline[] = "\n";
+
+/// @brief Recursive helper to dump a single value
+static void
+dumpYamlElement( QFile& f, const QVariant& value, int indent )
+{
+    if ( value.type() == QVariant::Type::Bool )
+        f.write( value.toBool() ? "true" : "false" );
+    else if ( value.type() == QVariant::Type::String )
+    {
+        f.write( quote );
+        f.write( value.toString().toUtf8() );
+        f.write( quote );
+    }
+    else if ( value.type() == QVariant::Type::Int )
+    {
+        f.write( QString::number( value.toInt() ).toUtf8() );
+    }
+    else if ( value.type() == QVariant::Type::Double )
+    {
+        f.write( QString::number( value.toDouble() ).toUtf8() );
+    }
+    else if ( value.type() == QVariant::Type::List )
+    {
+        int c = 0;
+        for ( const auto& it : value.toList() )
+        {
+            ++c;
+            f.write( newline );
+            writeIndent( f, indent+1 );
+            f.write( "- " );
+            dumpYamlElement( f, it, indent+1 );
+        }
+        if ( !c )  // i.e. list was empty
+            f.write( "[]" );
+    }
+    else if ( value.type() == QVariant::Type::Map )
+    {
+        f.write( newline );
+        dumpYaml( f, value.toMap(), indent+1 );
+    }
+    else
+    {
+        f.write( "<" );
+        f.write( value.typeName() );
+        f.write( ">" );
+    }
+}
+
+/// @brief Recursive helper to dump @p map to file
+static bool
+dumpYaml( QFile& f, const QVariantMap& map, int indent )
+{
+    for ( auto it = map.cbegin(); it != map.cend(); ++it )
+    {
+        writeIndent( f, indent );
+        f.write( quote );
+        f.write( it.key().toUtf8() );
+        f.write( quote );
+        f.write( ": " );
+        dumpYamlElement( f, it.value(), indent );
+        f.write( newline );
+    }
+    return true;
+}
+
+bool
+saveYaml( const QString& filename, const QVariantMap& map )
+{
+    QFile f( filename );
+    if ( !f.open( QFile::WriteOnly ) )
+        return false;
+
+    f.write( "# YAML dump\n---\n" );
+    return dumpYaml( f, map, 0 );
+}
+
+
+
 }  // namespace

src/libcalamares/utils/YamlUtils.h

@@ -51,6 +51,9 @@ QVariant yamlScalarToVariant( const YAML::Node& scalarNode );
 QVariant yamlSequenceToVariant( const YAML::Node& sequenceNode );
 QVariant yamlMapToVariant( const YAML::Node& mapNode );
 
+/// @brief Save a @p map to @p filename as YAML
+bool saveYaml( const QString& filename, const QVariantMap& map );
+
 /**
  * Given an exception from the YAML parser library, explain
  * what is going on in terms of the data passed to the parser.

src/modules/unpackfs/tests/8.global

@@ -0,0 +1,4 @@
+---
+rootMountPoint: /tmp/unpackfs-test-run-rootdir/
+localeConf:
+   - LANG: nl

src/modules/unpackfs/tests/8.job

@@ -0,0 +1,5 @@
+---
+unpack:
+   - source: .
+     sourcefs: ext4
+     destination: realdest