Merge branch 'rawfs-compat'

Improve testing framework while adding tests to rawfs to double-
check that the conversion (for Python 3.3 compatibility) of
capture_output is correct.

CMakeModules/CalamaresAddModuleSubdirectory.cmake

@@ -168,5 +168,24 @@ function( calamares_add_module_subdirectory )
             COMMAND loadmodule ${SUBDIRECTORY}
             WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
             )
+        # Try it with the tests/ configurations shipped with the module
+        set( _count 1 )
+        set( _testdir ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIRECTORY}/tests )
+        while ( EXISTS "${_testdir}/${_count}.global" OR EXISTS "${_testdir}/${_count}.job" )
+            set( _dash_g "" )
+            set( _dash_j "" )
+            if ( EXISTS "${_testdir}/${_count}.global" )
+                set( _dash_g -g ${_testdir}/${_count}.global )
+            endif()
+            if ( EXISTS "${_testdir}/${_count}.job" )
+                set( _dash_j -j ${_testdir}/${_count}.job )
+            endif()
+            add_test(
+                NAME load-${SUBDIRECTORY}-${_count}
+                COMMAND loadmodule ${_dash_g} ${_dash_j} ${SUBDIRECTORY}
+                WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+                )
+            math( EXPR _count "${_count} + 1" )
+        endwhile()
     endif()
 endfunction()

src/calamares/CMakeLists.txt

@@ -60,7 +60,10 @@ install( FILES ${CMAKE_SOURCE_DIR}/data/images/squid.svg
 )
 
 if( BUILD_TESTING )
+    # Don't install, these are just for enable_testing
     add_executable( loadmodule testmain.cpp )
     target_link_libraries( loadmodule PRIVATE Qt5::Core Qt5::Widgets calamares calamaresui )
-    # Don't install, it's just for enable_testing
+
+    add_executable( test_conf test_conf.cpp )
+    target_link_libraries( test_conf PUBLIC yamlcpp Qt5::Core )
 endif()

src/modules/CMakeLists.txt

@@ -10,12 +10,6 @@ include_directories(
     ${CMAKE_SOURCE_DIR}/src/libcalamaresui
 )
 
-if( BUILD_TESTING )
-    add_executable( test_conf test_conf.cpp )
-    target_link_libraries( test_conf yamlcpp Qt5::Core )
-    target_include_directories( test_conf PUBLIC ${YAMLCPP_INCLUDE_DIR} )
-endif()
-
 string( REPLACE " " ";" SKIP_LIST "${SKIP_MODULES}" )
 
 file( GLOB SUBDIRECTORIES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*" )

src/modules/globalStorage.yaml

@@ -1,18 +0,0 @@
----
-syntax: "YAML map of anything"
-example:
-    whats_this: "fake global storage contents"
-    from_where: "globalStorage.yaml"
-a_list:
-    - "item1"
-    - "item2"
-    - "item3"
-    - "item4"
-a_list_of_maps:
-    - name: "an Item"
-      contents:
-        - "an element"
-        - "another element"
-    - name: "another item"
-      contents:
-        - "not much"

src/modules/rawfs/main.py

@@ -96,6 +96,8 @@ class RawFSItem:
         count = 0
 
         libcalamares.utils.debug("Copying {} to {}".format(self.source, self.destination))
+        if libcalamares.job.configuration["bogus"]:
+            return
 
         srcsize, srcblksize = get_device_size(self.source)
         destsize, destblksize = get_device_size(self.destination)
@@ -171,6 +173,7 @@ def run():
         return (_("Configuration Error"),
                 _("No partitions are defined for <pre>{!s}</pre> to use." ).format("rawfs"))
 
+    libcalamares.utils.debug("Copying {!s} raw partitions.".format(len(partitions)))
     for partition in partitions:
         if partition["mountPoint"]:
             for src in libcalamares.job.configuration["targets"]:

src/modules/rawfs/rawfs.conf

@@ -22,3 +22,8 @@ targets:
       resize: true
     - mountPoint: /data
       source: /dev/mmcblk0p3
+
+# To support testing, set the *bogus* key to true. No actual work is done, but the
+# module's logic is exercised.
+
+# bogus: false

src/modules/rawfs/tests/1.global

@@ -0,0 +1,9 @@
+---
+rootMountPoint: /tmp/unpackfs-test-run-rootdir3/
+partitions:
+  - mountPoint: /
+    device: /dev/sda1
+    fs: ext4
+  - mountPoint: /home
+    device: /dev/sda2
+    fs: ext4

src/modules/rawfs/tests/1.job

@@ -0,0 +1,14 @@
+# Testing configuration for rawfs
+
+---
+
+targets:
+    - mountPoint: /
+      source: /
+    - mountPoint: /home
+      source: /images/home.img
+      resize: true
+    - mountPoint: /data
+      source: /dev/mmcblk0p3
+
+bogus: true

src/modules/testmodule.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# === This file is part of Calamares - <https://github.com/calamares> ===
-#
-#   Copyright 2014, Teo Mrnjavac <teo@kde.org>
-#   Copyright 2017, Adriaan de Groot <groot@kde.org>
-#
-#   Calamares is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
-#
-#   Calamares is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-#   GNU General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
-"""
-Testing tool to run a single Python module; optionally a global configuration
-and module configuration can be read from YAML files.
-"""
-argumentepilog = """
-moduledir may be a module name (e.g. "welcome") or a full path to the
-    module (e.g. "src/modules/welcome"). In the former case, an attempt
-    is made to find the module in several sensible places.
-globalstorage_yaml may be given as a full path to a YAML file containing
-    the global configuration, or as "" or "-" which will leave the
-    global storage empty.
-configuration_yaml may be given as a full path to a YAML file with the
-    module configuration, as "" or "-" to leave the configuration
-    empty, or as "+" to load the standard configuration from the
-    module-directory (e.g. welcome.conf if the welcome module is given).
-
-The simplest invocation to test a module, with its default configuration, is
-to call this program as follows (for, e.g., the welcome module):
-
-    testmodule.py welcome - +"""
-
-import argparse
-import os
-import sys
-
-import yaml
-
-calamaresimporterror = ("Can not import libcalamares. Ensure the PYTHONPATH "
-    "environment variable includes the dir where libcalamares.so is "
-    "installed.")
-try:
-    import libcalamares
-except ImportError:
-    print(calamaresimporterror)
-    print()
-    libcalamares = None
-
-
-class Job:
-    """
-
-    :param working_path:
-    :param doc:
-    :param cfg_doc:
-    """
-
-    def __init__(self, working_path, doc, cfg_doc):
-        self.module_name = doc["name"]
-        self.pretty_name = "Testing job " + doc["name"]
-        self.working_path = working_path
-        self.configuration = cfg_doc
-
-    def setprogress(self, progress):
-        """
-
-        :param progress:
-        """
-        print("Job set progress to {}%.".format(progress * 100))
-
-
-def test_module(moduledir, globalconfigfilename, moduleconfigfilename, lang):
-    print("Testing module in: " + moduledir)
-
-    confpath = os.path.join(moduledir, "module.desc")
-    with open(confpath) as f:
-        doc = yaml.load(f)
-
-    if doc["type"] != "job" or doc["interface"] != "python":
-        print("Only Python jobs can be tested.")
-        return 1
-
-    # Parameter None creates a new, empty GlobalStorage
-    libcalamares.globalstorage = libcalamares.GlobalStorage(None)
-    libcalamares.globalstorage.insert("testing", True)
-    if lang:
-        libcalamares.globalstorage.insert("locale", lang)
-        libcalamares.globalstorage.insert("localeConf", {"LANG": lang})
-
-    # if a file for simulating globalStorage contents is provided, load it
-    if globalconfigfilename:
-        with open(globalconfigfilename) as f:
-            gs_doc = yaml.load(f)
-        for key, value in gs_doc.items():
-            libcalamares.globalstorage.insert(key, value)
-        print("Global configuration '" + globalconfigfilename + "' loaded.")
-    else:
-        print("No global configuration loaded.")
-
-    cfg_doc = dict()
-    if moduleconfigfilename:
-        with open(moduleconfigfilename) as f:
-            cfg_doc = yaml.load(f)
-            print("Local configuration '" + moduleconfigfilename + "' loaded.")
-    else:
-        print("No module configuration loaded.")
-
-    libcalamares.job = Job(moduledir, doc, cfg_doc)
-
-    scriptpath = os.path.abspath(moduledir)
-    sys.path.append(scriptpath)
-    import main  # Assumed to import main from module itself
-
-    print("Output from module:")
-    print(main.run())
-
-    return 0
-
-
-def munge_filename(filename, module=None):
-    """
-    Maps files "" (empty) and "-" (just a dash) to None,
-    to simplify processing elsewhere.
-    """
-    if not filename or filename == "-":
-        return None
-    if filename == "+" and module is not None:
-        d, name = os.path.split(module)
-        if d and not name:
-            # Ended in a /
-            d, name = os.path.split(module)
-        if name:
-            return os.path.join(module, name + ".conf")
-
-    return filename
-
-
-def find_module(modulename):
-    if "/" in modulename:
-        return modulename
-    else:
-        for prefix in ("src/modules", "build/src/modules", "../src/modules"):
-            mp = os.path.join( prefix, modulename )
-            if os.path.exists( mp ):
-                return mp
-    # Not found? Bail out elsewhere
-    return modulename
-
-def main():
-    parser = argparse.ArgumentParser(description=globals()["__doc__"], epilog=argumentepilog, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("moduledir",
-                        help="Dir containing the Python module.")
-    parser.add_argument("globalstorage_yaml", nargs="?",
-                        help="A yaml file to initialize GlobalStorage.")
-    parser.add_argument("configuration_yaml", nargs="?",
-                        help="A yaml file to initialize the Job.")
-    parser.add_argument("--lang", "-l", nargs="?", default=None,
-                        help="Set translation language.")
-    args = parser.parse_args()
-
-    # If we get here, it wasn't a --help invocation, so complain
-    # if libcalamares wasn't found.
-    if not libcalamares:
-        parser.error(calamaresimporterror)
-
-    moduledir = find_module(args.moduledir)
-    return test_module(moduledir,
-                       munge_filename(args.globalstorage_yaml),
-                       munge_filename(args.configuration_yaml, moduledir),
-                       args.lang)
-
-
-if __name__ == "__main__":
-    sys.exit(main())

src/modules/testpythonrun.sh

@@ -1,55 +0,0 @@
-#! /bin/sh
-
-### Command-line validation
-#
-#
-SRCDIR=$( dirname "$0" )
-test -d "$SRCDIR" || { echo "! Can't find source directory." ; exit 1 ; }
-
-MODULE="$1"
-test -n "$MODULE" || { echo "! Usage: $0 <module>" ; exit 1 ; }
-
-### Run-time validation
-#
-# .. switch SRCDIR to the module that has been found
-BINDIR="$SRCDIR"  # Keep original SRCDIR
-SRCDIR="$SRCDIR/$MODULE"
-XSRCDIR="src/modules/$MODULE"  # In builddir
-TESTDIR="$SRCDIR/tests"
-
-test -x "$BINDIR/testmodule.py" || { echo "! No support script $BINDIR/testmodule.py" ; exit 1 ; }
-test -d "$SRCDIR" || { echo "! Source $SRCDIR is not a directory." ; exit 1 ; }
-test -f "$TESTDIR/1.global" || { echo "! Source $SRCDIR has no tests." ; exit 1 ; }
-
-test -f "libcalamares.so" || { echo "! Run the tests from the build-directory." ; exit 1 ; }
-test -d "$XSRCDIR" || { echo "! No module directory $XSRCDIR in build-dir." ; exit 1 ; }
-
-### Python setup
-#
-#
-export PYTHONPATH=".:$PYTHONPATH"
-PYTHON=$( which python3 2> /dev/null )
-if test -z "$PYTHON" ; then
-	PYTHON=$( which python 2> /dev/null )
-fi
-test -x "$PYTHON" || { echo "! No suitable Python executable found." ; exit 1 ; }
-
-### Test-execution
-#
-#
-C=0
-while true ; do
-	# Might use shell arithmetic, but need other shebang then
-	C=$( expr "$C" + 1 )
-
-	G_CFG="$TESTDIR/$C.global"
-	J_CFG="$TESTDIR/$C.job"
-
-	test -f "$G_CFG" || break
-	if test -f "$J_CFG" ; then
-		$PYTHON "$BINDIR/testmodule.py" "$XSRCDIR" "$G_CFG" "$J_CFG"
-	else
-		$PYTHON "$BINDIR/testmodule.py" "$XSRCDIR" "$G_CFG"
-	fi
-done
-