From dc87c11ac4a7aa9fed787dc93f497312fb9e9e82 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 8 May 2020 20:58:17 +0200 Subject: [PATCH 1/4] [rawfs] Slightly more debugging --- src/modules/rawfs/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/rawfs/main.py b/src/modules/rawfs/main.py index 670080f1c..d0f235372 100644 --- a/src/modules/rawfs/main.py +++ b/src/modules/rawfs/main.py @@ -171,6 +171,7 @@ def run(): return (_("Configuration Error"), _("No partitions are defined for 
{!s}
to use." ).format("rawfs")) + libcalamares.utils.debug("Copying {!s} raw partitions.".format(len(partitions))) for partition in partitions: if partition["mountPoint"]: for src in libcalamares.job.configuration["targets"]: From 74fb88f9ac674ccbe2aacf2e6a9e1298b692c50c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 8 May 2020 21:07:17 +0200 Subject: [PATCH 2/4] [rawfs] Minor testing support - set *bogus* to do no harm - add some test configurations --- src/modules/rawfs/main.py | 2 ++ src/modules/rawfs/rawfs.conf | 5 +++++ src/modules/rawfs/tests/1.global | 9 +++++++++ src/modules/rawfs/tests/1.job | 14 ++++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 src/modules/rawfs/tests/1.global create mode 100644 src/modules/rawfs/tests/1.job diff --git a/src/modules/rawfs/main.py b/src/modules/rawfs/main.py index d0f235372..1898d8fea 100644 --- a/src/modules/rawfs/main.py +++ b/src/modules/rawfs/main.py @@ -96,6 +96,8 @@ class RawFSItem: count = 0 libcalamares.utils.debug("Copying {} to {}".format(self.source, self.destination)) + if libcalamares.job.configuration["bogus"]: + return srcsize, srcblksize = get_device_size(self.source) destsize, destblksize = get_device_size(self.destination) diff --git a/src/modules/rawfs/rawfs.conf b/src/modules/rawfs/rawfs.conf index 10e654a7d..8d7da3ba3 100644 --- a/src/modules/rawfs/rawfs.conf +++ b/src/modules/rawfs/rawfs.conf @@ -22,3 +22,8 @@ targets: resize: true - mountPoint: /data source: /dev/mmcblk0p3 + +# To support testing, set the *bogus* key to true. No actual work is done, but the +# module's logic is exercised. + +# bogus: false diff --git a/src/modules/rawfs/tests/1.global b/src/modules/rawfs/tests/1.global new file mode 100644 index 000000000..c09df9375 --- /dev/null +++ b/src/modules/rawfs/tests/1.global @@ -0,0 +1,9 @@ +--- +rootMountPoint: /tmp/unpackfs-test-run-rootdir3/ +partitions: + - mountPoint: / + device: /dev/sda1 + fs: ext4 + - mountPoint: /home + device: /dev/sda2 + fs: ext4 diff --git a/src/modules/rawfs/tests/1.job b/src/modules/rawfs/tests/1.job new file mode 100644 index 000000000..6079c43ed --- /dev/null +++ b/src/modules/rawfs/tests/1.job @@ -0,0 +1,14 @@ +# Testing configuration for rawfs + +--- + +targets: + - mountPoint: / + source: / + - mountPoint: /home + source: /images/home.img + resize: true + - mountPoint: /data + source: /dev/mmcblk0p3 + +bogus: true From 91f88cebf09465c78c58b75145d1979e35e09e97 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 8 May 2020 21:20:22 +0200 Subject: [PATCH 3/4] [calamares] Move testing application out of intermediate dir - src/modules doesn't seem like a really good place for stray sources for a testing application. Move it. --- src/calamares/CMakeLists.txt | 5 ++++- src/{modules => calamares}/test_conf.cpp | 0 src/modules/CMakeLists.txt | 6 ------ 3 files changed, 4 insertions(+), 7 deletions(-) rename src/{modules => calamares}/test_conf.cpp (100%) diff --git a/src/calamares/CMakeLists.txt b/src/calamares/CMakeLists.txt index 336c27317..5c5a68a97 100644 --- a/src/calamares/CMakeLists.txt +++ b/src/calamares/CMakeLists.txt @@ -60,7 +60,10 @@ install( FILES ${CMAKE_SOURCE_DIR}/data/images/squid.svg ) if( BUILD_TESTING ) + # Don't install, these are just for enable_testing add_executable( loadmodule testmain.cpp ) target_link_libraries( loadmodule PRIVATE Qt5::Core Qt5::Widgets calamares calamaresui ) - # Don't install, it's just for enable_testing + + add_executable( test_conf test_conf.cpp ) + target_link_libraries( test_conf PUBLIC yamlcpp Qt5::Core ) endif() diff --git a/src/modules/test_conf.cpp b/src/calamares/test_conf.cpp similarity index 100% rename from src/modules/test_conf.cpp rename to src/calamares/test_conf.cpp diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt index b72b755d3..533c704f8 100644 --- a/src/modules/CMakeLists.txt +++ b/src/modules/CMakeLists.txt @@ -10,12 +10,6 @@ include_directories( ${CMAKE_SOURCE_DIR}/src/libcalamaresui ) -if( BUILD_TESTING ) - add_executable( test_conf test_conf.cpp ) - target_link_libraries( test_conf yamlcpp Qt5::Core ) - target_include_directories( test_conf PUBLIC ${YAMLCPP_INCLUDE_DIR} ) -endif() - string( REPLACE " " ";" SKIP_LIST "${SKIP_MODULES}" ) file( GLOB SUBDIRECTORIES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*" ) From 1ae38c8ebc84d3839fa518ba31d69e841f7e3680 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 8 May 2020 21:50:11 +0200 Subject: [PATCH 4/4] CMake: add tests when there are test-configs - If a module has tests/#.global or tests/#.job, these are used as arguments to a test-run of loadmodule (which reads them and runs the module with that configuration). - This makes the old python-loading test and test-runner entirely obsolete, so remove them too. --- .../CalamaresAddModuleSubdirectory.cmake | 19 ++ src/modules/globalStorage.yaml | 18 -- src/modules/testmodule.py | 183 ------------------ src/modules/testpythonrun.sh | 55 ------ 4 files changed, 19 insertions(+), 256 deletions(-) delete mode 100644 src/modules/globalStorage.yaml delete mode 100755 src/modules/testmodule.py delete mode 100644 src/modules/testpythonrun.sh diff --git a/CMakeModules/CalamaresAddModuleSubdirectory.cmake b/CMakeModules/CalamaresAddModuleSubdirectory.cmake index 53b952b54..1af520ca8 100644 --- a/CMakeModules/CalamaresAddModuleSubdirectory.cmake +++ b/CMakeModules/CalamaresAddModuleSubdirectory.cmake @@ -168,5 +168,24 @@ function( calamares_add_module_subdirectory ) COMMAND loadmodule ${SUBDIRECTORY} WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) + # Try it with the tests/ configurations shipped with the module + set( _count 1 ) + set( _testdir ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIRECTORY}/tests ) + while ( EXISTS "${_testdir}/${_count}.global" OR EXISTS "${_testdir}/${_count}.job" ) + set( _dash_g "" ) + set( _dash_j "" ) + if ( EXISTS "${_testdir}/${_count}.global" ) + set( _dash_g -g ${_testdir}/${_count}.global ) + endif() + if ( EXISTS "${_testdir}/${_count}.job" ) + set( _dash_j -j ${_testdir}/${_count}.job ) + endif() + add_test( + NAME load-${SUBDIRECTORY}-${_count} + COMMAND loadmodule ${_dash_g} ${_dash_j} ${SUBDIRECTORY} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + math( EXPR _count "${_count} + 1" ) + endwhile() endif() endfunction() diff --git a/src/modules/globalStorage.yaml b/src/modules/globalStorage.yaml deleted file mode 100644 index 1187fe93a..000000000 --- a/src/modules/globalStorage.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -syntax: "YAML map of anything" -example: - whats_this: "fake global storage contents" - from_where: "globalStorage.yaml" -a_list: - - "item1" - - "item2" - - "item3" - - "item4" -a_list_of_maps: - - name: "an Item" - contents: - - "an element" - - "another element" - - name: "another item" - contents: - - "not much" diff --git a/src/modules/testmodule.py b/src/modules/testmodule.py deleted file mode 100755 index 633d57d9f..000000000 --- a/src/modules/testmodule.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# === This file is part of Calamares - === -# -# Copyright 2014, Teo Mrnjavac -# Copyright 2017, Adriaan de Groot -# -# Calamares is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Calamares is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Calamares. If not, see . -""" -Testing tool to run a single Python module; optionally a global configuration -and module configuration can be read from YAML files. -""" -argumentepilog = """ -moduledir may be a module name (e.g. "welcome") or a full path to the - module (e.g. "src/modules/welcome"). In the former case, an attempt - is made to find the module in several sensible places. -globalstorage_yaml may be given as a full path to a YAML file containing - the global configuration, or as "" or "-" which will leave the - global storage empty. -configuration_yaml may be given as a full path to a YAML file with the - module configuration, as "" or "-" to leave the configuration - empty, or as "+" to load the standard configuration from the - module-directory (e.g. welcome.conf if the welcome module is given). - -The simplest invocation to test a module, with its default configuration, is -to call this program as follows (for, e.g., the welcome module): - - testmodule.py welcome - +""" - -import argparse -import os -import sys - -import yaml - -calamaresimporterror = ("Can not import libcalamares. Ensure the PYTHONPATH " - "environment variable includes the dir where libcalamares.so is " - "installed.") -try: - import libcalamares -except ImportError: - print(calamaresimporterror) - print() - libcalamares = None - - -class Job: - """ - - :param working_path: - :param doc: - :param cfg_doc: - """ - - def __init__(self, working_path, doc, cfg_doc): - self.module_name = doc["name"] - self.pretty_name = "Testing job " + doc["name"] - self.working_path = working_path - self.configuration = cfg_doc - - def setprogress(self, progress): - """ - - :param progress: - """ - print("Job set progress to {}%.".format(progress * 100)) - - -def test_module(moduledir, globalconfigfilename, moduleconfigfilename, lang): - print("Testing module in: " + moduledir) - - confpath = os.path.join(moduledir, "module.desc") - with open(confpath) as f: - doc = yaml.load(f) - - if doc["type"] != "job" or doc["interface"] != "python": - print("Only Python jobs can be tested.") - return 1 - - # Parameter None creates a new, empty GlobalStorage - libcalamares.globalstorage = libcalamares.GlobalStorage(None) - libcalamares.globalstorage.insert("testing", True) - if lang: - libcalamares.globalstorage.insert("locale", lang) - libcalamares.globalstorage.insert("localeConf", {"LANG": lang}) - - # if a file for simulating globalStorage contents is provided, load it - if globalconfigfilename: - with open(globalconfigfilename) as f: - gs_doc = yaml.load(f) - for key, value in gs_doc.items(): - libcalamares.globalstorage.insert(key, value) - print("Global configuration '" + globalconfigfilename + "' loaded.") - else: - print("No global configuration loaded.") - - cfg_doc = dict() - if moduleconfigfilename: - with open(moduleconfigfilename) as f: - cfg_doc = yaml.load(f) - print("Local configuration '" + moduleconfigfilename + "' loaded.") - else: - print("No module configuration loaded.") - - libcalamares.job = Job(moduledir, doc, cfg_doc) - - scriptpath = os.path.abspath(moduledir) - sys.path.append(scriptpath) - import main # Assumed to import main from module itself - - print("Output from module:") - print(main.run()) - - return 0 - - -def munge_filename(filename, module=None): - """ - Maps files "" (empty) and "-" (just a dash) to None, - to simplify processing elsewhere. - """ - if not filename or filename == "-": - return None - if filename == "+" and module is not None: - d, name = os.path.split(module) - if d and not name: - # Ended in a / - d, name = os.path.split(module) - if name: - return os.path.join(module, name + ".conf") - - return filename - - -def find_module(modulename): - if "/" in modulename: - return modulename - else: - for prefix in ("src/modules", "build/src/modules", "../src/modules"): - mp = os.path.join( prefix, modulename ) - if os.path.exists( mp ): - return mp - # Not found? Bail out elsewhere - return modulename - -def main(): - parser = argparse.ArgumentParser(description=globals()["__doc__"], epilog=argumentepilog, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("moduledir", - help="Dir containing the Python module.") - parser.add_argument("globalstorage_yaml", nargs="?", - help="A yaml file to initialize GlobalStorage.") - parser.add_argument("configuration_yaml", nargs="?", - help="A yaml file to initialize the Job.") - parser.add_argument("--lang", "-l", nargs="?", default=None, - help="Set translation language.") - args = parser.parse_args() - - # If we get here, it wasn't a --help invocation, so complain - # if libcalamares wasn't found. - if not libcalamares: - parser.error(calamaresimporterror) - - moduledir = find_module(args.moduledir) - return test_module(moduledir, - munge_filename(args.globalstorage_yaml), - munge_filename(args.configuration_yaml, moduledir), - args.lang) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/modules/testpythonrun.sh b/src/modules/testpythonrun.sh deleted file mode 100644 index be2b7800b..000000000 --- a/src/modules/testpythonrun.sh +++ /dev/null @@ -1,55 +0,0 @@ -#! /bin/sh - -### Command-line validation -# -# -SRCDIR=$( dirname "$0" ) -test -d "$SRCDIR" || { echo "! Can't find source directory." ; exit 1 ; } - -MODULE="$1" -test -n "$MODULE" || { echo "! Usage: $0 " ; exit 1 ; } - -### Run-time validation -# -# .. switch SRCDIR to the module that has been found -BINDIR="$SRCDIR" # Keep original SRCDIR -SRCDIR="$SRCDIR/$MODULE" -XSRCDIR="src/modules/$MODULE" # In builddir -TESTDIR="$SRCDIR/tests" - -test -x "$BINDIR/testmodule.py" || { echo "! No support script $BINDIR/testmodule.py" ; exit 1 ; } -test -d "$SRCDIR" || { echo "! Source $SRCDIR is not a directory." ; exit 1 ; } -test -f "$TESTDIR/1.global" || { echo "! Source $SRCDIR has no tests." ; exit 1 ; } - -test -f "libcalamares.so" || { echo "! Run the tests from the build-directory." ; exit 1 ; } -test -d "$XSRCDIR" || { echo "! No module directory $XSRCDIR in build-dir." ; exit 1 ; } - -### Python setup -# -# -export PYTHONPATH=".:$PYTHONPATH" -PYTHON=$( which python3 2> /dev/null ) -if test -z "$PYTHON" ; then - PYTHON=$( which python 2> /dev/null ) -fi -test -x "$PYTHON" || { echo "! No suitable Python executable found." ; exit 1 ; } - -### Test-execution -# -# -C=0 -while true ; do - # Might use shell arithmetic, but need other shebang then - C=$( expr "$C" + 1 ) - - G_CFG="$TESTDIR/$C.global" - J_CFG="$TESTDIR/$C.job" - - test -f "$G_CFG" || break - if test -f "$J_CFG" ; then - $PYTHON "$BINDIR/testmodule.py" "$XSRCDIR" "$G_CFG" "$J_CFG" - else - $PYTHON "$BINDIR/testmodule.py" "$XSRCDIR" "$G_CFG" - fi -done -