diff --git a/CHANGES-3.2 b/CHANGES-3.2 index 6cf5a99ba..c2b0416da 100644 --- a/CHANGES-3.2 +++ b/CHANGES-3.2 @@ -10,13 +10,21 @@ website will have to do for older versions. # 3.2.48 (unreleased) # This release contains contributions from (alphabetically by first name): - - No external contributors yet + - Evan James ## Core ## - - No core changes yet + - Python modules now have `warn()` and `error()` methods they can call, + alongside the existing `debug()` and `warning()` (all live in the + *libcalamares.utils* module). + - Python modules can load YAML files via `libcalamares.utils.load_yaml()`. + This may be the most useful for test-scripts. ## Modules ## - - No module changes yet + - The *packages* module now has some special settings for the `pacman` + package manager (generally used on Arch-derivatives). This allows + tweaking of the installation process, if downloads are slow or + packages may fail to install. See the `packages.conf` file for + details. (Thanks Evan) # 3.2.47 (2021-11-19) # diff --git a/src/libcalamares/PythonJob.cpp b/src/libcalamares/PythonJob.cpp index afeebbe07..291adbc54 100644 --- a/src/libcalamares/PythonJob.cpp +++ b/src/libcalamares/PythonJob.cpp @@ -79,13 +79,25 @@ BOOST_PYTHON_MODULE( libcalamares ) bp::scope utilsScope = utilsModule; Q_UNUSED( utilsScope ) + // .. Logging functions bp::def( "debug", &CalamaresPython::debug, bp::args( "s" ), "Writes the given string to the Calamares debug stream." ); bp::def( "warning", &CalamaresPython::warning, bp::args( "s" ), "Writes the given string to the Calamares warning stream." ); + bp::def( "warn", + &CalamaresPython::warning, + bp::args( "s" ), + "Writes the given string to the Calamares warning stream." ); + bp::def( + "error", &CalamaresPython::warning, bp::args( "s" ), "Writes the given string to the Calamares error stream." ); + + // .. YAML functions + bp::def( "load_yaml", &CalamaresPython::load_yaml, bp::args( "path" ), "Loads YAML from a file." ); + + // .. Filesystem functions bp::def( "mount", &CalamaresPython::mount, mount_overloads( bp::args( "device_path", "mount_point", "filesystem_name", "options" ), @@ -94,6 +106,8 @@ BOOST_PYTHON_MODULE( libcalamares ) "-1 = QProcess crash\n" "-2 = QProcess cannot start\n" "-3 = bad arguments" ) ); + + // .. Process functions bp::def( "target_env_call", static_cast< int ( * )( const std::string&, const std::string&, int ) >( &CalamaresPython::target_env_call ), @@ -152,6 +166,7 @@ BOOST_PYTHON_MODULE( libcalamares ) host_env_process_output_overloads( bp::args( "command", "callback", "stdin", "timeout" ), "Runs the specified command in the host system." ) ); + // .. String functions bp::def( "obscure", &CalamaresPython::obscure, bp::args( "s" ), @@ -160,7 +175,7 @@ BOOST_PYTHON_MODULE( libcalamares ) "Applying the function to a string obscured by this function will result " "in the original string." ); - + // .. Translation functions bp::def( "gettext_languages", &CalamaresPython::gettext_languages, "Returns list of languages (most to least-specific) for gettext." ); diff --git a/src/libcalamares/PythonJobApi.cpp b/src/libcalamares/PythonJobApi.cpp index 1713569a4..bb2b8749e 100644 --- a/src/libcalamares/PythonJobApi.cpp +++ b/src/libcalamares/PythonJobApi.cpp @@ -19,6 +19,7 @@ #include "utils/RAII.h" #include "utils/Runner.h" #include "utils/String.h" +#include "utils/Yaml.h" #include #include @@ -139,19 +140,44 @@ check_target_env_output( const bp::list& args, const std::string& stdin, int tim } static const char output_prefix[] = "[PYTHON JOB]:"; +static inline void +log_action( unsigned int level, const std::string& s ) +{ + Logger::CDebug( level ) << output_prefix << QString::fromStdString( s ); +} void debug( const std::string& s ) { - Logger::CDebug( Logger::LOGDEBUG ) << output_prefix << QString::fromStdString( s ); + log_action( Logger::LOGDEBUG, s ); } void warning( const std::string& s ) { - Logger::CDebug( Logger::LOGWARNING ) << output_prefix << QString::fromStdString( s ); + log_action( Logger::LOGWARNING, s ); } +void +error( const std::string& s ) +{ + log_action( Logger::LOGERROR, s ); +} + +boost::python::dict +load_yaml( const std::string& path ) +{ + const QString filePath = QString::fromStdString( path ); + bool ok = false; + auto map = CalamaresUtils::loadYaml( filePath, &ok ); + if ( !ok ) + { + cWarning() << "Loading YAML from" << filePath << "failed."; + } + return variantMapToPyDict( map ); +} + + PythonJobInterface::PythonJobInterface( Calamares::PythonJob* parent ) : m_parent( parent ) { diff --git a/src/libcalamares/PythonJobApi.h b/src/libcalamares/PythonJobApi.h index 48bd4f87c..62346ceda 100644 --- a/src/libcalamares/PythonJobApi.h +++ b/src/libcalamares/PythonJobApi.h @@ -60,6 +60,12 @@ boost::python::list gettext_languages(); void debug( const std::string& s ); void warning( const std::string& s ); +void error( const std::string& s ); + +/** @brief Loads YAML and returns (nested) dicts representing it + * + */ +boost::python::dict load_yaml( const std::string& path ); class PythonJobInterface { diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 518ff4907..10371777e 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -403,9 +403,9 @@ class PMPacman(PackageManager): if type(pacman) is not dict: libcalamares.utils.warning("Job configuration *pacman* will be ignored.") pacman = dict() - self.pacman_num_retries = pacman.get("pacman_num_retries", 0) - self.pacman_disable_timeout = pacman.get("pacman_disable_download_timeout", False) - self.pacman_needed_only = pacman.get("pacman_needed_only", False) + self.pacman_num_retries = pacman.get("num_retries", 0) + self.pacman_disable_timeout = pacman.get("disable_download_timeout", False) + self.pacman_needed_only = pacman.get("needed_only", False) def reset_progress(self): self.in_package_changes = False diff --git a/src/modules/packages/tests/1.global b/src/modules/packages/tests/1.global new file mode 100644 index 000000000..ee06ccfe1 --- /dev/null +++ b/src/modules/packages/tests/1.global @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +rootMountPoint: /tmp diff --git a/src/modules/packages/test.yaml b/src/modules/packages/tests/2.job similarity index 90% rename from src/modules/packages/test.yaml rename to src/modules/packages/tests/2.job index 130214dfd..ba205ed44 100644 --- a/src/modules/packages/test.yaml +++ b/src/modules/packages/tests/2.job @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: no # SPDX-License-Identifier: CC0-1.0 backend: dummy -rootMountPoint: /tmp/mount operations: - install: - pre-script: touch /tmp/foo diff --git a/src/modules/packages/tests/CMakeTests.txt b/src/modules/packages/tests/CMakeTests.txt new file mode 100644 index 000000000..4f7d6185f --- /dev/null +++ b/src/modules/packages/tests/CMakeTests.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +# +# We have tests to load (some) of the package-managers specifically, to +# test their configuration code and implementation. Those tests conventionally +# live in Python files here in the tests/ directory. Add them. + +# Pacman (Arch) tests +set(_pm pacman) +add_test( + NAME configure-packages-${_pm} + COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) +add_test( + NAME configure-packages-${_pm}-ops-1 + COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py ${CMAKE_CURRENT_LIST_DIR}/pm-pacman-1.yaml 4 1 1 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) +add_test( + NAME configure-packages-${_pm}-ops-2 + COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py ${CMAKE_CURRENT_LIST_DIR}/pm-pacman-2.yaml 3 0 0 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) + +if ( BUILD_TESTING AND BUILD_SCHEMA_TESTING AND PYTHONINTERP_FOUND AND PYTHON_EXECUTABLE ) + set( _module packages ) + set( _schema_file "${CMAKE_CURRENT_SOURCE_DIR}/${_module}/${_module}.schema.yaml" ) + message(STATUS "Schema ${_schema_file}") + foreach( _cf pm-pacman-1.yaml pm-pacman-2.yaml ) + set( _conf_file "${CMAKE_CURRENT_SOURCE_DIR}/${_module}/tests/${_cf}" ) + if ( EXISTS "${_schema_file}" AND EXISTS "${_conf_file}" ) + add_test( + NAME validate-packages-${_cf} + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/ci/configvalidator.py" "${_schema_file}" "${_conf_file}" + ) + else() + message(FATAL_ERROR "Missing ${_conf_file}") + endif() + endforeach() +endif() + diff --git a/src/modules/packages/tests/pm-pacman-1.yaml b/src/modules/packages/tests/pm-pacman-1.yaml new file mode 100644 index 000000000..e876bd0fe --- /dev/null +++ b/src/modules/packages/tests/pm-pacman-1.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +backend: pacman +operations: [] + +pacman: + num_retries: 4 + disable_download_timeout: yes + needed_only: true + diff --git a/src/modules/packages/tests/pm-pacman-2.yaml b/src/modules/packages/tests/pm-pacman-2.yaml new file mode 100644 index 000000000..8b0bda397 --- /dev/null +++ b/src/modules/packages/tests/pm-pacman-2.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +backend: pacman +operations: [] + +# Leave some things unspecified +pacman: + num_retries: 3 + diff --git a/src/modules/packages/tests/test-pm-pacman.py b/src/modules/packages/tests/test-pm-pacman.py new file mode 100644 index 000000000..ee814b620 --- /dev/null +++ b/src/modules/packages/tests/test-pm-pacman.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +# +# Calamares Boilerplate +import libcalamares +libcalamares.globalstorage = libcalamares.GlobalStorage(None) +libcalamares.globalstorage.insert("testing", True) + +# Module prep-work +from src.modules.packages import main + +# .. we don't have a job in this test, so fake one +class Job(object): + def __init__(self, filename): + self.configuration = libcalamares.utils.load_yaml(filename) if filename is not None else dict() + +import sys +if len(sys.argv) > 4: + filename = sys.argv[1] + retry = int(sys.argv[2]) + timeout = bool(int(sys.argv[3])) + needed = bool(int(sys.argv[4])) +else: + filename = None + retry = 0 + timeout = False + needed = False + +libcalamares.utils.warning("Expecting {!s} retry={!s} timeout={!s} needed={!s}".format(filename, retry, timeout, needed)) + +# Specific PM test +libcalamares.job = Job(filename) +p = main.PMPacman() +assert p.pacman_num_retries == retry, "{!r} vs {!r}".format(p.pacman_num_retries, retry) +assert p.pacman_disable_timeout == timeout, "{!r} vs {!r}".format(p.pacman_disable_timeout, timeout) +assert p.pacman_needed_only == needed, "{!r} vs {!r}".format(p.pacman_needed_only, needed)