Merge branch 'fixup-packages' into calamares

- Add a changelog entry for what has been done.

CHANGES-3.2

@@ -10,13 +10,21 @@ website will have to do for older versions.
 # 3.2.48 (unreleased) #
 
 This release contains contributions from (alphabetically by first name):
- - No external contributors yet
+ - Evan James
 
 ## Core ##
- - No core changes yet
+ - Python modules now have `warn()` and `error()` methods they can call,
+   alongside the existing `debug()` and `warning()` (all live in the
+   *libcalamares.utils* module).
+ - Python modules can load YAML files via `libcalamares.utils.load_yaml()`.
+   This may be the most useful for test-scripts.
 
 ## Modules ##
- - No module changes yet
+ - The *packages* module now has some special settings for the `pacman`
+   package manager (generally used on Arch-derivatives). This allows
+   tweaking of the installation process, if downloads are slow or
+   packages may fail to install. See the `packages.conf` file for
+   details. (Thanks Evan)
 
 
 # 3.2.47 (2021-11-19) #

src/libcalamares/PythonJob.cpp

@@ -79,13 +79,25 @@ BOOST_PYTHON_MODULE( libcalamares )
     bp::scope utilsScope = utilsModule;
     Q_UNUSED( utilsScope )
 
+    // .. Logging functions
     bp::def(
         "debug", &CalamaresPython::debug, bp::args( "s" ), "Writes the given string to the Calamares debug stream." );
     bp::def( "warning",
              &CalamaresPython::warning,
              bp::args( "s" ),
              "Writes the given string to the Calamares warning stream." );
+    bp::def( "warn",
+             &CalamaresPython::warning,
+             bp::args( "s" ),
+             "Writes the given string to the Calamares warning stream." );
+    bp::def(
+        "error", &CalamaresPython::warning, bp::args( "s" ), "Writes the given string to the Calamares error stream." );
 
+
+    // .. YAML functions
+    bp::def( "load_yaml", &CalamaresPython::load_yaml, bp::args( "path" ), "Loads YAML from a file." );
+
+    // .. Filesystem functions
     bp::def( "mount",
              &CalamaresPython::mount,
              mount_overloads( bp::args( "device_path", "mount_point", "filesystem_name", "options" ),
@@ -94,6 +106,8 @@ BOOST_PYTHON_MODULE( libcalamares )
                               "-1 = QProcess crash\n"
                               "-2 = QProcess cannot start\n"
                               "-3 = bad arguments" ) );
+
+    // .. Process functions
     bp::def(
         "target_env_call",
         static_cast< int ( * )( const std::string&, const std::string&, int ) >( &CalamaresPython::target_env_call ),
@@ -152,6 +166,7 @@ BOOST_PYTHON_MODULE( libcalamares )
              host_env_process_output_overloads( bp::args( "command", "callback", "stdin", "timeout" ),
                                                 "Runs the specified command in the host system." ) );
 
+    // .. String functions
     bp::def( "obscure",
              &CalamaresPython::obscure,
              bp::args( "s" ),
@@ -160,7 +175,7 @@ BOOST_PYTHON_MODULE( libcalamares )
              "Applying the function to a string obscured by this function will result "
              "in the original string." );
 
-
+    // .. Translation functions
     bp::def( "gettext_languages",
              &CalamaresPython::gettext_languages,
              "Returns list of languages (most to least-specific) for gettext." );

src/libcalamares/PythonJobApi.cpp

@@ -19,6 +19,7 @@
 #include "utils/RAII.h"
 #include "utils/Runner.h"
 #include "utils/String.h"
+#include "utils/Yaml.h"
 
 #include <QCoreApplication>
 #include <QDir>
@@ -139,19 +140,44 @@ check_target_env_output( const bp::list& args, const std::string& stdin, int tim
 }
 
 static const char output_prefix[] = "[PYTHON JOB]:";
+static inline void
+log_action( unsigned int level, const std::string& s )
+{
+    Logger::CDebug( level ) << output_prefix << QString::fromStdString( s );
+}
 
 void
 debug( const std::string& s )
 {
-    Logger::CDebug( Logger::LOGDEBUG ) << output_prefix << QString::fromStdString( s );
+    log_action( Logger::LOGDEBUG, s );
 }
 
 void
 warning( const std::string& s )
 {
-    Logger::CDebug( Logger::LOGWARNING ) << output_prefix << QString::fromStdString( s );
+    log_action( Logger::LOGWARNING, s );
 }
 
+void
+error( const std::string& s )
+{
+    log_action( Logger::LOGERROR, s );
+}
+
+boost::python::dict
+load_yaml( const std::string& path )
+{
+    const QString filePath = QString::fromStdString( path );
+    bool ok = false;
+    auto map = CalamaresUtils::loadYaml( filePath, &ok );
+    if ( !ok )
+    {
+        cWarning() << "Loading YAML from" << filePath << "failed.";
+    }
+    return variantMapToPyDict( map );
+}
+
+
 PythonJobInterface::PythonJobInterface( Calamares::PythonJob* parent )
     : m_parent( parent )
 {

src/libcalamares/PythonJobApi.h

@@ -60,6 +60,12 @@ boost::python::list gettext_languages();
 
 void debug( const std::string& s );
 void warning( const std::string& s );
+void error( const std::string& s );
+
+/** @brief Loads YAML and returns (nested) dicts representing it
+ *
+ */
+boost::python::dict load_yaml( const std::string& path );
 
 class PythonJobInterface
 {

src/modules/packages/main.py

@@ -403,9 +403,9 @@ class PMPacman(PackageManager):
         if type(pacman) is not dict:
             libcalamares.utils.warning("Job configuration *pacman* will be ignored.")
             pacman = dict()
-        self.pacman_num_retries = pacman.get("pacman_num_retries", 0)
+        self.pacman_num_retries = pacman.get("num_retries", 0)
-        self.pacman_disable_timeout = pacman.get("pacman_disable_download_timeout", False)
+        self.pacman_disable_timeout = pacman.get("disable_download_timeout", False)
-        self.pacman_needed_only = pacman.get("pacman_needed_only", False)
+        self.pacman_needed_only = pacman.get("needed_only", False)
 
     def reset_progress(self):
         self.in_package_changes = False

src/modules/packages/tests/1.global

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: no
+# SPDX-License-Identifier: CC0-1.0
+rootMountPoint: /tmp

src/modules/packages/tests/2.job

@@ -1,7 +1,6 @@
 # SPDX-FileCopyrightText: no
 # SPDX-License-Identifier: CC0-1.0
 backend: dummy
-rootMountPoint: /tmp/mount
 operations:
   - install:
     - pre-script: touch /tmp/foo

src/modules/packages/tests/CMakeTests.txt

@@ -0,0 +1,42 @@
+#   SPDX-FileCopyrightText: no
+#   SPDX-License-Identifier: CC0-1.0
+#
+# We have tests to load (some) of the package-managers specifically, to
+# test their configuration code and implementation. Those tests conventionally
+# live in Python files here in the tests/ directory. Add them.
+
+# Pacman (Arch) tests
+set(_pm pacman)
+add_test(
+    NAME configure-packages-${_pm}
+    COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+)
+add_test(
+    NAME configure-packages-${_pm}-ops-1
+    COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py ${CMAKE_CURRENT_LIST_DIR}/pm-pacman-1.yaml 4 1 1
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+)
+add_test(
+    NAME configure-packages-${_pm}-ops-2
+    COMMAND env PYTHONPATH=.: python3 ${CMAKE_CURRENT_LIST_DIR}/test-pm-${_pm}.py ${CMAKE_CURRENT_LIST_DIR}/pm-pacman-2.yaml 3 0 0
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+)
+
+if ( BUILD_TESTING AND BUILD_SCHEMA_TESTING AND PYTHONINTERP_FOUND AND PYTHON_EXECUTABLE )
+    set( _module packages )
+    set( _schema_file "${CMAKE_CURRENT_SOURCE_DIR}/${_module}/${_module}.schema.yaml" )
+    message(STATUS "Schema ${_schema_file}")
+    foreach( _cf pm-pacman-1.yaml pm-pacman-2.yaml )
+        set( _conf_file "${CMAKE_CURRENT_SOURCE_DIR}/${_module}/tests/${_cf}" )
+        if ( EXISTS "${_schema_file}" AND EXISTS "${_conf_file}" )
+            add_test(
+                NAME validate-packages-${_cf}
+                COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_SOURCE_DIR}/ci/configvalidator.py" "${_schema_file}" "${_conf_file}"
+            )
+        else()
+            message(FATAL_ERROR "Missing ${_conf_file}")
+        endif()
+    endforeach()
+endif()
+

src/modules/packages/tests/pm-pacman-1.yaml

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: no
+# SPDX-License-Identifier: CC0-1.0
+backend: pacman
+operations: []
+
+pacman:
+    num_retries: 4
+    disable_download_timeout: yes
+    needed_only: true
+

src/modules/packages/tests/pm-pacman-2.yaml

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: no
+# SPDX-License-Identifier: CC0-1.0
+backend: pacman
+operations: []
+
+# Leave some things unspecified
+pacman:
+    num_retries: 3
+

src/modules/packages/tests/test-pm-pacman.py

@@ -0,0 +1,36 @@
+#   SPDX-FileCopyrightText: no
+#   SPDX-License-Identifier: CC0-1.0
+#
+# Calamares Boilerplate
+import libcalamares
+libcalamares.globalstorage = libcalamares.GlobalStorage(None)
+libcalamares.globalstorage.insert("testing", True)
+
+# Module prep-work
+from src.modules.packages import main
+
+# .. we don't have a job in this test, so fake one
+class Job(object):
+    def __init__(self, filename):
+        self.configuration = libcalamares.utils.load_yaml(filename) if filename is not None else dict()
+
+import sys
+if len(sys.argv) > 4:
+    filename = sys.argv[1]
+    retry = int(sys.argv[2])
+    timeout = bool(int(sys.argv[3]))
+    needed = bool(int(sys.argv[4]))
+else:
+    filename = None
+    retry = 0
+    timeout = False
+    needed = False
+
+libcalamares.utils.warning("Expecting {!s} retry={!s} timeout={!s} needed={!s}".format(filename, retry, timeout, needed))
+
+# Specific PM test
+libcalamares.job = Job(filename)
+p = main.PMPacman()
+assert p.pacman_num_retries == retry, "{!r} vs {!r}".format(p.pacman_num_retries, retry)
+assert p.pacman_disable_timeout == timeout, "{!r} vs {!r}".format(p.pacman_disable_timeout, timeout)
+assert p.pacman_needed_only == needed, "{!r} vs {!r}".format(p.pacman_needed_only, needed)