From fa1c848faf77d317ffb72d38c425a6f1d5df8803 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 11:29:49 +0200 Subject: [PATCH 1/8] Docs: do not use freenode --- CONTRIBUTING.md | 10 ++++++---- README.md | 13 +++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c28163137..0be408ba0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,12 +29,14 @@ rules of decent behavior in both communities are pretty much the same). GitHub Issues are **one** place for discussing Calamares if there are concrete problems or a new feature to discuss. -Regular Calamares development chit-chat happens on old-school IRC -(no registration required). Responsiveness is best during the day -in Europe, but feel free to idle. **DO NOT** ask-and-leave. Keep +Regular Calamares development chit-chat happens in a [Matrix](https://matrix.org/) +room, `#calamares:kde.org`. The conversation is bridged with IRC +on [Libera.Chat](https://libera.chat/). +Responsiveness is best during the day +in Europe, but feel free to idle. If you use IRC, **DO NOT** ask-and-leave. Keep that chat window open because it can easily take a few hours for someone to notice a message. -The Matrix room is relatively new. +Matrix is persistent, and we'll see your message eventually. * [![Join us on Matrix](https://img.shields.io/badge/Matrix-%23calamares:kde.org-blue)](https://webchat.kde.org/#/room/%23calamares:kde.org) * [![Chat on IRC](https://img.shields.io/badge/IRC-Libera.Chat%20%23calamares-green)](https://kiwiirc.com/client/irc.libera.chat/#calamares) diff --git a/README.md b/README.md index 5f1095752..8fa2dfda7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![GitHub license](https://img.shields.io/github/license/calamares/calamares.svg)](https://github.com/calamares/calamares/blob/calamares/LICENSE) -| [Report a Bug](https://github.com/calamares/calamares/issues/new) | [Translate](https://www.transifex.com/projects/p/calamares/) | [Contribute](CONTRIBUTING.md) | [IRC: Libera.Chat #calamares](https://kiwiirc.com/client/irc.libera.chat/#calamares) | [Matrix: #calamares:kde.org](https://webchat.kde.org/#/room/%23calamares:kde.org) | [Wiki](https://github.com/calamares/calamares/wiki) | +| [Report a Bug](https://github.com/calamares/calamares/issues/new) | [Translate](https://www.transifex.com/projects/p/calamares/) | [Contribute](CONTRIBUTING.md) | [Matrix: #calamares:kde.org](https://webchat.kde.org/#/room/%23calamares:kde.org) | [IRC: Libera.Chat #calamares](https://kiwiirc.com/client/irc.libera.chat/#calamares) | [Wiki](https://github.com/calamares/calamares/wiki) | |:--:|:--:|:--:|:--:|:--:|:--:| @@ -56,9 +56,14 @@ There is more information in the [CONTRIBUTING.md](CONTRIBUTING.md) file. GitHub Issues are **one** place for discussing Calamares if there are concrete problems or a new feature to discuss. -Regular Calamares development chit-chat happens on old-school IRC -(no registration required). Responsiveness is best during the day -in Europe, but feel free to idle. The Matrix room is relatively new. +Regular Calamares development chit-chat happens in a [Matrix](https://matrix.org/) +room, `#calamares:kde.org`. The conversation is bridged with IRC +on [Libera.Chat](https://libera.chat/). +Responsiveness is best during the day +in Europe, but feel free to idle. If you use IRC, **DO NOT** ask-and-leave. Keep +that chat window open because it can easily take a few hours for +someone to notice a message. +Matrix is persistent, and we'll see your message eventually. * [![Join us on Matrix](https://img.shields.io/badge/Matrix-%23calamares:kde.org-blue)](https://webchat.kde.org/#/room/%23calamares:kde.org) * [![Chat on IRC](https://img.shields.io/badge/IRC-Libera.Chat%20%23calamares-green)](https://kiwiirc.com/client/irc.libera.chat/#calamares) From f0a25bd397615e5155838ce9fdde64bc23badcf7 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 11:41:17 +0200 Subject: [PATCH 2/8] [libcalamares] Avoid warnings from deprecated country codes. The Qt documentation does not mention @since for these values, so we're going to let CI catch them. --- src/libcalamares/locale/CountryData_p.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libcalamares/locale/CountryData_p.cpp b/src/libcalamares/locale/CountryData_p.cpp index f439de27c..455027ef8 100644 --- a/src/libcalamares/locale/CountryData_p.cpp +++ b/src/libcalamares/locale/CountryData_p.cpp @@ -31,6 +31,7 @@ * Edited anyway: * 20191211 India (IN) changed to AnyLanguage, since Hindi doesn't make sense. #1284 * 20210207 Belarus (BY) changed to Russian, as the more-common-language. #1634 + * 20210615 Tokelau and Tuvalu country enum values changed to avoid deprecation warning. * */ @@ -224,13 +225,13 @@ static const CountryData country_data_table[] = { { QLocale::Language::French, QLocale::Country::Togo, 'T', 'G' }, { QLocale::Language::Thai, QLocale::Country::Thailand, 'T', 'H' }, { QLocale::Language::Tajik, QLocale::Country::Tajikistan, 'T', 'J' }, -{ QLocale::Language::TokelauLanguage, QLocale::Country::Tokelau, 'T', 'K' }, +{ QLocale::Language::TokelauLanguage, QLocale::Country::TokelauCountry, 'T', 'K' }, { QLocale::Language::Portuguese, QLocale::Country::EastTimor, 'T', 'L' }, { QLocale::Language::Turkmen, QLocale::Country::Turkmenistan, 'T', 'M' }, { QLocale::Language::Arabic, QLocale::Country::Tunisia, 'T', 'N' }, { QLocale::Language::Tongan, QLocale::Country::Tonga, 'T', 'O' }, { QLocale::Language::Turkish, QLocale::Country::Turkey, 'T', 'R' }, -{ QLocale::Language::TuvaluLanguage, QLocale::Country::Tuvalu, 'T', 'V' }, +{ QLocale::Language::TuvaluLanguage, QLocale::Country::TuvaluCountry, 'T', 'V' }, { QLocale::Language::Chinese, QLocale::Country::Taiwan, 'T', 'W' }, { QLocale::Language::Swahili, QLocale::Country::Tanzania, 'T', 'Z' }, { QLocale::Language::Ukrainian, QLocale::Country::Ukraine, 'U', 'A' }, From 6ee0da41c89419204421c2e8e3239173e24ec5ed Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 11:45:29 +0200 Subject: [PATCH 3/8] [libcalamares] Avoid deprecation warning in network attributes --- src/libcalamares/network/Manager.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libcalamares/network/Manager.cpp b/src/libcalamares/network/Manager.cpp index d089551bb..79db1a587 100644 --- a/src/libcalamares/network/Manager.cpp +++ b/src/libcalamares/network/Manager.cpp @@ -27,10 +27,16 @@ namespace Network void RequestOptions::applyToRequest( QNetworkRequest* request ) const { +#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) + constexpr const auto RedirectPolicyAttribute = QNetworkRequest::FollowRedirectsAttribute; +#else + constexpr const auto RedirectPolicyAttribute = QNetworkRequest::RedirectPolicyAttribute; +#endif + if ( m_flags & Flag::FollowRedirect ) { // Follows all redirects except unsafe ones (https to http). - request->setAttribute( QNetworkRequest::FollowRedirectsAttribute, true ); + request->setAttribute( RedirectPolicyAttribute, true ); } if ( m_flags & Flag::FakeUserAgent ) From 846936357b6ff4ea298ae5443354bc7a2e2f985f Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 20:43:44 +0200 Subject: [PATCH 4/8] [mount] If no btrfs layout at all, use something reasonable This is related to https://invent.kde.org/neon/neon/calamares-settings/-/merge_requests/1 which adds .. the default things from the example configuration to the configuration file KDE neon ships. The default layout doesn't add any subvolumes at all, which seems to be non-functional. If nothing is configured, complain and use /@ as the lone subvolume. --- src/modules/mount/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/modules/mount/main.py b/src/modules/mount/main.py index 07166dd75..5e5233935 100644 --- a/src/modules/mount/main.py +++ b/src/modules/mount/main.py @@ -74,7 +74,14 @@ def mount_partition(root_mount_point, partition, partitions): # Special handling for btrfs subvolumes. Create the subvolumes listed in mount.conf if fstype == "btrfs" and partition["mountPoint"] == '/': # Root has been mounted to btrfs volume -> create subvolumes from configuration - btrfs_subvolumes = libcalamares.job.configuration.get("btrfsSubvolumes") or [] + btrfs_subvolumes = libcalamares.job.configuration.get("btrfsSubvolumes", None) + # Warn if there's no configuration at all, and empty configurations are + # replaced by a simple root-only layout. + if btrfs_subvolumes is None: + libcalamares.utils.warning("No configuration for btrfsSubvolumes") + if not btrfs_subvolumes: + btrfs_subvolumes = [ dict(mountPoint="/", subvolume="/@") ] + subvolume_mountpoints = [d['mountPoint'] for d in btrfs_subvolumes] # Check if listed mountpoints besides / are already present and don't create subvolume for those for p in partitions: From 7b297a0e16d36430f1370bf9ddc407cef1a0978c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 20:48:22 +0200 Subject: [PATCH 5/8] [mount] Rename test file, re-jig tests for mount --- src/modules/mount/{test.yaml => tests/1.global} | 0 src/modules/mount/tests/1.job | 6 ++++++ src/modules/mount/tests/2.global | 8 ++++++++ src/modules/mount/tests/2.job | 5 +++++ 4 files changed, 19 insertions(+) rename src/modules/mount/{test.yaml => tests/1.global} (100%) create mode 100644 src/modules/mount/tests/1.job create mode 100644 src/modules/mount/tests/2.global create mode 100644 src/modules/mount/tests/2.job diff --git a/src/modules/mount/test.yaml b/src/modules/mount/tests/1.global similarity index 100% rename from src/modules/mount/test.yaml rename to src/modules/mount/tests/1.global diff --git a/src/modules/mount/tests/1.job b/src/modules/mount/tests/1.job new file mode 100644 index 000000000..94b3a1492 --- /dev/null +++ b/src/modules/mount/tests/1.job @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +bogus: true + +# No configuration needed because the partitions are +# all filesystems that require no special handling. diff --git a/src/modules/mount/tests/2.global b/src/modules/mount/tests/2.global new file mode 100644 index 000000000..20aba89ec --- /dev/null +++ b/src/modules/mount/tests/2.global @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 +partitions: + - device: "/dev/sdb1" + mountPoint: "/" + fs: "btrfs" + +# Expect a complaint and a default btrfs layout diff --git a/src/modules/mount/tests/2.job b/src/modules/mount/tests/2.job new file mode 100644 index 000000000..40d8c6171 --- /dev/null +++ b/src/modules/mount/tests/2.job @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: no +# SPDX-License-Identifier: CC0-1.0 + +# Expect a complaint and a default btrfs layout because the +# partitions use btrfs From 69cad09a005388fa6c38c25a6ad2ef1f2e722b2c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 21:23:57 +0200 Subject: [PATCH 6/8] [calamares] Tighten debug-output from module-loader test executable --- src/calamares/testmain.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp index e038b5a2f..4cde80300 100644 --- a/src/calamares/testmain.cpp +++ b/src/calamares/testmain.cpp @@ -321,7 +321,7 @@ load_module( const ModuleConfig& moduleConfig ) QString configFile( moduleConfig.configFile().isEmpty() ? moduleDirectory + '/' + name + ".conf" : moduleConfig.configFile() ); - cDebug() << "Module" << moduleName << "job-configuration:" << configFile; + cDebug() << Logger::SubEntry << "Module" << moduleName << "job-configuration:" << configFile; Calamares::Module* module = Calamares::moduleFromDescriptor( Calamares::ModuleSystem::Descriptor::fromDescriptorData( descriptor ), name, configFile, moduleDirectory ); @@ -406,7 +406,7 @@ main( int argc, char* argv[] ) return 1; } - cDebug() << " .. got" << m->name() << m->typeString() << m->interfaceString(); + cDebug() << Logger::SubEntry << " .. got" << m->name() << m->typeString() << m->interfaceString(); if ( m->type() == Calamares::Module::Type::View ) { // If we forgot the --ui, any ViewModule will core dump as it @@ -455,7 +455,6 @@ main( int argc, char* argv[] ) cDebug() << "Module metadata" << TR( "name", m->name() ) << TR( "type", m->typeString() ) << TR( "interface", m->interfaceString() ); - cDebug() << "Job outputs:"; Calamares::JobList jobList = m->jobs(); unsigned int failure_count = 0; unsigned int count = 1; From 629fc83f210367b858c0f93a5ca220fadaab1973 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 21:50:59 +0200 Subject: [PATCH 7/8] [libcalamares] Add a pre-script for PythonJobs This allows injecting arbitrary Python code before the script of a module is even run. For testing purposes, that gives us a chance to modify existing (internal) modules before the script (e.g. to test subprocess calls). --- src/libcalamares/PythonJob.cpp | 14 ++++++++++++++ src/libcalamares/PythonJob.h | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/libcalamares/PythonJob.cpp b/src/libcalamares/PythonJob.cpp index 98f284ecc..1a4683c29 100644 --- a/src/libcalamares/PythonJob.cpp +++ b/src/libcalamares/PythonJob.cpp @@ -19,6 +19,8 @@ #include +static const char* s_preScript = nullptr; + namespace bp = boost::python; BOOST_PYTHON_FUNCTION_OVERLOADS( mount_overloads, CalamaresPython::mount, 2, 4 ); @@ -242,6 +244,11 @@ PythonJob::exec() calamaresNamespace[ "globalstorage" ] = CalamaresPython::GlobalStoragePythonWrapper( JobQueue::instance()->globalStorage() ); + if ( s_preScript ) + { + bp::exec( s_preScript, scriptNamespace, scriptNamespace ); + } + cDebug() << "Job file" << scriptFI.absoluteFilePath(); bp::object execResult = bp::exec_file( scriptFI.absoluteFilePath().toLocal8Bit().data(), scriptNamespace, scriptNamespace ); @@ -319,4 +326,11 @@ PythonJob::emitProgress( qreal progressValue ) emit progress( progressValue ); } +void +PythonJob::setInjectedPreScript( const char* preScript ) +{ + s_preScript = preScript; + cDebug() << "Python pre-script set to" << Logger::Pointer( preScript ); +} + } // namespace Calamares diff --git a/src/libcalamares/PythonJob.h b/src/libcalamares/PythonJob.h index 04a0645ea..af77d741d 100644 --- a/src/libcalamares/PythonJob.h +++ b/src/libcalamares/PythonJob.h @@ -41,6 +41,19 @@ public: QString prettyStatusMessage() const override; JobResult exec() override; + /** @brief Sets the pre-run Python code for all PythonJobs + * + * A PythonJob runs the code from the scriptFile parameter to + * the constructor; the pre-run code is **also** run, before + * even the scriptFile code. Use this in testing mode + * to modify Python internals. + * + * No ownership of @p script is taken: pass in a pointer to + * a character literal or something that lives longer than the + * job. Pass in @c nullptr to switch off pre-run code. + */ + static void setInjectedPreScript( const char* script ); + private: struct Private; From cc0e39db562af8b3a324505bea5767f820cda767 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 15 Jun 2021 23:21:32 +0200 Subject: [PATCH 8/8] [calamares] Inject a pre-script when testing Python modules Co-opt the subprocess module and replace call and check_call functions with something that logs the call and does nothing. --- src/calamares/testmain.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/calamares/testmain.cpp b/src/calamares/testmain.cpp index 4cde80300..6dae0af68 100644 --- a/src/calamares/testmain.cpp +++ b/src/calamares/testmain.cpp @@ -18,6 +18,7 @@ #include "GlobalStorage.h" #include "Job.h" #include "JobQueue.h" +#include "PythonJob.h" #include "Settings.h" #include "ViewManager.h" #include "modulesystem/Module.h" @@ -365,6 +366,27 @@ createApplication( int& argc, char* argv[] ) return new QCoreApplication( argc, argv ); } +static const char pythonPreScript[] = R"( +# This is Python code executed by Python modules *before* the +# script file (e.g. main.py) is executed. Beware " before ) +# because it's a C++ raw-string. +_calamares_subprocess = __import__("subprocess", globals(), locals(), [], 0) +import sys +import libcalamares +class fake_subprocess(object): + @staticmethod + def call(*args, **kwargs): + libcalamares.utils.debug("subprocess.call(%r,%r) X run normally" % (args, kwargs)) + return 0 + @staticmethod + def check_call(*args, **kwargs): + libcalamares.utils.debug("subprocess.check_call(%r,%r) X subverted to call" % (args, kwargs)) + return 0 +sys.modules["subprocess"] = fake_subprocess +libcalamares.utils.debug('pre-script for testing purposes injected') + +)"; + int main( int argc, char* argv[] ) { @@ -397,6 +419,7 @@ main( int argc, char* argv[] ) #ifdef WITH_QML CalamaresUtils::initQmlModulesDir(); // don't care if failed #endif + Calamares::PythonJob::setInjectedPreScript(pythonPreScript); cDebug() << "Calamares module-loader testing" << module.moduleName(); Calamares::Module* m = load_module( module );