Merge branch 'calamares' of https://github.com/calamares/calamares into development

.reuse/dep5

@@ -54,6 +54,13 @@ Files: data/example-root/usr/share/zoneinfo/Zulu data/example-root/usr/share/zon
 License: CC0-1.0
 Copyright: no
 
+# Test data
+#
+# These first files are mere lists of locale identifiers
+Files: src/modules/locale/tests/locale-data-neon src/modules/locale/tests/locale-data-freebsd
+License: CC0-1.0
+Copyright: no
+
 ### TRANSLATIONS
 #
 # .desktop files and template change only with translation
@@ -85,10 +92,3 @@ Files: lang/python/*/LC_MESSAGES/python.po
 License: GPL-3.0-or-later
 Copyright: 2020 Calamares authors and translators
 
-Files: src/modules/dummypythonqt/lang/dummypythonqt.pot
-License: GPL-3.0-or-later
-Copyright: 2020 Calamares authors and translators
-
-Files: src/modules/dummypythonqt/lang/*/LC_MESSAGES/dummypythonqt.po
-License: GPL-3.0-or-later
-Copyright: 2020 Calamares authors and translators

CHANGES-3.2

@@ -8,18 +8,36 @@ changelog -- this log starts with version 3.2.0. The release notes on the
 website will have to do for older versions.
 
 
-> Note that the 3.2 series is now in LTS / bug-fix-only mode.
+Calamares version 3.2.61 is the last one to have updated CHANGES-3.2
+in the *calamares* (e.g. development, or 3.3, branch). For changes
+in the stable release branch, see CHANGES-3.2 in that branch.
 
-# 3.2.61 (unreleased) #
+
+
+# 3.2.61 (2022-08-24) #
+
+This is the second community-maintainence release of Calamares 3.2.
+It corrects a handful of bugs foud in the stable release. There
+are also translation updates.
 
 This release contains contributions from (alphabetically by first name):
- - No external contributors yet
+ - Adriaan de Groot
+ - Anke Boersma
 
 ## Core ##
- - No core changes yet
+ - The "About" and "Debug" buttons in a QWidgets-based panel were no
+   longer translated. This has been fixed (by re-using translations
+   of the same buttons from the QML module. #2030 (Thanks Anke)
 
 ## Modules ##
- - No module changes yet
+ - *bootloader* Python code slipped in that was incompatible with
+   the minimum required Python version. #2033 (Thanks Adriaan)
+ - *locale* fixes a large regression introduced with 3.2.60, where
+   the location picked for many locales was not the same as in 3.2.59,
+   and generally peculiar (e.g. picking "English" led to "en_AG" which
+   is nice if you are in Bermuda, but not expected in the rest of the
+   world). #2008
+ - *luksopenswaphookcfg* Remove duplicate options. #1659 (Thanks Anke)
 
 
 # 3.2.60 (2022-06-19) #

CHANGES-3.3

@@ -5,9 +5,60 @@
 This is the changelog for Calamares. For each release, the major changes and
 contributors are listed. Note that Calamares does not have a historical
 changelog -- this log starts with version 3.3.0. See CHANGES-3.2 for
-the history of the 3.2 series (2018-05 - 2021-12).
+the history of the 3.2 series (2018-05 - 2022-08).
 
-# 3.3.0 (unreleased) #
+
+# 3.3.0-alpha3 (unreleased)
+
+This release contains contributions from (alphabetically by first name):
+ - Adriaan de Groot
+
+## Core ##
+ - No core changes yet
+
+## Modules ##
+ - No module changes yet
+
+
+# 3.3.0-alpha2 (2022-08-23)
+
+Second alpha release, with updated ABI compatibility checking,
+some 3.3.0 release goals, new features in modules and important bugfixes.
+
+This release contains contributions from (alphabetically by first name):
+ - Adriaan de Groot
+ - Anke Boersma
+ - Evan James
+ - Shivanand
+ - Vitor Lopes
+
+## Core ##
+
+A core **TODO** is moving all library code into the `Calamares` namespace,
+dropping the `CalamaresUtils` namespace. Modern C++ supports nested namespaces,
+so in some cases we can use those. This has a drastic effect on ABI compatibility,
+though, as functions move from one namespace to another. This needs to be
+completed before a 3.3.0 with ABI stability is released.
+
+## Modules ##
+
+Module schemas have been updated to reflect all the incompatible changes.
+
+
+# 3.3.0-alpha1 (2022-06-27)
+
+Initial 3.3.0 alpha release to check the release scripts &c.
+
+This release contains contributions from (alphabetically by first name):
+ - Adriaan de Groot
+ - Aleksey Samoilov
+ - Anke Boersma
+ - Dan Simmons
+ - Evan James
+ - Peter Jung
+
+
+# 3.3.0-pre-alpha (unreleased) #
 
 This release contains contributions from (alphabetically by first name):
  - Anubhav Choudhary
@@ -23,9 +74,6 @@ Users (distributions) are **strongly** advised to use the tools
 for configuration validation (`ci/configvalidator.py`) to check
 that the distribution configuration files follow the current schema.
 
-Pre-release versions:
- - 3.3.0-alpha1 (2022-06-27)
-   Initial 3.3.0 release to check the release scripts &c.
  - 3.3.0-alpha2 (unreleased)
    Incompatible module-configuration changes, see #1438.
 

CMakeLists.txt

@@ -43,8 +43,8 @@
 
 cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
 
-set(CALAMARES_VERSION 3.3.0-alpha1)
+set(CALAMARES_VERSION 3.3.0-alpha3)
-set(CALAMARES_RELEASE_MODE ON) # Set to ON during a release
+set(CALAMARES_RELEASE_MODE OFF) # Set to ON during a release
 
 if(CMAKE_SCRIPT_MODE_FILE)
     include(${CMAKE_CURRENT_LIST_DIR}/CMakeModules/ExtendedVersion.cmake)
@@ -65,6 +65,11 @@ project(CALAMARES VERSION ${CALAMARES_VERSION_SHORT} LANGUAGES C CXX HOMEPAGE_UR
 if(NOT CALAMARES_RELEASE_MODE AND CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
     message(FATAL_ERROR "Do not build development versions in the source-directory.")
 endif()
+# Calamares in the 3.3 series promises ABI compatbility, so it sets a
+# .so-version equal to the series number. We use ci/abicheck.sh to
+# keep track of this. Note that the **alpha** releases also have
+# such an .so-version, but are not ABI-stable yet.
+set(CALAMARES_SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}")
 
 ### OPTIONS
 #

ci/RELEASE.sh

@@ -86,6 +86,16 @@ KEY_ID="328D742D8807A435"
 rm -f CMakeLists.txt.gpg
 gpg -s -u $KEY_ID CMakeLists.txt
 
+### Get version number for this release
+#
+# Do this early, in a clean build-dir, since it doesn't cost much.
+# Redirect stderr from CMake script mode, because the message()
+# in CMakeLists.txt that prints the version, goes to stderr.
+rm -rf "$BUILDDIR"
+mkdir "$BUILDDIR" || { echo "Could not create build directory." ; exit 1 ; }
+V=$( cd "$BUILDDIR" && cmake -P ../CMakeLists.txt 2>&1 )
+test -n "$V" || { echo "Could not obtain version in $BUILDDIR ." ; exit 1 ; }
+
 ### Build with default compiler
 #
 #
@@ -124,12 +134,6 @@ else
     ( cd "$BUILDDIR" && cmake .. ) || { echo "Could not run cmake in $BUILDDIR ." ; exit 1 ; }
 fi
 
-### Get version number for this release
-#
-#
-V=$( cd "$BUILDDIR" && cmake -P ../CMakeLists.txt | grep ^CALAMARES_VERSION | sed s/^[A-Z_]*=// )
-test -n "$V" || { echo "Could not obtain version in $BUILDDIR ." ; exit 1 ; }
-
 ### Create signed tag
 #
 # This is the signing key ID associated with the GitHub account adriaandegroot,

ci/abicheck.sh

@@ -13,9 +13,10 @@
 # The base version can be a tag or git-hash; it will be checked-out
 # in a worktree.
 #
-# Note that the hash here now is the very start of 3.3, when ABI
+# Note that the hash here now is 3.3-alpha1, when ABI
-# compatibility was not expected yet at **all**.
+# compatibility was not expected much. From 3.3-beta,
-BASE_VERSION=419be4df25bc6fcc1958cb6e44afc1b9e64fce71
+# whenever that is, ABI compatibility should be more of a concern.
+BASE_VERSION=0c794183936b6d916a109784829e605cc4582e9f
 
 ### Build a tree and cache the ABI info into ci/
 #

ci/txcheck.sh

@@ -120,7 +120,7 @@ tx_sum()
 	WORKTREE_NAME="$1"
 	WORKTREE_TAG="$2"
 
-	git worktree add $WORKTREE_NAME $WORKTREE_TAG > /dev/null 2>&1 || { echo "! Could not create worktree." ; exit 1 ; }
+	git worktree add -d $WORKTREE_NAME $WORKTREE_TAG > /dev/null 2>&1 || { echo "! Could not create worktree." ; exit 1 ; }
 	( cd $WORKTREE_NAME && sh "$CURDIR"/ci/txpush.sh --no-tx ) > /dev/null 2>&1 || { echo "! Could not re-create translations." ; exit 1 ; }
 
 	# Remove linenumbers from .ts (XML) and .pot

ci/txpush.sh

@@ -124,8 +124,8 @@ tx push --source --no-interactive -r calamares.fdo
 PYGETTEXT="xgettext --keyword=_n:1,2 -L python"
 
 SHARED_PYTHON=""
-for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do
+for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d | sort) ; do
-	FILES=$(find "$MODULE_DIR" -name "*.py" -a -type f)
+	FILES=$(find "$MODULE_DIR" -name "*.py" -a -type f | sort)
 	if test -n "$FILES" ; then
 		MODULE_NAME=$(basename ${MODULE_DIR})
 		if [ -d ${MODULE_DIR}/lang ]; then

src/libcalamares/CMakeLists.txt

@@ -20,13 +20,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CalamaresVersionX.h.in ${CMAKE_CURREN
 # Map the available translations names into a suitable constexpr list
 # of names in C++. This gets us Calamares::Locale::availableLanguages,
 # a QStringList of names.
-set(_names_tu "#ifndef CALAMARES_TRANSLATIONS_H
+set(_names_tu
+    "
+#ifndef CALAMARES_TRANSLATIONS_H
 #define CALAMARES_TRANSLATIONS_H
 #include <QStringList>
 namespace {
 static const QStringList availableLanguageList{
-")
+"
-foreach( l ${CALAMARES_TRANSLATION_LANGUAGES})
+)
+foreach(l ${CALAMARES_TRANSLATION_LANGUAGES})
     string(APPEND _names_tu "\"${l}\",\n")
 endforeach()
 string(APPEND _names_tu "};\n} // namespace\n#endif\n\n")
@@ -94,7 +97,7 @@ set_target_properties(
     calamares
     PROPERTIES
         VERSION ${CALAMARES_VERSION_SHORT}
-        SOVERSION ${CALAMARES_VERSION_SHORT}
+        SOVERSION ${CALAMARES_SOVERSION}
         INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_INSTALL_FULL_INCLUDEDIR}/libcalamares
 )
 target_link_libraries(calamares LINK_PUBLIC yamlcpp::yamlcpp Qt5::Core KF5::CoreAddons)
@@ -256,7 +259,11 @@ calamares_add_test(libcalamaresnetworktest SOURCES network/Tests.cpp)
 calamares_add_test(libcalamarespackagestest SOURCES packages/Tests.cpp)
 
 if(KPMcore_FOUND)
-    calamares_add_test(libcalamarespartitiontest SOURCES partition/Global.cpp partition/Tests.cpp LIBRARIES calamares::kpmcore)
+    calamares_add_test(
+        libcalamarespartitiontest
+        SOURCES partition/Global.cpp partition/Tests.cpp
+        LIBRARIES calamares::kpmcore
+    )
     calamares_add_test(libcalamarespartitionkpmtest SOURCES partition/KPMTests.cpp LIBRARIES calamares::kpmcore)
 endif()
 

src/libcalamaresui/CMakeLists.txt

@@ -53,6 +53,7 @@ calamares_add_library(calamaresui
     UI
         utils/ErrorDialog/ErrorDialog.ui
     VERSION ${CALAMARES_VERSION_SHORT}
+    SOVERSION ${CALAMARES_SOVERSION}
 )
 target_link_libraries(calamaresui PRIVATE yamlcpp::yamlcpp)
 if(KF5CoreAddons_FOUND AND KF5CoreAddons_VERSION VERSION_GREATER_EQUAL 5.58)

src/modules/bootloader/main.py

@@ -471,7 +471,7 @@ def efi_boot_next():
     """
     boot_mgr = libcalamares.job.configuration["efiBootMgr"]
     boot_entry = None
-    efi_bootvars = subprocess.check_output([boot_mgr], text=True)
+    efi_bootvars = subprocess.check_output([boot_mgr], universal_newlines=True)
     for line in efi_bootvars.split('\n'):
         if not line:
             continue

src/modules/displaymanager/tests/test-dm-greetd.py

@@ -17,6 +17,14 @@ try:
 except FileNotFoundError as e:
     pass
 
+try:
+    import toml
+except ImportError:
+    # This is a failure of the test-environment.
+    import sys
+    print("Can't find module toml.", file=sys.stderr)
+    sys.exit(0)
+
 # Specific DM test
 d = main.DMgreetd("/tmp")
 d.set_autologin("d", True, default_desktop_environment)

src/modules/keyboardq/data/pan-end-symbolic.svg

@@ -0,0 +1,15 @@
+<?xml version='1.0' encoding='UTF-8' standalone='no'?>
+<svg height="16" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" version="1.1" width="16" xmlns="http://www.w3.org/2000/svg" enable-background="new">
+ <metadata id="metadata90"/>
+ <defs id="defs7386">
+  <linearGradient id="linearGradient5606" osb:paint="solid">
+   <stop id="stop5608"/>
+  </linearGradient>
+  <filter inkscape:collect="always" color-interpolation-filters="sRGB" id="filter7554">
+   <feBlend inkscape:collect="always" id="feBlend7556" in2="BackgroundImage" mode="darken"/>
+  </filter>
+ </defs>
+ <g inkscape:groupmode="layer" id="layer12" inkscape:label="actions" transform="translate(-445.0002,-129)">
+  <path inkscape:connector-curvature="0" d="m 451.0002,142 5,-5 -5,-5 z" id="path6412" sodipodi:nodetypes="cccc" fill="#555555"/>
+ </g>
+</svg>

src/modules/keyboardq/data/pan-end-symbolic.svg.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2022 demmm <anke62@gmail.com>
+SPDX-License-Identifier: GPL-3.0-or-later

src/modules/keyboardq/keyboardq.qml

@@ -12,7 +12,7 @@ import io.calamares.ui 1.0
 
 import QtQuick 2.15
 import QtQuick.Controls 2.15
-import QtQuick.Window 2.14
+import QtQuick.Window 2.15
 import QtQuick.Layouts 1.3
 
 import org.kde.kirigami 2.7 as Kirigami
@@ -52,6 +52,7 @@ Item {
     Rectangle {
         id: backgroundItem
         anchors.fill: parent
+        width: 800
         color: backgroundColor
 
         Label {
@@ -62,52 +63,105 @@ Item {
             font.bold: true
         }
 
+        Drawer {
+            id: drawer
+            width: 0.4 * backgroundItem.width
+            height: backgroundItem.height
+            edge: Qt.RightEdge
+
+            ScrollView {
+                id: scroll1
+                anchors.fill: parent
+                ScrollBar.horizontal.policy: ScrollBar.AlwaysOff
+
+                ListView {
+                    id: models
+                    focus: true
+                    clip: true
+                    boundsBehavior: Flickable.StopAtBounds
+                    width: parent.width
+
+                    model: config.keyboardModelsModel
+                    Component.onCompleted: positionViewAtIndex(model.currentIndex, ListView.Center)
+                    currentIndex: model.currentIndex
+                    delegate: ItemDelegate {
+
+                        property variant currentModel: model
+                        hoverEnabled: true
+                        width: 0.4 * backgroundItem.width
+                        implicitHeight: 24
+                        highlighted: ListView.isCurrentItem
                         Label {
-            id: intro
+                            Layout.fillHeight: true
-            anchors.horizontalCenter: parent.horizontalCenter
+                            Layout.fillWidth: true
-            anchors.top: header.bottom
-            color: textColor
                             horizontalAlignment: Text.AlignHCenter
-            width: parent.width / 1.2
+                            width: parent.width
-            wrapMode: Text.WordWrap
+                            height: 24
-            text: ( config.prettyStatus)
+                            color: highlighted ? "#eff0f1" : "#1F1F1F"
+                            text: model.label
+                            background: Rectangle {
+
+                                color: highlighted || hovered ? "#3498DB" : "#ffffff"
+                                opacity: highlighted || hovered ? 0.5 : 0.9
+                            }
+
+                            MouseArea {
+                                hoverEnabled: true
+                                anchors.fill: parent
+                                cursorShape: Qt.PointingHandCursor
+                                onClicked: {
+                                    models.currentIndex = index
+                                    drawer.close()
+                                }
+                            }
+                        }
+                    }
+                    onCurrentItemChanged: { config.keyboardModels = model[currentIndex] } /* This works because model is a stringlist */
+                }
+            }
+        }
+
+        Rectangle {
+            id: modelLabel
+            anchors.top: header.bottom
+            anchors.topMargin: 10
+            anchors.horizontalCenter: parent.horizontalCenter
+            width: parent.width / 1.5
+            height: 36
+            color: mouseBar.containsMouse ? "#eff0f1" : "transparent";
+
+            MouseArea {
+                id: mouseBar
+                anchors.fill: parent;
+                cursorShape: Qt.PointingHandCursor
+                hoverEnabled: true
+
+                onClicked: {
+                    drawer.open()
+                }
+                Text {
+                    anchors.centerIn: parent
+                    text: qsTr("<b>Keyboard Model:&nbsp;&nbsp;</b>") + models.currentItem.currentModel.label
+                    color: textColor
+                }
+                Image {
+                    source: "data/pan-end-symbolic.svg"
+                    anchors.centerIn: parent
+                    anchors.horizontalCenterOffset : parent.width / 2.5
+                    fillMode: Image.PreserveAspectFit
+                    height: 22
+                }
+            }
         }
 
         RowLayout {
-            id: models
+            id: stack
-            anchors.top: intro.bottom
+            anchors.top: modelLabel.bottom
             anchors.topMargin: 10
             anchors.horizontalCenter: parent.horizontalCenter
-            width: parent.width /1.5
+            width: parent.width / 1.1
             spacing: 10
 
-            Label {
-                Layout.alignment: Qt.AlignCenter
-                text: qsTr("Keyboard Model:")
-                color: textColor
-                font.bold: true
-            }
-
-            ComboBox {
-                Layout.fillWidth: true
-                textRole: "label"
-                model: config.keyboardModelsModel
-                currentIndex: model.currentIndex
-                onCurrentIndexChanged: config.keyboardModels = currentIndex
-            }
-        }
-
-        StackView {
-            id: stack
-            anchors.top: models.bottom
-            anchors.topMargin: 10
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.bottom: parent.bottom
-            clip: true
-
-            initialItem: Item {
-
             ListView {
                 id: layouts
 
@@ -115,13 +169,25 @@ Item {
                     active: true
                 }
 
-                    width: parent.width / 2
+                Layout.preferredWidth: parent.width / 2
-                    height: 200
+                height: 220
-                    anchors.horizontalCenter: parent.horizontalCenter
                 focus: true
                 clip: true
                 boundsBehavior: Flickable.StopAtBounds
                 spacing: 2
+                headerPositioning: ListView.OverlayHeader
+                header: Rectangle{
+                    height: 24
+                    width: parent.width
+                    z: 2
+                    color:backgroundColor
+                    Text {
+                        text: qsTr("Layout")
+                        anchors.centerIn: parent
+                        color: textColor
+                        font.bold: true
+                    }
+                }
 
                 Rectangle {
                     z: parent.z - 1
@@ -137,7 +203,7 @@ Item {
 
                     hoverEnabled: true
                     width: parent.width
-                        height: 18
+                    implicitHeight: 24
                     highlighted: ListView.isCurrentItem
 
                     RowLayout {
@@ -146,11 +212,11 @@ Item {
                         Label {
                             id: label1
                             text: model.label
+                            horizontalAlignment: Text.AlignHCenter
                             Layout.fillHeight: true
                             Layout.fillWidth: true
-                                padding: 10
                             width: parent.width
-                                height: 32
+                            height: 24
                             color: highlighted ? highlightedTextColor : textColor
 
                             background: Rectangle {
@@ -164,28 +230,10 @@ Item {
 
                         layouts.model.currentIndex = index
                         keyIndex = label1.text.substring(0,6)
-                            stack.push(variantsList)
                         layouts.positionViewAtIndex(index, ListView.Center)
                     }
                 }
             }
-                Button {
-
-                    Layout.fillWidth: true
-                    anchors.verticalCenter: parent.verticalCenter
-                    anchors.verticalCenterOffset: -parent.height / 3.5
-                    anchors.left: parent.left
-                    anchors.leftMargin: parent.width / 15
-                    icon.name: "go-next"
-                    text: qsTr("Variants")
-                    onClicked: stack.push(variantsList)
-                }
-            }
-
-            Component {
-                id: variantsList
-
-                Item {
 
             ListView {
                 id: variants
@@ -194,14 +242,25 @@ Item {
                     active: true
                 }
 
-                        width: parent.width / 2
+                Layout.preferredWidth: parent.width / 2
-                        height: 200
+                height: 220
-                        anchors.horizontalCenter: parent.horizontalCenter
-                        anchors.topMargin: 10
                 focus: true
                 clip: true
                 boundsBehavior: Flickable.StopAtBounds
                 spacing: 2
+                headerPositioning: ListView.OverlayHeader
+                header: Rectangle{
+                    height: 24
+                    width: parent.width
+                    z: 2
+                    color:backgroundColor
+                    Text {
+                        text: qsTr("Variant")
+                        anchors.centerIn: parent
+                        color: textColor
+                        font.bold: true
+                    }
+                }
 
                 Rectangle {
                     z: parent.z - 1
@@ -217,7 +276,7 @@ Item {
                 delegate: ItemDelegate {
                     hoverEnabled: true
                     width: parent.width
-                            height: 18
+                    implicitHeight: 24
                     highlighted: ListView.isCurrentItem
 
                     RowLayout {
@@ -225,11 +284,11 @@ Item {
 
                         Label {
                             text: model.label
+                            horizontalAlignment: Text.AlignHCenter
                             Layout.fillHeight: true
                             Layout.fillWidth: true
-                                    padding: 10
                             width: parent.width
-                                    height: 30
+                            height: 24
                             color: highlighted ? highlightedTextColor : textColor
 
                             background: Rectangle {
@@ -245,26 +304,13 @@ Item {
                     }
                 }
             }
-
-                    Button {
-                        Layout.fillWidth: true
-                        anchors.verticalCenter: parent.verticalCenter
-                        anchors.verticalCenterOffset: -parent.height / 3.5
-                        anchors.left: parent.left
-                        anchors.leftMargin: parent.width / 15
-                        icon.name: "go-previous"
-                        text: qsTr("Layouts")
-                        onClicked: stack.pop()
-                    }
-                }
-            }
         }
 
         TextField {
             id: textInput
             placeholderText: qsTr("Type here to test your keyboard")
             height: 36
-            width: parent.width / 1.5
+            width: parent.width / 1.6
             horizontalAlignment: TextInput.AlignHCenter
             anchors.horizontalCenter: parent.horizontalCenter
             anchors.bottom: keyboard.top

src/modules/keyboardq/keyboardq.qrc

@@ -24,5 +24,6 @@
         <file>data/button_bkg_center.png</file>
         <file>data/button_bkg_left.png</file>
         <file>data/button_bkg_right.png</file>
+        <file>data/pan-end-symbolic.svg</file>
     </qresource>
 </RCC>

src/modules/locale/CMakeLists.txt

@@ -22,6 +22,7 @@ calamares_add_plugin(locale
         Config.cpp
         LCLocaleDialog.cpp
         LocaleConfiguration.cpp
+        LocaleNames.cpp
         LocalePage.cpp
         LocaleViewStep.cpp
         SetTimezoneJob.cpp
@@ -39,7 +40,13 @@ calamares_add_plugin(locale
 
 calamares_add_test(
     localetest
-    SOURCES Tests.cpp Config.cpp LocaleConfiguration.cpp SetTimezoneJob.cpp timezonewidget/TimeZoneImage.cpp
+    SOURCES
+        Tests.cpp
+        Config.cpp
+        LocaleConfiguration.cpp
+        LocaleNames.cpp
+        SetTimezoneJob.cpp
+        timezonewidget/TimeZoneImage.cpp
     DEFINITIONS SOURCE_DIR="${CMAKE_CURRENT_LIST_DIR}/images" DEBUG_TIMEZONES=1
     LIBRARIES Qt5::Gui
 )

src/modules/locale/LocaleConfiguration.cpp

@@ -9,11 +9,13 @@
  */
 
 #include "LocaleConfiguration.h"
+#include "LocaleNames.h"
 
 #include "utils/Logger.h"
 
 #include <QLocale>
 #include <QRegularExpression>
+#include <QVector>
 
 LocaleConfiguration::LocaleConfiguration()
     : explicit_lang( false )
@@ -40,6 +42,106 @@ LocaleConfiguration::setLanguage( const QString& localeName )
     m_lang = localeName;
 }
 
+static LocaleNameParts
+updateCountry( LocaleNameParts p, const QString& country )
+{
+    p.country = country;
+    return p;
+}
+
+static QPair< int, LocaleNameParts >
+identifyBestLanguageMatch( const LocaleNameParts& referenceLocale, QVector< LocaleNameParts >& others )
+{
+    std::sort( others.begin(),
+               others.end(),
+               [ & ]( const LocaleNameParts& lhs, const LocaleNameParts& rhs )
+               { return referenceLocale.similarity( lhs ) < referenceLocale.similarity( rhs ); } );
+    // The best match is at the end
+    LocaleNameParts best_match = others.last();
+    if ( !( referenceLocale.similarity( best_match ) > LocaleNameParts::no_match ) )
+    {
+        cDebug() << Logger::SubEntry << "Got no good match for" << referenceLocale.name();
+        return { LocaleNameParts::no_match, LocaleNameParts {} };
+    }
+    else
+    {
+        cDebug() << Logger::SubEntry << "Got best match for" << referenceLocale.name() << "as" << best_match.name();
+        return { referenceLocale.similarity( best_match ), best_match };
+    }
+}
+
+/** @brief Returns the QString from @p availableLocales that best-matches.
+ */
+static LocaleNameParts
+identifyBestLanguageMatch( const QString& languageLocale,
+                           const QStringList& availableLocales,
+                           const QString& countryCode )
+{
+    const QString default_lang = QStringLiteral( "en_US.UTF-8" );
+
+    const LocaleNameParts self = LocaleNameParts::fromName( languageLocale );
+    if ( self.isValid() && !availableLocales.isEmpty() )
+    {
+        QVector< LocaleNameParts > others;
+        others.resize( availableLocales.length() );  // Makes default structs
+        std::transform( availableLocales.begin(), availableLocales.end(), others.begin(), LocaleNameParts::fromName );
+
+        // Keep track of the best match in various attempts
+        int best_score = LocaleNameParts::no_match;
+        LocaleNameParts best_match;
+
+        // Check with the unmodified language setting
+        {
+            auto [ score, match ] = identifyBestLanguageMatch( self, others );
+            if ( score >= LocaleNameParts::complete_match )
+            {
+                return match;
+            }
+            else if ( score > best_score )
+            {
+                best_match = match;
+            }
+        }
+
+
+        // .. but it might match **better** with the chosen location country Code
+        {
+            auto [ score, match ] = identifyBestLanguageMatch( updateCountry( self, countryCode ), others );
+            if ( score >= LocaleNameParts::complete_match )
+            {
+                return match;
+            }
+            else if ( score > best_score )
+            {
+                best_match = match;
+            }
+        }
+
+        // .. or better yet with the QLocale-derived country
+        {
+            const QString localeCountry = LocaleNameParts::fromName( QLocale( languageLocale ).name() ).country;
+            auto [ score, match ] = identifyBestLanguageMatch( updateCountry( self, localeCountry ), others );
+            if ( score >= LocaleNameParts::complete_match )
+            {
+                return match;
+            }
+            else if ( score > best_score )
+            {
+                best_match = match;
+            }
+        }
+
+        if ( best_match.isValid() )
+        {
+            cDebug() << Logger::SubEntry << "Matched best with" << best_match.name();
+            return best_match;
+        }
+    }
+
+    // Else we have an unrecognized or unsupported locale, all we can do is go with
+    // en_US.UTF-8 UTF-8. This completes all default language setting guesswork.
+    return LocaleNameParts::fromName( default_lang );
+}
 
 LocaleConfiguration
 LocaleConfiguration::fromLanguageAndLocation( const QString& languageLocale,
@@ -47,100 +149,7 @@ LocaleConfiguration::fromLanguageAndLocation( const QString& languageLocale,
                                               const QString& countryCode )
 {
     cDebug() << "Mapping" << languageLocale << "in" << countryCode << "to locale.";
-    QString language = languageLocale.split( '_' ).first();
+    const auto bestLocale = identifyBestLanguageMatch( languageLocale, availableLocales, countryCode );
-    QString region;
-    if ( language.contains( '@' ) )
-    {
-        auto r = language.split( '@' );
-        language = r.first();
-        region = r[ 1 ];  // second()
-    }
-
-    // Either an exact match, or the whole language part matches
-    // (followed by .<encoding> or _<country>
-    QStringList linesForLanguage = availableLocales.filter( QRegularExpression( language + "[._]" ) );
-    cDebug() << Logger::SubEntry << "Matching" << linesForLanguage;
-
-    QString lang;
-    if ( linesForLanguage.isEmpty() || languageLocale.isEmpty() )
-    {
-        lang = "en_US.UTF-8";
-    }
-    else if ( linesForLanguage.length() == 1 )
-    {
-        lang = linesForLanguage.first();
-    }
-
-    // lang could still be empty if we found multiple locales that satisfy myLanguage
-    const QString combinedLanguageAndCountry = QString( "%1_%2" ).arg( language ).arg( countryCode );
-    if ( lang.isEmpty() && region.isEmpty() )
-    {
-        auto l = linesForLanguage.filter(
-            QRegularExpression( combinedLanguageAndCountry + "[._]" ) );  // no regional variants
-        if ( l.length() == 1 )
-        {
-            lang = l.first();
-        }
-    }
-
-    // The following block was inspired by Ubiquity, scripts/localechooser-apply.
-    // No copyright statement found in file, assuming GPL v2 or later.
-    /*  # In the special cases of Portuguese and Chinese, selecting a
-        # different location may imply a different dialect of the language.
-        # In such cases, make LANG reflect the selected language (for
-        # messages, character types, and collation) and make the other
-        # locale categories reflect the selected location. */
-    if ( language == "pt" || language == "zh" )
-    {
-        cDebug() << Logger::SubEntry << "Special-case Portuguese and Chinese";
-        QString proposedLocale = QString( "%1_%2" ).arg( language ).arg( countryCode );
-        for ( const QString& line : linesForLanguage )
-        {
-            if ( line.contains( proposedLocale ) )
-            {
-                cDebug() << Logger::SubEntry << "Country-variant" << line << "chosen.";
-                lang = line;
-                break;
-            }
-        }
-    }
-    if ( lang.isEmpty() && !region.isEmpty() )
-    {
-        cDebug() << Logger::SubEntry << "Special-case region @" << region;
-        QString proposedRegion = QString( "@%1" ).arg( region );
-        for ( const QString& line : linesForLanguage )
-        {
-            if ( line.startsWith( language ) && line.contains( proposedRegion ) )
-            {
-                cDebug() << Logger::SubEntry << "Region-variant" << line << "chosen.";
-                lang = line;
-                break;
-            }
-        }
-    }
-
-
-    // If we found no good way to set a default lang, do a search with the whole
-    // language locale and pick the first result, if any.
-    if ( lang.isEmpty() )
-    {
-        for ( const QString& line : availableLocales )
-        {
-            if ( line.startsWith( languageLocale ) )
-            {
-                lang = line;
-                break;
-            }
-        }
-    }
-
-    // Else we have an unrecognized or unsupported locale, all we can do is go with
-    // en_US.UTF-8 UTF-8. This completes all default language setting guesswork.
-    if ( lang.isEmpty() )
-    {
-        lang = "en_US.UTF-8";
-    }
-
 
     // The following block was inspired by Ubiquity, scripts/localechooser-apply.
     // No copyright statement found in file, assuming GPL v2 or later.
@@ -188,34 +197,16 @@ LocaleConfiguration::fromLanguageAndLocation( const QString& languageLocale,
     // We make a proposed locale based on the UI language and the timezone's country. There is no
     // guarantee that this will be a valid, supported locale (often it won't).
     QString lc_formats;
-    const QString combined = QString( "%1_%2" ).arg( language ).arg( countryCode );
+    const QString combined = QString( "%1_%2" ).arg( bestLocale.language ).arg( countryCode );
-    if ( lang.isEmpty() )
+    if ( availableLocales.contains( bestLocale.language ) )
     {
-        cDebug() << Logger::SubEntry << "Looking up formats for" << combinedLanguageAndCountry;
+        cDebug() << Logger::SubEntry << "Exact formats match for language tag" << bestLocale.language;
-        // We look up if it's a supported locale.
+        lc_formats = bestLocale.language;
-        for ( const QString& line : availableLocales )
-        {
-            if ( line.startsWith( combinedLanguageAndCountry ) )
-            {
-                lang = line;
-                lc_formats = line;
-                break;
     }
-        }
+    else if ( availableLocales.contains( combined ) )
-    }
-    else
     {
-        if ( availableLocales.contains( lang ) )
+        cDebug() << Logger::SubEntry << "Exact formats match for combined" << combined;
-        {
+        lc_formats = combined;
-            cDebug() << Logger::SubEntry << "Exact formats match for language tag" << lang;
-            lc_formats = lang;
-        }
-        else if ( availableLocales.contains( combinedLanguageAndCountry ) )
-        {
-            cDebug() << Logger::SubEntry << "Exact formats match for combined" << combinedLanguageAndCountry;
-            lang = combinedLanguageAndCountry;
-            lc_formats = combinedLanguageAndCountry;
-        }
     }
 
     if ( lc_formats.isEmpty() )
@@ -303,12 +294,7 @@ LocaleConfiguration::fromLanguageAndLocation( const QString& languageLocale,
 
     // If we cannot make a good choice for a given country we go with the LANG
     // setting, which defaults to en_US.UTF-8 UTF-8 if all else fails.
-    if ( lc_formats.isEmpty() )
+    return LocaleConfiguration( bestLocale.name(), lc_formats.isEmpty() ? bestLocale.name() : lc_formats );
-    {
-        lc_formats = lang;
-    }
-
-    return LocaleConfiguration( lang, lc_formats );
 }
 
 

src/modules/locale/LocaleNames.cpp

@@ -0,0 +1,90 @@
+/* === This file is part of Calamares - <https://calamares.io> ===
+ *
+ *   SPDX-FileCopyrightText: 2022 Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#include "LocaleNames.h"
+
+#include "utils/Logger.h"
+
+#include <QRegularExpression>
+
+LocaleNameParts
+LocaleNameParts::fromName( const QString& name )
+{
+    auto requireAndRemoveLeadingChar = []( QChar c, QString s )
+    {
+        if ( s.startsWith( c ) )
+        {
+            return s.remove( 0, 1 );
+        }
+        else
+        {
+            return QString();
+        }
+    };
+
+    auto parts = QRegularExpression( "^([a-zA-Z]+)(_[a-zA-Z]+)?(\\.[-a-zA-Z0-9]+)?(@[a-zA-Z]+)?" ).match( name );
+    const QString calamaresLanguage = parts.captured( 1 );
+    const QString calamaresCountry = requireAndRemoveLeadingChar( '_', parts.captured( 2 ) );
+    const QString calamaresEncoding = requireAndRemoveLeadingChar( '.', parts.captured( 3 ) );
+    const QString calamaresRegion = requireAndRemoveLeadingChar( '@', parts.captured( 4 ) );
+
+    if ( calamaresLanguage.isEmpty() )
+    {
+        return LocaleNameParts {};
+    }
+    else
+    {
+        return LocaleNameParts { calamaresLanguage, calamaresCountry, calamaresRegion, calamaresEncoding };
+    }
+}
+
+QString
+LocaleNameParts::name() const
+{
+    // We don't want QStringView to a temporary; force conversion
+    auto insertLeadingChar = []( QChar c, QString s ) -> QString
+    {
+        if ( s.isEmpty() )
+        {
+            return QString();
+        }
+        else
+        {
+            return c + s;
+        }
+    };
+
+    if ( !isValid() )
+    {
+        return QString();
+    }
+    else
+    {
+        return language + insertLeadingChar( '_', country ) + insertLeadingChar( '.', encoding )
+            + insertLeadingChar( '@', region );
+    }
+}
+
+
+int
+LocaleNameParts::similarity( const LocaleNameParts& other ) const
+{
+    if ( !isValid() || !other.isValid() )
+    {
+        return 0;
+    }
+    if ( language != other.language )
+    {
+        return 0;
+    }
+    const auto matched_region = ( region == other.region ? 30 : 0 );
+    const auto matched_country = ( country == other.country ? ( country.isEmpty() ? 10 : 20 ) : 0 );
+    const auto no_other_country_given = ( ( country != other.country && other.country.isEmpty() ) ? 10 : 0 );
+    return 50 + matched_region + matched_country + no_other_country_given;
+}

src/modules/locale/LocaleNames.h

@@ -0,0 +1,46 @@
+/* === This file is part of Calamares - <https://calamares.io> ===
+ *
+ *   SPDX-FileCopyrightText: 2022 Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#ifndef LOCALENAMES_H
+#define LOCALENAMES_H
+
+#include <QString>
+
+/** @brief parts of a locale-name (e.g. "ar_LY.UTF-8", split apart)
+ *
+ * These are created from lines in `/usr/share/i18n/SUPPORTED`,
+ * which lists all the locales supported by the system (there
+ * are also other sources of the same).
+ *
+ */
+struct LocaleNameParts
+{
+    QString language;  // e.g. "ar"
+    QString country;  // e.g. "LY" (may be empty)
+    QString region;  // e.g. "@valencia" (may be empty)
+    QString encoding;  // e.g. "UTF-8" (may be empty)
+
+    bool isValid() const { return !language.isEmpty(); }
+    QString name() const;
+
+    static LocaleNameParts fromName( const QString& name );
+
+    static inline constexpr const int no_match = 0;
+    static inline constexpr const int complete_match = 100;
+
+    /** @brief Compute similarity-score with another locale-name.
+     *
+     * Similarity is driven by language and region, then country.
+     * Returns a number between 0 (no similarity, e.g. the
+     * language is different) and 100 (complete match).
+     */
+    int similarity( const LocaleNameParts& other ) const;
+};
+
+#endif

src/modules/locale/Tests.cpp

@@ -9,9 +9,12 @@
 
 #include "Config.h"
 #include "LocaleConfiguration.h"
+#include "LocaleNames.h"
 #include "timezonewidget/TimeZoneImage.h"
 
+#include "Settings.h"
 #include "locale/TimeZone.h"
+#include "locale/TranslationsModel.h"
 #include "utils/Logger.h"
 
 #include <QtTest/QtTest>
@@ -25,6 +28,9 @@ public:
     LocaleTests();
     ~LocaleTests() override;
 
+    // Implementation of data for MappingNeon and MappingFreeBSD
+    void MappingData();
+
 private Q_SLOTS:
     void initTestCase();
     // Check the sample config file is processed correctly
@@ -43,6 +49,21 @@ private Q_SLOTS:
     void testLanguageDetection_data();
     void testLanguageDetection();
     void testLanguageDetectionValencia();
+
+    // Check that the test-data is available and ok
+    void testKDENeonLanguageData();
+    void testLocaleNameParts();
+
+    // Check realistic language mapping for issue 2008
+    void testLanguageMappingNeon_data();
+    void testLanguageMappingNeon();
+    void testLanguageMappingFreeBSD_data();
+    void testLanguageMappingFreeBSD();
+    void testLanguageSimilarity();
+
+private:
+    QStringList m_KDEneonLocales;
+    QStringList m_FreeBSDLocales;
 };
 
 QTEST_MAIN( LocaleTests )
@@ -55,6 +76,12 @@ LocaleTests::~LocaleTests() {}
 void
 LocaleTests::initTestCase()
 {
+    Logger::setupLogLevel( Logger::LOGDEBUG );
+    const auto* settings = Calamares::Settings::instance();
+    if ( !settings )
+    {
+        (void)new Calamares::Settings( true );
+    }
 }
 
 void
@@ -280,10 +307,10 @@ LocaleTests::testLanguageDetection_data()
     QTest::newRow( "english (US)" ) << QStringLiteral( "en" ) << QStringLiteral( "US" )
                                     << QStringLiteral( "en_US.UTF-8" );
     QTest::newRow( "english (CA)" ) << QStringLiteral( "en" ) << QStringLiteral( "CA" )
-                                    << QStringLiteral( "en" );  // because it's first in the list
+                                    << QStringLiteral( "en_US.UTF-8" );
     QTest::newRow( "english (GB)" ) << QStringLiteral( "en" ) << QStringLiteral( "GB" )
                                     << QStringLiteral( "en_GB.UTF-8" );
-    QTest::newRow( "english (NL)" ) << QStringLiteral( "en" ) << QStringLiteral( "NL" ) << QStringLiteral( "en" );
+    QTest::newRow( "english (NL)" ) << QStringLiteral( "en" ) << QStringLiteral( "NL" ) << QStringLiteral( "en_US.UTF-8" );
 
     QTest::newRow( "portuguese (PT)" ) << QStringLiteral( "pt" ) << QStringLiteral( "PT" )
                                        << QStringLiteral( "pt_PT.UTF-8" );
@@ -293,11 +320,11 @@ LocaleTests::testLanguageDetection_data()
                                        << QStringLiteral( "pt_BR.UTF-8" );
 
     QTest::newRow( "catalan ()" ) << QStringLiteral( "ca" ) << QStringLiteral( "" )
-                                  << QStringLiteral( "ca_AD.UTF-8" );  // no country given? Matches first
+                                  << QStringLiteral( "ca_ES.UTF-8" );  // no country given? Matches QLocale-default
     QTest::newRow( "catalan (ES)" ) << QStringLiteral( "ca" ) << QStringLiteral( "ES" )
                                     << QStringLiteral( "ca_ES.UTF-8" );
     QTest::newRow( "catalan (NL)" ) << QStringLiteral( "ca" ) << QStringLiteral( "NL" )
-                                    << QStringLiteral( "ca_AD.UTF-8" );
+                                    << QStringLiteral( "ca_ES.UTF-8" );
     QTest::newRow( "catalan (@valencia)" ) << QStringLiteral( "ca@valencia" ) << QStringLiteral( "ES" )
                                            << QStringLiteral( "ca_ES@valencia" );  // Prefers regional variant
     QTest::newRow( "catalan (@valencia_NL)" )
@@ -344,7 +371,7 @@ LocaleTests::testLanguageDetectionValencia()
     {
         auto r = LocaleConfiguration::fromLanguageAndLocation(
             QStringLiteral( "sr" ), availableLocales, QStringLiteral( "NL" ) );
-        QCOMPARE( r.language(), "sr_ME" );  // Because that one is first in the list
+        QCOMPARE( r.language(), "sr_RS" );  // Because that one is first in the list
     }
     {
         auto r = LocaleConfiguration::fromLanguageAndLocation(
@@ -353,6 +380,206 @@ LocaleTests::testLanguageDetectionValencia()
     }
 }
 
+static QStringList
+splitTestFileIntoLines( const QString& filename )
+{
+    // BUILD_AS_TEST is the source-directory path
+    const QFileInfo fi( QString( "%1/tests/%2" ).arg( BUILD_AS_TEST, filename ) );
+    const QString path = fi.absoluteFilePath();
+    QFile testData( path );
+    if ( testData.open( QIODevice::ReadOnly ) )
+    {
+        return QString::fromUtf8( testData.readAll() ).split( '\n', Qt::SkipEmptyParts );
+    }
+    return QStringList {};
+}
+
+void
+LocaleTests::testKDENeonLanguageData()
+{
+    if ( !m_KDEneonLocales.isEmpty() )
+    {
+        return;
+    }
+    const QStringList neonLocales = splitTestFileIntoLines( QStringLiteral( "locale-data-neon" ) );
+    cDebug() << "Loaded KDE neon locales test data" << neonLocales.front() << "to" << neonLocales.back();
+    QCOMPARE( neonLocales.length(), 318 );  // wc -l tells me 318 lines
+    m_KDEneonLocales = neonLocales;
+
+    const QStringList bsdLocales = splitTestFileIntoLines( QStringLiteral( "locale-data-freebsd" ) );
+    cDebug() << "Loaded FreeBSD locales test data" << bsdLocales.front() << "to" << bsdLocales.back();
+    QCOMPARE( bsdLocales.length(), 79 );
+    m_FreeBSDLocales = bsdLocales;
+}
+
+void
+LocaleTests::MappingData()
+{
+    QTest::addColumn< QString >( "selectedLanguage" );
+    QTest::addColumn< QString >( "KDEneonLanguage" );
+    QTest::addColumn< QString >( "FreeBSDLanguage" );
+
+    // Tired of writing QString or QStringLiteral all the time.
+    auto l = []( const char* p ) { return QString::fromUtf8( p ); };
+    auto u = []() { return QString(); };
+
+    // The KDEneon columns include the .UTF-8 from the source data
+    // The FreeBSD columns may have u() to indicate "same as KDEneon",
+    //      that's an empty string.
+    //
+    // Each row shows how a language -- which can be selected from the
+    // welcome page, and is inserted into GS as the language key that
+    // Calamares knows -- should be mapped to a supported system locale.
+    //
+    // All the mappings are for ".. in NL", which can trigger minor variation
+    // if there are languages with a _NL variant (e.g. nl_NL and nl_BE).
+
+    // clang-format off
+    QTest::newRow( "en   " ) << l( "en" )           << l( "en_US.UTF-8" )       << u();
+    QTest::newRow( "en_GB" ) << l( "en_GB" )        << l( "en_GB.UTF-8" )       << u();
+    QTest::newRow( "ca   " ) << l( "ca" )           << l( "ca_ES.UTF-8" )       << u();
+    // FreeBSD has no Valencian variant
+    QTest::newRow( "ca@vl" ) << l( "ca@valencia" )  << l( "ca_ES@valencia" )    << l( "ca_ES.UTF-8" );
+    // FreeBSD has the UTF-8 marker before the @region part
+    QTest::newRow( "sr   " ) << l( "sr" )           << l( "sr_RS" )             << l( "sr_RS.UTF-8" );
+    QTest::newRow( "sr@lt" ) << l( "sr@latin" )     << l( "sr_RS@latin" )       << l( "sr_RS.UTF-8@latin" );
+    QTest::newRow( "pt_PT" ) << l( "pt_PT" )        << l( "pt_PT.UTF-8" )       << u();
+    QTest::newRow( "pt_BR" ) << l( "pt_BR" )        << l( "pt_BR.UTF-8" )       << u();
+    QTest::newRow( "nl   " ) << l( "nl" )           << l( "nl_NL.UTF-8" )       << u();
+    QTest::newRow( "zh_TW" ) << l( "zh_TW" )        << l( "zh_TW.UTF-8" )       << u();
+    // clang-format on
+}
+
+
+void
+LocaleTests::testLanguageMappingNeon_data()
+{
+    MappingData();
+}
+
+void
+LocaleTests::testLanguageMappingFreeBSD_data()
+{
+    MappingData();
+}
+
+void
+LocaleTests::testLanguageMappingNeon()
+{
+    testKDENeonLanguageData();
+    QVERIFY( !m_KDEneonLocales.isEmpty() );
+
+    QFETCH( QString, selectedLanguage );
+    QFETCH( QString, KDEneonLanguage );
+    QFETCH( QString, FreeBSDLanguage );
+
+    QVERIFY( Calamares::Locale::availableLanguages().contains( selectedLanguage ) );
+
+    const auto neon = LocaleConfiguration::fromLanguageAndLocation(
+        ( selectedLanguage ), m_KDEneonLocales, QStringLiteral( "NL" ) );
+    QCOMPARE( neon.language(), KDEneonLanguage );
+}
+
+void
+LocaleTests::testLanguageMappingFreeBSD()
+{
+    testKDENeonLanguageData();
+    QVERIFY( !m_FreeBSDLocales.isEmpty() );
+
+    QFETCH( QString, selectedLanguage );
+    QFETCH( QString, KDEneonLanguage );
+    QFETCH( QString, FreeBSDLanguage );
+
+    QVERIFY( Calamares::Locale::availableLanguages().contains( selectedLanguage ) );
+
+    const auto bsd = LocaleConfiguration::fromLanguageAndLocation(
+        ( selectedLanguage ), m_FreeBSDLocales, QStringLiteral( "NL" ) );
+    const auto expected = FreeBSDLanguage.isEmpty() ? KDEneonLanguage : FreeBSDLanguage;
+    QCOMPARE( bsd.language(), expected );
+}
+
+void
+LocaleTests::testLocaleNameParts()
+{
+    testKDENeonLanguageData();
+    QVERIFY( !m_FreeBSDLocales.isEmpty() );
+    QVERIFY( !m_KDEneonLocales.isEmpty() );
+
+    // Example constant locales
+    {
+        auto c_parts = LocaleNameParts::fromName( QStringLiteral( "nl_NL.UTF-8" ) );
+        QCOMPARE( c_parts.language, QStringLiteral( "nl" ) );
+        QCOMPARE( c_parts.country, QStringLiteral( "NL" ) );
+        QCOMPARE( c_parts.encoding, QStringLiteral( "UTF-8" ) );
+        QVERIFY( c_parts.region.isEmpty() );
+    }
+    {
+        auto c_parts = LocaleNameParts::fromName( QStringLiteral( "C.UTF-8" ) );
+        QCOMPARE( c_parts.language, QStringLiteral( "C" ) );
+        QVERIFY( c_parts.country.isEmpty() );
+        QCOMPARE( c_parts.encoding, QStringLiteral( "UTF-8" ) );
+        QVERIFY( c_parts.region.isEmpty() );
+    }
+
+    // Check all the loaded test locales
+    for ( const auto& s : m_FreeBSDLocales )
+    {
+        auto parts = LocaleNameParts::fromName( s );
+        QVERIFY( parts.isValid() );
+        QCOMPARE( parts.name(), s );
+    }
+
+    for ( const auto& s : m_KDEneonLocales )
+    {
+        auto parts = LocaleNameParts::fromName( s );
+        QVERIFY( parts.isValid() );
+        QCOMPARE( parts.name(), s );
+    }
+}
+
+void
+LocaleTests::testLanguageSimilarity()
+{
+    // Empty
+    {
+        QCOMPARE( LocaleNameParts().similarity( LocaleNameParts() ), 0 );
+    }
+    // Some simple Dutch situations
+    {
+        auto nl_parts = LocaleNameParts::fromName( QStringLiteral( "nl_NL.UTF-8" ) );
+        auto be_parts = LocaleNameParts::fromName( QStringLiteral( "nl_BE.UTF-8" ) );
+        auto nl_short_parts = LocaleNameParts::fromName( QStringLiteral( "nl" ) );
+        QCOMPARE( nl_parts.similarity( nl_parts ), 100 );
+        QCOMPARE( nl_parts.similarity( LocaleNameParts() ), 0 );
+        QCOMPARE( nl_parts.similarity( be_parts ), 80 );  // Language + (empty) region match
+        QCOMPARE( nl_parts.similarity( nl_short_parts ), 90 );
+    }
+
+    // Everything matches itself
+    {
+        if ( m_KDEneonLocales.isEmpty() )
+        {
+            testKDENeonLanguageData();
+        }
+        QVERIFY( !m_FreeBSDLocales.isEmpty() );
+        QVERIFY( !m_KDEneonLocales.isEmpty() );
+        for ( const auto& l : m_KDEneonLocales )
+        {
+            auto locale_name = LocaleNameParts::fromName( l );
+            auto self_similarity = locale_name.similarity( locale_name );
+            if ( self_similarity != 100 )
+            {
+                cDebug() << "Locale" << l << "is unusual.";
+                if ( l == QStringLiteral( "eo" ) )
+                {
+                    QEXPECT_FAIL( "", "Esperanto has no country to match", Continue );
+                }
+            }
+            QCOMPARE( self_similarity, 100 );
+        }
+    }
+}
+
 
 #include "utils/moc-warnings.h"
 

src/modules/locale/tests/locale-data-freebsd

@@ -0,0 +1,79 @@
+C.UTF-8
+af_ZA.UTF-8
+am_ET.UTF-8
+ar_AE.UTF-8
+ar_EG.UTF-8
+ar_JO.UTF-8
+ar_MA.UTF-8
+ar_QA.UTF-8
+ar_SA.UTF-8
+be_BY.UTF-8
+bg_BG.UTF-8
+ca_AD.UTF-8
+ca_ES.UTF-8
+ca_FR.UTF-8
+ca_IT.UTF-8
+cs_CZ.UTF-8
+da_DK.UTF-8
+de_AT.UTF-8
+de_CH.UTF-8
+de_DE.UTF-8
+el_GR.UTF-8
+en_AU.UTF-8
+en_CA.UTF-8
+en_GB.UTF-8
+en_HK.UTF-8
+en_IE.UTF-8
+en_NZ.UTF-8
+en_PH.UTF-8
+en_SG.UTF-8
+en_US.UTF-8
+en_ZA.UTF-8
+es_AR.UTF-8
+es_CR.UTF-8
+es_ES.UTF-8
+es_MX.UTF-8
+et_EE.UTF-8
+eu_ES.UTF-8
+fi_FI.UTF-8
+fr_BE.UTF-8
+fr_CA.UTF-8
+fr_CH.UTF-8
+fr_FR.UTF-8
+ga_IE.UTF-8
+he_IL.UTF-8
+hi_IN.UTF-8
+hr_HR.UTF-8
+hu_HU.UTF-8
+hy_AM.UTF-8
+is_IS.UTF-8
+it_CH.UTF-8
+it_IT.UTF-8
+ja_JP.UTF-8
+kk_KZ.UTF-8
+ko_KR.UTF-8
+lt_LT.UTF-8
+lv_LV.UTF-8
+mn_MN.UTF-8
+nb_NO.UTF-8
+nl_BE.UTF-8
+nl_NL.UTF-8
+nn_NO.UTF-8
+pl_PL.UTF-8
+pt_BR.UTF-8
+pt_PT.UTF-8
+ro_RO.UTF-8
+ru_RU.UTF-8
+se_FI.UTF-8
+se_NO.UTF-8
+sk_SK.UTF-8
+sl_SI.UTF-8
+sr_RS.UTF-8
+sr_RS.UTF-8@latin
+sv_FI.UTF-8
+sv_SE.UTF-8
+tr_TR.UTF-8
+uk_UA.UTF-8
+zh_CN.UTF-8
+zh_HK.UTF-8
+zh_TW.UTF-8

src/modules/locale/tests/locale-data-neon

@@ -0,0 +1,318 @@
+aa_DJ.UTF-8
+aa_ER
+aa_ER@saaho
+aa_ET
+af_ZA.UTF-8
+agr_PE
+ak_GH
+am_ET
+an_ES.UTF-8
+anp_IN
+ar_AE.UTF-8
+ar_BH.UTF-8
+ar_DZ.UTF-8
+ar_EG.UTF-8
+ar_IN
+ar_IQ.UTF-8
+ar_JO.UTF-8
+ar_KW.UTF-8
+ar_LB.UTF-8
+ar_LY.UTF-8
+ar_MA.UTF-8
+ar_OM.UTF-8
+ar_QA.UTF-8
+ar_SA.UTF-8
+ar_SD.UTF-8
+ar_SS
+ar_SY.UTF-8
+ar_TN.UTF-8
+ar_YE.UTF-8
+ayc_PE
+az_AZ
+az_IR
+as_IN
+ast_ES.UTF-8
+be_BY.UTF-8
+be_BY@latin
+bem_ZM
+ber_DZ
+ber_MA
+bg_BG.UTF-8
+bhb_IN.UTF-8
+bho_IN
+bho_NP
+bi_VU
+bn_BD
+bn_IN
+bo_CN
+bo_IN
+br_FR.UTF-8
+brx_IN
+bs_BA.UTF-8
+byn_ER
+ca_AD.UTF-8
+ca_ES.UTF-8
+ca_ES@valencia
+ca_FR.UTF-8
+ca_IT.UTF-8
+ce_RU
+ckb_IQ
+chr_US
+cmn_TW
+crh_UA
+cs_CZ.UTF-8
+csb_PL
+cv_RU
+cy_GB.UTF-8
+da_DK.UTF-8
+de_AT.UTF-8
+de_BE.UTF-8
+de_CH.UTF-8
+de_DE.UTF-8
+de_IT.UTF-8
+de_LI.UTF-8
+de_LU.UTF-8
+doi_IN
+dsb_DE
+dv_MV
+dz_BT
+el_GR.UTF-8
+el_CY.UTF-8
+en_AG
+en_AU.UTF-8
+en_BW.UTF-8
+en_CA.UTF-8
+en_DK.UTF-8
+en_GB.UTF-8
+en_HK.UTF-8
+en_IE.UTF-8
+en_IL
+en_IN
+en_NG
+en_NZ.UTF-8
+en_PH.UTF-8
+en_SC.UTF-8
+en_SG.UTF-8
+en_US.UTF-8
+en_ZA.UTF-8
+en_ZM
+en_ZW.UTF-8
+eo
+eo_US.UTF-8
+es_AR.UTF-8
+es_BO.UTF-8
+es_CL.UTF-8
+es_CO.UTF-8
+es_CR.UTF-8
+es_CU
+es_DO.UTF-8
+es_EC.UTF-8
+es_ES.UTF-8
+es_GT.UTF-8
+es_HN.UTF-8
+es_MX.UTF-8
+es_NI.UTF-8
+es_PA.UTF-8
+es_PE.UTF-8
+es_PR.UTF-8
+es_PY.UTF-8
+es_SV.UTF-8
+es_US.UTF-8
+es_UY.UTF-8
+es_VE.UTF-8
+et_EE.UTF-8
+eu_ES.UTF-8
+eu_FR.UTF-8
+fa_IR
+ff_SN
+fi_FI.UTF-8
+fil_PH
+fo_FO.UTF-8
+fr_BE.UTF-8
+fr_CA.UTF-8
+fr_CH.UTF-8
+fr_FR.UTF-8
+fr_LU.UTF-8
+fur_IT
+fy_NL
+fy_DE
+ga_IE.UTF-8
+gd_GB.UTF-8
+gez_ER
+gez_ER@abegede
+gez_ET
+gez_ET@abegede
+gl_ES.UTF-8
+gu_IN
+gv_GB.UTF-8
+ha_NG
+hak_TW
+he_IL.UTF-8
+hi_IN
+hif_FJ
+hne_IN
+hr_HR.UTF-8
+hsb_DE.UTF-8
+ht_HT
+hu_HU.UTF-8
+hy_AM
+ia_FR
+id_ID.UTF-8
+ig_NG
+ik_CA
+is_IS.UTF-8
+it_CH.UTF-8
+it_IT.UTF-8
+iu_CA
+ja_JP.UTF-8
+ka_GE.UTF-8
+kab_DZ
+kk_KZ.UTF-8
+kl_GL.UTF-8
+km_KH
+kn_IN
+ko_KR.UTF-8
+kok_IN
+ks_IN
+ks_IN@devanagari
+ku_TR.UTF-8
+kw_GB.UTF-8
+ky_KG
+lb_LU
+lg_UG.UTF-8
+li_BE
+li_NL
+lij_IT
+ln_CD
+lo_LA
+lt_LT.UTF-8
+lv_LV.UTF-8
+lzh_TW
+mag_IN
+mai_IN
+mai_NP
+mfe_MU
+mg_MG.UTF-8
+mhr_RU
+mi_NZ.UTF-8
+miq_NI
+mjw_IN
+mk_MK.UTF-8
+ml_IN
+mn_MN
+mni_IN
+mnw_MM
+mr_IN
+ms_MY.UTF-8
+mt_MT.UTF-8
+my_MM
+nan_TW
+nan_TW@latin
+nb_NO.UTF-8
+nds_DE
+nds_NL
+ne_NP
+nhn_MX
+niu_NU
+niu_NZ
+nl_AW
+nl_BE.UTF-8
+nl_NL.UTF-8
+nn_NO.UTF-8
+nr_ZA
+nso_ZA
+oc_FR.UTF-8
+om_ET
+om_KE.UTF-8
+or_IN
+os_RU
+pa_IN
+pa_PK
+pap_AW
+pap_CW
+pl_PL.UTF-8
+ps_AF
+pt_BR.UTF-8
+pt_PT.UTF-8
+quz_PE
+raj_IN
+ro_RO.UTF-8
+ru_RU.UTF-8
+ru_UA.UTF-8
+rw_RW
+sa_IN
+sah_RU
+sat_IN
+sc_IT
+sd_IN
+sd_IN@devanagari
+sd_PK
+se_NO
+sgs_LT
+shn_MM
+shs_CA
+si_LK
+sid_ET
+sk_SK.UTF-8
+sl_SI.UTF-8
+sm_WS
+so_DJ.UTF-8
+so_ET
+so_KE.UTF-8
+so_SO.UTF-8
+sq_AL.UTF-8
+sq_MK
+sr_ME
+sr_RS
+sr_RS@latin
+ss_ZA
+st_ZA.UTF-8
+sv_FI.UTF-8
+sv_SE.UTF-8
+sw_KE
+sw_TZ
+szl_PL
+ta_IN
+ta_LK
+tcy_IN.UTF-8
+te_IN
+tg_TJ.UTF-8
+th_TH.UTF-8
+the_NP
+ti_ER
+ti_ET
+tig_ER
+tk_TM
+tl_PH.UTF-8
+tn_ZA
+to_TO
+tpi_PG
+tr_CY.UTF-8
+tr_TR.UTF-8
+ts_ZA
+tt_RU
+tt_RU@iqtelif
+ug_CN
+ug_CN@latin
+uk_UA.UTF-8
+unm_US
+ur_IN
+ur_PK
+uz_UZ.UTF-8
+uz_UZ@cyrillic
+ve_ZA
+vi_VN
+wa_BE.UTF-8
+wae_CH
+wal_ET
+wo_SN
+xh_ZA.UTF-8
+yi_US.UTF-8
+yo_NG
+yue_HK
+yuw_PG
+zh_CN.UTF-8
+zh_HK.UTF-8
+zh_SG.UTF-8
+zh_TW.UTF-8
+zu_ZA.UTF-8

src/modules/localeq/CMakeLists.txt

@@ -31,8 +31,9 @@ calamares_add_plugin(localeq
     EXPORT_MACRO PLUGINDLLEXPORT_PRO
     SOURCES
         LocaleQmlViewStep.cpp
-        ${_locale}/LocaleConfiguration.cpp
         ${_locale}/Config.cpp
+        ${_locale}/LocaleConfiguration.cpp
+        ${_locale}/LocaleNames.cpp
         ${_locale}/SetTimezoneJob.cpp
     RESOURCES
         localeq.qrc

src/modules/luksbootkeyfile/CMakeLists.txt

@@ -12,9 +12,4 @@ calamares_add_plugin(luksbootkeyfile
     NO_CONFIG
 )
 
-calamares_add_test(
+calamares_add_test(luksbootkeyfiletest SOURCES Tests.cpp LuksBootKeyFileJob.cpp)
-    luksbootkeyfiletest
-    SOURCES
-        Tests.cpp
-        LuksBootKeyFileJob.cpp
-)

src/modules/partition/CMakeLists.txt

@@ -18,27 +18,27 @@
 # modules since the partitions on disk won't match GS, but it can be
 # useful for debugging simulated installations that don't need to
 # mount the target filesystems.
-option( DEBUG_PARTITION_UNSAFE "Allow unsafe partitioning choices." OFF )
+option(DEBUG_PARTITION_UNSAFE "Allow unsafe partitioning choices." OFF)
-option( DEBUG_PARTITION_BAIL_OUT "Unsafe partitioning will error out on exec." ON )
+option(DEBUG_PARTITION_BAIL_OUT "Unsafe partitioning will error out on exec." ON)
-option( DEBUG_PARTITION_SKIP "Don't actually do any partitioning." OFF)
+option(DEBUG_PARTITION_SKIP "Don't actually do any partitioning." OFF)
 
 # This is very chatty, useful mostly if you don't know what KPMCore offers.
 option(DEBUG_FILESYSTEMS "Log all available Filesystems from KPMCore." OFF)
 
 include_directories(${CMAKE_SOURCE_DIR}) # For 3rdparty
 
-set( _partition_defs )
+set(_partition_defs)
-if( DEBUG_PARTITION_UNSAFE )
+if(DEBUG_PARTITION_UNSAFE)
-    if( DEBUG_PARTITION_BAIL_OUT )
+    if(DEBUG_PARTITION_BAIL_OUT)
-        list( APPEND _partition_defs DEBUG_PARTITION_BAIL_OUT )
+        list(APPEND _partition_defs DEBUG_PARTITION_BAIL_OUT)
     endif()
     list(APPEND _partition_defs DEBUG_PARTITION_UNSAFE)
 endif()
 if(DEBUG_FILESYSTEMS)
     list(APPEND _partition_defs DEBUG_FILESYSTEMS)
 endif()
-if( DEBUG_PARTITION_SKIP )
+if(DEBUG_PARTITION_SKIP)
-    list( APPEND _partition_defs DEBUG_PARTITION_SKIP )
+    list(APPEND _partition_defs DEBUG_PARTITION_SKIP)
 endif()
 
 find_package(ECM ${ECM_VERSION} REQUIRED NO_MODULE)