From 63f9c256119fe4e9cd7df6bbf965703382bb604d Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 23 Aug 2017 06:57:11 -0400 Subject: [PATCH 01/19] Update unpackfs.conf examples and documentation --- src/modules/unpackfs/unpackfs.conf | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index 34e7bc48d..9720f19a1 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -13,12 +13,26 @@ unpack: # Each list item is unpacked, in order, to the target system. # Each list item has the following attributes: # source: path relative to the live / intstalling system to the image -# sourcefs: ext4 or squashfs (may be others if mount supports is) +# sourcefs: ext4 or squashfs (may be others if mount supports it) # destination: path relative to rootMountPoint (so in the target # system) where this filesystem is unpacked. - - source: "/path/to/filesystem.img" - sourcefs: "ext4" - destination: "" - - source: "/path/to/another/filesystem.sqfs" - sourcefs: "squashfs" + +# Usually you list a filesystem image to unpack; you can use +# squashfs or an ext4 image. +# +# - source: "/path/to/filesystem.sqfs" +# sourcefs: "squashfs" +# destination: "" + +# You can list more than one filesystem. +# +# - source: "/path/to/another/filesystem.img" +# sourcefs: "ext4" +# destination: "" +# + +# You can list filesystem source paths relative to the Calamares run +# directory, if you use -d (this is only useful for testing, though). + - source: ./example.sqfs + sourcefs: squashfs destination: "" From 186f6cd1e2296f0227e77f91740170fdd661a7c4 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 23 Aug 2017 10:45:04 -0400 Subject: [PATCH 02/19] initcpiocfg: accept (but warn) for missing config file in the host --- src/modules/initcpiocfg/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/initcpiocfg/main.py b/src/modules/initcpiocfg/main.py index eb29d7def..ca9455d20 100644 --- a/src/modules/initcpiocfg/main.py +++ b/src/modules/initcpiocfg/main.py @@ -65,8 +65,12 @@ def write_mkinitcpio_lines(hooks, modules, files, root_mount_point): :param files: :param root_mount_point: """ - with open("/etc/mkinitcpio.conf", "r") as mkinitcpio_file: - mklins = [x.strip() for x in mkinitcpio_file.readlines()] + try: + with open("/etc/mkinitcpio.conf", "r") as mkinitcpio_file: + mklins = [x.strip() for x in mkinitcpio_file.readlines()] + except FileNotFoundError: + libcalamares.utils.debug("Could not open host file /etc/mkinitcpio.conf") + mklins = [] for i in range(len(mklins)): if mklins[i].startswith("HOOKS"): From bba965185027a4517d37880da4a91805e0f718f2 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 23 Aug 2017 09:31:15 -0400 Subject: [PATCH 03/19] Create example distro squashfs (from build host) - Add a target 'example-distro' which must be manually invoked This creates an example.sqfs with a minimal binary distro based on the build hosts's /bin and /lib. The purpose is to provide a simple test image which the default configuration of the unpackfs module can use to create a system within which the *other* steps of the installation can run. Example files are some zoneinfo's (remember to choose an existing zone when using the example distro), groups and sudoers files, etc .. The example distro has a special /xbin which contains bogus binaries for many system-administration tasks (e.g. useradd which would otherwise come from /usr/sbin). --- CMakeLists.txt | 44 ++ data/example-root/README.md | 11 + data/example-root/etc/bash.bashrc | 2 + data/example-root/etc/group | 1 + data/example-root/etc/issue | 1 + data/example-root/etc/locale.gen | 486 ++++++++++++++++++ data/example-root/etc/profile | 2 + data/example-root/etc/sudoers.d/10-installer | 0 data/example-root/usr/share/zoneinfo/.dummy | 0 .../usr/share/zoneinfo/America/New_York | Bin 0 -> 3545 bytes data/example-root/usr/share/zoneinfo/UTC | Bin 0 -> 127 bytes data/example-root/usr/share/zoneinfo/Zulu | Bin 0 -> 127 bytes data/example-root/var/lib/dbus/.dummy | 0 data/example-root/var/lib/initramfs-tools | 0 data/example-root/xbin/linux-version | 1 + data/example-root/xbin/useradd | 1 + 16 files changed, 549 insertions(+) create mode 100644 data/example-root/README.md create mode 100644 data/example-root/etc/bash.bashrc create mode 100644 data/example-root/etc/group create mode 100644 data/example-root/etc/issue create mode 100644 data/example-root/etc/locale.gen create mode 100644 data/example-root/etc/profile create mode 100644 data/example-root/etc/sudoers.d/10-installer create mode 100644 data/example-root/usr/share/zoneinfo/.dummy create mode 100644 data/example-root/usr/share/zoneinfo/America/New_York create mode 100644 data/example-root/usr/share/zoneinfo/UTC create mode 100644 data/example-root/usr/share/zoneinfo/Zulu create mode 100644 data/example-root/var/lib/dbus/.dummy create mode 100644 data/example-root/var/lib/initramfs-tools create mode 100755 data/example-root/xbin/linux-version create mode 100755 data/example-root/xbin/useradd diff --git a/CMakeLists.txt b/CMakeLists.txt index 8358060ba..f98b25c99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,6 +185,50 @@ set( CALAMARES_LIBRARIES calamares ) set( THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty" ) +### Example Distro +# +# For testing purposes Calamares includes a very, very, limited sample +# distro called "Generic". The root filesystem of "Generic" lives in +# data/example-root and can be squashed up as part of the build, so +# that a pure-upstream run of ./calamares -d from the build directory +# (with all the default settings and configurations) can actually +# do an complete example run. +# +# Some binaries from the build host (e.g. /bin and /lib) are also +# squashed into the example filesystem. +# +# To build the example distro (for use by the default, example, +# unsquashfs module), build the target 'example-distro', eg.: +# +# make example-distro +# +find_program( mksquashfs_PROGRAM mksquashfs ) +macro_log_feature( mksquashfs_PROGRAM "mksquashfs" "Create example distro" "http://tldp.org/HOWTO/SquashFS-HOWTO/creatingandusing.html") +if( mksquashfs_PROGRAM ) + set( src_fs ${CMAKE_SOURCE_DIR}/data/example-root/ ) + set( dst_fs ${CMAKE_BINARY_DIR}/example.sqfs ) + if( EXISTS ${src_fs} ) + # Collect directories needed for a minimal binary distro, + # based on the build host. If /lib64 exists, assume it is needed. + # Note that the last path component is added to the root, so + # if you add /usr/sbin here, it will be put into /sbin_1. + # Add such paths to /etc/profile under ${src_fs}. + set( candidate_fs /sbin /bin /lib /lib64 ) + set( host_fs "" ) + foreach( c_fs ${candidate_fs} ) + if( EXISTS ${c_fs} ) + list( APPEND host_fs ${c_fs} ) + endif() + endforeach() + add_custom_command( + OUTPUT ${dst_fs} + COMMAND ${mksquashfs_PROGRAM} ${src_fs} ${dst_fs} -all-root + COMMAND ${mksquashfs_PROGRAM} ${host_fs} ${dst_fs} -all-root + ) + add_custom_target(example-distro DEPENDS ${dst_fs}) + endif() +endif() + add_subdirectory( thirdparty ) add_subdirectory( src ) diff --git a/data/example-root/README.md b/data/example-root/README.md new file mode 100644 index 000000000..7173b3eaf --- /dev/null +++ b/data/example-root/README.md @@ -0,0 +1,11 @@ +# Example Filesystem + +This is a filesystem that will be used as / in an example distro, +*if* you build the `example-distro` target and use the default +unpackfs configuration. It should hold files and configuration +bits that need to be on the target system for example purposes. + +It should *not* have a bin/, lib/, sbin/ or lib64/ directory, +since those are copied into the example-distro filesystem +from the build host. + diff --git a/data/example-root/etc/bash.bashrc b/data/example-root/etc/bash.bashrc new file mode 100644 index 000000000..42bd3d007 --- /dev/null +++ b/data/example-root/etc/bash.bashrc @@ -0,0 +1,2 @@ +# Global .profile -- add /sbin_1 +PATH=$PATH:/sbin_1:/xbin diff --git a/data/example-root/etc/group b/data/example-root/etc/group new file mode 100644 index 000000000..1dbf9013e --- /dev/null +++ b/data/example-root/etc/group @@ -0,0 +1 @@ +root:x:0: diff --git a/data/example-root/etc/issue b/data/example-root/etc/issue new file mode 100644 index 000000000..ce0ac58e3 --- /dev/null +++ b/data/example-root/etc/issue @@ -0,0 +1 @@ +This is an example /etc/issue file. diff --git a/data/example-root/etc/locale.gen b/data/example-root/etc/locale.gen new file mode 100644 index 000000000..5e729a18d --- /dev/null +++ b/data/example-root/etc/locale.gen @@ -0,0 +1,486 @@ +# This file lists locales that you wish to have built. You can find a list +# of valid supported locales at /usr/share/i18n/SUPPORTED, and you can add +# user defined locales to /usr/local/share/i18n/SUPPORTED. If you change +# this file, you need to rerun locale-gen. + + +# aa_DJ ISO-8859-1 +# aa_DJ.UTF-8 UTF-8 +# aa_ER UTF-8 +# aa_ER@saaho UTF-8 +# aa_ET UTF-8 +# af_ZA ISO-8859-1 +# af_ZA.UTF-8 UTF-8 +# ak_GH UTF-8 +# am_ET UTF-8 +# an_ES ISO-8859-15 +# an_ES.UTF-8 UTF-8 +# anp_IN UTF-8 +# ar_AE ISO-8859-6 +# ar_AE.UTF-8 UTF-8 +# ar_BH ISO-8859-6 +# ar_BH.UTF-8 UTF-8 +# ar_DZ ISO-8859-6 +# ar_DZ.UTF-8 UTF-8 +# ar_EG ISO-8859-6 +# ar_EG.UTF-8 UTF-8 +# ar_IN UTF-8 +# ar_IQ ISO-8859-6 +# ar_IQ.UTF-8 UTF-8 +# ar_JO ISO-8859-6 +# ar_JO.UTF-8 UTF-8 +# ar_KW ISO-8859-6 +# ar_KW.UTF-8 UTF-8 +# ar_LB ISO-8859-6 +# ar_LB.UTF-8 UTF-8 +# ar_LY ISO-8859-6 +# ar_LY.UTF-8 UTF-8 +# ar_MA ISO-8859-6 +# ar_MA.UTF-8 UTF-8 +# ar_OM ISO-8859-6 +# ar_OM.UTF-8 UTF-8 +# ar_QA ISO-8859-6 +# ar_QA.UTF-8 UTF-8 +# ar_SA ISO-8859-6 +# ar_SA.UTF-8 UTF-8 +# ar_SD ISO-8859-6 +# ar_SD.UTF-8 UTF-8 +# ar_SS UTF-8 +# ar_SY ISO-8859-6 +# ar_SY.UTF-8 UTF-8 +# ar_TN ISO-8859-6 +# ar_TN.UTF-8 UTF-8 +# ar_YE ISO-8859-6 +# ar_YE.UTF-8 UTF-8 +# as_IN UTF-8 +# ast_ES ISO-8859-15 +# ast_ES.UTF-8 UTF-8 +# ayc_PE UTF-8 +# az_AZ UTF-8 +# be_BY CP1251 +# be_BY.UTF-8 UTF-8 +# be_BY@latin UTF-8 +# bem_ZM UTF-8 +# ber_DZ UTF-8 +# ber_MA UTF-8 +# bg_BG CP1251 +# bg_BG.UTF-8 UTF-8 +# bhb_IN.UTF-8 UTF-8 +# bho_IN UTF-8 +# bn_BD UTF-8 +# bn_IN UTF-8 +# bo_CN UTF-8 +# bo_IN UTF-8 +# br_FR ISO-8859-1 +# br_FR.UTF-8 UTF-8 +# br_FR@euro ISO-8859-15 +# brx_IN UTF-8 +# bs_BA ISO-8859-2 +# bs_BA.UTF-8 UTF-8 +# byn_ER UTF-8 +# ca_AD ISO-8859-15 +# ca_AD.UTF-8 UTF-8 +# ca_ES ISO-8859-1 +# ca_ES.UTF-8 UTF-8 +# ca_ES.UTF-8@valencia UTF-8 +# ca_ES@euro ISO-8859-15 +# ca_ES@valencia ISO-8859-15 +# ca_FR ISO-8859-15 +# ca_FR.UTF-8 UTF-8 +# ca_IT ISO-8859-15 +# ca_IT.UTF-8 UTF-8 +# ce_RU UTF-8 +# ckb_IQ UTF-8 +# cmn_TW UTF-8 +# crh_UA UTF-8 +# cs_CZ ISO-8859-2 +# cs_CZ.UTF-8 UTF-8 +# csb_PL UTF-8 +# cv_RU UTF-8 +# cy_GB ISO-8859-14 +# cy_GB.UTF-8 UTF-8 +# da_DK ISO-8859-1 +# da_DK.UTF-8 UTF-8 +# de_AT ISO-8859-1 +# de_AT.UTF-8 UTF-8 +# de_AT@euro ISO-8859-15 +# de_BE ISO-8859-1 +# de_BE.UTF-8 UTF-8 +# de_BE@euro ISO-8859-15 +# de_CH ISO-8859-1 +# de_CH.UTF-8 UTF-8 +# de_DE ISO-8859-1 +# de_DE.UTF-8 UTF-8 +# de_DE@euro ISO-8859-15 +# de_LI.UTF-8 UTF-8 +# de_LU ISO-8859-1 +# de_LU.UTF-8 UTF-8 +# de_LU@euro ISO-8859-15 +# doi_IN UTF-8 +# dv_MV UTF-8 +# dz_BT UTF-8 +# el_CY ISO-8859-7 +# el_CY.UTF-8 UTF-8 +# el_GR ISO-8859-7 +# el_GR.UTF-8 UTF-8 +# en_AG UTF-8 +# en_AU ISO-8859-1 +# en_AU.UTF-8 UTF-8 +# en_BW ISO-8859-1 +# en_BW.UTF-8 UTF-8 +# en_CA ISO-8859-1 +en_CA.UTF-8 UTF-8 +# en_DK ISO-8859-1 +# en_DK.ISO-8859-15 ISO-8859-15 +# en_DK.UTF-8 UTF-8 +# en_GB ISO-8859-1 +# en_GB.ISO-8859-15 ISO-8859-15 +# en_GB.UTF-8 UTF-8 +# en_HK ISO-8859-1 +# en_HK.UTF-8 UTF-8 +# en_IE ISO-8859-1 +# en_IE.UTF-8 UTF-8 +# en_IE@euro ISO-8859-15 +# en_IN UTF-8 +# en_NG UTF-8 +# en_NZ ISO-8859-1 +# en_NZ.UTF-8 UTF-8 +# en_PH ISO-8859-1 +# en_PH.UTF-8 UTF-8 +# en_SG ISO-8859-1 +# en_SG.UTF-8 UTF-8 +# en_US ISO-8859-1 +# en_US.ISO-8859-15 ISO-8859-15 +en_US.UTF-8 UTF-8 +# en_ZA ISO-8859-1 +# en_ZA.UTF-8 UTF-8 +# en_ZM UTF-8 +# en_ZW ISO-8859-1 +# en_ZW.UTF-8 UTF-8 +# eo ISO-8859-3 +# eo.UTF-8 UTF-8 +# eo_US.UTF-8 UTF-8 +# es_AR ISO-8859-1 +# es_AR.UTF-8 UTF-8 +# es_BO ISO-8859-1 +# es_BO.UTF-8 UTF-8 +# es_CL ISO-8859-1 +# es_CL.UTF-8 UTF-8 +# es_CO ISO-8859-1 +# es_CO.UTF-8 UTF-8 +# es_CR ISO-8859-1 +# es_CR.UTF-8 UTF-8 +# es_CU UTF-8 +# es_DO ISO-8859-1 +# es_DO.UTF-8 UTF-8 +# es_EC ISO-8859-1 +# es_EC.UTF-8 UTF-8 +# es_ES ISO-8859-1 +# es_ES.UTF-8 UTF-8 +# es_ES@euro ISO-8859-15 +# es_GT ISO-8859-1 +# es_GT.UTF-8 UTF-8 +# es_HN ISO-8859-1 +# es_HN.UTF-8 UTF-8 +# es_MX ISO-8859-1 +# es_MX.UTF-8 UTF-8 +# es_NI ISO-8859-1 +# es_NI.UTF-8 UTF-8 +# es_PA ISO-8859-1 +# es_PA.UTF-8 UTF-8 +# es_PE ISO-8859-1 +# es_PE.UTF-8 UTF-8 +# es_PR ISO-8859-1 +# es_PR.UTF-8 UTF-8 +# es_PY ISO-8859-1 +# es_PY.UTF-8 UTF-8 +# es_SV ISO-8859-1 +# es_SV.UTF-8 UTF-8 +# es_US ISO-8859-1 +# es_US.UTF-8 UTF-8 +# es_UY ISO-8859-1 +# es_UY.UTF-8 UTF-8 +# es_VE ISO-8859-1 +# es_VE.UTF-8 UTF-8 +# et_EE ISO-8859-1 +# et_EE.ISO-8859-15 ISO-8859-15 +# et_EE.UTF-8 UTF-8 +# eu_ES ISO-8859-1 +# eu_ES.UTF-8 UTF-8 +# eu_ES@euro ISO-8859-15 +# eu_FR ISO-8859-1 +# eu_FR.UTF-8 UTF-8 +# eu_FR@euro ISO-8859-15 +# fa_IR UTF-8 +# ff_SN UTF-8 +# fi_FI ISO-8859-1 +# fi_FI.UTF-8 UTF-8 +# fi_FI@euro ISO-8859-15 +# fil_PH UTF-8 +# fo_FO ISO-8859-1 +# fo_FO.UTF-8 UTF-8 +# fr_BE ISO-8859-1 +# fr_BE.UTF-8 UTF-8 +# fr_BE@euro ISO-8859-15 +# fr_CA ISO-8859-1 +# fr_CA.UTF-8 UTF-8 +# fr_CH ISO-8859-1 +# fr_CH.UTF-8 UTF-8 +# fr_FR ISO-8859-1 +# fr_FR.UTF-8 UTF-8 +# fr_FR@euro ISO-8859-15 +# fr_LU ISO-8859-1 +# fr_LU.UTF-8 UTF-8 +# fr_LU@euro ISO-8859-15 +# fur_IT UTF-8 +# fy_DE UTF-8 +# fy_NL UTF-8 +# ga_IE ISO-8859-1 +# ga_IE.UTF-8 UTF-8 +# ga_IE@euro ISO-8859-15 +# gd_GB ISO-8859-15 +# gd_GB.UTF-8 UTF-8 +# gez_ER UTF-8 +# gez_ER@abegede UTF-8 +# gez_ET UTF-8 +# gez_ET@abegede UTF-8 +# gl_ES ISO-8859-1 +# gl_ES.UTF-8 UTF-8 +# gl_ES@euro ISO-8859-15 +# gu_IN UTF-8 +# gv_GB ISO-8859-1 +# gv_GB.UTF-8 UTF-8 +# ha_NG UTF-8 +# hak_TW UTF-8 +# he_IL ISO-8859-8 +# he_IL.UTF-8 UTF-8 +# hi_IN UTF-8 +# hne_IN UTF-8 +# hr_HR ISO-8859-2 +# hr_HR.UTF-8 UTF-8 +# hsb_DE ISO-8859-2 +# hsb_DE.UTF-8 UTF-8 +# ht_HT UTF-8 +# hu_HU ISO-8859-2 +# hu_HU.UTF-8 UTF-8 +# hy_AM UTF-8 +# hy_AM.ARMSCII-8 ARMSCII-8 +# ia_FR UTF-8 +# id_ID ISO-8859-1 +# id_ID.UTF-8 UTF-8 +# ig_NG UTF-8 +# ik_CA UTF-8 +# is_IS ISO-8859-1 +# is_IS.UTF-8 UTF-8 +# it_CH ISO-8859-1 +# it_CH.UTF-8 UTF-8 +# it_IT ISO-8859-1 +# it_IT.UTF-8 UTF-8 +# it_IT@euro ISO-8859-15 +# iu_CA UTF-8 +# iw_IL ISO-8859-8 +# iw_IL.UTF-8 UTF-8 +# ja_JP.EUC-JP EUC-JP +# ja_JP.UTF-8 UTF-8 +# ka_GE GEORGIAN-PS +# ka_GE.UTF-8 UTF-8 +# kk_KZ PT154 +# kk_KZ RK1048 +# kk_KZ.UTF-8 UTF-8 +# kl_GL ISO-8859-1 +# kl_GL.UTF-8 UTF-8 +# km_KH UTF-8 +# kn_IN UTF-8 +# ko_KR.EUC-KR EUC-KR +# ko_KR.UTF-8 UTF-8 +# kok_IN UTF-8 +# ks_IN UTF-8 +# ks_IN@devanagari UTF-8 +# ku_TR ISO-8859-9 +# ku_TR.UTF-8 UTF-8 +# kw_GB ISO-8859-1 +# kw_GB.UTF-8 UTF-8 +# ky_KG UTF-8 +# lb_LU UTF-8 +# lg_UG ISO-8859-10 +# lg_UG.UTF-8 UTF-8 +# li_BE UTF-8 +# li_NL UTF-8 +# lij_IT UTF-8 +# ln_CD UTF-8 +# lo_LA UTF-8 +# lt_LT ISO-8859-13 +# lt_LT.UTF-8 UTF-8 +# lv_LV ISO-8859-13 +# lv_LV.UTF-8 UTF-8 +# lzh_TW UTF-8 +# mag_IN UTF-8 +# mai_IN UTF-8 +# mg_MG ISO-8859-15 +# mg_MG.UTF-8 UTF-8 +# mhr_RU UTF-8 +# mi_NZ ISO-8859-13 +# mi_NZ.UTF-8 UTF-8 +# mk_MK ISO-8859-5 +# mk_MK.UTF-8 UTF-8 +# ml_IN UTF-8 +# mn_MN UTF-8 +# mni_IN UTF-8 +# mr_IN UTF-8 +# ms_MY ISO-8859-1 +# ms_MY.UTF-8 UTF-8 +# mt_MT ISO-8859-3 +# mt_MT.UTF-8 UTF-8 +# my_MM UTF-8 +# nan_TW UTF-8 +# nan_TW@latin UTF-8 +# nb_NO ISO-8859-1 +# nb_NO.UTF-8 UTF-8 +# nds_DE UTF-8 +# nds_NL UTF-8 +# ne_NP UTF-8 +# nhn_MX UTF-8 +# niu_NU UTF-8 +# niu_NZ UTF-8 +# nl_AW UTF-8 +# nl_BE ISO-8859-1 +# nl_BE.UTF-8 UTF-8 +# nl_BE@euro ISO-8859-15 +# nl_NL ISO-8859-1 +# nl_NL.UTF-8 UTF-8 +# nl_NL@euro ISO-8859-15 +# nn_NO ISO-8859-1 +# nn_NO.UTF-8 UTF-8 +# nr_ZA UTF-8 +# nso_ZA UTF-8 +# oc_FR ISO-8859-1 +# oc_FR.UTF-8 UTF-8 +# om_ET UTF-8 +# om_KE ISO-8859-1 +# om_KE.UTF-8 UTF-8 +# or_IN UTF-8 +# os_RU UTF-8 +# pa_IN UTF-8 +# pa_PK UTF-8 +# pap_AN UTF-8 +# pap_AW UTF-8 +# pap_CW UTF-8 +# pl_PL ISO-8859-2 +# pl_PL.UTF-8 UTF-8 +# ps_AF UTF-8 +# pt_BR ISO-8859-1 +# pt_BR.UTF-8 UTF-8 +# pt_PT ISO-8859-1 +# pt_PT.UTF-8 UTF-8 +# pt_PT@euro ISO-8859-15 +# quz_PE UTF-8 +# raj_IN UTF-8 +# ro_RO ISO-8859-2 +# ro_RO.UTF-8 UTF-8 +# ru_RU ISO-8859-5 +# ru_RU.CP1251 CP1251 +# ru_RU.KOI8-R KOI8-R +# ru_RU.UTF-8 UTF-8 +# ru_UA KOI8-U +# ru_UA.UTF-8 UTF-8 +# rw_RW UTF-8 +# sa_IN UTF-8 +# sat_IN UTF-8 +# sc_IT UTF-8 +# sd_IN UTF-8 +# sd_IN@devanagari UTF-8 +# sd_PK UTF-8 +# se_NO UTF-8 +# shs_CA UTF-8 +# si_LK UTF-8 +# sid_ET UTF-8 +# sk_SK ISO-8859-2 +# sk_SK.UTF-8 UTF-8 +# sl_SI ISO-8859-2 +# sl_SI.UTF-8 UTF-8 +# so_DJ ISO-8859-1 +# so_DJ.UTF-8 UTF-8 +# so_ET UTF-8 +# so_KE ISO-8859-1 +# so_KE.UTF-8 UTF-8 +# so_SO ISO-8859-1 +# so_SO.UTF-8 UTF-8 +# sq_AL ISO-8859-1 +# sq_AL.UTF-8 UTF-8 +# sq_MK UTF-8 +# sr_ME UTF-8 +# sr_RS UTF-8 +# sr_RS@latin UTF-8 +# ss_ZA UTF-8 +# st_ZA ISO-8859-1 +# st_ZA.UTF-8 UTF-8 +# sv_FI ISO-8859-1 +# sv_FI.UTF-8 UTF-8 +# sv_FI@euro ISO-8859-15 +# sv_SE ISO-8859-1 +# sv_SE.ISO-8859-15 ISO-8859-15 +# sv_SE.UTF-8 UTF-8 +# sw_KE UTF-8 +# sw_TZ UTF-8 +# szl_PL UTF-8 +# ta_IN UTF-8 +# ta_LK UTF-8 +# tcy_IN.UTF-8 UTF-8 +# te_IN UTF-8 +# tg_TJ KOI8-T +# tg_TJ.UTF-8 UTF-8 +# th_TH TIS-620 +# th_TH.UTF-8 UTF-8 +# the_NP UTF-8 +# ti_ER UTF-8 +# ti_ET UTF-8 +# tig_ER UTF-8 +# tk_TM UTF-8 +# tl_PH ISO-8859-1 +# tl_PH.UTF-8 UTF-8 +# tn_ZA UTF-8 +# tr_CY ISO-8859-9 +# tr_CY.UTF-8 UTF-8 +# tr_TR ISO-8859-9 +# tr_TR.UTF-8 UTF-8 +# ts_ZA UTF-8 +# tt_RU UTF-8 +# tt_RU@iqtelif UTF-8 +# ug_CN UTF-8 +# ug_CN@latin UTF-8 +# uk_UA KOI8-U +# uk_UA.UTF-8 UTF-8 +# unm_US UTF-8 +# ur_IN UTF-8 +# ur_PK UTF-8 +# uz_UZ ISO-8859-1 +# uz_UZ.UTF-8 UTF-8 +# uz_UZ@cyrillic UTF-8 +# ve_ZA UTF-8 +# vi_VN UTF-8 +# wa_BE ISO-8859-1 +# wa_BE.UTF-8 UTF-8 +# wa_BE@euro ISO-8859-15 +# wae_CH UTF-8 +# wal_ET UTF-8 +# wo_SN UTF-8 +# xh_ZA ISO-8859-1 +# xh_ZA.UTF-8 UTF-8 +# yi_US CP1255 +# yi_US.UTF-8 UTF-8 +# yo_NG UTF-8 +# yue_HK UTF-8 +# zh_CN GB2312 +# zh_CN.GB18030 GB18030 +# zh_CN.GBK GBK +# zh_CN.UTF-8 UTF-8 +# zh_HK BIG5-HKSCS +# zh_HK.UTF-8 UTF-8 +# zh_SG GB2312 +# zh_SG.GBK GBK +# zh_SG.UTF-8 UTF-8 +# zh_TW BIG5 +# zh_TW.EUC-TW EUC-TW +# zh_TW.UTF-8 UTF-8 +# zu_ZA ISO-8859-1 +# zu_ZA.UTF-8 UTF-8 diff --git a/data/example-root/etc/profile b/data/example-root/etc/profile new file mode 100644 index 000000000..42bd3d007 --- /dev/null +++ b/data/example-root/etc/profile @@ -0,0 +1,2 @@ +# Global .profile -- add /sbin_1 +PATH=$PATH:/sbin_1:/xbin diff --git a/data/example-root/etc/sudoers.d/10-installer b/data/example-root/etc/sudoers.d/10-installer new file mode 100644 index 000000000..e69de29bb diff --git a/data/example-root/usr/share/zoneinfo/.dummy b/data/example-root/usr/share/zoneinfo/.dummy new file mode 100644 index 000000000..e69de29bb diff --git a/data/example-root/usr/share/zoneinfo/America/New_York b/data/example-root/usr/share/zoneinfo/America/New_York new file mode 100644 index 0000000000000000000000000000000000000000..7553fee37a5d03e9163ee19b1ced730a02345cfb GIT binary patch literal 3545 zcmd_sSy0tw7{~FW;u1<|in&Fm6{0L|xKd(jgnGmUQpxbGKnAsVN+m4AN|bP>tkJ=? z!Q9a_a=|1E(F`4%HgPS*S5s0GdzBW_I;Z#h-geP=(N%xve?Dg%818=GCn+U!T5r!k zp2qfnczG_{m+x&}v>!$5LS@CrKdJW?d1U3=U#eB{j*B;NN9u6QjyHo{+MdLuyyRuVz=}cJ;}*W9HM6Z*=*-GP8ThR$Z~? z9kVBEnckcCg83{lTklJoYCeyiq$|DiWPk7=eIPPb4%AOn2ZQ3|;PHX#i&u;s>iUZu zQa5zfoO9-I+$nt|xzZf%yjvfODK^JFEA@$x#pZ-wpuh92m+vdm^~vf2Ikn+sRb4(q zP8XypUF4NBnGdS7xzX}NLN|3TwUwNo7^Q3CBh8QfTj~p8!RBJyYx+`?tLD;ghxJc2 zRp#>19lEx%)LhwJrG73sBxXgay1Hb$T${gK)nygRFH`5LUlViWw;_+H-=kBczT30< zkKkCj-fXhIUO&m)xG-4%d3=!h>%bk_x3iP+ulH-ua-V6Ce?~WaR+~oRQvvEPX*^b| zCUK{wY0tf?>8tJKmX>SOEt{8_K(k0S*9)b^iB&qNB13L1%hSOd7MPZAP1CIk(#>si zAJVNe<4v2%-E~MpxM@4Eg}yz!xoOuWT(xgjYdSP+t~y)`l#XX=Ri|$+%N={ZR-s$I zk~>#!QJu3r=B}5PsxHZAP1orq`tF#0=AMyn=zBxfnXvA&beQim2@g!x;ni!U`=$Q6 zM|r+PR3)j%qD+a})=x#}j*^~B+o@g|8K(C$*HxeR1k-o?Nfi^;!}RN2uKG6(G6On( zrw7#hYzE%=L=UR`)(rl>NXM33k^6SNsPA9$jSP9`aUGYnRfguxR}UmElVNF(so~Mt zGGh2JHKMNA#79om@l}gWLeNm1ux+LpS=&{QdbdDEAB|Jqc{60pjxH*3idV)K2B>kd z(K3EcjhfJ@l_Vt}P)RrHf!UjW>RRSp0w|(nd~dpDQl|CBh`!bl)O^&X!%T? znzr0bEgGYhce^~6KSMnpStw6rcvV_Zj->ozrL!U%(GouMi>H9_XT=}`?E+~mJT0XO*zH~R_6OYt*7Fr~ zUy+SPb{5%MWN(qpMRph2USxlf4R+ccMz+{#_ZZn^WS5a`M)n!mXk@36tw#15*=%ID zk?ltI8`*HD-Em~gop#TWO-FX!X}2BOcVy#{okzAF*?VO3k==LN?ML<>X#mmzqyMS(m14ZNb8W^Aedx|s_=_=Azq_0S0ka;NP%(sZZoI?{He??~g3 z&LgcydXF?8=|0kar2ohb;IwxDatk=^J%HQ1axWk^19CSY zw*zuNAU6bZMZQ``|338(#cM5W=AomJ#vmkd1a=UnL`V#q{9xs9Rrirn)O@y~k SRPU&s5#C Date: Tue, 29 Aug 2017 05:46:17 -0400 Subject: [PATCH 04/19] Testing: set more locale-globals in testing-script. When testing Python modules, passing option --lang should also set the global 'locale' (to a BCP47 string, but hey) like it already sets localeConf.LANG. --- src/modules/testmodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/testmodule.py b/src/modules/testmodule.py index 955db1830..d115694eb 100755 --- a/src/modules/testmodule.py +++ b/src/modules/testmodule.py @@ -88,6 +88,7 @@ def main(): libcalamares.globalstorage = libcalamares.GlobalStorage(None) libcalamares.globalstorage.insert("testing", True) if args.lang: + libcalamares.globalstorage.insert("locale", args.lang) libcalamares.globalstorage.insert("localeConf", {"LANG": args.lang}) # if a file for simulating globalStorage contents is provided, load it From d66434985e7c247fc26ef513587974b887c2d164 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 28 Aug 2017 16:26:26 -0400 Subject: [PATCH 05/19] Package module: refactor package-manager into multiple classes --- src/modules/packages/main.py | 269 ++++++++++++++++++++++++----------- 1 file changed, 182 insertions(+), 87 deletions(-) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 93fcc9eb7..97d97738e 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -21,114 +21,207 @@ # You should have received a copy of the GNU General Public License # along with Calamares. If not, see . +import abc +from string import Template import subprocess + import libcalamares from libcalamares.utils import check_target_env_call, target_env_call -from string import Template -class PackageManager: +class PackageManager(metaclass=abc.ABCMeta): """ - Package manager class. + Package manager base class. A subclass implements package management + for a specific backend, and must have a class property `backend` + with the string identifier for that backend. - :param backend: + Subclasses are collected below to populate the list of possible + backends. """ - def __init__(self, backend): - self.backend = backend + backend = None + @abc.abstractmethod def install(self, pkgs, from_local=False): - """ Installs packages. - - :param pkgs: - :param from_local: - """ - if self.backend == "packagekit": - for pkg in pkgs: - check_target_env_call(["pkcon", "-py", "install", pkg]) - elif self.backend == "zypp": - check_target_env_call(["zypper", "--non-interactive", - "--quiet-install", "install", - "--auto-agree-with-licenses", - "install"] + pkgs) - elif self.backend == "yum": - check_target_env_call(["yum", "install", "-y"] + pkgs) - elif self.backend == "dnf": - check_target_env_call(["dnf", "install", "-y"] + pkgs) - elif self.backend == "urpmi": - check_target_env_call(["urpmi", "--download-all", "--no-suggests", - "--no-verify-rpm", "--fastunsafe", - "--ignoresize", "--nolock", - "--auto"] + pkgs) - elif self.backend == "apt": - check_target_env_call(["apt-get", "-q", "-y", "install"] + pkgs) - elif self.backend == "pacman": - if from_local: - pacman_flags = "-U" - else: - pacman_flags = "-Sy" - - check_target_env_call(["pacman", pacman_flags, - "--noconfirm"] + pkgs) - elif self.backend == "portage": - check_target_env_call(["emerge", "-v"] + pkgs) - elif self.backend == "entropy": - check_target_env_call(["equo", "i"] + pkgs) + pass + @abc.abstractmethod def remove(self, pkgs): """ Removes packages. :param pkgs: """ - if self.backend == "packagekit": - for pkg in pkgs: - check_target_env_call(["pkcon", "-py", "remove", pkg]) - elif self.backend == "zypp": - check_target_env_call(["zypper", "--non-interactive", - "remove"] + pkgs) - elif self.backend == "yum": - check_target_env_call(["yum", "--disablerepo=*", "-C", "-y", - "remove"] + pkgs) - elif self.backend == "dnf": - # ignore the error code for now because dnf thinks removing a - # nonexistent package is an error - target_env_call(["dnf", "--disablerepo=*", "-C", "-y", - "remove"] + pkgs) - elif self.backend == "urpmi": - check_target_env_call(["urpme", "--auto"] + pkgs) - elif self.backend == "apt": - check_target_env_call(["apt-get", "--purge", "-q", "-y", - "remove"] + pkgs) - check_target_env_call(["apt-get", "--purge", "-q", "-y", - "autoremove"]) - elif self.backend == "pacman": - check_target_env_call(["pacman", "-Rs", "--noconfirm"] + pkgs) - elif self.backend == "portage": - check_target_env_call(["emerge", "-C"] + pkgs) - check_target_env_call(["emerge", "--depclean", "-q"]) - elif self.backend == "entropy": - check_target_env_call(["equo", "rm"] + pkgs) + pass + @abc.abstractmethod def update_db(self): - if self.backend == "packagekit": - check_target_env_call(["pkcon", "refresh"]) - elif self.backend == "zypp": - check_target_env_call(["zypper", "--non-interactive", "update"]) - elif self.backend == "urpmi": - check_target_env_call(["urpmi.update", "-a"]) - elif self.backend == "apt": - check_target_env_call(["apt-get", "update"]) - elif self.backend == "pacman": - check_target_env_call(["pacman", "-Sy"]) - elif self.backend == "portage": - check_target_env_call(["emerge", "--sync"]) - elif self.backend == "entropy": - check_target_env_call(["equo", "update"]) + pass def run(self, script): if script != "": check_target_env_call(script.split(" ")) +class PMPackageKit(PackageManager): + backend = "packagekit" + + def install(self, pkgs, from_local=False): + for pkg in pkgs: + check_target_env_call(["pkcon", "-py", "install", pkg]) + + def remove(self, pkgs): + for pkg in pkgs: + check_target_env_call(["pkcon", "-py", "remove", pkg]) + + def update_db(self): + check_target_env_call(["pkcon", "refresh"]) + + +class PMZypp(PackageManager): + backend = "zypp" + + def install(self, pkgs, from_local=False): + check_target_env_call(["zypper", "--non-interactive", + "--quiet-install", "install", + "--auto-agree-with-licenses", + "install"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["zypper", "--non-interactive", + "remove"] + pkgs) + + def update_db(self): + check_target_env_call(["zypper", "--non-interactive", "update"]) + + +class PMYum(PackageManager): + backend = "yum" + + def install(self, pkgs, from_local=False): + check_target_env_call(["yum", "install", "-y"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["yum", "--disablerepo=*", "-C", "-y", + "remove"] + pkgs) + + def update_db(self): + # Doesn't need updates + pass + + +class PMDnf(PackageManager): + backend = "dnf" + + def install(self, pkgs, from_local=False): + check_target_env_call(["dnf", "install", "-y"] + pkgs) + + def remove(self, pkgs): + # ignore the error code for now because dnf thinks removing a + # nonexistent package is an error + target_env_call(["dnf", "--disablerepo=*", "-C", "-y", + "remove"] + pkgs) + + def update_db(self): + # Doesn't need to update explicitly + pass + + +class PMUrpmi(PackageManager): + backend = "urpmi"; + + def install(self, pkgs, from_local=False): + check_target_env_call(["urpmi", "--download-all", "--no-suggests", + "--no-verify-rpm", "--fastunsafe", + "--ignoresize", "--nolock", + "--auto"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["urpme", "--auto"] + pkgs) + + def update_db(self): + check_target_env_call(["urpmi.update", "-a"]) + + +class PMApt(PackageManager): + backend = "apt" + + def install(self, pkgs, from_local=False): + check_target_env_call(["apt-get", "-q", "-y", "install"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["apt-get", "--purge", "-q", "-y", + "remove"] + pkgs) + check_target_env_call(["apt-get", "--purge", "-q", "-y", + "autoremove"]) + + def update_db(self): + check_target_env_call(["apt-get", "update"]) + + +class PMPacman(PackageManager): + backend = "pacman" + + def install(self, pkgs, from_local=False): + if from_local: + pacman_flags = "-U" + else: + pacman_flags = "-Sy" + + check_target_env_call(["pacman", pacman_flags, + "--noconfirm"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["pacman", "-Rs", "--noconfirm"] + pkgs) + + def update_db(self): + check_target_env_call(["pacman", "-Sy"]) + + +class PMPortage(PackageManager): + backend = "portage" + + def install(self, pkgs, from_local=False): + check_target_env_call(["emerge", "-v"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["emerge", "-C"] + pkgs) + check_target_env_call(["emerge", "--depclean", "-q"]) + + def update_db(self): + check_target_env_call(["emerge", "--sync"]) + + +class PMEntropy(PackageManager): + backend = "entropy" + + def install(self, pkgs, from_local=False): + check_target_env_call(["equo", "i"] + pkgs) + + def remove(self, pkgs): + check_target_env_call(["equo", "rm"] + pkgs) + + def update_db(self): + check_target_env_call(["equo", "update"]) + + +class PMDummy(PackageManager): + backend = "dummy" + + def install(self, pkgs, from_local=False): + libcalamares.utils.debug("Installing " + str(pkgs)) + + def remove(self, pkgs): + libcalamares.utils.debug("Removing " + str(pkgs)) + + def update_db(self): + libcalamares.utils.debug("Updating DB") + + +backend_managers = [ + (c.backend, c) + for c in globals().values() + if type(c) is abc.ABCMeta and issubclass(c, PackageManager) and c.backend ] + + def subst_locale(list): ret = [] locale = libcalamares.globalstorage.value("locale") @@ -204,11 +297,13 @@ def run(): """ backend = libcalamares.job.configuration.get("backend") - if backend not in ("packagekit", "zypp", "yum", "dnf", "urpmi", "apt", - "pacman", "portage", "entropy"): + for identifier, impl in backend_managers: + if identifier == backend: + pkgman = impl() + break + else: return "Bad backend", "backend=\"{}\"".format(backend) - pkgman = PackageManager(backend) operations = libcalamares.job.configuration.get("operations", []) update_db = libcalamares.job.configuration.get("update_db", False) From 6c36534206bb1f850b1a957cbb3206e5af9373a1 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 29 Aug 2017 04:18:47 -0400 Subject: [PATCH 06/19] Package module: fix packages-could-be-objects code - Check for 'list' when it's actually a 'dict' is strange. Reverse logic to consider 'str' a package name and everything else is special. - Refactor to handle the difference between package names and packages-with-script-data in one place. - Add code and config documentation. - Switch sample configurations to dummy-backend. --- src/modules/packages/main.py | 96 ++++++++++++++++++------------ src/modules/packages/packages.conf | 10 +++- src/modules/packages/test.yaml | 3 +- 3 files changed, 66 insertions(+), 43 deletions(-) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 97d97738e..dcd999727 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -42,13 +42,26 @@ class PackageManager(metaclass=abc.ABCMeta): @abc.abstractmethod def install(self, pkgs, from_local=False): + """ + Install a list of packages (named) into the system. + Although this handles lists, in practice it is called + with one package at a time. + + @param pkgs: list[str] + list of package names + @param from_local: bool + if True, then these are local packages (on disk) and the + pkgs names are paths. + """ pass @abc.abstractmethod def remove(self, pkgs): - """ Removes packages. + """ + Removes packages. - :param pkgs: + @param pkgs: list[str] + list of package names """ pass @@ -60,6 +73,23 @@ class PackageManager(metaclass=abc.ABCMeta): if script != "": check_target_env_call(script.split(" ")) + def install_package(self, packagedata, from_local=False): + """ + Install a package from a single entry in the install list. + This can be either a single package name, or an object + with pre- and post-scripts. + + @param packagedata: str|dict + @param from_local: bool + see install.from_local + """ + if isinstance(packagedata, str): + self.install([packagedata], from_local=from_local) + else: + self.run(packagedata["pre-script"]) + self.install([packagedata["package"]], from_local=from_local) + self.run(packagedata["post-script"]) + class PMPackageKit(PackageManager): backend = "packagekit" @@ -81,13 +111,13 @@ class PMZypp(PackageManager): def install(self, pkgs, from_local=False): check_target_env_call(["zypper", "--non-interactive", - "--quiet-install", "install", - "--auto-agree-with-licenses", - "install"] + pkgs) + "--quiet-install", "install", + "--auto-agree-with-licenses", + "install"] + pkgs) def remove(self, pkgs): check_target_env_call(["zypper", "--non-interactive", - "remove"] + pkgs) + "remove"] + pkgs) def update_db(self): check_target_env_call(["zypper", "--non-interactive", "update"]) @@ -101,7 +131,7 @@ class PMYum(PackageManager): def remove(self, pkgs): check_target_env_call(["yum", "--disablerepo=*", "-C", "-y", - "remove"] + pkgs) + "remove"] + pkgs) def update_db(self): # Doesn't need updates @@ -118,7 +148,7 @@ class PMDnf(PackageManager): # ignore the error code for now because dnf thinks removing a # nonexistent package is an error target_env_call(["dnf", "--disablerepo=*", "-C", "-y", - "remove"] + pkgs) + "remove"] + pkgs) def update_db(self): # Doesn't need to update explicitly @@ -126,13 +156,13 @@ class PMDnf(PackageManager): class PMUrpmi(PackageManager): - backend = "urpmi"; + backend = "urpmi" def install(self, pkgs, from_local=False): check_target_env_call(["urpmi", "--download-all", "--no-suggests", - "--no-verify-rpm", "--fastunsafe", - "--ignoresize", "--nolock", - "--auto"] + pkgs) + "--no-verify-rpm", "--fastunsafe", + "--ignoresize", "--nolock", + "--auto"] + pkgs) def remove(self, pkgs): check_target_env_call(["urpme", "--auto"] + pkgs) @@ -149,9 +179,9 @@ class PMApt(PackageManager): def remove(self, pkgs): check_target_env_call(["apt-get", "--purge", "-q", "-y", - "remove"] + pkgs) + "remove"] + pkgs) check_target_env_call(["apt-get", "--purge", "-q", "-y", - "autoremove"]) + "autoremove"]) def update_db(self): check_target_env_call(["apt-get", "update"]) @@ -167,7 +197,7 @@ class PMPacman(PackageManager): pacman_flags = "-Sy" check_target_env_call(["pacman", pacman_flags, - "--noconfirm"] + pkgs) + "--noconfirm"] + pkgs) def remove(self, pkgs): check_target_env_call(["pacman", "-Rs", "--noconfirm"] + pkgs) @@ -215,11 +245,14 @@ class PMDummy(PackageManager): def update_db(self): libcalamares.utils.debug("Updating DB") + def run(self, script): + libcalamares.utils.debug("Running script '" + str(script) + "'") + backend_managers = [ (c.backend, c) for c in globals().values() - if type(c) is abc.ABCMeta and issubclass(c, PackageManager) and c.backend ] + if type(c) is abc.ABCMeta and issubclass(c, PackageManager) and c.backend] def subst_locale(list): @@ -247,33 +280,18 @@ def run_operations(pkgman, entry): for key in entry.keys(): entry[key] = subst_locale(entry[key]) if key == "install": - if isinstance(entry[key], list): - for package in entry[key]: - pkgman.run(package["pre-script"]) - pkgman.install([package["package"]]) - pkgman.run(package["post-script"]) - else: - pkgman.install(entry[key]) + for package in entry[key]: + pkgman.install_package(package) elif key == "try_install": # we make a separate package manager call for each package so a # single failing package won't stop all of them for package in entry[key]: - if isinstance(package, str): - try: - pkgman.install([package]) - except subprocess.CalledProcessError: - warn_text = "WARNING: could not install package " - warn_text += package - libcalamares.utils.debug(warn_text) - else: - try: - pkgman.run(package["pre-script"]) - pkgman.install([package["package"]]) - pkgman.run(package["post-script"]) - except subprocess.CalledProcessError: - warn_text = "WARNING: could not install packages " - warn_text += package["package"] - libcalamares.utils.debug(warn_text) + try: + pkgman.install_package(package) + except subprocess.CalledProcessError: + warn_text = "WARNING: could not install package " + warn_text += str(package) + libcalamares.utils.debug(warn_text) elif key == "remove": pkgman.remove(entry[key]) elif key == "try_remove": diff --git a/src/modules/packages/packages.conf b/src/modules/packages/packages.conf index 4039278b3..da47cec25 100644 --- a/src/modules/packages/packages.conf +++ b/src/modules/packages/packages.conf @@ -8,11 +8,15 @@ # - urpmi - Mandriva package manager # - apt - APT frontend for DEB and RPM # - pacman - Pacman -# - portage - Gentoo package manager -# - entropy - Sabayon package manager +# - portage - Gentoo package manager +# - entropy - Sabayon package manager +# - dummy - Dummy manager, only logs # -backend: packagekit +backend: dummy +# If set to true, a package-manager specific update procedure +# is run first (only if there is internet) to update the list +# of packages and dependencies. update_db: true # diff --git a/src/modules/packages/test.yaml b/src/modules/packages/test.yaml index 345172622..49c1a5197 100644 --- a/src/modules/packages/test.yaml +++ b/src/modules/packages/test.yaml @@ -1,5 +1,6 @@ +backend: dummy rootMountPoint: /tmp/mount -packageOperations: +operations: - install: - pre-script: touch /tmp/foo package: vi From 3799a26b3ca97c83214ebb282363b05e88e3e289 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 29 Aug 2017 06:33:07 -0400 Subject: [PATCH 07/19] Package module: optimize & fix - Expand example configurations - Optimize commoon case of just listing package names - Do locale substitution in both kinds of cases --- src/modules/packages/main.py | 58 +++++++++++++++++++++++------- src/modules/packages/packages.conf | 9 +++++ src/modules/packages/test.yaml | 1 + 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index dcd999727..9eb0a8ea0 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -249,24 +249,53 @@ class PMDummy(PackageManager): libcalamares.utils.debug("Running script '" + str(script) + "'") +# Collect all the subclasses of PackageManager defined above, +# and index them based on the backend property of each class. backend_managers = [ (c.backend, c) for c in globals().values() if type(c) is abc.ABCMeta and issubclass(c, PackageManager) and c.backend] -def subst_locale(list): - ret = [] +def subst_locale(plist): + """ + Returns a locale-aware list of packages, based on @p plist. + Package names that contain LOCALE are localized with the + BCP47 name of the chosen system locale; if the system + locale is 'en' (e.g. English, US) then these localized + packages are dropped from the list. + + @param plist: list[str|dict] + Candidate packages to install. + @return: list[str|dict] + """ locale = libcalamares.globalstorage.value("locale") - if locale: - for e in list: - if locale != "en": - entry = Template(e) - ret.append(entry.safe_substitute(LOCALE=locale)) - elif 'LOCALE' not in e: - ret.append(e) - else: - ret = list + if not locale: + return plist + + ret = [] + for packagedata in plist: + if isinstance(packagedata, str): + packagename = packagedata + else: + packagename = packagedata["package"] + + # Update packagename: substitute LOCALE, and drop packages + # if locale is en and LOCALE is in the package name. + if locale != "en": + packagename = Template(packagename).safe_substitute(LOCALE=locale) + elif 'LOCALE' in packagename: + packagename = None + + if packagename is not None: + # Put it back in packagedata + if isinstance(packagedata, str): + packagedata = packagename + else: + packagedata["package"] = packagename + + ret.append(packagedata) + return ret @@ -280,8 +309,11 @@ def run_operations(pkgman, entry): for key in entry.keys(): entry[key] = subst_locale(entry[key]) if key == "install": - for package in entry[key]: - pkgman.install_package(package) + if all([isinstance(x, str) for x in entry[key]]): + pkgman.install(entry[key]) + else: + for package in entry[key]: + pkgman.install_package(package) elif key == "try_install": # we make a separate package manager call for each package so a # single failing package won't stop all of them diff --git a/src/modules/packages/packages.conf b/src/modules/packages/packages.conf index da47cec25..7bf589243 100644 --- a/src/modules/packages/packages.conf +++ b/src/modules/packages/packages.conf @@ -50,3 +50,12 @@ update_db: true # - pkg7 # - localInstall: # - /path/to/pkg8 +operations: + - install: + - vi + - wget + - binutils + - remove: + - vi + - wget + - binutils diff --git a/src/modules/packages/test.yaml b/src/modules/packages/test.yaml index 49c1a5197..8902b657a 100644 --- a/src/modules/packages/test.yaml +++ b/src/modules/packages/test.yaml @@ -6,6 +6,7 @@ operations: package: vi post-script: rm /tmp/foo - wget + - binutils - remove: - vi - wget From 162de207c8c507f123d176fe88016dfa2a0d4e77 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 29 Aug 2017 10:34:38 -0400 Subject: [PATCH 08/19] Package module: extensive documentation of options --- src/modules/packages/packages.conf | 95 ++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/src/modules/packages/packages.conf b/src/modules/packages/packages.conf index 7bf589243..6e3af05a8 100644 --- a/src/modules/packages/packages.conf +++ b/src/modules/packages/packages.conf @@ -33,23 +33,84 @@ update_db: true # storage called "packageOperations" and it is processed # after the static list in the job configuration. # -#operations: -# - install: -# - pkg1 -# - pkg2 -# - remove: -# - pkg3 -# - pkg4 -# - try_install: # no system install failure if a package cannot be installed -# - pkg5 -# - try_remove: # no system install failure if a package cannot be removed -# - pkg2 -# - pkg1 -# - install: -# - pkgs6 -# - pkg7 -# - localInstall: -# - /path/to/pkg8 +# Allowed package operations are: +# - install, try_install: will call the package manager to +# install one or more packages. The install target will +# abort the whole installation if package-installation +# fails, while try_install carries on. Packages may be +# listed as (localized) names, or as (localized) package-data. +# See below for the description of the format. +# - localInstall: this is used to call the package manager +# to install a package from a path-to-a-package. This is +# useful if you have a static package archive on the install media. +# - remove, try_remove: will call the package manager to +# remove one or more packages. The remove target will +# abort the whole installation if package-removal fails, +# while try_remove carries on. Packages may be listed as +# (localized) names. +# +# There are two formats for naming packages: as a name # or as package-data, +# which is an object notation providing package-name, as well as pre- and +# post-install scripts. +# +# Here are both formats, for installing vi. The first one just names the +# package for vi (using the naming of the installed package manager), while +# the second contains three data-items; the pre-script is run before invoking +# the package manager, and the post-script runs once it is done. +# +# - install +# - vi +# - package: vi +# pre-script: touch /tmp/installing-vi +# post-script: rm -f /tmp/installing-vi +# +# The pre- and post-scripts are optional, but not both optional: using +# "package: vi" with neither script option will trick Calamares into +# trying to install a package named "package: vi", which is unlikely to work. +# +# Any package name may be localized; this is used to install localization +# packages for software based on the selected system locale. By including +# the string LOCALE in the package name, the following happens: +# +# - if the system locale is English (generally US English; en_GB is a valid +# localization), then the package is not installed at all, +# - otherwise LOCALE is replaced by the Bcp47 name of the selected system +# locale, e.g. nl_BE. +# +# The following installs localizations for vi, if they are relevant; if +# there is no localization, installation continues normally. +# +# - install +# - vi-LOCALE +# - package: vi-LOCALE +# pre-script: touch /tmp/installing-vi +# post-script: rm -f /tmp/installing-vi +# +# When installing packages, Calamares will invoke the package manager +# with a list of package names if it can; package-data prevents this because +# of the scripts that need to run. In other words, this: +# +# - install: +# - vi +# - binutils +# - package: wget +# pre-script: touch /tmp/installing-wget +# +# This will invoke the package manager three times, once for each package, +# because not all of them are simple package names. You can speed up the +# process if you have only a few pre-scriots, by using multiple install targets: +# +# - install: +# - vi +# - binutils +# - install: +# - package: wget +# pre-script: touch /tmp/installing-wget +# +# This will call the package manager once with the package-names "vi" and +# "binutils", and then a second time for "wget". When installing large numbers +# of packages, this can lead to a considerable time savings. +# operations: - install: - vi From 1ea79efce82852cf0085fab7424860bcf16ff877 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 29 Aug 2017 14:00:48 -0400 Subject: [PATCH 09/19] Uninitialized value --- src/modules/partition/gui/DeviceInfoWidget.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/partition/gui/DeviceInfoWidget.cpp b/src/modules/partition/gui/DeviceInfoWidget.cpp index 1fadb9566..abe5c7a49 100644 --- a/src/modules/partition/gui/DeviceInfoWidget.cpp +++ b/src/modules/partition/gui/DeviceInfoWidget.cpp @@ -33,6 +33,7 @@ DeviceInfoWidget::DeviceInfoWidget( QWidget* parent ) : QWidget( parent ) , m_ptIcon( new QLabel ) , m_ptLabel( new QLabel ) + , m_tableType( PartitionTable::unknownTableType ) { QHBoxLayout* mainLayout = new QHBoxLayout; setLayout( mainLayout ); From e1a93987d01536369e41159ed779d11d48e794ad Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 10:08:44 -0400 Subject: [PATCH 10/19] Packages module: add progress reporting Adds i18n to the module (but these strings are not yet extracted), and reports progress as each group of packages is installed. FIXES #781 --- src/modules/packages/main.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 9eb0a8ea0..623344384 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -28,6 +28,22 @@ import subprocess import libcalamares from libcalamares.utils import check_target_env_call, target_env_call +import gettext +_ = gettext.translation("calamares-python", + localedir=libcalamares.utils.gettext_path(), + languages=libcalamares.utils.gettext_languages(), + fallback=True).gettext + + +total_packages = 0 +completed_packages = 0 + +def pretty_name(): + if (not (total_packages and completed_packages)) + return _("Install packages.") + else + return _("Installing packages, {} of {}.").format(str(completed_packages), str(total_packages)) + class PackageManager(metaclass=abc.ABCMeta): """ @@ -306,6 +322,15 @@ def run_operations(pkgman, entry): :param pkgman: :param entry: """ + global total_packages, completed_packages + total_packages = 0 + completed_packages = 0 + for packagelist in entry.values(): + total_packages += len(packagelist) + if not total_packages: + # Avoids potential divide-by-zero in progress reporting + return + for key in entry.keys(): entry[key] = subst_locale(entry[key]) if key == "install": @@ -337,6 +362,9 @@ def run_operations(pkgman, entry): elif key == "localInstall": pkgman.install(entry[key], from_local=True) + completed_packages += len(entry[key]) + libcalamares.job.setprogress(completed_packages * 1.0 / total_packages) + def run(): """ From a31c4b4cb35a7bec7b4d6326c07ce611507b06a7 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 11:03:52 -0400 Subject: [PATCH 11/19] Packages module: better progress reporting - introduce multiple modes (remove, install) to distinguish progress messages - handle plurals via gettext - fix PEP8 whining from previous --- src/modules/packages/main.py | 95 ++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 623344384..0839d2a24 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -27,22 +27,64 @@ import subprocess import libcalamares from libcalamares.utils import check_target_env_call, target_env_call +from libcalamares.utils import gettext_path, gettext_languages import gettext -_ = gettext.translation("calamares-python", - localedir=libcalamares.utils.gettext_path(), - languages=libcalamares.utils.gettext_languages(), - fallback=True).gettext +_translation = gettext.translation("calamares-python", + localedir=gettext_path(), + languages=gettext_languages(), + fallback=True) +_ = _translation.gettext +_n = _translation.ngettext total_packages = 0 completed_packages = 0 +INSTALL = object() +REMOVE = object() +mode_packages = None # Changes to INSTALL or REMOVE + + +def _change_mode(mode): + global mode_packages + mode_packages = mode + libcalamares.job.setprogress(completed_packages * 1.0 / total_packages) + + def pretty_name(): - if (not (total_packages and completed_packages)) + if mode_packages is INSTALL: + if not total_packages: + return _("Install packages.") + elif not completed_packages: + return _n("Installing package.", + "Installing %(num)d packages.", total_packages) % \ + {"num": total_packages} + elif completed_packages < total_packages: + return _n("Installing package, %(count)d of one.", + "Installing packages, %(count)d of %(total)d.") % \ + {"total": total_packages, "count": completed_packages} + else: + return _n("Installed one package.", + "Installed %(num)d packages.", total_packages) % \ + {"num": total_packages} + elif mode_packages is REMOVE: + if not total_packages: + return _("Remove packages.") + elif not completed_packages: + return _n("Removing package.", + "Removing %(num)d packages.", total_packages) % \ + {"num": total_packages} + elif completed_packages < total_packages: + return _("Removing packages, %(count)d of %(total)d.") % \ + {"total": total_packages, "count": completed_packages} + else: + return _n("Removed one package.", + "Removed %(num)d packages.", total_packages) % \ + {"num": total_packages} + else: + # No mode, generic description return _("Install packages.") - else - return _("Installing packages, {} of {}.").format(str(completed_packages), str(total_packages)) class PackageManager(metaclass=abc.ABCMeta): @@ -322,24 +364,19 @@ def run_operations(pkgman, entry): :param pkgman: :param entry: """ - global total_packages, completed_packages - total_packages = 0 - completed_packages = 0 - for packagelist in entry.values(): - total_packages += len(packagelist) - if not total_packages: - # Avoids potential divide-by-zero in progress reporting - return + global completed_packages, mode_packages for key in entry.keys(): entry[key] = subst_locale(entry[key]) if key == "install": + _change_mode(INSTALL) if all([isinstance(x, str) for x in entry[key]]): pkgman.install(entry[key]) else: for package in entry[key]: pkgman.install_package(package) elif key == "try_install": + _change_mode(INSTALL) # we make a separate package manager call for each package so a # single failing package won't stop all of them for package in entry[key]: @@ -350,8 +387,10 @@ def run_operations(pkgman, entry): warn_text += str(package) libcalamares.utils.debug(warn_text) elif key == "remove": + _change_mode(REMOVE) pkgman.remove(entry[key]) elif key == "try_remove": + _change_mode(REMOVE) for package in entry[key]: try: pkgman.remove([package]) @@ -360,10 +399,12 @@ def run_operations(pkgman, entry): warn_text += package libcalamares.utils.debug(warn_text) elif key == "localInstall": + _change_mode(INSTALL) pkgman.install(entry[key], from_local=True) completed_packages += len(entry[key]) libcalamares.job.setprogress(completed_packages * 1.0 / total_packages) + libcalamares.utils.debug(pretty_name()) def run(): @@ -373,6 +414,8 @@ def run(): :return: """ + global mode_packages, total_packages, completed_packages + backend = libcalamares.job.configuration.get("backend") for identifier, impl in backend_managers: @@ -382,17 +425,29 @@ def run(): else: return "Bad backend", "backend=\"{}\"".format(backend) - operations = libcalamares.job.configuration.get("operations", []) - update_db = libcalamares.job.configuration.get("update_db", False) if update_db and libcalamares.globalstorage.value("hasInternet"): pkgman.update_db() + operations = libcalamares.job.configuration.get("operations", []) + if libcalamares.globalstorage.contains("packageOperations"): + operations += libcalamares.globalstorage.value("packageOperations") + + mode_packages = None + total_packages = 0 + completed_packages = 0 + for op in operations: + for packagelist in op.values(): + total_packages += len(packagelist) + + if not total_packages: + # Avoids potential divide-by-zero in progress reporting + return None + for entry in operations: run_operations(pkgman, entry) - if libcalamares.globalstorage.contains("packageOperations"): - run_operations(pkgman, - libcalamares.globalstorage.value("packageOperations")) + mode_packages = None + libcalamares.job.setprogress(1.0) return None From 44262951a1b87d802da30c203aef51f863571d08 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 17:36:48 -0400 Subject: [PATCH 12/19] Python-i18n: add _n() as a gettext keyword, for plurals --- ci/txpush.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/txpush.sh b/ci/txpush.sh index 5ec2c0614..3633c1664 100755 --- a/ci/txpush.sh +++ b/ci/txpush.sh @@ -59,7 +59,7 @@ for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do if test -n "$FILES" ; then MODULE_NAME=$(basename ${MODULE_DIR}) if [ -d ${MODULE_DIR}/lang ]; then - ${PYGETTEXT} -p ${MODULE_DIR}/lang -d ${MODULE_NAME} ${MODULE_DIR}/*.py + ${PYGETTEXT} -k _n -p ${MODULE_DIR}/lang -d ${MODULE_NAME} ${MODULE_DIR}/*.py if [ -f ${MODULE_DIR}/lang/${MODULE_NAME}.pot ]; then tx set -r calamares.${MODULE_NAME} --source -l en ${MODULE_DIR}/lang/${MODULE_NAME}.pot tx push --source --no-interactive -r calamares.${MODULE_NAME} @@ -71,7 +71,7 @@ for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do done if test -n "$SHARED_PYTHON" ; then - ${PYGETTEXT} -p lang -d python $SHARED_PYTHON + ${PYGETTEXT} -k _n -p lang -d python $SHARED_PYTHON tx set -r calamares.python --source -l en lang/python.pot tx push --source --no-interactive -r calamares.python fi From a4f4d417a2967bc1fc3d3a7d60d674bde7b2f287 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 31 Aug 2017 03:51:18 -0400 Subject: [PATCH 13/19] Packages module: improve translated progress reporting - Reduce number of strings a bit - Less confusing translation requirements (I hope) - Report on progress between groups --- src/modules/packages/main.py | 59 ++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/modules/packages/main.py b/src/modules/packages/main.py index 0839d2a24..48caae6bd 100644 --- a/src/modules/packages/main.py +++ b/src/modules/packages/main.py @@ -38,8 +38,9 @@ _ = _translation.gettext _n = _translation.ngettext -total_packages = 0 -completed_packages = 0 +total_packages = 0 # For the entire job +completed_packages = 0 # Done so far for this job +group_packages = 0 # One group of packages from an -install or -remove entry INSTALL = object() REMOVE = object() @@ -53,38 +54,22 @@ def _change_mode(mode): def pretty_name(): - if mode_packages is INSTALL: - if not total_packages: - return _("Install packages.") - elif not completed_packages: - return _n("Installing package.", - "Installing %(num)d packages.", total_packages) % \ - {"num": total_packages} - elif completed_packages < total_packages: - return _n("Installing package, %(count)d of one.", - "Installing packages, %(count)d of %(total)d.") % \ - {"total": total_packages, "count": completed_packages} - else: - return _n("Installed one package.", - "Installed %(num)d packages.", total_packages) % \ - {"num": total_packages} + if not group_packages: + # Outside the context of an operation + s = _("Processing packages (%(count)d / %(total)d)") + elif mode_packages is INSTALL: + s = _n("Installing one package.", + "Installing %(num)d packages.", group_packages) elif mode_packages is REMOVE: - if not total_packages: - return _("Remove packages.") - elif not completed_packages: - return _n("Removing package.", - "Removing %(num)d packages.", total_packages) % \ - {"num": total_packages} - elif completed_packages < total_packages: - return _("Removing packages, %(count)d of %(total)d.") % \ - {"total": total_packages, "count": completed_packages} - else: - return _n("Removed one package.", - "Removed %(num)d packages.", total_packages) % \ - {"num": total_packages} + s = _n("Removing one package.", + "Removing %(num)d packages.", group_packages) else: # No mode, generic description - return _("Install packages.") + s = _("Install packages.") + + return s % {"num": group_packages, + "count": completed_packages, + "total": total_packages} class PackageManager(metaclass=abc.ABCMeta): @@ -364,10 +349,11 @@ def run_operations(pkgman, entry): :param pkgman: :param entry: """ - global completed_packages, mode_packages + global group_packages, completed_packages, mode_packages for key in entry.keys(): entry[key] = subst_locale(entry[key]) + group_packages = len(entry[key]) if key == "install": _change_mode(INSTALL) if all([isinstance(x, str) for x in entry[key]]): @@ -406,6 +392,9 @@ def run_operations(pkgman, entry): libcalamares.job.setprogress(completed_packages * 1.0 / total_packages) libcalamares.utils.debug(pretty_name()) + group_packages = 0 + _change_mode(None) + def run(): """ @@ -414,7 +403,7 @@ def run(): :return: """ - global mode_packages, total_packages, completed_packages + global mode_packages, total_packages, completed_packages, group_packages backend = libcalamares.job.configuration.get("backend") @@ -445,9 +434,13 @@ def run(): return None for entry in operations: + group_packages = 0 + libcalamares.utils.debug(pretty_name()) run_operations(pkgman, entry) mode_packages = None + libcalamares.job.setprogress(1.0) + libcalamares.utils.debug(pretty_name()) return None From 71fe0f6f0301676b290087f10cf142d35912dee3 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 31 Aug 2017 04:09:48 -0400 Subject: [PATCH 14/19] Python-i18n: pygettext is deprecated - Use xgettext -L python instead - Mark _n as a plural-forms translation function - Explicit output to .pot files (instead of default .po) --- ci/txpush.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ci/txpush.sh b/ci/txpush.sh index 3633c1664..cb2499f3e 100755 --- a/ci/txpush.sh +++ b/ci/txpush.sh @@ -47,11 +47,7 @@ tx push --source --no-interactive -r calamares.fdo # - python modules without lang/, which use one shared catalog # -# Arch -# PYGETTEXT=/usr/lib/python3.5/Tools/i18n/pygettext.py - -# Ubuntu -PYGETTEXT=pygettext3 +PYGETTEXT="xgettext --keyword=_n:1,2 -L python" SHARED_PYTHON="" for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do @@ -59,7 +55,7 @@ for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do if test -n "$FILES" ; then MODULE_NAME=$(basename ${MODULE_DIR}) if [ -d ${MODULE_DIR}/lang ]; then - ${PYGETTEXT} -k _n -p ${MODULE_DIR}/lang -d ${MODULE_NAME} ${MODULE_DIR}/*.py + ${PYGETTEXT} -p ${MODULE_DIR}/lang -d ${MODULE_NAME} -o ${MODULE_NAME}.pot ${MODULE_DIR}/*.py if [ -f ${MODULE_DIR}/lang/${MODULE_NAME}.pot ]; then tx set -r calamares.${MODULE_NAME} --source -l en ${MODULE_DIR}/lang/${MODULE_NAME}.pot tx push --source --no-interactive -r calamares.${MODULE_NAME} @@ -71,7 +67,7 @@ for MODULE_DIR in $(find src/modules -maxdepth 1 -mindepth 1 -type d) ; do done if test -n "$SHARED_PYTHON" ; then - ${PYGETTEXT} -k _n -p lang -d python $SHARED_PYTHON + ${PYGETTEXT} -p lang -d python -o python.pot $SHARED_PYTHON tx set -r calamares.python --source -l en lang/python.pot tx push --source --no-interactive -r calamares.python fi From d5dca07e22852541888d90f20f4c1111bb5781cd Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 06:12:14 -0400 Subject: [PATCH 15/19] Fix uninitialized values (valgrind report) --- src/modules/partition/gui/EncryptWidget.cpp | 1 + src/modules/welcome/checker/partman_devices.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/partition/gui/EncryptWidget.cpp b/src/modules/partition/gui/EncryptWidget.cpp index 9fc7be59d..198f2ebe1 100644 --- a/src/modules/partition/gui/EncryptWidget.cpp +++ b/src/modules/partition/gui/EncryptWidget.cpp @@ -23,6 +23,7 @@ EncryptWidget::EncryptWidget( QWidget* parent ) : QWidget( parent ) + , m_state( EncryptionDisabled ) { setupUi( this ); diff --git a/src/modules/welcome/checker/partman_devices.c b/src/modules/welcome/checker/partman_devices.c index d417f3c0f..e2c2bb98f 100644 --- a/src/modules/welcome/checker/partman_devices.c +++ b/src/modules/welcome/checker/partman_devices.c @@ -112,7 +112,7 @@ process_device(PedDevice *dev) int check_big_enough(long long required_space) { - PedDevice *dev; + PedDevice *dev = NULL; ped_exception_fetch_all(); ped_device_probe_all(); From e26d5ab20636c4c67b01a202db4c47b96ca112ed Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 07:46:01 -0400 Subject: [PATCH 16/19] Don't leak memory for allocated modules --- src/libcalamaresui/modulesystem/ModuleManager.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libcalamaresui/modulesystem/ModuleManager.cpp b/src/libcalamaresui/modulesystem/ModuleManager.cpp index dd7abdf29..d08cdf8dd 100644 --- a/src/libcalamaresui/modulesystem/ModuleManager.cpp +++ b/src/libcalamaresui/modulesystem/ModuleManager.cpp @@ -57,7 +57,13 @@ ModuleManager::ModuleManager( const QStringList& paths, QObject* parent ) ModuleManager::~ModuleManager() -{} +{ + // The map is populated with Module::fromDescriptor(), which allocates on the heap. + for( auto moduleptr : m_loadedModulesByInstanceKey ) + { + delete moduleptr; + } +} void From 0e96621b941e175f3ee1c2247511ba44b3e0bdf4 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 30 Aug 2017 07:50:24 -0400 Subject: [PATCH 17/19] Don't leak memory when winnowing disk devices - Improve logging a little - Don't leak Device*, but delete the raw pointer when erasing - Document that DeviceInfo takes ownership and doesn't leak --- src/modules/partition/core/DeviceList.cpp | 24 +++++++++++++------ .../partition/core/PartitionCoreModule.cpp | 1 + 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/modules/partition/core/DeviceList.cpp b/src/modules/partition/core/DeviceList.cpp index 5bce4a0e3..055c18dbc 100644 --- a/src/modules/partition/core/DeviceList.cpp +++ b/src/modules/partition/core/DeviceList.cpp @@ -106,12 +106,22 @@ operator <<( QDebug& s, QList< Device* >::iterator& it ) return s; } +using DeviceList = QList< Device* >; + +static inline DeviceList::iterator +erase(DeviceList& l, DeviceList::iterator& it) +{ + Device* p = *it; + auto r = l.erase( it ); + if (p) + delete p; + return r; +} + QList< Device* > getDevices( DeviceType which, qint64 minimumSize ) { bool writableOnly = (which == DeviceType::WritableOnly); - using DeviceList = QList< Device* >; - CoreBackend* backend = CoreBackendManager::self()->backend(); DeviceList devices = backend->scanDevices( true ); @@ -123,8 +133,8 @@ QList< Device* > getDevices( DeviceType which, qint64 minimumSize ) ( *it )->deviceNode().startsWith( "/dev/zram" ) ) { - cDebug() << " .. Removing" << it; - it = devices.erase( it ); + cDebug() << " .. Removing zram" << it; + it = erase(devices, it ); } else if ( writableOnly && ( @@ -132,13 +142,13 @@ QList< Device* > getDevices( DeviceType which, qint64 minimumSize ) isIso9660( *it ) ) ) { - cDebug() << " .. Removing" << it; - it = devices.erase( it ); + cDebug() << " .. Removing root-or-CD" << it; + it = erase(devices, it ); } else if ( (minimumSize >= 0) && !( (*it)->capacity() > minimumSize ) ) { cDebug() << " .. Removing too-small" << it; - it = devices.erase( it ); + it = erase(devices, it ); } else ++it; diff --git a/src/modules/partition/core/PartitionCoreModule.cpp b/src/modules/partition/core/PartitionCoreModule.cpp index ac22b2b7a..07c09c6d7 100644 --- a/src/modules/partition/core/PartitionCoreModule.cpp +++ b/src/modules/partition/core/PartitionCoreModule.cpp @@ -122,6 +122,7 @@ PartitionCoreModule::doInit() cDebug() << "node\tcapacity\tname\tprettyName"; for ( auto device : devices ) { + // Gives ownership of the Device* to the DeviceInfo object auto deviceInfo = new DeviceInfo( device ); m_deviceInfos << deviceInfo; cDebug() << device->deviceNode() << device->capacity() << device->name() << device->prettyName(); From 798640be0d74c550c5729578d126a32acb951692 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 31 Aug 2017 04:51:34 -0400 Subject: [PATCH 18/19] PEP8 whining --- src/modules/initcpiocfg/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/modules/initcpiocfg/main.py b/src/modules/initcpiocfg/main.py index ca9455d20..a18cd0c4f 100644 --- a/src/modules/initcpiocfg/main.py +++ b/src/modules/initcpiocfg/main.py @@ -65,11 +65,12 @@ def write_mkinitcpio_lines(hooks, modules, files, root_mount_point): :param files: :param root_mount_point: """ + hostfile = "/etc/mkinitcpio.conf" try: - with open("/etc/mkinitcpio.conf", "r") as mkinitcpio_file: + with open(hostfile, "r") as mkinitcpio_file: mklins = [x.strip() for x in mkinitcpio_file.readlines()] except FileNotFoundError: - libcalamares.utils.debug("Could not open host file /etc/mkinitcpio.conf") + libcalamares.utils.debug("Could not open host file '%s'" % hostfile) mklins = [] for i in range(len(mklins)): From b22bd67a5fe400ba3da3be4a121487071598f4ad Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 31 Aug 2017 05:28:58 -0400 Subject: [PATCH 19/19] Avoid race condition around libparted device use. FIXES #782 --- src/modules/welcome/checker/partman_devices.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/modules/welcome/checker/partman_devices.c b/src/modules/welcome/checker/partman_devices.c index e2c2bb98f..2cc97557a 100644 --- a/src/modules/welcome/checker/partman_devices.c +++ b/src/modules/welcome/checker/partman_devices.c @@ -126,7 +126,15 @@ check_big_enough(long long required_space) break; } } - ped_device_free_all(); + + // We would free the devices to release allocated memory, + // but other modules might be using partman use well, + // and they can hold pointers to libparted structures in + // other threads. + // + // So prefer to leak memory, instead. + // + // ped_device_free_all(); return big_enough; }