From d58fdaa2cde86feeeb6865b9ecbdae70c94dff3a Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 8 Oct 2019 15:00:25 +0200 Subject: [PATCH 01/11] [unpackfs] Stub of special handling for a single file --- src/modules/unpackfs/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index fd3f97353..b6cdd76a0 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -303,6 +303,10 @@ class UnpackOperation: try: if entry.is_file(): source = entry.source + if entry.total <= 1: + # If there is one file, *and* the target does not exist (as a file or dir) + # but the dirname of the target does, we should copy just one file and rename. + pass else: source = imgmountdir From 05c039812689666d3386f39f211a3f5e11308239 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 8 Oct 2019 16:19:39 +0200 Subject: [PATCH 02/11] [unpackfs] Support multiple items better - it's ok if item one creates directories where item two will write, so don't check for existence of all directories on start-up. Reported by ArcoLinux. --- src/modules/unpackfs/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index b6cdd76a0..0c511b959 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -378,6 +378,7 @@ def run(): unpack = list() + is_first = True for entry in job.configuration["unpack"]: source = os.path.abspath(entry["source"]) sourcefs = entry["sourcefs"] @@ -385,10 +386,14 @@ def run(): if not os.path.isdir(destination): utils.warning(("The destination \"{}\" in the target system is not a directory").format(destination)) - return (_("Bad unsquash configuration"), - _("The destination \"{}\" in the target system is not a directory").format(destination)) + if is_first: + return (_("Bad unsquash configuration"), + _("The destination \"{}\" in the target system is not a directory").format(destination)) + else: + utils.debug(".. assuming that the previous targets will create that directory.") unpack.append(UnpackEntry(source, sourcefs, destination)) + is_first = False unpackop = UnpackOperation(unpack) From 4acf63fd9adbca56e6431bc1d317db2e110f179a Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 8 Oct 2019 17:31:51 +0200 Subject: [PATCH 03/11] [unpackfs] Document fields of the UnpackEntry - Document fields - While here, remove code for special-casing single-file copies. Those already work, based on rsync. --- src/modules/unpackfs/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 0c511b959..3a64c6284 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -52,6 +52,17 @@ class UnpackEntry: __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total'] def __init__(self, source, sourcefs, destination): + """ + @p source is the source file name (might be an image file, or + a directory, too) + @p sourcefs is a type indication; "file" is special, as is + "squashfs". + @p destination is where the files from the source go. This is + **already** prefixed by rootMountPoint, so should be a + valid absolute path within the host system. + + The members copied and total are filled in by the copying process. + """ self.source = source self.sourcefs = sourcefs self.destination = destination @@ -85,7 +96,6 @@ def list_excludes(destination): return lst - def file_copy(source, dest, progress_cb): """ Extract given image using rsync. @@ -303,10 +313,6 @@ class UnpackOperation: try: if entry.is_file(): source = entry.source - if entry.total <= 1: - # If there is one file, *and* the target does not exist (as a file or dir) - # but the dirname of the target does, we should copy just one file and rename. - pass else: source = imgmountdir From 57c0b2088c266d7d9feed7a9a0ebe3a35f0494ef Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 15:36:39 +0200 Subject: [PATCH 04/11] [unpackfs] Allow naming a non-directory for single files - this enables renaming files on copy --- src/modules/unpackfs/main.py | 6 +++--- src/modules/unpackfs/unpackfs.conf | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 3a64c6284..a9e6ea067 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -58,9 +58,9 @@ class UnpackEntry: @p sourcefs is a type indication; "file" is special, as is "squashfs". @p destination is where the files from the source go. This is - **already** prefixed by rootMountPoint, so should be a + **already** prefixed by rootMountPoint, so should be a valid absolute path within the host system. - + The members copied and total are filled in by the copying process. """ self.source = source @@ -390,7 +390,7 @@ def run(): sourcefs = entry["sourcefs"] destination = os.path.abspath(root_mount_point + entry["destination"]) - if not os.path.isdir(destination): + if not os.path.isdir(destination) and sourcefs != "file": utils.warning(("The destination \"{}\" in the target system is not a directory").format(destination)) if is_first: return (_("Bad unsquash configuration"), diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index d994e351a..e81a90b0a 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -60,8 +60,13 @@ # - source: ../src/modules/dummycpp # sourcefs: file # destination: "/tmp/derp" +# +# The *destination* and *source* are handed off to rsync, so the semantics +# of trailing slashes apply. In order to *rename* a file as it is +# copied, specify one single file (e.g. CHANGES) and a full pathname +# for its destination name, as in the example below. unpack: - source: ../CHANGES sourcefs: file - destination: "/tmp" + destination: "/tmp/changes.txt" From 371ae598a0ef1433b17f0100ab1b7cb769e25f80 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 16:18:46 +0200 Subject: [PATCH 05/11] Changes: document unpackfs --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index de07562f3..711709922 100644 --- a/CHANGES +++ b/CHANGES @@ -6,8 +6,10 @@ website will have to do for older versions. # 3.2.15 (unreleased) # This release contains contributions from (alphabetically by first name): + - No other contributors this time around. ## Core ## + - No changes to core functionality ## Modules ## @@ -16,6 +18,9 @@ This release contains contributions from (alphabetically by first name): shorthand to force **only** that entry in the list. #1253 - *machineid* module has been re-written in C++ and extended with a new configuration key to generate urandom pool data. #1252 + - *unpackfs* now supports a special *sourcefs* value of `file` + for copying single files (optionally with renaming) or directory + trees to the target system. # 3.2.14 (2019-09-30) # From fe1ddb8ead8b5e974093a3a8d9fbaeaeded54d32 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 16:32:39 +0200 Subject: [PATCH 06/11] [unpackfs] Expand on the documentation --- src/modules/unpackfs/unpackfs.conf | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index e81a90b0a..96f628c11 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -11,14 +11,14 @@ --- # Each list item is unpacked, in order, to the target system. # -# Each list item has the following attributes: -# source: path relative to the live / intstalling system to the image -# sourcefs: the type of the source files; valid entries are -# - *ext4* (copies the filesystem contents) -# - *squashfs* (unsquashes) -# - *file* (copies a file or directory) +# Each list item has the following **mandatory** attributes: +# - *source* path relative to the live / intstalling system to the image +# - *sourcefs* the type of the source files; valid entries are +# - `ext4` (copies the filesystem contents) +# - `squashfs` (unsquashes) +# - `file` (copies a file or directory) # - (may be others if mount supports it) -# destination: path relative to rootMountPoint (so in the target +# - *destination* path relative to rootMountPoint (so in the target # system) where this filesystem is unpacked. It may be an # empty string, which effectively is / (the root) of the target # system. @@ -26,13 +26,18 @@ # EXAMPLES # # Usually you list a filesystem image to unpack; you can use -# squashfs or an ext4 image. +# squashfs or an ext4 image. An empty destination is equivalent to "/", +# the root of the target system. The destination directory must exist +# in the target system. # # - source: "/path/to/filesystem.sqfs" # sourcefs: "squashfs" # destination: "" # -# Multiple entries are unpacked in-order +# Multiple entries are unpacked in-order; if there is more than one +# item then only the first must exist beforehand -- it's ok to +# create directories with one unsquash and then to use those +# directories as a target from a second unsquash. # # - source: "/path/to/another/filesystem.img" # sourcefs: "ext4" From 311c65737fd41a061fe3fe4b8822075e704f636c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 16:46:21 +0200 Subject: [PATCH 07/11] [unpackfs] Remove unused parameter - list_excludes() turns the extra mounts from global storage into --exclude parameters for rsync; it doesn't do anything with the destination parameter. - while here rename to something more descriptive --- src/modules/unpackfs/main.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index a9e6ea067..035912d48 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -76,12 +76,9 @@ class UnpackEntry: ON_POSIX = 'posix' in sys.builtin_module_names -def list_excludes(destination): +def global_excludes(): """ List excludes for rsync. - - :param destination: - :return: """ lst = [] extra_mounts = globalstorage.value("extraMounts") @@ -120,7 +117,7 @@ def file_copy(source, dest, progress_cb): num_files_copied = 0 # Gets updated through rsync output args = ['rsync', '-aHAXr'] - args.extend(list_excludes(dest)) + args.extend(global_excludes()) args.extend(['--progress', source, dest]) process = subprocess.Popen( args, env=at_env, bufsize=1, stdout=subprocess.PIPE, close_fds=ON_POSIX From 4552631086bbd4bfcf6305f8704fa0c9a16378ab Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 16:56:04 +0200 Subject: [PATCH 08/11] [unpackfs] Prep for more information in file_copy - Give the whole entry to file_copy, not just the destination. This will allow file_copy to work with local excludes. - Pluck entry.destination out immediately, to keep code changes minimal. - Document the parameters. --- src/modules/unpackfs/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 035912d48..8c074738a 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -93,15 +93,18 @@ def global_excludes(): return lst -def file_copy(source, dest, progress_cb): +def file_copy(source, entry, progress_cb): """ Extract given image using rsync. - :param source: - :param dest: - :param progress_cb: - :return: + :param source: Source file. This may be the place the entry's + image is mounted, or if it's a single file, the entry's source value. + :param entry: The UnpackEntry being copied. + :param progress_cb: A callback function for progress reporting. + Takes a number and a total-number. """ + dest = entry.destination + # Environment used for executing rsync properly # Setting locale to C (fix issue with tr_TR locale) at_env = os.environ @@ -313,7 +316,7 @@ class UnpackOperation: else: source = imgmountdir - return file_copy(source, entry.destination, progress_cb) + return file_copy(source, entry, progress_cb) finally: if not entry.is_file(): subprocess.check_call(["umount", "-l", imgmountdir]) From 9d47716e5dc56300d8a882fd322d75b6a9fc703e Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 17:19:56 +0200 Subject: [PATCH 09/11] [unpackfs] Add settings per-item for exclude and excludeFile - support excluding specific files / lists when rsyncing, in case the squashfs has more in it than you really want to copy over. --- src/modules/unpackfs/main.py | 10 +++++++++- src/modules/unpackfs/unpackfs.conf | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 8c074738a..bfd291e3c 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -49,7 +49,7 @@ class UnpackEntry: :param sourcefs: :param destination: """ - __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total'] + __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile'] def __init__(self, source, sourcefs, destination): """ @@ -66,6 +66,8 @@ class UnpackEntry: self.source = source self.sourcefs = sourcefs self.destination = destination + self.exclude = None + self.excludeFile = None self.copied = 0 self.total = 0 @@ -399,6 +401,12 @@ def run(): utils.debug(".. assuming that the previous targets will create that directory.") unpack.append(UnpackEntry(source, sourcefs, destination)) + # Optional settings + if entry.get("exclude", None): + unpack[-1].exclude = entry["exclude"] + if entry.get("excludeFile", None): + unpack[-1].excludeFile = entry["excludeFile"] + is_first = False unpackop = UnpackOperation(unpack) diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index 96f628c11..98569b477 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -23,6 +23,13 @@ # empty string, which effectively is / (the root) of the target # system. # +# Each list item **optionally** can include the following attributes: +# - *exclude* is a list of values that is expanded into --exclude +# arguments for rsync (each entry in exclude gets its own --exclude). +# - *excludeFile* is a single file that is passed to rsync as an +# --exclude-file argument. This should be a full pathname +# inside the **host** filesystem. +# # EXAMPLES # # Usually you list a filesystem image to unpack; you can use @@ -75,3 +82,7 @@ unpack: - source: ../CHANGES sourcefs: file destination: "/tmp/changes.txt" + - source: src/qml/calamares/slideshow/ + sourcefs: file + destination: "/tmp/" + exclude: [ "*.qmlc", "qmldir" ] From 4370669fc39a93dc80ea35aa907a3d528996d78c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 17:46:01 +0200 Subject: [PATCH 10/11] [unpackfs] Put exclusion settings into the rsync command line --- src/modules/unpackfs/main.py | 5 +++++ src/modules/unpackfs/unpackfs.conf | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index bfd291e3c..53ffd37df 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -123,6 +123,11 @@ def file_copy(source, entry, progress_cb): args = ['rsync', '-aHAXr'] args.extend(global_excludes()) + if entry.excludeFile: + args.extend(["--exclude-from=" + entry.excludeFile]) + if entry.exclude: + for f in entry.exclude: + args.extend(["--exclude", f]) args.extend(['--progress', source, dest]) process = subprocess.Popen( args, env=at_env, bufsize=1, stdout=subprocess.PIPE, close_fds=ON_POSIX diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index 98569b477..454e82e66 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -82,7 +82,8 @@ unpack: - source: ../CHANGES sourcefs: file destination: "/tmp/changes.txt" - - source: src/qml/calamares/slideshow/ + - source: src/qml/calamares/slideshow sourcefs: file - destination: "/tmp/" + destination: "/tmp/slideshow/" exclude: [ "*.qmlc", "qmldir" ] + # excludeFile: /etc/calamares/modules/unpackfs/exclude-list.txt From 3a7d0d4ee7e170ca974266f1a7b5692d43221f5c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Fri, 11 Oct 2019 17:53:24 +0200 Subject: [PATCH 11/11] Changes: pre-release housekeeping --- CHANGES | 4 +++- CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 711709922..5ade0ba1b 100644 --- a/CHANGES +++ b/CHANGES @@ -3,7 +3,7 @@ contributors are listed. Note that Calamares does not have a historical changelog -- this log starts with version 3.2.0. The release notes on the website will have to do for older versions. -# 3.2.15 (unreleased) # +# 3.2.15 (2019-10-11) # This release contains contributions from (alphabetically by first name): - No other contributors this time around. @@ -21,6 +21,8 @@ This release contains contributions from (alphabetically by first name): - *unpackfs* now supports a special *sourcefs* value of `file` for copying single files (optionally with renaming) or directory trees to the target system. + - *unpackfs* now support an *exclude* and *excludeFile* setting for + excluding particular files or patters from unpacking. #1229 # 3.2.14 (2019-09-30) # diff --git a/CMakeLists.txt b/CMakeLists.txt index d13e6a398..9d61ebbc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ project( CALAMARES VERSION 3.2.15 LANGUAGES C CXX ) -set( CALAMARES_VERSION_RC 1 ) # Set to 0 during release cycle, 1 during development +set( CALAMARES_VERSION_RC 0 ) # Set to 0 during release cycle, 1 during development ### OPTIONS #