Merge branch 'master' of https://github.com/calamares/calamares into development

CHANGES

@@ -3,11 +3,13 @@ contributors are listed. Note that Calamares does not have a historical
 changelog -- this log starts with version 3.2.0. The release notes on the
 website will have to do for older versions.
 
-# 3.2.15 (unreleased) #
+# 3.2.15 (2019-10-11) #
 
 This release contains contributions from (alphabetically by first name):
+ - No other contributors this time around.
 
 ## Core ##
+ - No changes to core functionality
 
 ## Modules ##
 
@@ -16,6 +18,11 @@ This release contains contributions from (alphabetically by first name):
    shorthand to force **only** that entry in the list. #1253
  - *machineid* module has been re-written in C++ and extended with
    a new configuration key to generate urandom pool data. #1252
+ - *unpackfs* now supports a special *sourcefs* value of `file`
+   for copying single files (optionally with renaming) or directory
+   trees to the target system.
+ - *unpackfs* now support an *exclude* and *excludeFile* setting for
+   excluding particular files or patters from unpacking. #1229
 
 
 # 3.2.14 (2019-09-30) #

CMakeLists.txt

@@ -43,7 +43,7 @@ project( CALAMARES
     VERSION 3.2.15
     LANGUAGES C CXX )
 
-set( CALAMARES_VERSION_RC 1 )  # Set to 0 during release cycle, 1 during development
+set( CALAMARES_VERSION_RC 0 )  # Set to 0 during release cycle, 1 during development
 
 ### OPTIONS
 #

src/modules/unpackfs/main.py

@@ -49,12 +49,25 @@ class UnpackEntry:
     :param sourcefs:
     :param destination:
     """
-    __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total']
+    __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile']
 
     def __init__(self, source, sourcefs, destination):
+        """
+        @p source is the source file name (might be an image file, or
+            a directory, too)
+        @p sourcefs is a type indication; "file" is special, as is
+            "squashfs".
+        @p destination is where the files from the source go. This is
+            **already** prefixed by rootMountPoint, so should be a
+            valid absolute path within the host system.
+
+        The members copied and total are filled in by the copying process.
+        """
         self.source = source
         self.sourcefs = sourcefs
         self.destination = destination
+        self.exclude = None
+        self.excludeFile = None
         self.copied = 0
         self.total = 0
 
@@ -65,12 +78,9 @@ class UnpackEntry:
 ON_POSIX = 'posix' in sys.builtin_module_names
 
 
-def list_excludes(destination):
+def global_excludes():
     """
     List excludes for rsync.
-
-    :param destination:
-    :return:
     """
     lst = []
     extra_mounts = globalstorage.value("extraMounts")
@@ -85,16 +95,18 @@ def list_excludes(destination):
 
     return lst
 
-
+def file_copy(source, entry, progress_cb):
-def file_copy(source, dest, progress_cb):
     """
     Extract given image using rsync.
 
-    :param source:
+    :param source: Source file. This may be the place the entry's
-    :param dest:
+        image is mounted, or if it's a single file, the entry's source value.
-    :param progress_cb:
+    :param entry: The UnpackEntry being copied.
-    :return:
+    :param progress_cb: A callback function for progress reporting.
+        Takes a number and a total-number.
     """
+    dest = entry.destination
+
     # Environment used for executing rsync properly
     # Setting locale to C (fix issue with tr_TR locale)
     at_env = os.environ
@@ -110,7 +122,12 @@ def file_copy(source, dest, progress_cb):
     num_files_copied = 0  # Gets updated through rsync output
 
     args = ['rsync', '-aHAXr']
-    args.extend(list_excludes(dest))
+    args.extend(global_excludes())
+    if entry.excludeFile:
+        args.extend(["--exclude-from=" + entry.excludeFile])
+    if entry.exclude:
+        for f in entry.exclude:
+            args.extend(["--exclude", f])
     args.extend(['--progress', source, dest])
     process = subprocess.Popen(
         args, env=at_env, bufsize=1, stdout=subprocess.PIPE, close_fds=ON_POSIX
@@ -306,7 +323,7 @@ class UnpackOperation:
             else:
                 source = imgmountdir
 
-            return file_copy(source, entry.destination, progress_cb)
+            return file_copy(source, entry, progress_cb)
         finally:
             if not entry.is_file():
                 subprocess.check_call(["umount", "-l", imgmountdir])
@@ -374,17 +391,28 @@ def run():
 
     unpack = list()
 
+    is_first = True
     for entry in job.configuration["unpack"]:
         source = os.path.abspath(entry["source"])
         sourcefs = entry["sourcefs"]
         destination = os.path.abspath(root_mount_point + entry["destination"])
 
-        if not os.path.isdir(destination):
+        if not os.path.isdir(destination) and sourcefs != "file":
             utils.warning(("The destination \"{}\" in the target system is not a directory").format(destination))
-            return (_("Bad unsquash configuration"),
+            if is_first:
-                    _("The destination \"{}\" in the target system is not a directory").format(destination))
+                return (_("Bad unsquash configuration"),
+                        _("The destination \"{}\" in the target system is not a directory").format(destination))
+            else:
+                utils.debug(".. assuming that the previous targets will create that directory.")
 
         unpack.append(UnpackEntry(source, sourcefs, destination))
+        # Optional settings
+        if entry.get("exclude", None):
+            unpack[-1].exclude = entry["exclude"]
+        if entry.get("excludeFile", None):
+            unpack[-1].excludeFile = entry["excludeFile"]
+
+        is_first = False
 
     unpackop = UnpackOperation(unpack)
 

src/modules/unpackfs/unpackfs.conf

@@ -11,28 +11,40 @@
 ---
 # Each list item is unpacked, in order, to the target system.
 #
-# Each list item has the following attributes:
+# Each list item has the following **mandatory** attributes:
-#   source: path relative to the live / intstalling system to the image
+#   - *source* path relative to the live / intstalling system to the image
-#   sourcefs: the type of the source files; valid entries are
+#   - *sourcefs* the type of the source files; valid entries are
-#       - *ext4* (copies the filesystem contents)
+#       - `ext4` (copies the filesystem contents)
-#       - *squashfs* (unsquashes)
+#       - `squashfs` (unsquashes)
-#       - *file* (copies a file or directory)
+#       - `file` (copies a file or directory)
 #       - (may be others if mount supports it)
-#   destination: path relative to rootMountPoint (so in the target
+#   - *destination* path relative to rootMountPoint (so in the target
 #       system) where this filesystem is unpacked. It may be an
 #       empty string, which effectively is / (the root) of the target
 #       system.
 #
+# Each list item **optionally** can include the following attributes:
+#   - *exclude* is a list of values that is expanded into --exclude
+#       arguments for rsync (each entry in exclude gets its own --exclude).
+#   - *excludeFile* is a single file that is passed to rsync as an
+#       --exclude-file argument. This should be a full pathname
+#       inside the **host** filesystem.
+#
 # EXAMPLES
 #
 # Usually you list a filesystem image to unpack; you can use
-# squashfs or an ext4 image.
+# squashfs or an ext4 image. An empty destination is equivalent to "/",
+# the root of the target system. The destination directory must exist
+# in the target system.
 #
 #   -   source: "/path/to/filesystem.sqfs"
 #       sourcefs: "squashfs"
 #       destination: ""
 #
-# Multiple entries are unpacked in-order
+# Multiple entries are unpacked in-order; if there is more than one
+# item then only the first must exist beforehand -- it's ok to
+# create directories with one unsquash and then to use those
+# directories as a target from a second unsquash.
 #
 #   -   source: "/path/to/another/filesystem.img"
 #       sourcefs: "ext4"
@@ -60,8 +72,18 @@
 #    -   source: ../src/modules/dummycpp
 #        sourcefs: file
 #        destination: "/tmp/derp"
+#
+# The *destination* and *source* are handed off to rsync, so the semantics
+# of trailing slashes apply. In order to *rename* a file as it is
+# copied, specify one single file (e.g. CHANGES) and a full pathname
+# for its destination name, as in the example below.
 
 unpack:
     -   source: ../CHANGES
         sourcefs: file
-        destination: "/tmp"
+        destination: "/tmp/changes.txt"
+    -   source: src/qml/calamares/slideshow
+        sourcefs: file
+        destination: "/tmp/slideshow/"
+        exclude: [ "*.qmlc", "qmldir" ]
+        # excludeFile: /etc/calamares/modules/unpackfs/exclude-list.txt