From 784bc5b86eba62393e085243728da2fabac40adb Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 30 Sep 2019 16:23:07 +0200 Subject: [PATCH 1/5] [unpackfs] Expand documentation - comment out the squashfs example, since it's not readily available even on developer systems - add entries for the upcoming "file" type. --- src/modules/unpackfs/unpackfs.conf | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index 9720f19a1..b58b2f1d2 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -9,13 +9,18 @@ # target dir relative to rootMountPoint. --- -unpack: # Each list item is unpacked, in order, to the target system. +# # Each list item has the following attributes: # source: path relative to the live / intstalling system to the image -# sourcefs: ext4 or squashfs (may be others if mount supports it) +# sourcefs: the type of the source files; valid entries are +# - *ext4* (copies the filesystem contents) +# - *squashfs* (unsquashes) +# - *file* (copies a file or directory) +# - (may be others if mount supports it) # destination: path relative to rootMountPoint (so in the target # system) where this filesystem is unpacked. +unpack: # Usually you list a filesystem image to unpack; you can use # squashfs or an ext4 image. @@ -33,6 +38,14 @@ unpack: # You can list filesystem source paths relative to the Calamares run # directory, if you use -d (this is only useful for testing, though). - - source: ./example.sqfs - sourcefs: squashfs - destination: "" +# - source: ./example.sqfs +# sourcefs: squashfs +# destination: "" +# You can list individual files + - source: ../CHANGES + sourcefs: file + destination: "/tmp/derp" +# .. or directories + - source: ../src/modules/dummycpp + sourcefs: file + destination: "/tmp/derp" From 7856c1a6a8a0dd2ccc3618f92d867a7f94cd1ba3 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 30 Sep 2019 16:27:57 +0200 Subject: [PATCH 2/5] [unpackfs] Do more checks before doing any work --- src/modules/unpackfs/main.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 90c258cd7..940f9152a 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -330,8 +330,7 @@ def run(): supported_filesystems = get_supported_filesystems() - unpack = list() - + # Bail out before we start when there are obvious problems for entry in job.configuration["unpack"]: source = os.path.abspath(entry["source"]) sourcefs = entry["sourcefs"] @@ -340,14 +339,18 @@ def run(): utils.warning("The filesystem for \"{}\" ({}) is not supported".format(source, sourcefs)) return (_("Bad unsquash configuration"), _("The filesystem for \"{}\" ({}) is not supported").format(source, sourcefs)) - - destination = os.path.abspath(root_mount_point + entry["destination"]) - if not os.path.exists(source): utils.warning("The source filesystem \"{}\" does not exist".format(source)) return (_("Bad unsquash configuration"), _("The source filesystem \"{}\" does not exist").format(source)) + unpack = list() + + for entry in job.configuration["unpack"]: + source = os.path.abspath(entry["source"]) + sourcefs = entry["sourcefs"] + destination = os.path.abspath(root_mount_point + entry["destination"]) + if not os.path.isdir(destination): utils.warning(("The destination \"{}\" in the target system is not a directory").format(destination)) return (_("Bad unsquash configuration"), From 27cdaba8b2b57665b762fac1b1ae5d05c10197ef Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 30 Sep 2019 16:47:39 +0200 Subject: [PATCH 3/5] [unpackfs] Special-case file sources --- src/modules/unpackfs/main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 940f9152a..12aaa5b10 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -58,6 +58,9 @@ class UnpackEntry: self.copied = 0 self.total = 0 + def is_file(self): + return self.sourcefs == "file" + ON_POSIX = 'posix' in sys.builtin_module_names @@ -247,9 +250,15 @@ class UnpackOperation: """ Mount given image as loop device. + A *file* entry (e.g. one with *sourcefs* set to *file*) + is not mounted and just ignored. + :param entry: :param imgmountdir: """ + if entry.is_file(): + return + if os.path.isdir(entry.source): subprocess.check_call(["mount", "--bind", entry.source, @@ -289,10 +298,11 @@ class UnpackOperation: try: return file_copy(imgmountdir, entry.destination, progress_cb) finally: - subprocess.check_call(["umount", "-l", imgmountdir]) + if not entry.is_file(): + subprocess.check_call(["umount", "-l", imgmountdir]) -def get_supported_filesystems(): +def get_supported_filesystems_kernel(): """ Reads /proc/filesystems (the list of supported filesystems for the current kernel) and returns a list of (names of) @@ -310,6 +320,14 @@ def get_supported_filesystems(): return [] +def get_supported_filesystems(): + """ + Returns a list of all the supported filesystems + (valid values for the *sourcefs* key in an item. + """ + return ["file"] + get_supported_filesystems_kernel() + + def run(): """ Unsquash filesystem. From 6fb2563c754f08d91442823f26c657d2c76ed0bb Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 30 Sep 2019 17:12:48 +0200 Subject: [PATCH 4/5] [unpackfs] Copy files when sourcefs = "file" - Just use the existing rsync code, which can do both files and directory trees. - The existing code assumed we were always copying directories. Now double-check beforehand. --- src/modules/unpackfs/main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py index 12aaa5b10..fd3f97353 100644 --- a/src/modules/unpackfs/main.py +++ b/src/modules/unpackfs/main.py @@ -103,7 +103,7 @@ def file_copy(source, dest, progress_cb): # `source` *must* end with '/' otherwise a directory named after the source # will be created in `dest`: ie if `source` is "/foo/bar" and `dest` is # "/dest", then files will be copied in "/dest/bar". - if not source.endswith("/"): + if not source.endswith("/") and not os.path.isfile(source): source += "/" num_files_total_local = 0 @@ -228,11 +228,16 @@ class UnpackOperation: ["unsquashfs", "-l", entry.source] ) - if entry.sourcefs == "ext4": + elif entry.sourcefs == "ext4": fslist = subprocess.check_output( ["find", imgmountdir, "-type", "f"] ) + elif entry.is_file(): + # Hasn't been mounted, copy directly; find handles both + # files and directories. + fslist = subprocess.check_output(["find", entry.source, "-type", "f"]) + entry.total = len(fslist.splitlines()) self.report_progress() @@ -296,7 +301,12 @@ class UnpackOperation: self.report_progress() try: - return file_copy(imgmountdir, entry.destination, progress_cb) + if entry.is_file(): + source = entry.source + else: + source = imgmountdir + + return file_copy(source, entry.destination, progress_cb) finally: if not entry.is_file(): subprocess.check_call(["umount", "-l", imgmountdir]) From 95f725831ff898872273ef6f29c5ad25849dfb4a Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Mon, 30 Sep 2019 17:31:29 +0200 Subject: [PATCH 5/5] [unpackfs] Shuffle the documentation - move all the examples to an EXAMPLES section - leave one single-file example enabled in the configuration --- src/modules/unpackfs/unpackfs.conf | 40 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf index b58b2f1d2..d994e351a 100644 --- a/src/modules/unpackfs/unpackfs.conf +++ b/src/modules/unpackfs/unpackfs.conf @@ -19,33 +19,49 @@ # - *file* (copies a file or directory) # - (may be others if mount supports it) # destination: path relative to rootMountPoint (so in the target -# system) where this filesystem is unpacked. -unpack: - +# system) where this filesystem is unpacked. It may be an +# empty string, which effectively is / (the root) of the target +# system. +# +# EXAMPLES +# # Usually you list a filesystem image to unpack; you can use # squashfs or an ext4 image. # # - source: "/path/to/filesystem.sqfs" # sourcefs: "squashfs" # destination: "" - -# You can list more than one filesystem. +# +# Multiple entries are unpacked in-order # # - source: "/path/to/another/filesystem.img" # sourcefs: "ext4" # destination: "" +# - source: "/path/to/another/filesystem2.img" +# sourcefs: "ext4" +# destination: "/usr/lib/extra" # - # You can list filesystem source paths relative to the Calamares run # directory, if you use -d (this is only useful for testing, though). +# # - source: ./example.sqfs # sourcefs: squashfs # destination: "" -# You can list individual files +# +# You can list individual files (copied one-by-one), or directories +# (the files inside this directory are copied directly to the destination, +# so no "dummycpp/" subdirectory is created in this example). +# Do note that the target directory must exist already (e.g. from +# extracting some other filesystem). +# +# - source: ../CHANGES +# sourcefs: file +# destination: "/tmp/derp" +# - source: ../src/modules/dummycpp +# sourcefs: file +# destination: "/tmp/derp" + +unpack: - source: ../CHANGES sourcefs: file - destination: "/tmp/derp" -# .. or directories - - source: ../src/modules/dummycpp - sourcefs: file - destination: "/tmp/derp" + destination: "/tmp"