From 4e6492de28032f25fc53e4dc3862dc79c3cc42b6 Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Wed, 23 Jan 2019 16:36:31 +0100 Subject: [PATCH] Add a raw filesystem copy module In some cases, we might want to copy a filesystem as if we were using a simple 'dd' command, in order to create an exact copy, down to the block level. This can be useful in particular when working with dm-verity for checking the rootfs integrity: that way, we can make a direct copy of the rootfs and its verity partition and keep the system usable. This patch adds a new 'rawfs' module to calamares, making possible to block-copy a filesystem to a block device. Signed-off-by: Arnaud Ferraris --- src/modules/rawfs/main.py | 184 ++++++++++++++++++++++++++++++++++ src/modules/rawfs/module.desc | 7 ++ src/modules/rawfs/rawfs.conf | 24 +++++ 3 files changed, 215 insertions(+) create mode 100644 src/modules/rawfs/main.py create mode 100644 src/modules/rawfs/module.desc create mode 100644 src/modules/rawfs/rawfs.conf diff --git a/src/modules/rawfs/main.py b/src/modules/rawfs/main.py new file mode 100644 index 000000000..6ff258918 --- /dev/null +++ b/src/modules/rawfs/main.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# === This file is part of Calamares - === +# +# Copyright 2014, Teo Mrnjavac +# Copyright 2017, Alf Gaida +# Copyright 2017, Adriaan de Groot +# Copyright 2019, Collabora Ltd +# +# Calamares is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Calamares is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Calamares. If not, see . + +import libcalamares +import os +import stat +import subprocess +from time import gmtime, strftime, sleep +from math import gcd + +import gettext +_ = gettext.translation("calamares-python", + localedir=libcalamares.utils.gettext_path(), + languages=libcalamares.utils.gettext_languages(), + fallback=True).gettext + +def pretty_name(): + return _("Installing data.") + +def lcm(a, b): + """ + Computes the Least Common Multiple of 2 numbers + """ + return a * b / gcd(a, b) + +def get_device_size(device): + """ + Returns a filesystem's total size and block size in bytes. + For block devices, block size is the device's block size. + For other files (fs images), block size is 1 byte. + + @param device: str + Absolute path to the device or filesystem image. + @return: tuple(int, int) + The filesystem's size and its block size. + """ + mode = os.stat(device).st_mode + if stat.S_ISBLK(mode): + basedevice = "" + partition = os.path.basename(device) + tmp = partition + while len(tmp) > 0: + tmp = tmp[:-1] + if os.path.exists("/sys/block/" + tmp): + basedevice = tmp + break + # Get device block size + file = open("/sys/block/" + basedevice + "/queue/hw_sector_size") + blocksize = int(file.readline()) + file.close() + # Get partition size + file = open("/sys/block/" + basedevice + "/" + partition + "/size") + size = int(file.readline()) * blocksize + file.close() + else: + size = os.path.getsize(device) + blocksize = 1 + + return size, blocksize + +class RawFSLowSpaceError(Exception): + pass + +class RawFSItem: + __slots__ = ['source', 'destination', 'filesystem', 'resize'] + + def copy(self, current=0, total=1): + """ + Copies a raw filesystem on a disk partition, and grow it to the full destination + partition's size if required. + + @param current: int + The index of the current item in the filesystems list + (used for progress reporting) + @param total: int + The number of items in the filesystems list + (used for progress reporting) + """ + count = 0 + + libcalamares.utils.debug("Copying {} to {}".format(self.source, self.destination)) + + srcsize, srcblksize = get_device_size(self.source) + destsize, destblksize = get_device_size(self.destination) + + if destsize < srcsize: + raise RawFSLowSpaceError + return + + # Compute transfer block size (100x the LCM of the block sizes seems a good fit) + blksize = int(100 * lcm(srcblksize, destblksize)) + + # Execute copy + src = open(self.source, "rb") + dest = open(self.destination, "wb") + buffer = src.read(blksize) + while len(buffer) > 0: + dest.write(buffer) + count += len(buffer) + # Compute job progress + progress = ((count / srcsize) + (current)) / total + libcalamares.job.setprogress(progress) + # Read next data block + buffer = src.read(blksize) + src.close() + dest.close() + + if self.resize: + if "ext" in self.filesystem: + libcalamares.utils.debug("Resizing filesystem on {}".format(self.destination)) + subprocess.run(["e2fsck", "-f", "-y", self.destination]) + subprocess.run(["resize2fs", self.destination]) + + def __init__(self, config, device, fs): + libcalamares.utils.debug("Adding an entry for raw copy of {} to {}".format( + config["source"], device)) + self.source = config["source"] + # If source is a mount point, look for the actual device mounted on it + if os.path.ismount(self.source): + procmounts = open("/proc/mounts", "r") + for line in procmounts: + if self.source in line.split(): + self.source = line.split()[0] + break + + self.destination = device + self.filesystem = fs + try: + self.resize = bool(config["resize"]) + except KeyError: + self.resize = False + +def update_global_storage(item, gs): + for partition in gs: + if partition["device"] == item.destination: + ret = subprocess.run(["blkid", "-s", "UUID", "-o", "value", item.destination], + capture_output=True, text=True) + if ret.returncode == 0: + libcalamares.utils.debug("Setting {} UUID to {}".format(item.destination, + ret.stdout.rstrip())) + gs[gs.index(partition)]["uuid"] = ret.stdout.rstrip() + libcalamares.globalstorage.remove("partitions") + libcalamares.globalstorage.insert("partitions", gs) + +def run(): + """Raw filesystem copy module""" + filesystems = list() + partitions = libcalamares.globalstorage.value("partitions") + + for partition in partitions: + if partition["mountPoint"]: + for src in libcalamares.job.configuration["targets"]: + if src["mountPoint"] == partition["mountPoint"]: + filesystems.append(RawFSItem(src, partition["device"], partition["fs"])) + + for item in filesystems: + try: + item.copy(filesystems.index(item), len(filesystems)) + except RawFSLowSpaceError: + return ("Not enough free space", + "{} partition is too small to copy {} on it".format(item.destination, item.source)) + update_global_storage(item, partitions) + + return None diff --git a/src/modules/rawfs/module.desc b/src/modules/rawfs/module.desc new file mode 100644 index 000000000..aaf65c183 --- /dev/null +++ b/src/modules/rawfs/module.desc @@ -0,0 +1,7 @@ +# Module metadata file for block-copy jobmodule +# Syntax is YAML 1.2 +--- +type: "job" +name: "rawfs" +interface: "python" +script: "main.py" diff --git a/src/modules/rawfs/rawfs.conf b/src/modules/rawfs/rawfs.conf new file mode 100644 index 000000000..6a314ce1b --- /dev/null +++ b/src/modules/rawfs/rawfs.conf @@ -0,0 +1,24 @@ +# Configuration for the rawfs module: raw filesystem copy to a block device + +--- + +# To apply a custom partition layout, it has to be defined this way : +# +# targets: +# - mountPoint: / +# source: / +# - mountPoint: /home +# source: /images/home.img +# resize: true +# - mountPoint: /data +# source: /dev/mmcblk0p3 +# +# For each target, the following attributes must be defined: +# * mountPoint: The mount point of the destination device on the installed system +# The corresponding block device will automatically be identified and used as the +# destination for the operation +# * source: The source filesystem; it can be the mount point of a locally (on the +# live system) mounted filesystem, a disk image, or a block device +# * resize (optional): Expand the destination filesystem to fill the whole +# partition at the end of the operation; this works only with ext filesystems +# for now