2014-07-25 16:49:16 +02:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
# encoding: utf-8
|
|
|
|
# === This file is part of Calamares - <http://github.com/calamares> ===
|
|
|
|
#
|
|
|
|
# Copyright 2014, Teo Mrnjavac <teo@kde.org>
|
2014-08-19 11:30:59 +02:00
|
|
|
# Copyright 2014, Daniel Hillenbrand <codeworkx@bbqlinux.org>
|
|
|
|
# Copyright 2014, Philip Müller <philm@manjaro.org>
|
2014-07-25 16:49:16 +02:00
|
|
|
#
|
|
|
|
# Calamares is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
#
|
|
|
|
# Calamares is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with Calamares. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
import os
|
2014-07-28 17:57:53 +02:00
|
|
|
import re
|
2014-07-28 17:46:56 +02:00
|
|
|
import shutil
|
2014-07-25 16:49:16 +02:00
|
|
|
import subprocess
|
2014-07-28 17:57:53 +02:00
|
|
|
import sys
|
2014-07-28 11:35:24 +02:00
|
|
|
import tempfile
|
2014-07-28 12:17:06 +02:00
|
|
|
from collections import namedtuple
|
2014-07-25 16:49:16 +02:00
|
|
|
|
|
|
|
from libcalamares import *
|
|
|
|
|
2014-08-01 11:59:44 +02:00
|
|
|
class UnpackEntry:
|
2014-08-19 11:30:59 +02:00
|
|
|
__slots__= ['source', 'sourcefs', 'destination', 'copied', 'total']
|
2014-07-30 15:10:23 +02:00
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
def __init__(self, source, sourcefs, destination):
|
2014-08-01 11:59:44 +02:00
|
|
|
self.source = source
|
2014-08-19 11:30:59 +02:00
|
|
|
self.sourcefs = sourcefs
|
2014-08-01 11:59:44 +02:00
|
|
|
self.destination = destination
|
2014-07-30 15:10:23 +02:00
|
|
|
self.copied = 0
|
|
|
|
self.total = 0
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-28 17:57:53 +02:00
|
|
|
ON_POSIX = 'posix' in sys.builtin_module_names
|
|
|
|
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
def file_copy(source, dest, progress_cb):
|
2014-07-28 17:57:53 +02:00
|
|
|
# Environment used for executing rsync properly
|
|
|
|
# Setting locale to C (fix issue with tr_TR locale)
|
|
|
|
at_env = os.environ
|
|
|
|
at_env["LC_ALL"] = "C"
|
|
|
|
|
2014-08-01 11:46:29 +02:00
|
|
|
# `source` *must* end with '/' otherwise a directory named after the source
|
|
|
|
# will be created in `dest`: ie if `source` is "/foo/bar" and `dest` is
|
|
|
|
# "/dest", then files will be copied in "/dest/bar".
|
|
|
|
source += "/"
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
process = subprocess.Popen(['rsync', '-ar', '--progress', source, dest],
|
|
|
|
env=at_env,
|
|
|
|
bufsize=1,
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
close_fds=ON_POSIX)
|
2014-07-28 17:57:53 +02:00
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
for line in iter(process.stdout.readline, b''):
|
2014-07-28 17:57:53 +02:00
|
|
|
# small comment on this regexp.
|
|
|
|
# rsync outputs three parameters in the progress.
|
|
|
|
# xfer#x => i try to interpret it as 'file copy try no. x'
|
|
|
|
# to-check=x/y, where:
|
|
|
|
# - x = number of files yet to be checked
|
|
|
|
# - y = currently calculated total number of files.
|
2014-07-29 15:10:18 +02:00
|
|
|
# but if you're copying directory with some links in it, the xfer#
|
|
|
|
# might not be a reliable counter (for one increase of xfer, many
|
|
|
|
# files may be created).
|
2014-07-28 17:57:53 +02:00
|
|
|
# In case of manjaro, we pre-compute the total number of files.
|
2014-07-29 15:10:18 +02:00
|
|
|
# therefore we can easily subtract x from y in order to get real files
|
|
|
|
# copied / processed count.
|
|
|
|
m = re.findall(r'xfr#(\d+), ir-chk=(\d+)/(\d+)', line.decode())
|
2014-07-28 17:57:53 +02:00
|
|
|
if m:
|
|
|
|
# we've got a percentage update
|
|
|
|
num_files_remaining = int(m[0][1])
|
|
|
|
num_files_total_local = int(m[0][2])
|
|
|
|
# adjusting the offset so that progressbar can be continuesly drawn
|
|
|
|
num_files_copied = num_files_total_local - num_files_remaining
|
|
|
|
|
|
|
|
# I guess we're updating every 100 files...
|
|
|
|
if num_files_copied % 100 == 0:
|
2014-07-29 15:10:18 +02:00
|
|
|
progress_cb(num_files_copied)
|
2014-08-06 15:10:51 +02:00
|
|
|
process.wait()
|
|
|
|
if process.returncode != 0:
|
|
|
|
return "rsync failed with error code {}.".format(process.returncode)
|
|
|
|
return None
|
2014-07-28 17:57:53 +02:00
|
|
|
|
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
class UnpackOperation:
|
2014-07-29 15:10:18 +02:00
|
|
|
|
2014-08-01 11:59:44 +02:00
|
|
|
def __init__(self, entries):
|
|
|
|
self.entries = entries
|
|
|
|
self.entry_for_source = dict((x.source, x) for x in self.entries)
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
def report_progress(self):
|
|
|
|
progress = float(0)
|
2014-08-01 11:59:44 +02:00
|
|
|
for entry in self.entries:
|
|
|
|
if entry.total == 0:
|
2014-07-25 16:49:16 +02:00
|
|
|
continue
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
partialprogress = 0.05 # Having a total !=0 gives 5%
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
partialprogress += 0.95 * \
|
2014-08-01 11:59:44 +02:00
|
|
|
(entry.copied / float(entry.total))
|
|
|
|
progress += partialprogress / len(self.entries)
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
job.setprogress(progress)
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
def run(self):
|
|
|
|
source_mount_path = tempfile.mkdtemp()
|
2014-07-28 11:35:24 +02:00
|
|
|
try:
|
2014-08-01 11:59:44 +02:00
|
|
|
for entry in self.entries:
|
2014-07-29 15:10:18 +02:00
|
|
|
imgbasename = os.path.splitext(
|
|
|
|
os.path.basename(entry.source))[0]
|
2014-07-30 15:06:59 +02:00
|
|
|
imgmountdir = os.path.join(source_mount_path, imgbasename)
|
2014-07-29 15:10:18 +02:00
|
|
|
os.mkdir(imgmountdir)
|
2014-08-19 11:30:59 +02:00
|
|
|
|
|
|
|
self.mount_image(entry, imgmountdir)
|
|
|
|
|
|
|
|
if entry.sourcefs == "squashfs":
|
|
|
|
sqfslist = subprocess.check_output(["unsquashfs",
|
|
|
|
"-l",
|
|
|
|
entry.source])
|
|
|
|
entry.total = len(sqfslist.splitlines())
|
|
|
|
if entry.sourcefs == "ext4":
|
|
|
|
fslist = subprocess.check_output(["ls", "-1",
|
|
|
|
imgmountdir])
|
|
|
|
entry.total = len(fslist.splitlines())
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
self.report_progress()
|
2014-08-19 11:30:59 +02:00
|
|
|
error_msg = self.unpack_image(entry, imgmountdir)
|
2014-08-06 15:10:51 +02:00
|
|
|
if error_msg:
|
2014-08-19 11:30:59 +02:00
|
|
|
return ("Failed to unpack image {}".format(entry.source),
|
2014-08-06 15:10:51 +02:00
|
|
|
error_msg)
|
|
|
|
return None
|
2014-07-28 11:35:24 +02:00
|
|
|
finally:
|
2014-07-29 15:10:18 +02:00
|
|
|
shutil.rmtree(source_mount_path)
|
2014-07-28 12:17:06 +02:00
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
def mount_image(self, entry, imgmountdir):
|
|
|
|
subprocess.check_call(["mount",
|
|
|
|
entry.source,
|
|
|
|
imgmountdir,
|
2014-08-19 12:02:03 +02:00
|
|
|
"-t",
|
|
|
|
entry.sourcefs,
|
|
|
|
"-o", "loop"])
|
2014-08-19 11:30:59 +02:00
|
|
|
|
|
|
|
def unpack_image(self, entry, imgmountdir):
|
2014-07-30 15:35:51 +02:00
|
|
|
def progress_cb(copied):
|
2014-08-01 11:59:44 +02:00
|
|
|
entry.copied = copied
|
2014-07-30 15:35:51 +02:00
|
|
|
self.report_progress()
|
|
|
|
|
2014-07-28 11:35:24 +02:00
|
|
|
try:
|
2014-08-06 15:10:51 +02:00
|
|
|
return file_copy(imgmountdir,
|
2014-07-29 15:10:18 +02:00
|
|
|
entry.destination,
|
2014-07-30 15:35:51 +02:00
|
|
|
progress_cb)
|
2014-07-28 11:35:24 +02:00
|
|
|
finally:
|
2014-07-30 15:06:39 +02:00
|
|
|
subprocess.check_call(["umount", "-l", imgmountdir])
|
2014-07-25 16:49:16 +02:00
|
|
|
|
|
|
|
|
|
|
|
def run():
|
2014-07-29 15:10:18 +02:00
|
|
|
# from globalstorage: rootMountPoint
|
2014-07-25 16:49:16 +02:00
|
|
|
# from job.configuration:
|
|
|
|
# the path to where to mount the source image(s) for copying
|
2014-08-19 11:30:59 +02:00
|
|
|
# an ordered list of unpack mappings for image file <-> target dir relative
|
2014-07-25 16:49:16 +02:00
|
|
|
# to rootMountPoint, e.g.:
|
|
|
|
# configuration:
|
|
|
|
# unpack:
|
2014-08-19 11:30:59 +02:00
|
|
|
# - source: "/path/to/filesystem.img"
|
|
|
|
# sourcefs: "ext4"
|
2014-07-25 16:49:16 +02:00
|
|
|
# destination: ""
|
2014-08-19 11:30:59 +02:00
|
|
|
# - source: "/path/to/another/filesystem.sqfs"
|
|
|
|
# sourcefs: "squashfs"
|
2014-07-25 16:49:16 +02:00
|
|
|
# destination: ""
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
root_mount_point = globalstorage.value("rootMountPoint")
|
|
|
|
if not root_mount_point:
|
|
|
|
return ("No mount point for root partition in globalstorage",
|
|
|
|
"globalstorage does not contain a \"rootMountPoint\" key, "
|
|
|
|
"doing nothing")
|
|
|
|
if not os.path.exists(root_mount_point):
|
|
|
|
return ("Bad mount point for root partition in globalstorage",
|
|
|
|
"globalstorage[\"rootMountPoint\"] is \"{}\", which does not "
|
|
|
|
"exist, doing nothing".format(root_mount_point))
|
2014-07-25 16:49:16 +02:00
|
|
|
unpack = list()
|
|
|
|
|
2014-07-29 15:10:18 +02:00
|
|
|
for entry in job.configuration["unpack"]:
|
|
|
|
source = os.path.abspath(entry["source"])
|
2014-08-19 12:02:03 +02:00
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
sourcefs = entry["sourcefs"]
|
2014-08-19 12:02:03 +02:00
|
|
|
if sourcefs not in ["ext4", "squashfs"]:
|
|
|
|
return "Bad filesystem", "sourcefs=\"{}\"".format(sourcefs)
|
|
|
|
|
2014-07-30 15:05:19 +02:00
|
|
|
destination = os.path.abspath(root_mount_point + entry["destination"])
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-07-30 15:05:44 +02:00
|
|
|
if not os.path.isfile(source):
|
|
|
|
return ("Bad source", "source=\"{}\"".format(source))
|
2014-08-19 11:30:59 +02:00
|
|
|
# Add test for supported filesystems
|
2014-07-30 15:05:44 +02:00
|
|
|
if not os.path.isdir(destination):
|
|
|
|
return ("Bad destination",
|
|
|
|
"destination=\"{}\"".format(destination))
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
unpack.append(UnpackEntry(source, sourcefs, destination))
|
2014-07-25 16:49:16 +02:00
|
|
|
|
2014-08-19 11:30:59 +02:00
|
|
|
unpackop = UnpackOperation(unpack)
|
|
|
|
return unpackop.run()
|