diff --git a/src/modules/unsquashfs/__init__.py b/src/modules/unsquashfs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/modules/unsquashfs/filecopy.py b/src/modules/unsquashfs/filecopy.py new file mode 100644 index 000000000..e97ef660b --- /dev/null +++ b/src/modules/unsquashfs/filecopy.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# === This file is part of Calamares - === +# +# Copyright 2014, Teo Mrnjavac +# +# Originally from Cnchi and Thus, +# Copyright 2013 Antergos (http://antergos.com/) +# Copyright 2013 Manjaro (http://manjaro.org) +# +# Calamares is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Calamares is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Calamares. If not, see . + +import sys +import os +import subprocess +import re + +from threading import Thread + + +COPY_CMD = 'rsync -ar --progress %(source)s %(dest)s' +ON_POSIX = 'posix' in sys.builtin_module_names + + +class FileCopyThread( Thread ): + """ Update the value of the progress bar so that we get some movement """ + def __init__( self, source, dest, progress_cb ): + # Environment used for executing rsync properly + # Setting locale to C (fix issue with tr_TR locale) + self.at_env = os.environ + self.at_env["LC_ALL"] = "C" + + self.process = subprocess.Popen( + ( COPY_CMD % { + 'source': source, + 'dest': dest, + } ).split(), + env=self.at_env, + bufsize=1, + stdout=subprocess.PIPE, + close_fds=ON_POSIX + ) + + self.progress_cb = progress_cb + + super( FileCopyThread, self ).__init__() + + + def kill( self ): + if self.process.poll() is None: + self.process.kill() + + + def run( self ): + num_files_copied = 0 + for line in iter( self.process.stdout.readline, b'' ): + # small comment on this regexp. + # rsync outputs three parameters in the progress. + # xfer#x => i try to interpret it as 'file copy try no. x' + # to-check=x/y, where: + # - x = number of files yet to be checked + # - y = currently calculated total number of files. + # but if you're copying directory with some links in it, the xfer# might not be a + # reliable counter. ( for one increase of xfer, many files may be created) + # In case of manjaro, we pre-compute the total number of files. + # therefore we can easily subtract x from y in order to get real files copied / processed count. + m = re.findall( r'xfr#(\d+), ir-chk=(\d+)/(\d+)', line.decode() ) + if m: + # we've got a percentage update + num_files_remaining = int(m[0][1]) + num_files_total_local = int(m[0][2]) + # adjusting the offset so that progressbar can be continuesly drawn + num_files_copied = num_files_total_local - num_files_remaining + + + # I guess we're updating every 100 files... + if num_files_copied % 100 == 0: + self.progress_cb( num_files_copied ) diff --git a/src/modules/unsquashfs/main.py b/src/modules/unsquashfs/main.py new file mode 100644 index 000000000..1366333b6 --- /dev/null +++ b/src/modules/unsquashfs/main.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# === This file is part of Calamares - === +# +# Copyright 2014, Teo Mrnjavac +# +# Calamares is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Calamares is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Calamares. If not, see . + +import os +import subprocess + +from libcalamares import * +from filecopy import FileCopyThread + + +class UnsquashOperation: + def __init__( self, unpack ): + self.unpacklist = unpack + self.unpackstatus = dict() + for entry in unpack: + self.unpackstatus[ entry[ "source" ] ] = { 'copied': 0, 'total': 0 } + + + def updateCopyProgress( self, source, nfiles ): + if source in self.unpackstatus: + self.unpackstatus[ source ][ 'copied' ] = nfiles + self.reportProgress() + + + def reportProgress( self ): + progress = float( 0 ) + for entry in self.unpackstatus: + partialProgress = float( 0 ) + if entry[ 'total' ] is not 0: + partialProgress += 0.05 + else: + continue + + partialProgress += 0.95 * ( entry[ 'copied' ] / float( entry[ 'total' ] ) ) + progress += partialProgress / len( self.unpackstatus ) + + job.setprogress( progress ) + + + def run( self ): + sourceMountPath = job.configuration[ "sourceMountPath" ] + for entry in self.unpacklist: + unsqfsProcess = subprocess.Popen( [ "unsquashfs", "-l", entry[ "source" ] ], stdout = subprocess.PIPE ) + wcProcess = subprocess.Popen( [ "wc", "-l" ], stdin = unsqfsProcess.stdout, stdout = subprocess.PIPE ) + countString = wcProcess.communicate()[ 0 ] + filesCount = int( float( countString ) ) + self.unpackstatus[ entry[ "source" ] ][ 'total' ] = filesCount + + imgBaseName = os.path.splitext( os.path.basename( entry[ "source" ] ) )[ 0 ] + imgMountDir = sourceMountPath + os.sep + imgBaseName + os.mkdir( imgMountDir ) + + entry[ "sourceDir" ] = imgMountDir + + self.reportProgress() + + self.unsquashImage( entry ) + + + def unsquashImage( self, entry ): + subprocess.check_call( [ "mount", entry[ "source" ], entry[ "sourceDir" ], "-t", "squashfs", "-o", "loop" ] ) + + t = FileCopyThread( entry[ "sourceDir" ], entry[ "destination" ], self.reportProgress ) + t.start() + t.join() + + +def run(): + # from globalStorage: rootMountPoint + # from job.configuration: + # the path to where to mount the source image(s) for copying + # an ordered list of unpack mappings for sqfs file <-> target dir relative + # to rootMountPoint, e.g.: + # configuration: + # sourceMountPath: "/some/path" + # unpack: + # - source: "/path/to/squashfs/image.sqfs" + # destination: "" + # - source: "/path/to/another/image.sqfs" + # destination: "" + + rootMountPoint = globalStorage.value( "rootMountPoint" ) + + unpack = list() + + sourceTempPath = os.path.abspath( job.configuration[ "sourceMountPath" ] ) + if not os.path.exists( sourceTempPath ): + os.mkdir( sourceTempPath ) + if not os.path.exists( sourceTempPath ): + return "Error: cannot create temporary mount directory for source images" + + for entry in job.configuration[ "unpack" ]: + source = os.path.abspath( entry[ "source" ] ) + destination = os.path.abspath( os.path.join( rootMountPoint, entry[ "destination" ] ) ) + + if not os.path.isfile( source ) or not os.path.isdir( destination ): + return "Error: bad source or destination" + + unpack.append( { 'source': source, 'destination': destination } ) + + unsquashop = UnsquashOperation( unpack ) + return unsquashop.run() diff --git a/src/modules/unsquashfs/module.conf b/src/modules/unsquashfs/module.conf new file mode 100644 index 000000000..15e961cbe --- /dev/null +++ b/src/modules/unsquashfs/module.conf @@ -0,0 +1,12 @@ +# Syntax is YAML 1.2 +--- +type: "job" +name: "unsquashfs" +interface: "python" +requires: [] +script: "main.py" #assumed relative to the current directory +configuration: + sourceMountPath: "/some/path" + unpack: + - source: "/path/to/squashfs/image.sqfs" + destination: ""