Merge branch 'master' of https://github.com/calamares/calamares into development

CHANGES

@@ -7,6 +7,7 @@ website will have to do for older versions.
 
 This release contains contributions from (alphabetically by first name):
  - Anke Boersma
+ - Camilo Higuita
 
 ## Core ##
  - Both the sidebar (on the left) and the navigation buttons (along the
@@ -23,6 +24,11 @@ This release contains contributions from (alphabetically by first name):
 ## Modules ##
  - The *welcomeq* module has been improved with better layout and
    nicer buttons in the example QML form. (Thanks to Anke Boersma)
+ - The *keyboardq* and *localeq* modules now provide some QML for
+   configuring these parts, although they are still very primitive.
+ - *netinstall* has had some minor layout fixes.
+ - *unpackfs* has much more detailed progress reporting and no
+   longer jumps around strangely in overall progress.
 
 
 # 3.2.21 (2020-03-27) #

src/libcalamares/utils/CalamaresUtilsSystem.cpp

@@ -124,7 +124,11 @@ System::runCommand( System::RunLocation location,
                     const QString& stdInput,
                     std::chrono::seconds timeoutSec )
 {
-    QString output;
+    if ( args.isEmpty() )
+    {
+        cWarning() << "Cannot run an empty program list";
+        return ProcessResult::Code::FailedToStart;
+    }
 
     Calamares::GlobalStorage* gs
         = Calamares::JobQueue::instance() ? Calamares::JobQueue::instance()->globalStorage() : nullptr;
@@ -135,9 +139,8 @@ System::runCommand( System::RunLocation location,
         return ProcessResult::Code::NoWorkingDirectory;
     }
 
-    QProcess process;
     QString program;
-    QStringList arguments;
+    QStringList arguments( args );
 
     if ( location == System::RunLocation::RunInTarget )
     {
@@ -149,15 +152,14 @@ System::runCommand( System::RunLocation location,
         }
 
         program = "chroot";
-        arguments = QStringList( { destDir } );
+        arguments.prepend( destDir );
-        arguments << args;
     }
     else
     {
         program = "env";
-        arguments << args;
     }
 
+    QProcess process;
     process.setProgram( program );
     process.setArguments( arguments );
     process.setProcessChannelMode( QProcess::MergedChannels );
@@ -179,7 +181,7 @@ System::runCommand( System::RunLocation location,
     process.start();
     if ( !process.waitForStarted() )
     {
-        cWarning() << "Process failed to start" << process.error();
+        cWarning() << "Process" << args.first() << "failed to start" << process.error();
         return ProcessResult::Code::FailedToStart;
     }
 
@@ -193,15 +195,15 @@ System::runCommand( System::RunLocation location,
                                        ? ( static_cast< int >( std::chrono::milliseconds( timeoutSec ).count() ) )
                                        : -1 ) )
     {
-        cWarning().noquote().nospace() << "Timed out. Output so far:\n" << process.readAllStandardOutput();
+        ( cWarning() << "Process" << args.first() << "timed out after" << timeoutSec.count() << "s. Output so far:\n" ).noquote().nospace() << process.readAllStandardOutput();
         return ProcessResult::Code::TimedOut;
     }
 
-    output.append( QString::fromLocal8Bit( process.readAllStandardOutput() ).trimmed() );
+    QString output = QString::fromLocal8Bit( process.readAllStandardOutput() ).trimmed();
 
     if ( process.exitStatus() == QProcess::CrashExit )
     {
-        cWarning().noquote().nospace() << "Process crashed. Output so far:\n" << output;
+        ( cWarning() << "Process" << args.first() << "crashed. Output so far:\n" ).noquote().nospace() << output;
         return ProcessResult::Code::Crashed;
     }
 
@@ -210,8 +212,7 @@ System::runCommand( System::RunLocation location,
     bool showDebug = ( !Calamares::Settings::instance() ) || ( Calamares::Settings::instance()->debugMode() );
     if ( ( r != 0 ) || showDebug )
     {
-        cDebug() << "Target cmd:" << RedactedList( args );
+        ( cDebug() << "Target cmd:" << RedactedList( args ) << "output:\n" ).noquote().nospace() << output;
-        cDebug().noquote().nospace() << "Target output:\n" << output;
     }
     return ProcessResult( r, output );
 }

src/modules/netinstall/NetInstallViewStep.cpp

@@ -71,6 +71,14 @@ NetInstallViewStep::prettyName() const
     tr( "Login" );
     tr( "Desktop" );
     tr( "Applications" );
+    tr( "Communication" );
+    tr( "Development" );
+    tr( "Office" );
+    tr( "Multimedia" );
+    tr( "Internet" );
+    tr( "Theming" );
+    tr( "Gaming" );
+    tr( "Utilities" );
 #endif
 }
 

src/modules/unpackfs/main.py

@@ -43,6 +43,11 @@ _ = gettext.translation("calamares-python",
 def pretty_name():
     return _("Filling up filesystems.")
 
+# This is going to be changed from various methods
+status = pretty_name()
+
+def pretty_status_message():
+    return status
 
 class UnpackEntry:
     """
@@ -52,7 +57,8 @@ class UnpackEntry:
     :param sourcefs:
     :param destination:
     """
-    __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile']
+    __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile',
+                 'mountPoint']
 
     def __init__(self, source, sourcefs, destination):
         """
@@ -73,10 +79,67 @@ class UnpackEntry:
         self.excludeFile = None
         self.copied = 0
         self.total = 0
+        self.mountPoint = None
 
     def is_file(self):
         return self.sourcefs == "file"
 
+    def do_count(self):
+        """
+        Counts the number of files this entry has.
+        """
+        fslist = ""
+
+        if self.sourcefs == "squashfs":
+            fslist = subprocess.check_output(
+                ["unsquashfs", "-l", self.source]
+                )
+
+        elif self.sourcefs == "ext4":
+            fslist = subprocess.check_output(
+                ["find", self.mountPoint, "-type", "f"]
+                )
+
+        elif self.is_file():
+            # Hasn't been mounted, copy directly; find handles both
+            # files and directories.
+            fslist = subprocess.check_output(["find", self.source, "-type", "f"])
+
+        self.total = len(fslist.splitlines())
+        return self.total
+
+    def do_mount(self, base):
+        """
+        Mount given @p entry as loop device underneath @p base
+
+        A *file* entry (e.g. one with *sourcefs* set to *file*)
+        is not mounted and just ignored.
+
+        :param base: directory to place all the mounts in.
+
+        :returns: None, but throws if the mount failed
+        """
+        imgbasename = os.path.splitext(
+            os.path.basename(self.source))[0]
+        imgmountdir = os.path.join(base, imgbasename)
+        os.makedirs(imgmountdir, exist_ok=True)
+
+        # This is where it *would* go (files bail out before actually mounting)
+        self.mountPoint = imgmountdir
+
+        if self.is_file():
+            return
+
+        if os.path.isdir(self.source):
+            r = mount(self.source, imgmountdir, "", "--bind")
+        elif os.path.isfile(self.source):
+            r = mount(self.source, imgmountdir, self.sourcefs, "loop")
+        else: # self.source is a device
+            r = mount(self.source, imgmountdir, self.sourcefs, "")
+
+        if r != 0:
+            raise subprocess.CalledProcessError(r, "mount")
+
 
 ON_POSIX = 'posix' in sys.builtin_module_names
 
@@ -139,6 +202,9 @@ def file_copy(source, entry, progress_cb):
     # last_num_files_copied trails num_files_copied, and whenever at least 100 more
     # files have been copied, progress is reported and last_num_files_copied is updated.
     last_num_files_copied = 0
+    file_count_chunk = entry.total / 100
+    if file_count_chunk < 100:
+        file_count_chunk = 100
 
     for line in iter(process.stdout.readline, b''):
         # rsync outputs progress in parentheses. Each line will have an
@@ -163,14 +229,17 @@ def file_copy(source, entry, progress_cb):
             # adjusting the offset so that progressbar can be continuesly drawn
             num_files_copied = num_files_total_local - num_files_remaining
 
-            # I guess we're updating every 100 files...
+            # Update about once every 1% of this entry
-            if num_files_copied - last_num_files_copied >= 100:
+            if num_files_copied - last_num_files_copied >= file_count_chunk:
                 last_num_files_copied = num_files_copied
                 progress_cb(num_files_copied, num_files_total_local)
 
     process.wait()
     progress_cb(num_files_copied, num_files_total_local)  # Push towards 100%
 
+    # Mark this entry as really done
+    entry.copied = entry.total
+
     # 23 is the return code rsync returns if it cannot write extended
     # attributes (with -X) because the target file system does not support it,
     # e.g., the FAT EFI system partition. We need -X because distributions
@@ -207,20 +276,30 @@ class UnpackOperation:
         """
         progress = float(0)
 
-        done = 0
+        done = 0  # Done and total apply to the entry now-unpacking
         total = 0
-        complete = 0
+        complete = 0  # This many are already finished
         for entry in self.entries:
             if entry.total == 0:
+                # Total 0 hasn't counted yet
                 continue
-            total += entry.total
-            done += entry.copied
             if entry.total == entry.copied:
                 complete += 1
+            else:
+                # There is at most *one* entry in-progress
+                total = entry.total
+                done = entry.copied
+                break
 
-        if done > 0 and total > 0:
+        if total > 0:
-            progress = 0.05 + (0.90 * done / total) + (0.05 * complete / len(self.entries))
+            # Pretend that each entry represents an equal amount of work;
+            # the complete ones count as 100% of their own fraction
+            # (and have *not* been counted in total or done), while
+            # total/done represents the fraction of the current fraction.
+            progress = ( ( 1.0 * complete ) / len(self.entries) ) + ( ( 1.0 / len(self.entries) ) * ( 1.0 * done / total ) )
 
+        global status
+        status = _("Unpacking image {}/{}, file {}/{}").format((complete+1),len(self.entries),done, total)
         job.setprogress(progress)
 
     def run(self):
@@ -229,78 +308,29 @@ class UnpackOperation:
 
         :return:
         """
+        global status
         source_mount_path = tempfile.mkdtemp()
 
         try:
+            complete = 0
             for entry in self.entries:
-                imgbasename = os.path.splitext(
+                status = _("Starting to unpack {}").format(entry.source)
-                    os.path.basename(entry.source))[0]
+                job.setprogress( ( 1.0 * complete ) / len(self.entries) )
-                imgmountdir = os.path.join(source_mount_path, imgbasename)
+                entry.do_mount(source_mount_path)
-                os.makedirs(imgmountdir, exist_ok=True)
+                entry.do_count()  # Fill in the entry.total
-
-                self.mount_image(entry, imgmountdir)
-
-                fslist = ""
-
-                if entry.sourcefs == "squashfs":
-                    if shutil.which("unsquashfs") is None:
-                        utils.warning("Failed to find unsquashfs")
-
-                        return (_("Failed to unpack image \"{}\"").format(entry.source),
-                                _("Failed to find unsquashfs, make sure you have the squashfs-tools package installed"))
-
-                    fslist = subprocess.check_output(
-                        ["unsquashfs", "-l", entry.source]
-                        )
-
-                elif entry.sourcefs == "ext4":
-                    fslist = subprocess.check_output(
-                        ["find", imgmountdir, "-type", "f"]
-                        )
-
-                elif entry.is_file():
-                    # Hasn't been mounted, copy directly; find handles both
-                    # files and directories.
-                    fslist = subprocess.check_output(["find", entry.source, "-type", "f"])
-
-                entry.total = len(fslist.splitlines())
 
                 self.report_progress()
-                error_msg = self.unpack_image(entry, imgmountdir)
+                error_msg = self.unpack_image(entry, entry.mountPoint)
 
                 if error_msg:
                     return (_("Failed to unpack image \"{}\"").format(entry.source),
                             error_msg)
+                complete += 1
 
             return None
         finally:
             shutil.rmtree(source_mount_path, ignore_errors=True, onerror=None)
 
-    def mount_image(self, entry, imgmountdir):
-        """
-        Mount given @p entry as loop device on @p imgmountdir.
-
-        A *file* entry (e.g. one with *sourcefs* set to *file*)
-        is not mounted and just ignored.
-
-        :param entry: the entry to mount (source is the important property)
-        :param imgmountdir: where to mount it
-
-        :returns: None, but throws if the mount failed
-        """
-        if entry.is_file():
-            return
-
-        if os.path.isdir(entry.source):
-            r = mount(entry.source, imgmountdir, "", "--bind")
-        elif os.path.isfile(entry.source):
-            r = mount(entry.source, imgmountdir, entry.sourcefs, "loop")
-        else: # entry.source is a device
-            r = mount(entry.source, imgmountdir, entry.sourcefs, "")
-
-        if r != 0:
-            raise subprocess.CalledProcessError(r, "mount")
-
 
     def unpack_image(self, entry, imgmountdir):
         """
@@ -379,6 +409,9 @@ def run():
     supported_filesystems = get_supported_filesystems()
 
     # Bail out before we start when there are obvious problems
+    #   - unsupported filesystems
+    #   - non-existent sources
+    #   - missing tools for specific FS
     for entry in job.configuration["unpack"]:
         source = os.path.abspath(entry["source"])
         sourcefs = entry["sourcefs"]
@@ -392,6 +425,12 @@ def run():
             utils.warning("The source filesystem \"{}\" does not exist".format(source))
             return (_("Bad unsquash configuration"),
                     _("The source filesystem \"{}\" does not exist").format(source))
+        if sourcefs == "squashfs":
+            if shutil.which("unsquashfs") is None:
+                utils.warning("Failed to find unsquashfs")
+
+                return (_("Failed to unpack image \"{}\"").format(self.source),
+                        _("Failed to find unsquashfs, make sure you have the squashfs-tools package installed"))
 
     unpack = list()