From 842a90e02644d08ecf6d7a76a2183cef1247e42f Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 29 Jan 2020 16:03:50 +0100 Subject: [PATCH 1/8] [libcalamares] Add an Entropy service for getting random data - Tries to get the "best" random data - Reports the quality of the random data it got --- src/libcalamares/CMakeLists.txt | 3 +- src/libcalamares/utils/Entropy.cpp | 75 ++++++++++++++++++++++++++++++ src/libcalamares/utils/Entropy.h | 44 ++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 src/libcalamares/utils/Entropy.cpp create mode 100644 src/libcalamares/utils/Entropy.h diff --git a/src/libcalamares/CMakeLists.txt b/src/libcalamares/CMakeLists.txt index 8b4233659..0e7f7c6e7 100644 --- a/src/libcalamares/CMakeLists.txt +++ b/src/libcalamares/CMakeLists.txt @@ -39,7 +39,7 @@ set( libSources # Modules modulesystem/InstanceKey.cpp - + # Network service network/Manager.cpp @@ -50,6 +50,7 @@ set( libSources utils/CalamaresUtilsSystem.cpp utils/CommandList.cpp utils/Dirs.cpp + utils/Entropy.cpp utils/Logger.cpp utils/PluginFactory.cpp utils/Retranslator.cpp diff --git a/src/libcalamares/utils/Entropy.cpp b/src/libcalamares/utils/Entropy.cpp new file mode 100644 index 000000000..adba7478c --- /dev/null +++ b/src/libcalamares/utils/Entropy.cpp @@ -0,0 +1,75 @@ +/* === This file is part of Calamares - === + * + * Copyright 2019-2020, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#include "Entropy.h" + +#include + +#include + +CalamaresUtils::EntropySource +CalamaresUtils::getEntropy( int size, QByteArray& b ) +{ + b.clear(); + b.resize( size ); + char* buffer = b.data(); + std::fill( buffer, buffer + size, 0xcb ); + + int readSize = 0; + QFile urandom( "/dev/urandom" ); + if ( urandom.exists() && urandom.open( QIODevice::ReadOnly ) ) + { + readSize = urandom.read( buffer, size ); + urandom.close(); + } + + if ( readSize >= size ) + { + return EntropySource::URandom; + } + + // If it wasn't available, or did not return enough bytes, + // complete it with twister (and tell the client). + std::random_device r; + std::seed_seq seed { r(), r(), r(), r(), r(), r(), r(), r() }; + std::mt19937_64 twister( seed ); + + std::uint64_t next = 0; + + do + { + next = twister(); + // Eight times, for a 64-bit next +#define GET_ONE_BYTE \ + if ( readSize < size ) \ + { \ + buffer[ readSize++ ] = next & 0xff; \ + next = next >> 8; \ + } + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + GET_ONE_BYTE + } while ( readSize < size ); + + return EntropySource::Twister; +} diff --git a/src/libcalamares/utils/Entropy.h b/src/libcalamares/utils/Entropy.h new file mode 100644 index 000000000..da2236266 --- /dev/null +++ b/src/libcalamares/utils/Entropy.h @@ -0,0 +1,44 @@ +/* === This file is part of Calamares - === + * + * Copyright 2019-2020, Adriaan de Groot + * + * Calamares is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Calamares is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Calamares. If not, see . + */ + +#ifndef UTILS_ENTROPY_H +#define UTILS_ENTROPY_H + +#include "DllMacro.h" + +#include + +namespace CalamaresUtils +{ +/// @brief Which entropy source was actually used for the entropy. +enum class EntropySource +{ + URandom, ///< Read from /dev/urandom + Twister ///< Generated by pseudo-random +}; + +/** @brief Fill buffer @p b with exactly @p size random bytes + * + * The array is cleared and resized, then filled with 0xcb + * "just in case", after which it is filled with random + * bytes from a suitable source. Returns which source was used. + */ +DLLEXPORT EntropySource getEntropy( int size, QByteArray& b ); +} // namespace CalamaresUtils + +#endif From a574b43eb81a7792f2e7cdba1051e69f699f414e Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 29 Jan 2020 16:08:33 +0100 Subject: [PATCH 2/8] [libcalamares] Also report empty buffer as no-entropy --- src/libcalamares/utils/Entropy.cpp | 5 +++++ src/libcalamares/utils/Entropy.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/libcalamares/utils/Entropy.cpp b/src/libcalamares/utils/Entropy.cpp index adba7478c..21ee25115 100644 --- a/src/libcalamares/utils/Entropy.cpp +++ b/src/libcalamares/utils/Entropy.cpp @@ -26,6 +26,11 @@ CalamaresUtils::EntropySource CalamaresUtils::getEntropy( int size, QByteArray& b ) { b.clear(); + if ( size < 1) + { + return EntropySource::None; + } + b.resize( size ); char* buffer = b.data(); std::fill( buffer, buffer + size, 0xcb ); diff --git a/src/libcalamares/utils/Entropy.h b/src/libcalamares/utils/Entropy.h index da2236266..2983e0e24 100644 --- a/src/libcalamares/utils/Entropy.h +++ b/src/libcalamares/utils/Entropy.h @@ -28,6 +28,7 @@ namespace CalamaresUtils /// @brief Which entropy source was actually used for the entropy. enum class EntropySource { + None, ///< Buffer is empty, no random data URandom, ///< Read from /dev/urandom Twister ///< Generated by pseudo-random }; From 8947f9c00c3a397a11f44ebdb3651f48196ea96c Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 29 Jan 2020 16:14:17 +0100 Subject: [PATCH 3/8] [libcalamares] Test the Entropy service --- src/libcalamares/utils/Tests.cpp | 24 ++++++++++++++++++++++++ src/libcalamares/utils/Tests.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/src/libcalamares/utils/Tests.cpp b/src/libcalamares/utils/Tests.cpp index 97edd0bc4..1ea50b2f2 100644 --- a/src/libcalamares/utils/Tests.cpp +++ b/src/libcalamares/utils/Tests.cpp @@ -19,6 +19,7 @@ #include "Tests.h" #include "CalamaresUtilsSystem.h" +#include "Entropy.h" #include "Logger.h" #include "UMask.h" #include "Yaml.h" @@ -176,3 +177,26 @@ LibCalamaresTests::testUmask() QCOMPARE( CalamaresUtils::setUMask( 022 ), m ); QCOMPARE( CalamaresUtils::setUMask( m ), 022 ); } + +void +LibCalamaresTests::testEntropy() +{ + QByteArray data; + + auto r0 = CalamaresUtils::getEntropy( 0, data ); + QCOMPARE( CalamaresUtils::EntropySource::None, r0 ); + QCOMPARE( 0, data.size() ); + + auto r1 = CalamaresUtils::getEntropy( 16, data ); + QVERIFY( r1 != CalamaresUtils::EntropySource::None ); + QCOMPARE( 16, data.size() ); + // This can randomly fail (but not often) + QVERIFY( data.at( data.size() - 1 ) != char( 0xcb ) ); + + auto r2 = CalamaresUtils::getEntropy( 8, data ); + QVERIFY( r2 != CalamaresUtils::EntropySource::None ); + QCOMPARE( 8, data.size() ); + QCOMPARE( r1, r2 ); + // This can randomly fail (but not often) + QVERIFY( data.at( data.size() - 1 ) != char( 0xcb ) ); +} diff --git a/src/libcalamares/utils/Tests.h b/src/libcalamares/utils/Tests.h index d9140e0d0..d142152b1 100644 --- a/src/libcalamares/utils/Tests.h +++ b/src/libcalamares/utils/Tests.h @@ -39,6 +39,9 @@ private Q_SLOTS: /** @brief Test that all the UMask objects work correctly. */ void testUmask(); + + /** @brief Tests the entropy functions. */ + void testEntropy(); }; #endif From be0831ee11f10e17dd512a4727b8ded141f0d3f1 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 29 Jan 2020 16:39:25 +0100 Subject: [PATCH 4/8] [libcalamares] Add printable-entropy (e.g. for password salt) --- src/libcalamares/utils/Entropy.cpp | 48 ++++++++++++++++++++++++++++-- src/libcalamares/utils/Entropy.h | 8 +++++ src/libcalamares/utils/Tests.cpp | 25 ++++++++++++++-- src/libcalamares/utils/Tests.h | 1 + 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/libcalamares/utils/Entropy.cpp b/src/libcalamares/utils/Entropy.cpp index 21ee25115..643346855 100644 --- a/src/libcalamares/utils/Entropy.cpp +++ b/src/libcalamares/utils/Entropy.cpp @@ -26,7 +26,7 @@ CalamaresUtils::EntropySource CalamaresUtils::getEntropy( int size, QByteArray& b ) { b.clear(); - if ( size < 1) + if ( size < 1 ) { return EntropySource::None; } @@ -55,7 +55,6 @@ CalamaresUtils::getEntropy( int size, QByteArray& b ) std::mt19937_64 twister( seed ); std::uint64_t next = 0; - do { next = twister(); @@ -78,3 +77,48 @@ CalamaresUtils::getEntropy( int size, QByteArray& b ) return EntropySource::Twister; } + +CalamaresUtils::EntropySource +CalamaresUtils::getPrintableEntropy( int size, QString& s ) +{ + s.clear(); + if ( size < 1 ) + { + return EntropySource::None; + } + + static const char salt_chars[] = { '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', + 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' }; + static_assert( sizeof( salt_chars ) == 64, "Missing salt_chars" ); + + // Number of bytes we're going to need + int byteSize = ( ( size * 6 ) / 8 ) + 1; + QByteArray b; + EntropySource r = getEntropy( byteSize, b ); + + int bitsLeft = 0; + int byteOffset = 0; + qint64 next = 0; + do + { + if ( bitsLeft < 6 ) + { + next = ( next << 8 ) | b.at( byteOffset++ ); + bitsLeft += 8; + } + char c = salt_chars[ next & 0b0111111 ]; + next >>= 6; + bitsLeft -= 6; + s.append( c ); + } while ( ( s.length() < size ) && ( byteOffset < b.size() ) ); + + if ( s.length() < size ) + { + // It's incomplete, not really no-entropy + return EntropySource::None; + } + + return r; +} diff --git a/src/libcalamares/utils/Entropy.h b/src/libcalamares/utils/Entropy.h index 2983e0e24..2ab7a609d 100644 --- a/src/libcalamares/utils/Entropy.h +++ b/src/libcalamares/utils/Entropy.h @@ -40,6 +40,14 @@ enum class EntropySource * bytes from a suitable source. Returns which source was used. */ DLLEXPORT EntropySource getEntropy( int size, QByteArray& b ); + +/** @brief Fill string @p s with exactly @p size random printable ASCII characters + * + * The characters are picked from a set of 64 (2^6). The string + * contains 6 * size bits of entropy. * Returns which source was used. + * @see getEntropy + */ +DLLEXPORT EntropySource getPrintableEntropy( int size, QString& s ); } // namespace CalamaresUtils #endif diff --git a/src/libcalamares/utils/Tests.cpp b/src/libcalamares/utils/Tests.cpp index 1ea50b2f2..81e1a9af6 100644 --- a/src/libcalamares/utils/Tests.cpp +++ b/src/libcalamares/utils/Tests.cpp @@ -185,18 +185,37 @@ LibCalamaresTests::testEntropy() auto r0 = CalamaresUtils::getEntropy( 0, data ); QCOMPARE( CalamaresUtils::EntropySource::None, r0 ); - QCOMPARE( 0, data.size() ); + QCOMPARE( data.size(), 0 ); auto r1 = CalamaresUtils::getEntropy( 16, data ); QVERIFY( r1 != CalamaresUtils::EntropySource::None ); - QCOMPARE( 16, data.size() ); + QCOMPARE( data.size(), 16 ); // This can randomly fail (but not often) QVERIFY( data.at( data.size() - 1 ) != char( 0xcb ) ); auto r2 = CalamaresUtils::getEntropy( 8, data ); QVERIFY( r2 != CalamaresUtils::EntropySource::None ); - QCOMPARE( 8, data.size() ); + QCOMPARE( data.size(), 8 ); QCOMPARE( r1, r2 ); // This can randomly fail (but not often) QVERIFY( data.at( data.size() - 1 ) != char( 0xcb ) ); } + +void +LibCalamaresTests::testPrintableEntropy() +{ + QString s; + + auto r0 = CalamaresUtils::getPrintableEntropy( 0, s ); + QCOMPARE( CalamaresUtils::EntropySource::None, r0 ); + QCOMPARE( s.length(), 0 ); + + auto r1 = CalamaresUtils::getPrintableEntropy( 16, s ); + QVERIFY( r1 != CalamaresUtils::EntropySource::None ); + QCOMPARE( s.length(), 16 ); + for ( QChar c : s ) + { + QVERIFY( c.isPrint() ); + QCOMPARE( c.cell(), 0 ); + } +} diff --git a/src/libcalamares/utils/Tests.h b/src/libcalamares/utils/Tests.h index d142152b1..d369ed4cb 100644 --- a/src/libcalamares/utils/Tests.h +++ b/src/libcalamares/utils/Tests.h @@ -42,6 +42,7 @@ private Q_SLOTS: /** @brief Tests the entropy functions. */ void testEntropy(); + void testPrintableEntropy(); }; #endif From e56948cefaf1cdd4ac0ffae16481671b901f68d2 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 29 Jan 2020 16:41:17 +0100 Subject: [PATCH 5/8] [libcalamares] Fix tests (cell is the lower unicode byte) --- src/libcalamares/utils/Tests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libcalamares/utils/Tests.cpp b/src/libcalamares/utils/Tests.cpp index 81e1a9af6..16faec9a1 100644 --- a/src/libcalamares/utils/Tests.cpp +++ b/src/libcalamares/utils/Tests.cpp @@ -216,6 +216,8 @@ LibCalamaresTests::testPrintableEntropy() for ( QChar c : s ) { QVERIFY( c.isPrint() ); - QCOMPARE( c.cell(), 0 ); + QCOMPARE( c.row(), 0 ); + QVERIFY( c.cell() > 32 ); // ASCII SPACE + QVERIFY( c.cell() < 127 ); } } From c971127b17aa5a4c3c34810890e93a6f90cd8ce3 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 30 Jan 2020 10:08:55 +0100 Subject: [PATCH 6/8] [machineid] Fix entropy-file return - If the file was created and written, it would drop out of the if() and return an error anyway. --- src/modules/machineid/Workers.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/machineid/Workers.cpp b/src/modules/machineid/Workers.cpp index 178f03de5..cfee6f179 100644 --- a/src/modules/machineid/Workers.cpp +++ b/src/modules/machineid/Workers.cpp @@ -127,6 +127,7 @@ createNewEntropy( int poolSize, const QString& rootMountPoint, const QString& fi { cWarning() << "Entropy data is" << data.length() << "bytes, rather than poolSize" << poolSize; } + return Calamares::JobResult::ok(); } return Calamares::JobResult::error( QObject::tr( "File not found" ), From 5b987d4f3316a86e4cff8f4c475e7306a429b92f Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 30 Jan 2020 10:16:15 +0100 Subject: [PATCH 7/8] [machineid] Use entropy service - Most of the code was error-checking, just replace the open-read with a call to the service instead. - It's not an error if /dev/urandom doesn't exist in the source system (there may be other good random sources, and otherwise we have the low-quality random fallback). --- src/modules/machineid/Workers.cpp | 66 +++++++++++++++---------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/modules/machineid/Workers.cpp b/src/modules/machineid/Workers.cpp index cfee6f179..fefaf24b9 100644 --- a/src/modules/machineid/Workers.cpp +++ b/src/modules/machineid/Workers.cpp @@ -3,7 +3,7 @@ * Copyright 2014, Kevin Kofler * Copyright 2016, Philip Müller * Copyright 2017, Alf Gaida - * Copyright 2019, Adriaan de Groot + * Copyright 2019-2020, Adriaan de Groot * * Calamares is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,6 +22,7 @@ #include "Workers.h" #include "utils/CalamaresUtilsSystem.h" +#include "utils/Entropy.h" #include "utils/Logger.h" #include @@ -83,7 +84,7 @@ getUrandomPoolSize() { if ( v.endsWith( '\n' ) ) { - v.chop(1); + v.chop( 1 ); } bool ok = false; poolSize = v.toInt( &ok ); @@ -93,45 +94,42 @@ getUrandomPoolSize() } } } - return (poolSize >= minimumPoolSize) ? poolSize : minimumPoolSize; + return ( poolSize >= minimumPoolSize ) ? poolSize : minimumPoolSize; } Calamares::JobResult createNewEntropy( int poolSize, const QString& rootMountPoint, const QString& fileName ) { - QFile urandom( "/dev/urandom" ); - if ( urandom.exists() && urandom.open( QIODevice::ReadOnly ) ) + QFile entropyFile( rootMountPoint + fileName ); + if ( entropyFile.exists() ) { - QByteArray data = urandom.read( poolSize ); - urandom.close(); - - QFile entropyFile( rootMountPoint + fileName ); - if ( entropyFile.exists() ) - { - cWarning() << "Entropy file" << ( rootMountPoint + fileName ) << "already exists."; - return Calamares::JobResult::ok(); // .. anyway - } - if ( !entropyFile.open( QIODevice::WriteOnly ) ) - { - return Calamares::JobResult::error( - QObject::tr( "File not found" ), - QObject::tr( "Could not create new random file
%1
." ).arg( fileName ) ); - } - entropyFile.write( data ); - entropyFile.close(); - if ( entropyFile.size() < data.length() ) - { - cWarning() << "Entropy file is" << entropyFile.size() << "bytes, random data was" << data.length(); - } - if ( data.length() < poolSize ) - { - cWarning() << "Entropy data is" << data.length() << "bytes, rather than poolSize" << poolSize; - } - return Calamares::JobResult::ok(); + cWarning() << "Entropy file" << ( rootMountPoint + fileName ) << "already exists."; + return Calamares::JobResult::ok(); // .. anyway } - return Calamares::JobResult::error( - QObject::tr( "File not found" ), - QObject::tr( "Could not read random file
%1
." ).arg( QStringLiteral( "/dev/urandom" ) ) ); + if ( !entropyFile.open( QIODevice::WriteOnly ) ) + { + return Calamares::JobResult::error( + QObject::tr( "File not found" ), + QObject::tr( "Could not create new random file
%1
." ).arg( fileName ) ); + } + + QByteArray data; + CalamaresUtils::EntropySource source = CalamaresUtils::getEntropy( poolSize, data ); + entropyFile.write( data ); + entropyFile.close(); + if ( entropyFile.size() < data.length() ) + { + cWarning() << "Entropy file is" << entropyFile.size() << "bytes, random data was" << data.length(); + } + if ( data.length() < poolSize ) + { + cWarning() << "Entropy data is" << data.length() << "bytes, rather than poolSize" << poolSize; + } + if ( source != CalamaresUtils::EntropySource::URandom ) + { + cWarning() << "Entropy data for pool is low-quality."; + } + return Calamares::JobResult::ok(); } From 02e5e0de5ee43d33b860a00df490dc32137d604b Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Thu, 30 Jan 2020 10:32:34 +0100 Subject: [PATCH 8/8] [users] Use entropy service. FIXES #1254 --- src/modules/users/SetPasswordJob.cpp | 40 +++++++--------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/src/modules/users/SetPasswordJob.cpp b/src/modules/users/SetPasswordJob.cpp index 325d24fe3..363a04ccc 100644 --- a/src/modules/users/SetPasswordJob.cpp +++ b/src/modules/users/SetPasswordJob.cpp @@ -22,6 +22,7 @@ #include "GlobalStorage.h" #include "JobQueue.h" #include "utils/CalamaresUtilsSystem.h" +#include "utils/Entropy.h" #include "utils/Logger.h" #include @@ -63,41 +64,18 @@ SetPasswordJob::make_salt( int length ) Q_ASSERT( length >= 8 ); Q_ASSERT( length <= 128 ); - static const char salt_chars[] = { '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', - 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', - 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' }; - - static_assert( sizeof( salt_chars ) == 64, "Missing salt_chars" ); - - std::random_device r; - std::seed_seq seed { r(), r(), r(), r(), r(), r(), r(), r() }; - std::mt19937_64 twister( seed ); - - std::uint64_t next; - int current_length = 0; - QString salt_string; - salt_string.reserve( length + 10 ); - - while ( current_length < length ) + CalamaresUtils::EntropySource source = CalamaresUtils::getPrintableEntropy( length, salt_string ); + if ( salt_string.length() != length ) { - next = twister(); - // In 64 bits, we have 10 blocks of 6 bits; map each block of 6 bits - // to a single salt character. - for ( unsigned int char_count = 0; char_count < 10; ++char_count ) - { - char c = salt_chars[ next & 0b0111111 ]; - next >>= 6; - salt_string.append( c ); - if ( ++current_length >= length ) - { - break; - } - } + cWarning() << "getPrintableEntropy returned string of length" << salt_string.length() << "expected" << length; + salt_string.truncate( length ); + } + if ( source != CalamaresUtils::EntropySource::URandom ) + { + cWarning() << "Entropy data for salt is low-quality."; } - salt_string.truncate( length ); salt_string.insert( 0, "$6$" ); salt_string.append( '$' ); return salt_string;