Merge branch 'networking-service'

This commit is contained in:
Adriaan de Groot 2019-08-26 15:49:09 +02:00
commit 6fb909a799
11 changed files with 455 additions and 121 deletions

View File

@ -36,6 +36,9 @@ set( libSources
locale/Lookup.cpp
locale/TranslatableConfiguration.cpp
# Network service
network/Manager.cpp
# Partition service
partition/PartitionSize.cpp
@ -194,6 +197,16 @@ if ( ECM_FOUND AND BUILD_TESTING )
Qt5::Test
)
calamares_automoc( libcalamareslocaletest )
ecm_add_test(
network/Tests.cpp
TEST_NAME
libcalamaresnetworktest
LINK_LIBRARIES
calamares
Qt5::Test
)
calamares_automoc( libcalamaresnetworktest )
endif()
if( BUILD_TESTING )

View File

@ -24,9 +24,7 @@
#endif
#include "Handler.h"
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include "network/Manager.h"
#include <QtTest/QtTest>
@ -197,27 +195,9 @@ GeoIPTests::testSplitTZ()
}
static QByteArray
synchronous_get( const char* urlstring )
{
QUrl url( urlstring );
QNetworkAccessManager manager;
QEventLoop loop;
qDebug() << "Fetching" << url;
QObject::connect( &manager, &QNetworkAccessManager::finished, &loop, &QEventLoop::quit );
QNetworkRequest request( url );
QNetworkReply* reply = manager.get( request );
loop.exec();
reply->deleteLater();
return reply->readAll();
}
#define CHECK_GET( t, selector, url ) \
{ \
auto tz = GeoIP##t( selector ).processReply( synchronous_get( url ) ); \
auto tz = GeoIP##t( selector ).processReply( CalamaresUtils::Network::Manager::instance().synchronousGet( QUrl( url ) ) ); \
qDebug() << tz; \
QCOMPARE( default_tz, tz ); \
auto tz2 = CalamaresUtils::GeoIP::Handler( "" #t, url, selector ).get(); \
@ -236,7 +216,7 @@ GeoIPTests::testGet()
GeoIPJSON default_handler;
// Call the KDE service the definitive source.
auto default_tz = default_handler.processReply( synchronous_get( "https://geoip.kde.org/v1/calamares" ) );
auto default_tz = default_handler.processReply( CalamaresUtils::Network::Manager::instance().synchronousGet( QUrl( "https://geoip.kde.org/v1/calamares" ) ) );
// This is bogus, because the test isn't always run by me
// QCOMPARE( default_tz.first, QStringLiteral("Europe") );

View File

@ -23,14 +23,11 @@
#include "GeoIPXML.h"
#endif
#include "network/Manager.h"
#include "utils/Logger.h"
#include "utils/NamedEnum.h"
#include "utils/Variant.h"
#include <QEventLoop>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <memory>
static const NamedEnumTable< CalamaresUtils::GeoIP::Handler::Type >&
@ -87,22 +84,6 @@ Handler::Handler( const QString& implementation, const QString& url, const QStri
Handler::~Handler() {}
static QByteArray
synchronous_get( const QString& urlstring )
{
QUrl url( urlstring );
QNetworkAccessManager manager;
QEventLoop loop;
QObject::connect( &manager, &QNetworkAccessManager::finished, &loop, &QEventLoop::quit );
QNetworkRequest request( url );
QNetworkReply* reply = manager.get( request );
loop.exec();
reply->deleteLater();
return reply->readAll();
}
static std::unique_ptr< Interface >
create_interface( Handler::Type t, const QString& selector )
{
@ -131,7 +112,7 @@ do_query( Handler::Type type, const QString& url, const QString& selector )
return RegionZonePair();
}
return interface->processReply( synchronous_get( url ) );
return interface->processReply( CalamaresUtils::Network::Manager::instance().synchronousGet( url ) );
}
static QString
@ -143,7 +124,7 @@ do_raw_query( Handler::Type type, const QString& url, const QString& selector )
return QString();
}
return interface->rawReply( synchronous_get( url ) );
return interface->rawReply( CalamaresUtils::Network::Manager::instance().synchronousGet( url ) );
}
RegionZonePair

View File

@ -0,0 +1,177 @@
/* === This file is part of Calamares - <https://github.com/calamares> ===
*
* Copyright 2019, Adriaan de Groot <groot@kde.org>
*
* Calamares is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Calamares is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Calamares. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Manager.h"
#include <QEventLoop>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QTimer>
namespace CalamaresUtils
{
namespace Network
{
void
RequestOptions::applyToRequest( QNetworkRequest* request ) const
{
if ( m_flags & Flag::FollowRedirect )
{
// Follows all redirects except unsafe ones (https to http).
request->setAttribute( QNetworkRequest::FollowRedirectsAttribute, true );
}
if ( m_flags & Flag::FakeUserAgent )
{
// Not everybody likes the default User Agent used by this class (looking at you,
// sourceforge.net), so let's set a more descriptive one.
request->setRawHeader( "User-Agent", "Mozilla/5.0 (compatible; Calamares)" );
}
}
struct Manager::Private
{
std::unique_ptr< QNetworkAccessManager > m_nam;
QUrl m_hasInternetUrl;
bool m_hasInternet;
Private();
};
Manager::Private::Private()
: m_nam( std::make_unique< QNetworkAccessManager >() )
, m_hasInternet( false )
{
}
Manager::Manager()
: d( std::make_unique< Private >() )
{
}
Manager::~Manager() {}
Manager&
Manager::instance()
{
static auto* s_manager = new Manager();
return *s_manager;
}
bool
Manager::hasInternet()
{
return d->m_hasInternet;
}
bool
Manager::checkHasInternet()
{
bool hasInternet = d->m_nam->networkAccessible() == QNetworkAccessManager::Accessible;
if ( !hasInternet && ( d->m_nam->networkAccessible() == QNetworkAccessManager::UnknownAccessibility ) )
{
hasInternet = synchronousPing( d->m_hasInternetUrl );
}
d->m_hasInternet = hasInternet;
return hasInternet;
}
void
Manager::setCheckHasInternetUrl( const QUrl& url )
{
d->m_hasInternetUrl = url;
}
/** @brief Does a request synchronously, returns the request itself
*
* The extra options for the request are taken from @p options,
* including the timeout setting.
*
* On failure, returns nullptr (e.g. bad URL, timeout). The request
* is marked for later automatic deletion, so don't store the pointer.
*/
static QPair< RequestStatus, QNetworkReply* >
synchronousRun( const std::unique_ptr< QNetworkAccessManager >& nam, const QUrl& url, const RequestOptions& options )
{
QNetworkRequest request = QNetworkRequest( url );
QNetworkReply* reply = nam->get( request );
QEventLoop loop;
QTimer timer;
// Bail out early if the request is bad
if ( reply->error() )
{
reply->deleteLater();
return qMakePair( RequestStatus( RequestStatus::Failed ), nullptr );
}
options.applyToRequest( &request );
if ( options.hasTimeout() )
{
timer.setSingleShot( true );
QObject::connect( &timer, &QTimer::timeout, &loop, &QEventLoop::quit );
timer.start( options.timeout() );
}
QObject::connect( reply, &QNetworkReply::finished, &loop, &QEventLoop::quit );
loop.exec();
if ( options.hasTimeout() && !timer.isActive() )
{
reply->deleteLater();
return qMakePair( RequestStatus( RequestStatus::Timeout ), nullptr );
}
reply->deleteLater();
return qMakePair( RequestStatus( RequestStatus::Ok ), reply );
}
RequestStatus
Manager::synchronousPing( const QUrl& url, const RequestOptions& options )
{
if ( !url.isValid() )
{
return RequestStatus::Failed;
}
auto reply = synchronousRun( d->m_nam, url, options );
if ( reply.first )
{
return reply.second->bytesAvailable() ? RequestStatus::Ok : RequestStatus::Empty;
}
else
{
return reply.first;
}
}
QByteArray
Manager::synchronousGet( const QUrl& url, const RequestOptions& options )
{
if ( !url.isValid() )
{
return QByteArray();
}
auto reply = synchronousRun( d->m_nam, url, options );
return reply.first ? reply.second->readAll() : QByteArray();
}
} // namespace Network
} // namespace CalamaresUtils

View File

@ -0,0 +1,148 @@
/* === This file is part of Calamares - <https://github.com/calamares> ===
*
* Copyright 2019, Adriaan de Groot <groot@kde.org>
*
* Calamares is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Calamares is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Calamares. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef LIBCALAMARES_NETWORK_MANAGER_H
#define LIBCALAMARES_NETWORK_MANAGER_H
#include "DllMacro.h"
#include <QByteArray>
#include <QObject>
#include <QUrl>
#include <chrono>
#include <memory>
class QNetworkRequest;
namespace CalamaresUtils
{
namespace Network
{
class DLLEXPORT RequestOptions
{
public:
using milliseconds = std::chrono::milliseconds;
enum Flag
{
FollowRedirect = 0x1,
FakeUserAgent = 0x100
};
Q_DECLARE_FLAGS( Flags, Flag )
RequestOptions()
: m_flags( 0 )
, m_timeout( -1 )
{
}
RequestOptions( Flags f, milliseconds timeout = milliseconds( -1 ) )
: m_flags( f )
, m_timeout( timeout )
{
}
void applyToRequest( QNetworkRequest* ) const;
bool hasTimeout() const { return m_timeout > milliseconds( 0 ); }
auto timeout() const { return m_timeout; }
private:
Flags m_flags;
milliseconds m_timeout;
};
Q_DECLARE_OPERATORS_FOR_FLAGS( RequestOptions::Flags );
struct RequestStatus
{
enum State
{
Ok,
Timeout, // Timeout exceeded
Failed, // bad Url
Empty // for ping(), response is empty
};
RequestStatus( State s = Ok )
: status( s )
{
}
operator bool() const { return status == Ok; }
State status;
};
class DLLEXPORT Manager : QObject
{
Q_OBJECT
Manager();
public:
/** @brief Gets the single Manager instance.
*
* Typical code will use `auto& nam = Manager::instance();`
* to keep the reference.
*/
static Manager& instance();
virtual ~Manager();
/** @brief Checks if the given @p url returns data.
*
* Returns a RequestStatus, which converts to @c true if the ping
* was successful. Other status reasons convert to @c false,
* typically because of no data, a Url error or no network access.
*
* May return Empty if the request was successful but returned
* no data at all.
*/
RequestStatus synchronousPing( const QUrl& url, const RequestOptions& options = RequestOptions() );
/** @brief Downloads the data from a given @p url
*
* Returns the data as a QByteArray, or an empty
* array if any error occurred (or no data was returned).
*/
QByteArray synchronousGet( const QUrl& url, const RequestOptions& options = RequestOptions() );
/// @brief Set the URL which is used for the general "is there internet" check.
void setCheckHasInternetUrl( const QUrl& url );
/** @brief Do an explicit check for internet connectivity.
*
* This **may** do a ping to the configured check URL, but can also
* use other mechanisms.
*/
bool checkHasInternet();
/** @brief Is there internet connectivity?
*
* This returns the result of the last explicit check, or if there
* is other information about the state of the internet connection,
* whatever is known. @c true means you can expect (all) internet
* connectivity to be present.
*/
bool hasInternet();
private:
struct Private;
std::unique_ptr< Private > d;
};
} // namespace Network
} // namespace CalamaresUtils
#endif // LIBCALAMARES_NETWORK_MANAGER_H

View File

@ -0,0 +1,48 @@
/* === This file is part of Calamares - <https://github.com/calamares> ===
*
* Copyright 2019, Adriaan de Groot <groot@kde.org>
*
* Calamares is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Calamares is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Calamares. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Tests.h"
#include "Manager.h"
#include <QtTest/QtTest>
QTEST_GUILESS_MAIN( NetworkTests )
NetworkTests::NetworkTests() {}
NetworkTests::~NetworkTests() {}
void
NetworkTests::initTestCase()
{
}
void
NetworkTests::testInstance()
{
auto& nam = CalamaresUtils::Network::Manager::instance();
QVERIFY( !nam.hasInternet() );
}
void
NetworkTests::testPing()
{
auto& nam = CalamaresUtils::Network::Manager::instance();
QVERIFY( nam.synchronousPing( QUrl( "https://www.kde.org" ) ) );
}

View File

@ -0,0 +1,38 @@
/* === This file is part of Calamares - <https://github.com/calamares> ===
*
* Copyright 2019, Adriaan de Groot <groot@kde.org>
*
* Calamares is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Calamares is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Calamares. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef LIBCALAMARES_NETWORK_TESTS_H
#define LIBCALAMARES_NETWORK_TESTS_H
#include <QObject>
class NetworkTests : public QObject
{
Q_OBJECT
public:
NetworkTests();
~NetworkTests() override;
private Q_SLOTS:
void initTestCase();
void testInstance();
void testPing();
};
#endif

View File

@ -18,12 +18,10 @@
#include "TrackingJobs.h"
#include "network/Manager.h"
#include "utils/CalamaresUtilsSystem.h"
#include "utils/Logger.h"
#include <QEventLoop>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QSemaphore>
#include <QTimer>
@ -31,13 +29,11 @@
TrackingInstallJob::TrackingInstallJob( const QString& url )
: m_url( url )
, m_networkManager( nullptr )
{
}
TrackingInstallJob::~TrackingInstallJob()
{
delete m_networkManager;
}
QString
@ -61,48 +57,23 @@ TrackingInstallJob::prettyStatusMessage() const
Calamares::JobResult
TrackingInstallJob::exec()
{
m_networkManager = new QNetworkAccessManager();
using CalamaresUtils::Network::Manager;
using CalamaresUtils::Network::RequestOptions;
using CalamaresUtils::Network::RequestStatus;
QNetworkRequest request;
request.setUrl( QUrl( m_url ) );
// Follows all redirects except unsafe ones (https to http).
request.setAttribute( QNetworkRequest::FollowRedirectsAttribute, true );
// Not everybody likes the default User Agent used by this class (looking at you,
// sourceforge.net), so let's set a more descriptive one.
request.setRawHeader( "User-Agent", "Mozilla/5.0 (compatible; Calamares)" );
QTimer timeout;
timeout.setSingleShot( true );
QEventLoop loop;
connect( m_networkManager, &QNetworkAccessManager::finished, this, &TrackingInstallJob::dataIsHere );
connect( m_networkManager, &QNetworkAccessManager::finished, &loop, &QEventLoop::quit );
connect( &timeout, &QTimer::timeout, &loop, &QEventLoop::quit );
m_networkManager->get( request ); // The semaphore is released when data is received
timeout.start( std::chrono::milliseconds( 5000 ) );
loop.exec();
if ( !timeout.isActive() )
auto result = Manager::instance().synchronousPing(
QUrl( m_url ),
RequestOptions( RequestOptions::FollowRedirect | RequestOptions::FakeUserAgent,
RequestOptions::milliseconds( 5000 ) ) );
if ( result.status == RequestStatus::Timeout )
{
cWarning() << "install-tracking request timed out.";
return Calamares::JobResult::error( tr( "Internal error in install-tracking." ),
tr( "HTTP request timed out." ) );
}
timeout.stop();
return Calamares::JobResult::ok();
}
void
TrackingInstallJob::dataIsHere( QNetworkReply* reply )
{
cDebug() << "Installation feedback request OK";
reply->deleteLater();
}
QString
TrackingMachineNeonJob::prettyName() const
{

View File

@ -21,13 +21,10 @@
#include "Job.h"
class QNetworkAccessManager;
class QNetworkReply;
class QSemaphore;
class TrackingInstallJob : public Calamares::Job
{
Q_OBJECT
public:
TrackingInstallJob( const QString& url );
~TrackingInstallJob() override;
@ -37,13 +34,8 @@ public:
QString prettyStatusMessage() const override;
Calamares::JobResult exec() override;
public slots:
void dataIsHere( QNetworkReply* );
private:
const QString m_url;
QNetworkAccessManager* m_networkManager;
};
class TrackingMachineNeonJob : public Calamares::Job

View File

@ -25,12 +25,14 @@
#include "partman_devices.h"
#include "modulesystem/Requirement.h"
#include "network/Manager.h"
#include "widgets/WaitingWidget.h"
#include "utils/CalamaresUtilsGui.h"
#include "utils/Logger.h"
#include "utils/Retranslator.h"
#include "utils/CalamaresUtilsSystem.h"
#include "utils/Units.h"
#include "utils/Variant.h"
#include "Settings.h"
#include "JobQueue.h"
@ -42,13 +44,9 @@
#include <QDBusInterface>
#include <QDesktopWidget>
#include <QDir>
#include <QEventLoop>
#include <QFile>
#include <QFileInfo>
#include <QLabel>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QProcess>
#include <QTimer>
@ -245,16 +243,16 @@ GeneralRequirements::setConfigurationMap( const QVariantMap& configurationMap )
incompleteConfiguration = true;
}
if ( configurationMap.contains( "internetCheckUrl" ) &&
configurationMap.value( "internetCheckUrl" ).type() == QVariant::String )
QUrl checkInternetUrl;
QString checkInternetSetting = CalamaresUtils::getString( configurationMap, "internetCheckUrl" );
if ( !checkInternetSetting.isEmpty() )
{
m_checkHasInternetUrl = configurationMap.value( "internetCheckUrl" ).toString().trimmed();
if ( m_checkHasInternetUrl.isEmpty() ||
!QUrl( m_checkHasInternetUrl ).isValid() )
checkInternetUrl = QUrl( checkInternetSetting.trimmed() );
if ( !checkInternetUrl.isValid() )
{
cWarning() << "GeneralRequirements entry 'internetCheckUrl' is invalid in welcome.conf" << m_checkHasInternetUrl
cWarning() << "GeneralRequirements entry 'internetCheckUrl' is invalid in welcome.conf" << checkInternetSetting
<< "reverting to default (http://example.com).";
m_checkHasInternetUrl = "http://example.com";
checkInternetUrl = QUrl( "http://example.com" );
incompleteConfiguration = true;
}
}
@ -262,10 +260,13 @@ GeneralRequirements::setConfigurationMap( const QVariantMap& configurationMap )
{
cWarning() << "GeneralRequirements entry 'internetCheckUrl' is undefined in welcome.conf,"
"reverting to default (http://example.com).";
m_checkHasInternetUrl = "http://example.com";
checkInternetUrl = "http://example.com";
incompleteConfiguration = true;
}
if ( checkInternetUrl.isValid() )
{
CalamaresUtils::Network::Manager::instance().setCheckHasInternetUrl( checkInternetUrl );
}
if ( incompleteConfiguration )
{
@ -357,22 +358,8 @@ GeneralRequirements::checkHasPower()
bool
GeneralRequirements::checkHasInternet()
{
// default to true in the QNetworkAccessManager::UnknownAccessibility case
QNetworkAccessManager qnam;
bool hasInternet = qnam.networkAccessible() == QNetworkAccessManager::Accessible;
if ( !hasInternet && qnam.networkAccessible() == QNetworkAccessManager::UnknownAccessibility )
{
QNetworkRequest req = QNetworkRequest( QUrl( m_checkHasInternetUrl ) );
QNetworkReply* reply = qnam.get( req );
QEventLoop loop;
connect( reply, &QNetworkReply::finished,
&loop, &QEventLoop::quit );
loop.exec();
if( reply->bytesAvailable() )
hasInternet = true;
}
auto& nam = CalamaresUtils::Network::Manager::instance();
bool hasInternet = nam.checkHasInternet();
Calamares::JobQueue::instance()->globalStorage()->insert( "hasInternet", hasInternet );
return hasInternet;
}

View File

@ -48,7 +48,6 @@ private:
qreal m_requiredStorageGiB;
qreal m_requiredRamGiB;
QString m_checkHasInternetUrl;
};
#endif // REQUIREMENTSCHECKER_H