quicktestresult.cpp

Go to the documentation of this file.

// Copyright (C) 2017 Crimson AS <info@crimson.no>
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "quicktestresult_p.h"
#include "quicktest.h"
#include "quicktest_p.h"
#include <QtTest/qtestcase.h>
#include <QtTest/qtestsystem.h>
#include <QtTest/private/qtestblacklist_p.h>
#include <QtTest/private/qtestresult_p.h>
#include <QtTest/private/qtesttable_p.h>
#include <QtTest/private/qtestlog_p.h>
#include "qtestoptions_p.h"
#include <QtTest/qbenchmark.h>
#include <QtTest/private/qbenchmark_p.h>
#include <QtCore/qset.h>
#include <QtCore/qmap.h>
#include <QtCore/qbytearray.h>
#include <QtCore/qcoreapplication.h>
#include <QtCore/qdatetime.h>
#include <QtCore/qdebug.h>
#include <QtCore/QUrl>
#include <QtCore/QDir>
#if QT_CONFIG(regularexpression)
#include <QtCore/qregularexpression.h>
#endif
#include <QtQuick/qquickwindow.h>
#include <QtGui/qvector3d.h>
#include <QtGui/qimagewriter.h>
#include <QtQml/private/qqmlglobal_p.h>
#include <QtQml/QQmlEngine>
#include <QtQml/QQmlContext>
#include <private/qv4qobjectwrapper_p.h>
 
#include <algorithm>
 
QT_BEGIN_NAMESPACE
 
static const char *globalProgramName = nullptr;
static bool loggingStarted = false;
static QBenchmarkGlobalData globalBenchmarkData;
 
class Q_QMLTEST_EXPORT QuickTestImageObject : public QObject
{
    Q_OBJECT
 
    Q_PROPERTY(int width READ width CONSTANT)
    Q_PROPERTY(int height READ height CONSTANT)
    Q_PROPERTY(QSize size READ size CONSTANT)
 
public:
    QuickTestImageObject(const QImage& img, QObject *parent = nullptr)
        : QObject(parent)
        , m_image(img)
    {
    }
 
    ~QuickTestImageObject() {}
 
public Q_SLOTS:
    int red(int x, int y) const
    {
        return pixel(x, y).value<QColor>().red();
    }
 
    int green(int x, int y) const
    {
        return pixel(x, y).value<QColor>().green();
    }
 
    int blue(int x, int y) const
    {
        return pixel(x, y).value<QColor>().blue();
    }
 
    int alpha(int x, int y) const
    {
        return pixel(x, y).value<QColor>().alpha();
    }
 
    QVariant pixel(int x, int y) const
    {
        if (m_image.isNull()
         || x >= m_image.width()
         || y >= m_image.height()
         || x < 0
         || y < 0
         || x * y >= m_image.width() * m_image.height())
            return QVariant();
 
        return QColor::fromRgba(m_image.pixel(QPoint(x, y)));
    }
 
    bool equals(QuickTestImageObject *other) const
    {
        if (!other)
            return m_image.isNull();
 
        return m_image == other->m_image;
    }
 
    void save(const QString &filePath)
    {
        QImageWriter writer(filePath);
        if (!writer.write(m_image)) {
            QQmlEngine *engine = qmlContext(this)->engine();
            QV4::ExecutionEngine *v4 = engine->handle();
            v4->throwError(QStringLiteral("Can't save to %1: %2").arg(filePath, writer.errorString()));
        }
    }
 
public:
    int width() const
    {
        return m_image.width();
    }
 
    int height() const
    {
        return m_image.height();
    }
 
    QSize size() const
    {
        return m_image.size();
    }
 
private:
    QImage m_image;
};
 
class QuickTestResultPrivate
{
public:
    QuickTestResultPrivate()
        : table(nullptr)
        , benchmarkIter(nullptr)
        , benchmarkData(nullptr)
        , iterCount(0)
    {
    }
    ~QuickTestResultPrivate()
    {
        delete table;
        delete benchmarkIter;
        delete benchmarkData;
    }
 
    QByteArray intern(const QString &str);
 
    QString testCaseName;
    QString functionName;
    QSet<QByteArray> internedStrings;
    QTestTable *table;
    QTest::QBenchmarkIterationController *benchmarkIter;
    QBenchmarkTestMethodData *benchmarkData;
    int iterCount;
    QList<QList<QBenchmarkResult>> resultsList;
};
 
QByteArray QuickTestResultPrivate::intern(const QString &str)
{
    QByteArray bstr = str.toUtf8();
    return *(internedStrings.insert(bstr));
}
 
QuickTestResult::QuickTestResult(QObject *parent)
    : QObject(parent), d_ptr(new QuickTestResultPrivate)
{
    if (!QBenchmarkGlobalData::current)
        QBenchmarkGlobalData::current = &globalBenchmarkData;
}
 
QuickTestResult::~QuickTestResult()
{
}
 
QString QuickTestResult::testCaseName() const
{
    Q_D(const QuickTestResult);
    return d->testCaseName;
}
 
void QuickTestResult::setTestCaseName(const QString &name)
{
    Q_D(QuickTestResult);
    d->testCaseName = name;
    emit testCaseNameChanged();
}
 
QString QuickTestResult::functionName() const
{
    Q_D(const QuickTestResult);
    return d->functionName;
}
 
void QuickTestResult::setFunctionName(const QString &name)
{
    Q_D(QuickTestResult);
    if (!name.isEmpty()) {
        if (d->testCaseName.isEmpty()) {
            QTestResult::setCurrentTestFunction
                (d->intern(name).constData());
        } else {
            QString fullName = d->testCaseName + QLatin1String("::") + name;
            QTestResult::setCurrentTestFunction
                (d->intern(fullName).constData());
            if (QTestPrivate::checkBlackLists(fullName.toUtf8().constData(), nullptr))
                QTestResult::setBlacklistCurrentTest(true);
        }
    } else {
        QTestResult::setCurrentTestFunction(nullptr);
    }
    d->functionName = name;
    emit functionNameChanged();
}
 
QString QuickTestResult::dataTag() const
{
    const char *tag = QTestResult::currentDataTag();
    if (tag)
        return QString::fromUtf8(tag);
    else
        return QString();
}
 
void QuickTestResult::setDataTag(const QString &tag)
{
    if (!tag.isEmpty()) {
        QTestData *data = &(QTest::newRow(tag.toUtf8().constData()));
        QTestResult::setCurrentTestData(data);
        if (QTestPrivate::checkBlackLists((testCaseName() + QLatin1String("::")
            + functionName()).toUtf8().constData(), tag.toUtf8().constData())) {
            QTestResult::setBlacklistCurrentTest(true);
        }
        emit dataTagChanged();
    } else {
        QTestResult::setCurrentTestData(nullptr);
    }
}
 
bool QuickTestResult::isFailed() const
{
    return QTestResult::currentTestFailed();
}
 
bool QuickTestResult::isSkipped() const
{
    return QTestResult::skipCurrentTest();
}
 
void QuickTestResult::setSkipped(bool skip)
{
    QTestResult::setSkipCurrentTest(skip);
    if (!skip)
        QTestResult::setBlacklistCurrentTest(false);
    emit skippedChanged();
}
 
int QuickTestResult::passCount() const
{
    return QTestLog::passCount();
}
 
int QuickTestResult::failCount() const
{
    return QTestLog::failCount();
}
 
int QuickTestResult::skipCount() const
{
    return QTestLog::skipCount();
}
 
QStringList QuickTestResult::functionsToRun() const
{
    return QTest::testFunctions;
}
 
QStringList QuickTestResult::tagsToRun() const
{
    return QTest::testTags;
}
 
void QuickTestResult::reset()
{
    if (!globalProgramName)     // Only if run via qmlviewer.
        QTestResult::reset();
}
 
void QuickTestResult::startLogging()
{
    // The program name is used for logging headers and footers if it
    // is set.  Otherwise the test case name is used.
    if (loggingStarted)
        return;
    QTestLog::startLogging();
    loggingStarted = true;
}
 
void QuickTestResult::stopLogging()
{
    Q_D(QuickTestResult);
    if (globalProgramName)
        return;     // Logging will be stopped by setProgramName(0).
    QTestResult::setCurrentTestObject(d->intern(d->testCaseName).constData());
    QTestLog::stopLogging();
}
 
void QuickTestResult::initTestTable()
{
    Q_D(QuickTestResult);
    delete d->table;
    d->table = new QTestTable;
    //qmltest does not really need the column for data driven test
    //add this to avoid warnings.
    d->table->addColumn(qMetaTypeId<QString>(), "qmltest_dummy_data_column");
}
 
void QuickTestResult::clearTestTable()
{
    Q_D(QuickTestResult);
    delete d->table;
    d->table = nullptr;
}
 
void QuickTestResult::finishTestData()
{
    QTestResult::finishedCurrentTestData();
}
 
void QuickTestResult::finishTestDataCleanup()
{
    QTestResult::finishedCurrentTestDataCleanup();
}
 
void QuickTestResult::finishTestFunction()
{
    QTestResult::finishedCurrentTestFunction();
}
 
static QString qtestFixUrl(const QUrl &location)
{
    if (location.isLocalFile()) // Use QUrl's logic for Windows drive letters.
        return QDir::toNativeSeparators(location.toLocalFile());
    return location.toString();
}
 
void QuickTestResult::fail
    (const QString &message, const QUrl &location, int line)
{
    QTestResult::addFailure(message.toUtf8().constData(),
                            qtestFixUrl(location).toLatin1().constData(), line);
}
 
bool QuickTestResult::verify
    (bool success, const QString &message, const QUrl &location, int line)
{
    if (!success && message.isEmpty()) {
        return QTestResult::verify
            (success, "verify()", "",
             qtestFixUrl(location).toLatin1().constData(), line);
    } else {
        return QTestResult::verify
            (success, message.toUtf8().constData(), "",
             qtestFixUrl(location).toLatin1().constData(), line);
    }
}
 
bool QuickTestResult::fuzzyCompare(const QVariant &actual, const QVariant &expected, qreal delta)
{
    if (actual.userType() == QMetaType::QColor || expected.userType() == QMetaType::QColor) {
        if (!actual.canConvert(QMetaType(QMetaType::QColor))
                || !expected.canConvert(QMetaType(QMetaType::QColor))) {
            return false;
        }
 
        //fuzzy color comparison
        QColor act;
        QColor exp;
        bool ok(false);
 
        QVariant var = QQml_colorProvider()->colorFromString(actual.toString(), &ok);
        if (!ok)
            return false;
        act = var.value<QColor>();
 
        var = QQml_colorProvider()->colorFromString(expected.toString(), &ok);
        if (!ok)
            return false;
        exp = var.value<QColor>();
 
        return ( qAbs(act.red() - exp.red()) <= delta
              && qAbs(act.green() - exp.green()) <= delta
              && qAbs(act.blue() - exp.blue()) <= delta
              && qAbs(act.alpha() - exp.alpha()) <= delta);
    } else {
        //number comparison
        bool ok = true;
        qreal act = actual.toFloat(&ok);
        if (!ok)
            return false;
 
        qreal exp = expected.toFloat(&ok);
        if (!ok)
            return false;
 
        return (qAbs(act - exp) <= delta);
    }
 
    return false;
}
 
void QuickTestResult::stringify(QQmlV4FunctionPtr args)
{
    if (args->length() < 1)
        args->setReturnValue(QV4::Encode::null());
 
    QV4::Scope scope(args->v4engine());
    QV4::ScopedValue value(scope, (*args)[0]);
 
    QString result;
 
    //Check for Object Type
    if (value->isObject()
        && !value->as<QV4::FunctionObject>()
        && !value->as<QV4::ArrayObject>()) {
        QVariant v = QV4::ExecutionEngine::toVariant(value, QMetaType {});
        if (v.isValid()) {
            switch (v.userType()) {
            case QMetaType::QVector3D:
            {
                QVector3D v3d = v.value<QVector3D>();
                result = QString::fromLatin1("Qt.vector3d(%1, %2, %3)").arg(v3d.x()).arg(v3d.y()).arg(v3d.z());
                break;
            }
            case QMetaType::QUrl:
            {
                QUrl url = v.value<QUrl>();
                result = QString::fromLatin1("Qt.url(%1)").arg(url.toString());
                break;
            }
            case QMetaType::QDateTime:
            {
                QDateTime dt = v.value<QDateTime>();
                result = dt.toString(Qt::ISODateWithMs);
                break;
            }
            default:
                result = v.toString();
            }
 
        } else {
            result = QLatin1String("Object");
        }
    }
 
    if (result.isEmpty()) {
        QString tmp = value->toQStringNoThrow();
        if (value->as<QV4::ArrayObject>())
            result += QLatin1Char('[') + tmp + QLatin1Char(']');
        else
            result.append(tmp);
    }
 
    args->setReturnValue(QV4::Encode(args->v4engine()->newString(result)));
}
 
bool QuickTestResult::compare
    (bool success, const QString &message,
     const QVariant &val1, const QVariant &val2,
     const QUrl &location, int line)
{
    return QTestResult::compare
        (success, message.toUtf8().constData(),
         QTest::toString(val1.toString().toLatin1().constData()),
         QTest::toString(val2.toString().toLatin1().constData()),
         "", "",
         qtestFixUrl(location).toLatin1().constData(), line);
}
 
void QuickTestResult::skip
    (const QString &message, const QUrl &location, int line)
{
    QTestResult::addSkip(message.toUtf8().constData(),
                         qtestFixUrl(location).toLatin1().constData(), line);
    QTestResult::setSkipCurrentTest(true);
}
 
bool QuickTestResult::expectFail
    (const QString &tag, const QString &comment, const QUrl &location, int line)
{
    return QTestResult::expectFail
        (tag.toLatin1().constData(),
         QTest::toString(comment.toLatin1().constData()),
         QTest::Abort, qtestFixUrl(location).toLatin1().constData(), line);
}
 
bool QuickTestResult::expectFailContinue
    (const QString &tag, const QString &comment, const QUrl &location, int line)
{
    return QTestResult::expectFail
        (tag.toLatin1().constData(),
         QTest::toString(comment.toUtf8().constData()),
         QTest::Continue, qtestFixUrl(location).toLatin1().constData(), line);
}
 
void QuickTestResult::warn(const QString &message, const QUrl &location, int line)
{
    QTestLog::warn(message.toUtf8().constData(), qtestFixUrl(location).toLatin1().constData(), line);
}
 
void QuickTestResult::ignoreWarning(const QJSValue &message)
{
    if (message.isRegExp()) {
#if QT_CONFIG(regularexpression)
        QTestLog::ignoreMessage(QtWarningMsg, qjsvalue_cast<QRegularExpression>(message));
#endif
    } else {
        QTestLog::ignoreMessage(QtWarningMsg, message.toString().toUtf8());
    }
}
 
void QuickTestResult::failOnWarning(const QJSValue &message)
{
    if (message.isRegExp()) {
#if QT_CONFIG(regularexpression)
        QTestLog::failOnWarning(qjsvalue_cast<QRegularExpression>(message));
#endif
    } else {
        QTestLog::failOnWarning(message.toString().toUtf8());
    }
}
 
void QuickTestResult::wait(int ms)
{
    QTest::qWait(ms);
}
 
void QuickTestResult::sleep(int ms)
{
    QTest::qSleep(ms);
}
 
bool QuickTestResult::waitForRendering(QQuickItem *item, int timeout)
{
    Q_ASSERT(item);
 
    return qWaitForSignal(item->window(), SIGNAL(frameSwapped()), timeout);
}
 
void QuickTestResult::startMeasurement()
{
    Q_D(QuickTestResult);
    delete d->benchmarkData;
    d->benchmarkData = new QBenchmarkTestMethodData();
    QBenchmarkTestMethodData::current = d->benchmarkData;
    d->iterCount = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
    d->resultsList.clear();
}
 
void QuickTestResult::beginDataRun()
{
    QBenchmarkTestMethodData::current->beginDataRun();
}
 
void QuickTestResult::endDataRun()
{
    Q_D(QuickTestResult);
    QBenchmarkTestMethodData::current->endDataRun();
    const QList<QBenchmarkResult> &results = QBenchmarkTestMethodData::current->results;
    if (results.isEmpty())
        return;     // shouldn't happen
    if (d->iterCount > -1)  // iteration -1 is the warmup iteration.
        d->resultsList.append(results);
 
    if (QBenchmarkGlobalData::current->verboseOutput) {
        if (d->iterCount == -1) {
            qDebug() << "warmup stage result      :" << results.first().measurement.value;
        } else {
            qDebug() << "accumulation stage result:" << results.first().measurement.value;
        }
    }
}
 
bool QuickTestResult::measurementAccepted()
{
    return QBenchmarkTestMethodData::current->resultsAccepted();
}
 
static QList<QBenchmarkResult> qMedian(const QList<QList<QBenchmarkResult>> &container)
{
    const int count = container.size();
    if (count == 0)
        return {};
 
    if (count == 1)
        return container.at(0);
 
    QList<QList<QBenchmarkResult>> containerCopy = container;
    std::sort(containerCopy.begin(), containerCopy.end(),
              [](const QList<QBenchmarkResult> &a, const QList<QBenchmarkResult> &b) {
        return a.first() < b.first();
    });
 
    const int middle = count / 2;
 
    // ### handle even-sized containers here by doing an aritmetic mean of the two middle items.
    return containerCopy.at(middle);
}
 
bool QuickTestResult::needsMoreMeasurements()
{
    Q_D(QuickTestResult);
    ++(d->iterCount);
    if (d->iterCount < QBenchmarkGlobalData::current->adjustMedianIterationCount())
        return true;
    if (QBenchmarkTestMethodData::current->resultsAccepted())
        QTestLog::addBenchmarkResults(qMedian(d->resultsList));
    return false;
}
 
void QuickTestResult::startBenchmark(RunMode runMode, const QString &tag)
{
    QBenchmarkTestMethodData::current->results = {};
    QBenchmarkTestMethodData::current->resultAccepted = false;
    QBenchmarkGlobalData::current->context.tag = tag;
    QBenchmarkGlobalData::current->context.slotName = functionName();
 
    Q_D(QuickTestResult);
    delete d->benchmarkIter;
    d->benchmarkIter = new QTest::QBenchmarkIterationController
        (QTest::QBenchmarkIterationController::RunMode(runMode));
}
 
bool QuickTestResult::isBenchmarkDone() const
{
    Q_D(const QuickTestResult);
    if (d->benchmarkIter)
        return d->benchmarkIter->isDone();
    else
        return true;
}
 
void QuickTestResult::nextBenchmark()
{
    Q_D(QuickTestResult);
    if (d->benchmarkIter)
        d->benchmarkIter->next();
}
 
void QuickTestResult::stopBenchmark()
{
    Q_D(QuickTestResult);
    delete d->benchmarkIter;
    d->benchmarkIter = nullptr;
}
 
QObject *QuickTestResult::grabImage(QQuickItem *item)
{
    if (item && item->window()) {
        QQuickWindow *window = item->window();
        QImage grabbed = window->grabWindow();
        const auto dpi = grabbed.devicePixelRatio();
        QRectF rf(item->x() * dpi, item->y() * dpi, item->width() * dpi, item->height() * dpi);
        rf = rf.intersected(QRectF(0, 0, grabbed.width(), grabbed.height()));
        QObject *o = new QuickTestImageObject(grabbed.copy(rf.toAlignedRect()));
        QQmlEngine::setContextForObject(o, qmlContext(this));
        return o;
    }
    return nullptr;
}
 
QObject *QuickTestResult::findChild(QObject *parent, const QString &objectName)
{
    return parent ? parent->findChild<QObject*>(objectName) : 0;
}
 
bool QuickTestResult::isPolishScheduled(QObject *itemOrWindow) const
{
    if (auto item = qobject_cast<QQuickItem*>(itemOrWindow))
        return QQuickTest::qIsPolishScheduled(item);
 
    if (auto window = qobject_cast<QQuickWindow*>(itemOrWindow))
        return QQuickTest::qIsPolishScheduled(window);
 
    qmlWarning(this) << "isPolishScheduled() expects either an Item or Window, but got"
        << QDebug::toString(itemOrWindow);
    return false;
}
 
bool QuickTestResult::waitForPolish(QObject *itemOrWindow, int timeout) const
{
    if (auto item = qobject_cast<QQuickItem*>(itemOrWindow))
        return QQuickTest::qWaitForPolish(item, timeout);
 
    if (auto window = qobject_cast<QQuickWindow*>(itemOrWindow))
        return QQuickTest::qWaitForPolish(window, timeout);
 
    qmlWarning(this) << "waitForItemPolish() expects either an Item or Window, but got"
        << QDebug::toString(itemOrWindow);
    return false;
}
 
namespace QTest {
    void qtest_qParseArgs(int argc, char *argv[], bool qml);
};
 
void QuickTestResult::parseArgs(int argc, char *argv[])
{
    if (!QBenchmarkGlobalData::current)
        QBenchmarkGlobalData::current = &globalBenchmarkData;
    QTest::qtest_qParseArgs(argc, argv, true);
}
 
void QuickTestResult::setProgramName(const char *name)
{
    if (name) {
        QTestPrivate::parseBlackList();
        QTestResult::reset();
    } else if (!name && loggingStarted) {
        QTestResult::setCurrentTestObject(globalProgramName);
        QTestLog::stopLogging();
        QTestResult::setCurrentTestObject(nullptr);
    }
    globalProgramName = name;
    QTestResult::setCurrentTestObject(globalProgramName);
}
 
void QuickTestResult::setCurrentAppname(const char *appname)
{
    QTestResult::setCurrentAppName(appname);
}
 
int QuickTestResult::exitCode()
{
#if defined(QTEST_NOEXITCODE)
    return 0;
#else
    // make sure our exit code is never going above 127
    // since that could wrap and indicate 0 test fails
    return qMin(QTestLog::failCount(), 127);
#endif
}
 
QT_END_NAMESPACE
 
#include "quicktestresult.moc"
#include "moc_quicktestresult_p.cpp"