From 130e0a3115cd66e38ec57846e06f69e7fd284400 Mon Sep 17 00:00:00 2001 From: Arno Date: Tue, 3 Sep 2013 06:12:33 +0200 Subject: Switch to QRunnable + QThreadPool Get rid of SmDataCollector and do its job in small, QRunnable tasks and let QThreadPool manage the treads. Works well with a local Filesystem. Yet to see how it works over networked Filesystems. Ah, before I forget: NEVER, EVER USE QPixmap in THREADS -> Random crashes! (Yes, I know, it's documented...) --- smdirwatcher.cpp | 255 +++++++++++++++++++++++++------------------------------ 1 file changed, 118 insertions(+), 137 deletions(-) (limited to 'smdirwatcher.cpp') diff --git a/smdirwatcher.cpp b/smdirwatcher.cpp index 2b03179..d825c2c 100644 --- a/smdirwatcher.cpp +++ b/smdirwatcher.cpp @@ -5,56 +5,105 @@ 2 of the License, or (at your option) any later version. */ -#include -#include #include #include -#include #include #include -#include -#include #include -#include +#include "smdirmodel.h" #include "smdirwatcher.h" #include "smtreeitem.h" #include "helper.h" -extern int errno; - -SmDirWatcher::SmDirWatcher(int numFields, QObject *parent) : QThread(parent), mFd(0), mDescr(0) { +SmDirWatcher::SmDirWatcher(int numFields, QObject *parent) : QThread(parent), mFd(0), mDescr(0), mNumFields(numFields) { mBufLen = 1024 * (sizeof(struct inotify_event) + 16); mINdata = new char[mBufLen]; mFd = inotify_init(); - - mCollector = new SmDataColletor(numFields, this); - mSemFree = new QSemaphore(1024); - mSemUsed = new QSemaphore; - mQueueMx = new QMutex; - mDataQueue = new QQueue >(); - mCollector->init(mSemFree, mSemUsed, mDataQueue, mQueueMx); - mCollector->start(); + mAsyncPool = new QThreadPool(this); } void SmDirWatcher::setDir(const QString &dir){ if(mDescr){ inotify_rm_watch(mFd, mDescr); //generates IN_IGNORE ??? } + foreach(AsyncTask *s, mAsyncTasks){ + s->skipMe(); + } + mAsyncPool->waitForDone(); + foreach(AsyncTask *s, mAsyncTasks){ + s->deleteLater(); + } + mAsyncTasks.clear(); + mCurrent = dir; + QDir d(mCurrent); + SmTreeItem *rootItem = new SmTreeItem(mNumFields); + foreach(QFileInfo fi, d.entryInfoList()){ + if(fi.fileName() == "." || fi.fileName() == ".."){ + continue; + } + QList data = generalData(fi.absoluteFilePath()); + QString mime = data.at(SmDirModel::Type).toString(); + SmTreeItem *newItem = new SmTreeItem(data, rootItem); + rootItem->appendChild(newItem); + Md5Summer *s = new Md5Summer(fi.absoluteFilePath()); + s->setAutoDelete(false); + connect(s, SIGNAL(md5sumDone(QString,QString)), this, SIGNAL(setMd5Sum(QString,QString))); + mAsyncTasks.append(s); + if(mime.startsWith("video")){ + FfmpegGatherer *g = new FfmpegGatherer(fi.absoluteFilePath()); + g->setAutoDelete(false); + connect(g, SIGNAL(ffmpegDone(QString,QVariantMap)), this, SIGNAL(setFfmpeg(QString,QVariantMap))); + mAsyncTasks.append(g); + }else if(mime.startsWith("image")){ + PicSizeGatherer *pg = new PicSizeGatherer(fi.absoluteFilePath()); + pg->setAutoDelete(false); + connect(pg, SIGNAL(picSizeDone(QString,QVariant)), this, SIGNAL(setPicSize(QString,QVariant))); + mAsyncTasks.append(pg); + } + } + emit population(rootItem); /* mask rationale: * IN_DELETE_SELF cannot happen since we're only * watching one directory at all time. * We don't care about the other events */ mDescr = inotify_add_watch(mFd, qPrintable(dir), IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | IN_MODIFY | IN_MOVED_FROM | IN_MOVED_TO); - mCurrent = dir; - mSemFree->acquire(); - QPair c = qMakePair(dir, Populate); - mQueueMx->lock(); - mDataQueue->enqueue(c); - mQueueMx->unlock(); - mSemUsed->release(); +} + +void SmDirWatcher::startAsyncJobs(){ + foreach(AsyncTask *s, mAsyncTasks){ + mAsyncPool->start(s); + } +} + +void SmDirWatcher::gatherAsync(const QString &path){ + QString mimeType = Helper::mimeType(path); + Md5Summer *s = new Md5Summer(path); + connect(s, SIGNAL(md5sumDone(QString,QString)), this, SIGNAL(setMd5Sum(QString,QString))); + mAsyncPool->start(s); + if(mimeType.startsWith("video")){ + FfmpegGatherer *g = new FfmpegGatherer(path); + connect(g, SIGNAL(ffmpegDone(QString,QVariantMap)), this, SIGNAL(setFfmpeg(QString,QVariantMap))); + mAsyncPool->start(g); + }else if(mimeType.startsWith("image")){ + PicSizeGatherer *ps = new PicSizeGatherer(path); + connect(ps, SIGNAL(picSizeDone(QString,QVariant)), this, SIGNAL(setPicSize(QString,QVariant))); + mAsyncPool->start(ps); + } +} + +QList SmDirWatcher::generalData(const QString &path){ + QFileInfo fi(path); + QList data; + data << fi.fileName() << fi.size(); + QString mime = Helper::mimeType(fi.absoluteFilePath()); + data << mime; + data << fi.lastModified(); + data << QVariant() << QVariant() << QVariant(); + data << fi.absoluteFilePath() << 0; + return data; } void SmDirWatcher::run(){ @@ -69,142 +118,74 @@ void SmDirWatcher::run(){ i += sizeof(inotify_event) + e->len; continue; } - mSemFree->acquire(); quint32 mask = e->mask; QString name = QString("%1/%2").arg(mCurrent).arg(e->name); - DWEvent curEvent = None; - QPair c; + QList d = generalData(name); if(mask & IN_CREATE || mask & IN_MOVED_TO){ - curEvent = Added; + emit newData(d, Added); }else if(mask & IN_DELETE || mask & IN_MOVED_FROM){ - curEvent = Deleted; - }else if(mask & IN_CLOSE_WRITE || e->mask & IN_MODIFY){ - curEvent = Modified; - }else{ - goto out; + emit newData(d, Deleted); + }else if(mask & IN_MODIFY){ + emit newData(d, Modified); + }else if(mask & IN_CLOSE_WRITE){ + emit newData(d, CloseWrite); } - c = qMakePair(name, curEvent); - mQueueMx->lock(); - mDataQueue->enqueue(c); - mQueueMx->unlock(); - out: - mSemUsed->release(); i += sizeof(inotify_event) + e->len; } } void SmDirWatcher::stop(){ - mSemFree->acquire(); - mCollector->stop(); - mSemUsed->release(); - if(mCollector->isRunning()){ - mCollector->wait(); - } quit(); } -SmDataColletor::SmDataColletor(const int numFields, QObject *parent) : QThread(parent), mSemFree(0), mSemUsed(0), mDataQueue(0), mNumFields(numFields), mCheckForPresent(true), mCancel(false) { - QSqlDatabase db = QSqlDatabase::cloneDatabase(QSqlDatabase::database("treedb"), "collectordb"); - db.open(); - mPicPresentQ = new QSqlQuery(db); - mPicPresentQ->prepare("SELECT COUNT(cmd5sum) FROM pics WHERE cmd5sum = :md5"); - mMovPresentQ = new QSqlQuery(db); - mMovPresentQ->prepare("SELECT COUNT(cmd5sum) FROM files WHERE cmd5sum = :md5"); -} +AsyncTask::AsyncTask(const QString &path) : mSkip(false), mPath(path) {} -void SmDataColletor::init(QSemaphore *set, QSemaphore *get, QQueue > *data, QMutex *queueMx){ - mSemFree = set; - mSemUsed = get; - mDataQueue = data; - mQueueMx = queueMx; +bool AsyncTask::skipMe(){ + if(mStatusMx.tryLock()){ + mSkip = true; + mStatusMx.unlock(); + return true; + } + return false; } -void SmDataColletor::setCheckForPresent(bool present){ - QMutexLocker l(&mCheckForPresentMx); - mCheckForPresent = present; -} +#include -void SmDataColletor::run(){ - forever { - mSemUsed->acquire(); - if(mCancel){ - QSqlDatabase::database("collectordb").close(); - mSemFree->release(); - break; - } - mQueueMx->lock(); - QPair cur = mDataQueue->dequeue(); - mQueueMx->unlock(); - if(cur.second == SmDirWatcher::Populate){ - SmTreeItem *i = populate(cur.first); - SmTreeItem *copy = new SmTreeItem(*i); - emit population(copy); - mSemFree->release(); - continue; +Md5Summer::Md5Summer(const QString &path) : AsyncTask(path) {} - } - QFileInfo fi(cur.first); - QList fd = fileData(fi); - emit newData(fd, cur.second); - mSemFree->release(); +void Md5Summer::run(){ + mStatusMx.lock(); + bool skip = mSkip; + mStatusMx.unlock(); + if(skip){ + return; } - quit(); + QString md5 = Helper::md5Sum(mPath); + emit md5sumDone(mPath, md5); } -SmTreeItem * SmDataColletor::populate(const QString &dir){ - SmTreeItem *retval = new SmTreeItem(mNumFields); - QDir d = QDir(dir); - emit totalFiles(d.entryInfoList().size()); - foreach(QFileInfo fi, d.entryInfoList()){ - emit progress(); - if(fi.fileName() == "."){ - continue; - } - QList data = fileData(fi); - SmTreeItem *newItem = new SmTreeItem(data, retval); - retval->appendChild(newItem); +FfmpegGatherer::FfmpegGatherer(const QString &path) : AsyncTask(path) {} + +void FfmpegGatherer::run(){ + mStatusMx.lock(); + bool skip = mSkip; + mStatusMx.unlock(); + if(skip){ + return; } - return retval; + QVariantMap retval = Helper::ffmpegData(mPath); + emit ffmpegDone(mPath, retval); } -const QList SmDataColletor::fileData(const QFileInfo &fi){ - QList data; - data << fi.fileName() << fi.size(); - QString mime = Helper::mimeType(fi.absoluteFilePath()); - data << mime; - data << fi.lastModified(); - QList si = QList() << QVariant() << QVariant(); - if(mime.startsWith("video")){ - QVariantMap ffData = Helper::ffmpegData(fi.absoluteFilePath()); - si[0] = ffData.value("duration").toDouble(); - si[1] = ffData.value("bit_rate").toInt(); - }else if(mime.startsWith("image")){ - si[0] = Helper::picSize(fi.absoluteFilePath()); - } - data << Helper::md5Sum(fi.absoluteFilePath()); - data << si << fi.absoluteFilePath(); - data << 0; - mCheckForPresentMx.lock(); - bool p = mCheckForPresent; - mCheckForPresentMx.unlock(); - if(p){ - QSqlQuery *curQuery = 0; - if(mime.startsWith("video")){ - curQuery = mMovPresentQ; - } - if(mime.startsWith("image")){ - curQuery = mPicPresentQ; - } - if(curQuery){ - curQuery->bindValue(":md5", data.at(4)); - if(curQuery->exec()){ - QVariant res; - while(curQuery->next()){ - res = curQuery->value(0); - } - data[8] = res; - } - } +PicSizeGatherer::PicSizeGatherer(const QString &path) : AsyncTask(path) {} + +void PicSizeGatherer::run(){ + mStatusMx.lock(); + bool skip = mSkip; + mStatusMx.unlock(); + if(skip){ + return; } - return data; + QVariant retval = Helper::picSize(mPath); + emit picSizeDone(mPath, retval); } -- cgit v1.2.3-70-g09d2