Implement download caching. Set the folder for the cache data in the configuration dialog. Caching is disabled per default and defaults to the systems temp folder.

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@14476 a1c6a512-1295-4272-9138-f99709370657

rbutil/rbutilqt/configure.cpp

@@ -66,14 +66,9 @@ Config::Config(QWidget *parent) : QDialog(parent)
     connect(ui.radioSystemProxy, SIGNAL(toggled(bool)), this, SLOT(setSystemProxy(bool)));
     connect(ui.browseMountPoint, SIGNAL(clicked()), this, SLOT(browseFolder()));
     connect(ui.buttonAutodetect,SIGNAL(clicked()),this,SLOT(autodetect()));
-    
+    connect(ui.buttonCacheBrowse, SIGNAL(clicked()), this, SLOT(browseCache()));
-    // disable unimplemented stuff
+    connect(ui.buttonCacheClear, SIGNAL(clicked()), this, SLOT(cacheClear()));
-    ui.buttonCacheBrowse->setEnabled(false);
-    ui.cacheDisable->setEnabled(false);
-    ui.cacheOfflineMode->setEnabled(false);
-    ui.buttonCacheClear->setEnabled(false);
 
-    //ui.buttonAutodetect->setEnabled(false);
 }
 
 
@@ -116,6 +111,14 @@ void Config::accept()
         userSettings->setValue("defaults/platform", nplat);
     }
 
+    // cache settings
+    if(QFileInfo(ui.cachePath->text()).isDir())
+        userSettings->setValue("defaults/cachepath", ui.cachePath->text());
+    else // default to system temp path
+        userSettings->setValue("defaults/cachepath", QDir::tempPath());
+    userSettings->setValue("defaults/cachedisable", ui.cacheDisable->isChecked());
+    userSettings->setValue("defaults/offline", ui.cacheOfflineMode->isChecked());
+
     // sync settings
     userSettings->sync();
     this->close();
@@ -169,6 +172,20 @@ void Config::setUserSettings(QSettings *user)
     // devices tab
     ui.mountPoint->setText(userSettings->value("defaults/mountpoint").toString());
 
+    // cache tab
+    if(!QFileInfo(userSettings->value("defaults/cachepath").toString()).isDir())
+        userSettings->setValue("defaults/cachepath", QDir::tempPath());
+    ui.cachePath->setText(userSettings->value("defaults/cachepath").toString());
+    ui.cacheDisable->setChecked(userSettings->value("defaults/cachedisable").toBool());
+    ui.cacheOfflineMode->setChecked(userSettings->value("defaults/offline").toBool());
+    QList<QFileInfo> fs = QDir(userSettings->value("defaults/cachepath").toString() + "/rbutil-cache/").entryInfoList(QDir::Files | QDir::NoDotAndDotDot);
+    qint64 sz = 0;
+    for(int i = 0; i < fs.size(); i++) {
+        sz += fs.at(i).size();
+        qDebug() << fs.at(i).fileName() << fs.at(i).size();
+    }
+    ui.cacheSize->setText(tr("Current cache size is %1 kiB.")
+            .arg(sz/1024));
 }
 
 
@@ -344,12 +361,32 @@ void Config::browseFolder()
 }
 
 
+void Config::browseCache()
+{
+    cbrowser = new BrowseDirtree(this);
+#if defined(Q_OS_LINUX) || defined(Q_OS_MACX)
+    cbrowser->setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
+#elif defined(Q_OS_WIN32)
+    cbrowser->setFilter(QDir::Drives);
+#endif
+    QDir d(ui.cachePath->text());
+    cbrowser->setDir(d);
+    cbrowser->show();
+    connect(cbrowser, SIGNAL(itemChanged(QString)), this, SLOT(setCache(QString)));
+}
+
 void Config::setMountpoint(QString m)
 {
     ui.mountPoint->setText(m);
 }
 
 
+void Config::setCache(QString c)
+{
+    ui.cachePath->setText(c);
+}
+
+
 void Config::autodetect()
 {
     Autodetection detector(this);
@@ -403,3 +440,29 @@ void Config::autodetect()
     }
 }
 
+void Config::cacheClear()
+{
+    if(QMessageBox::critical(this, tr("Really delete cache?"),
+       tr("Do you really want to delete the cache? "
+         "Make absolutely sure this setting is correct as it will "
+         "remove <b>all</b> files in this folder!").arg(ui.cachePath->text()),
+       QMessageBox::Yes | QMessageBox::No) != QMessageBox::Yes)
+        return;
+    
+    QString cache = ui.cachePath->text() + "/rbutil-cache/";
+    if(!QFileInfo(cache).isDir()) {
+        QMessageBox::critical(this, tr("Path wrong!"),
+            tr("The cache path is invalid. Aborting."), QMessageBox::Ok);
+        return;
+    }
+    QDir dir(cache);
+    QStringList fn;
+    fn = dir.entryList(QStringList("*"), QDir::Files, QDir::Name);
+    qDebug() << fn;
+
+    for(int i = 0; i < fn.size(); i++) {
+        QString f = cache + fn.at(i);
+        QFile::remove(f);
+        qDebug() << "removed:" << f;
+    }
+}

rbutil/rbutilqt/configure.h

@@ -51,14 +51,18 @@ class Config : public QDialog
         QUrl proxy;
         
         BrowseDirtree *browser;
+        BrowseDirtree *cbrowser;
         
     private slots:
         void setNoProxy(bool);
         void setSystemProxy(bool);
         void updateLanguage(void);
         void browseFolder(void);
+        void browseCache(void);
         void autodetect(void);
         void setMountpoint(QString);
+        void setCache(QString);
+        void cacheClear(void);
 };
 
 #endif

rbutil/rbutilqt/configurefrm.ui

@@ -270,6 +270,9 @@
       <attribute name="title" >
        <string>Cac&amp;he</string>
       </attribute>
+      <attribute name="icon" >
+       <iconset resource="rbutilqt.qrc" >:/icons/icons/package-x-generic.png</iconset>
+      </attribute>
       <attribute name="toolTip" >
        <string>Download cache settings</string>
       </attribute>
@@ -304,7 +307,11 @@
           </widget>
          </item>
          <item>
-          <widget class="QLineEdit" name="cachePath" />
+          <widget class="QLineEdit" name="cachePath" >
+           <property name="toolTip" >
+            <string>Entering an invalid folder will reset the path to the systems temporary path.</string>
+           </property>
+          </widget>
          </item>
          <item>
           <widget class="QPushButton" name="buttonCacheBrowse" >
@@ -329,6 +336,9 @@
          </item>
          <item>
           <widget class="QCheckBox" name="cacheOfflineMode" >
+           <property name="toolTip" >
+            <string>This will try to use all information from the cache, even information about updates. Only use this option if you want to install without network connection. Note: you need to do the same install you want to perform later with network access first to download all required files to the cache.</string>
+           </property>
            <property name="text" >
             <string>O&amp;ffline mode</string>
            </property>
@@ -367,6 +377,9 @@
          <property name="text" >
           <string>Clean cache &amp;now</string>
          </property>
+         <property name="icon" >
+          <iconset resource="rbutilqt.qrc" >:/icons/icons/user-trash-full.png</iconset>
+         </property>
         </widget>
        </item>
       </layout>

rbutil/rbutilqt/httpget.cpp

@@ -27,8 +27,10 @@
 HttpGet::HttpGet(QObject *parent)
     : QObject(parent)
 {
+    m_usecache = false;
     qDebug() << "--> HttpGet::HttpGet()";
     outputToBuffer = true;
+    cached = false;
     getRequest = -1;
     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
     connect(&http, SIGNAL(dataReadProgress(int, int)), this, SLOT(httpProgress(int, int)));
@@ -42,6 +44,27 @@ HttpGet::HttpGet(QObject *parent)
 }
 
 
+void HttpGet::setCache(QDir d)
+{
+    m_cachedir = d;
+    bool result = true;
+    
+    QString p = m_cachedir.absolutePath() + "/rbutil-cache";
+    if(QFileInfo(m_cachedir.absolutePath()).isDir())
+        if(!QFileInfo(p).isDir())
+            result = m_cachedir.mkdir("rbutil-cache");
+    else result = false;
+    qDebug() << "HttpGet::setCache(QDir)" << result;
+    m_usecache = !result;
+}
+
+
+void HttpGet::setCache(bool c)
+{
+    m_usecache = c;
+}
+
+
 QByteArray HttpGet::readAll()
 {
     return dataBuffer;
@@ -108,6 +131,40 @@ bool HttpGet::getFile(const QUrl &url)
             return false;
         }
     }
+    // put hash generation here so it can get reused later
+    QString hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
+    cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + hash;
+    if(m_usecache) {
+        // check if the file is present in cache
+        qDebug() << "[HTTP] cache ENABLED for" << url.toEncoded();
+        if(QFileInfo(cachefile).isReadable() && QFileInfo(cachefile).size() > 0) {
+            qDebug() << "[HTTP] cached file found!" << cachefile;
+            getRequest = -1;
+            QFile c(cachefile);
+            if(!outputToBuffer) {
+                qDebug() << outputFile->fileName();
+                c.open(QIODevice::ReadOnly);
+                outputFile->open(QIODevice::ReadWrite);
+                outputFile->write(c.readAll());
+                outputFile->close();
+                c.close();
+            }
+            else {
+                c.open(QIODevice::ReadOnly);
+                dataBuffer = c.readAll();
+                c.close();
+            }
+            response = 200; // fake "200 OK" HTTP response
+            cached = true;
+            httpDone(false); // we're done now. This will emit the correct signal too.
+            return true;
+        }
+        else qDebug() << "[HTTP] file not cached, downloading to" << cachefile;
+
+    }
+    else {
+        qDebug() << "[HTTP] cache DISABLED";
+    }
     http.setHost(url.host(), url.port(80));
     // construct query (if any)
     QList<QPair<QString, QString> > qitems = url.queryItems();
@@ -119,14 +176,14 @@ bool HttpGet::getFile(const QUrl &url)
     }
 
     if(outputToBuffer) {
-        qDebug() << "downloading to buffer:" << url.toString();
+        qDebug() << "[HTTP] downloading to buffer:" << url.toString();
         getRequest = http.get(url.path() + query);
     }
     else {
-        qDebug() << "downloading to file:" << url.toString() << qPrintable(outputFile->fileName());
+        qDebug() << "[HTTP] downloading to file:" << url.toString() << qPrintable(outputFile->fileName());
         getRequest = http.get(url.path() + query, outputFile);
     }
-    qDebug() << "request scheduled: GET" << getRequest;
+    qDebug() << "[HTTP] request scheduled: GET" << getRequest;
     
     return true;
 }
@@ -135,11 +192,25 @@ bool HttpGet::getFile(const QUrl &url)
 void HttpGet::httpDone(bool error)
 {
     if (error) {
-        qDebug() << "Error: " << qPrintable(http.errorString()) << httpResponse();
+        qDebug() << "[HTTP] Error: " << qPrintable(http.errorString()) << httpResponse();
     }
     if(!outputToBuffer)
         outputFile->close();
-    
+
+    if(m_usecache && !cached) {
+        qDebug() << "[HTTP] creating cache file" << cachefile;
+        QFile c(cachefile);
+        c.open(QIODevice::ReadWrite);
+        if(!outputToBuffer) {
+            outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
+            c.write(outputFile->readAll());
+            outputFile->close();
+        }
+        else
+            c.write(dataBuffer);
+
+        c.close();
+    }
     emit done(error);
 }
 

rbutil/rbutilqt/httpget.h

@@ -21,8 +21,8 @@
 #ifndef HTTPGET_H
 #define HTTPGET_H
 
-#include <QFile>
+#include <QtCore>
-#include <QHttp>
+#include <QtNetwork>
 
 class QUrl;
 
@@ -38,8 +38,11 @@ class HttpGet : public QObject
         QHttp::Error error(void);
         QString errorString(void);
         void setFile(QFile*);
+        void setCache(QDir);
+        void setCache(bool);
         int httpResponse(void);
         QByteArray readAll(void);
+        bool isCached() { return cached; }
 
     public slots:
         void abort(void);
@@ -65,6 +68,10 @@ class HttpGet : public QObject
         QByteArray dataBuffer;
         bool outputToBuffer;
         QString query;
+        bool m_usecache;
+        QDir m_cachedir;
+        QString cachefile;
+        bool cached;
 };
 
 #endif

rbutil/rbutilqt/install.cpp

@@ -99,6 +99,8 @@ void Install::accept()
     installer->setUrl(file);
     installer->setProxy(proxy);
     installer->setLogSection("rockboxbase");
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->setLogVersion(myversion);
     installer->setMountPoint(mountPoint);
     installer->install(logger);

rbutil/rbutilqt/installthemes.cpp

@@ -77,6 +77,8 @@ void ThemesInstallWindow::downloadInfo()
     qDebug() << "downloadInfo()" << url;
     qDebug() << url.queryItems();
     getter->setProxy(proxy);
+    if(userSettings->value("defaults/offline").toBool())
+        getter->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     getter->setFile(&themesInfo);
     getter->getFile(url);
 }
@@ -182,6 +184,8 @@ void ThemesInstallWindow::updateDetails(int row)
 
     igetter.abort();
     igetter.setProxy(proxy);
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        igetter.setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     igetter.getFile(img);
     connect(&igetter, SIGNAL(done(bool)), this, SLOT(updateImage(bool)));
 }
@@ -294,6 +298,8 @@ void ThemesInstallWindow::accept()
     installer->setLogSection(names);
     installer->setLogVersion(version);
     installer->setMountPoint(mountPoint);
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->install(logger);
     connect(logger, SIGNAL(closed()), this, SLOT(close()));
 }

rbutil/rbutilqt/installzip.cpp

@@ -25,6 +25,7 @@
 ZipInstaller::ZipInstaller(QObject* parent): QObject(parent) 
 {
     m_unzip = true;
+    m_cache = "";
 }
 
 
@@ -86,6 +87,10 @@ void ZipInstaller::installStart()
     // get the real file.
     getter = new HttpGet(this);
     getter->setProxy(m_proxy);
+    if(m_cache.exists()) {
+        getter->setCache(m_cache);
+        qDebug() << "installzip: setting cache to" << m_cache;
+    }
     getter->setFile(downloadFile);
     getter->getFile(QUrl(m_url));
 
@@ -117,12 +122,13 @@ void ZipInstaller::downloadDone(bool error)
         m_dp->setProgressMax(max);
     }
     m_dp->setProgressValue(max);
-    if(getter->httpResponse() != 200) {
+    if(getter->httpResponse() != 200 && !getter->isCached()) {
         m_dp->addItem(tr("Download error: received HTTP error %1.").arg(getter->httpResponse()),LOGERROR);
         m_dp->abort();
         emit done(true);
         return;
     }
+    if(getter->isCached()) m_dp->addItem(tr("Cached file used."), LOGINFO);
     if(error) {
         m_dp->addItem(tr("Download error: %1").arg(getter->errorString()),LOGERROR);
         m_dp->abort();
@@ -205,3 +211,4 @@ void ZipInstaller::updateDataReadProgress(int read, int total)
 
 }
 
+

rbutil/rbutilqt/installzip.h

@@ -46,6 +46,8 @@ public:
     void setLogVersion(QStringList v) { m_verlist = v; qDebug() << m_verlist;}
     void setUnzip(bool i) { m_unzip = i; }
     void setTarget(QString t) { m_target = t; }
+    void setCache(QDir c) { m_cache = c; };
+    void setCache(QString c) { m_cache = QDir(c); qDebug() << "!!!set cache:" << m_cache;}
     
 signals:
     void done(bool error);
@@ -66,6 +68,7 @@ private:
     bool m_unzip;
     QString m_target;
     int runner;
+    QDir m_cache;
     
     HttpGet *getter;
     QTemporaryFile *downloadFile;

rbutil/rbutilqt/rbutilqt.cpp

@@ -128,11 +128,11 @@ void RbUtilQt::downloadInfo()
     connect(daily, SIGNAL(done(bool)), this, SLOT(downloadDone(bool)));
     connect(daily, SIGNAL(requestFinished(int, bool)), this, SLOT(downloadDone(int, bool)));
     daily->setProxy(proxy());
-
+    if(userSettings->value("defaults/offline").toBool())
+        daily->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     qDebug() << "downloading build info";
     daily->setFile(&buildInfo);
     daily->getFile(QUrl(devices->value("server_conf_url").toString()));
-
 }
 
 
@@ -154,7 +154,8 @@ void RbUtilQt::downloadDone(bool error)
     connect(bleeding, SIGNAL(done(bool)), this, SLOT(downloadBleedingDone(bool)));
     connect(bleeding, SIGNAL(requestFinished(int, bool)), this, SLOT(downloadDone(int, bool)));
     bleeding->setProxy(proxy());
-
+    if(userSettings->value("defaults/offline").toBool())
+        bleeding->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     bleeding->setFile(&bleedingInfo);
     bleeding->getFile(QUrl(devices->value("bleeding_info").toString()));
 }
@@ -462,6 +463,8 @@ bool RbUtilQt::installAuto()
     installer->setProxy(proxy());
     installer->setLogSection("rockboxbase");
     installer->setLogVersion(myversion);
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->setMountPoint(userSettings->value("defaults/mountpoint").toString());
     installer->install(logger);
            
@@ -623,6 +626,8 @@ void RbUtilQt::installFonts()
     installer->setLogSection("Fonts");
     installer->setLogVersion(versmap.value("arch_date"));
     installer->setMountPoint(userSettings->value("defaults/mountpoint").toString());
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->install(logger);    
 }
 
@@ -651,6 +656,8 @@ void RbUtilQt::installVoice()
     installer->setLogVersion(versmap.value("arch_date"));
     installer->setMountPoint(userSettings->value("defaults/mountpoint").toString());
     installer->setTarget("/.rockbox/langs/english.voice");
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->install(logger);
     
     //connect(installer, SIGNAL(done(bool)), this, SLOT(done(bool)));
@@ -684,6 +691,8 @@ void RbUtilQt::installDoom()
     installer->setLogSection("Game Addons");
     installer->setLogVersion(versmap.value("arch_date"));
     installer->setMountPoint(userSettings->value("defaults/mountpoint").toString());
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->install(logger);
 
 }
@@ -783,6 +792,8 @@ void RbUtilQt::downloadManual(void)
     logger->show();
     installer = new ZipInstaller(this);
     installer->setMountPoint(userSettings->value("defaults/mountpoint").toString());
+    if(!userSettings->value("defaults/cachedisable").toBool())
+        installer->setCache(userSettings->value("defaults/cachepath", QDir::tempPath()).toString());
     installer->setProxy(proxy());
     installer->setLogSection(section);
     installer->setUrl(manualurl);