gamedownloader.cpp 13.8 KB
Newer Older
Gerhard Stein's avatar
Gerhard Stein committed
1

2

3 4
#include "gamedownloader.h"

5 6
#include <base/utils/FindFile.h>
#include <base/GsLogging.h>
7 8
#include <cstdio>
#include <curl/curl.h>
9
#include <SDL_image.h>
10 11
#include <fileio/KeenFiles.h>

12 13
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
14
#include "../version.h"
15

16 17
extern "C"
{
18 19
int unzipFile(const char *input,
              const char *outputDir);
20
}
21

22 23 24 25 26
// Limit to max 1 GB
const curl_off_t  STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES = 1024 * 1024 * 1024;
#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL     3

struct myprogress {
Gerhard Stein's avatar
Gerhard Stein committed
27 28
    double lastruntime;
    CURL *curl;
29 30 31
};

int *progressPtr;
32
bool *pCancelDownload;
33

34 35
int gDlto, gDlfrom;

36 37 38 39 40
/* this is how the CURLOPT_XFERINFOFUNCTION callback works */
static int xferinfo(void *p,
                    curl_off_t dltotal, curl_off_t dlnow,
                    curl_off_t ultotal, curl_off_t ulnow)
{
41
    if(*pCancelDownload)
Gerhard Stein's avatar
Gerhard Stein committed
42
    {
43
        return 2;
Gerhard Stein's avatar
Gerhard Stein committed
44
    }
45

Gerhard Stein's avatar
Gerhard Stein committed
46 47 48
    struct myprogress *myp = (struct myprogress *)p;
    CURL *curl = myp->curl;
    double curtime = 0;
49

Gerhard Stein's avatar
Gerhard Stein committed
50
    curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime);
51

Gerhard Stein's avatar
Gerhard Stein committed
52
    /* under certain circumstances it may be desirable for certain functionality
53 54
     to only run every N seconds, in order to do this the transaction time can
     be used */
Gerhard Stein's avatar
Gerhard Stein committed
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
    if((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL)
    {
        myp->lastruntime = curtime;
    }

    if(dltotal > 0)
    {
        const int newProgress = gDlfrom + ((gDlto-gDlfrom)*dlnow)/dltotal;
        *progressPtr = newProgress;
    }

    if(dlnow > STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES)
    {
        return 1;
    }
    return 0;
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
}

/* for libcurl older than 7.32.0 (CURLOPT_PROGRESSFUNCTION) */
static int older_progress(void *p,
                          double dltotal, double dlnow,
                          double ultotal, double ulnow)
{
  return xferinfo(p,
                  (curl_off_t)dltotal,
                  (curl_off_t)dlnow,
                  (curl_off_t)ultotal,
                  (curl_off_t)ulnow);
}


size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
    size_t written = fwrite(ptr, size, nmemb, stream);
    return written;
}



94 95
int downloadFile(const std::string &filename, int &progress,
                 const std::string &downloadDirPath)
96 97 98 99
{
    progressPtr = &progress;

    const std::string urlString = "http://downloads.sourceforge.net/project/clonekeenplus/Downloads/" + filename;
100
    const std::string outputPath = JoinPaths(downloadDirPath, filename);
101 102

    CURLcode res = CURLE_OK;
Gerhard Stein's avatar
Gerhard Stein committed
103 104 105
    struct myprogress prog;                    

    CURL *curl = curl_easy_init();
106 107 108 109 110 111 112 113 114 115

    if(curl)
    {
      prog.lastruntime = 0;
      prog.curl = curl;

      curl_easy_setopt(curl, CURLOPT_URL, urlString.c_str());

      curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

Gerhard Stein's avatar
Gerhard Stein committed
116 117 118 119
      curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);

      curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_NONE);

120 121 122 123 124
      curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, older_progress);

      /* pass the struct pointer into the progress function */
      curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog);

Gerhard Stein's avatar
Gerhard Stein committed
125
      FILE *fp = OpenGameFile(outputPath, "wb");
Gerhard Stein's avatar
Gerhard Stein committed
126 127 128 129
      if(fp != nullptr)
      {
          curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
          curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
130

Gerhard Stein's avatar
Gerhard Stein committed
131 132 133
      #if LIBCURL_VERSION_NUM >= 0x072000
          /* xferinfo was introduced in 7.32.0, no earlier libcurl versions will
             compile as they won't have the symbols around.
134

Gerhard Stein's avatar
Gerhard Stein committed
135 136 137
             If built with a newer libcurl, but running with an older libcurl:
             curl_easy_setopt() will fail in run-time trying to set the new
             callback, making the older callback get used.
138

Gerhard Stein's avatar
Gerhard Stein committed
139 140
             New libcurls will prefer the new callback and instead use that one even
             if both callbacks are set. */
141

Gerhard Stein's avatar
Gerhard Stein committed
142 143 144 145 146
          curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferinfo);
          /* pass the struct pointer into the xferinfo function, note that this is
             an alias to CURLOPT_PROGRESSDATA */
          curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &prog);
      #endif
147

Gerhard Stein's avatar
Gerhard Stein committed
148 149
          curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
          res = curl_easy_perform(curl);
150

151
          gLogging.ftextOut( FONTCOLORS::GREEN, "Finished downloading from \"%s\", destination: \"%s\"", urlString.c_str(), outputPath.c_str());
152 153

          progress = 1000;
Gerhard Stein's avatar
Gerhard Stein committed
154 155
      }
      else
156
      {                    
Gerhard Stein's avatar
Gerhard Stein committed
157 158 159
          /* always cleanup */
          curl_easy_cleanup(curl);

160
          gLogging.ftextOut( FONTCOLORS::RED, "Error creating path \"%s\" for writing", outputPath.c_str());
Gerhard Stein's avatar
Gerhard Stein committed
161 162
          return 1;
      }
163 164


165

Gerhard Stein's avatar
Gerhard Stein committed
166
      // output any error to the central CG Log
167
      if(res != CURLE_OK)          
Gerhard Stein's avatar
Gerhard Stein committed
168
      {
169
          gLogging.textOut(FONTCOLORS::RED,"%s<br>", curl_easy_strerror(res));
Gerhard Stein's avatar
Gerhard Stein committed
170
      }
171 172 173 174 175 176 177 178 179 180 181

      /* always cleanup */
      curl_easy_cleanup(curl);

      fclose(fp);
    }

    return (int)res;

}

182 183 184
#define TRACE_NODE(x) gLogging << #x"=" << x;

bool GameDownloader::readGamesNode(const boost::property_tree::ptree &pt)
185 186 187
{
    try
    {
188 189 190 191 192
        for( auto &gameNode : pt.get_child("Games") )
        {
            // Filter the comments ...
            if(gameNode.first == "<xmlcomment>")
                continue;
193

194
            GameCatalogueEntry gce;            
195

196
            gce.mVersionCode = gameNode.second.get<int>("<xmlattr>.versioncode");
197
            TRACE_NODE(gce.mVersionCode);
198 199

            gce.mName = gameNode.second.get<std::string>("<xmlattr>.name");
200
            TRACE_NODE(gce.mName);
201
            gce.mLink = gameNode.second.get<std::string>("<xmlattr>.link");
202
            TRACE_NODE(gce.mLink);
203

204

205 206 207
            if(gce.mVersionCode > CGVERSIONCODE)
            {
                gLogging.ftextOut("Game %s not supported. Required Version code %d, got %d.\n<br>",
208
                                  gce.mName.c_str(), CGVERSIONCODE, gce.mVersionCode );
209 210 211 212 213
                continue;
            }


            gce.mDescription = gameNode.second.get<std::string>("<xmlattr>.description");
214
            TRACE_NODE(gce.mDescription);
215
            gce.mPictureFile = gameNode.second.get<std::string>("<xmlattr>.picture");
216
            TRACE_NODE(gce.mPictureFile);
217 218 219 220 221 222 223 224 225 226 227 228

            const auto filePath = JoinPaths("cache", gce.mPictureFile);

            const auto fullfname = GetFullFileName(filePath);

            gce.mBmpPtr.reset(new GsBitmap);

            gce.mBmpPtr->loadImg(fullfname);

            mGameCatalogue.push_back(gce);
        }
    }
229 230 231 232 233
    catch(std::exception const&  ex)
    {
        gLogging << "Exception while reading game node: " << ex.what() << "\n";
        return false;
    }
234 235
    catch(...)
    {
236
        gLogging << "Unknown Exception while reading game node\n.";
237 238
        return false;
    }
239 240

    return true;
241 242
}

243
bool GameDownloader::readLegacyCatalogue(const boost::property_tree::ptree &pt)
244 245 246
{
    try
    {
247 248
        for( auto &gameNode : pt.get_child("Catalogue") )
        {
Gerhard Stein's avatar
Gerhard Stein committed
249
            // Filter the comments ...
250 251 252 253 254 255 256 257 258 259
            if(gameNode.first == "<xmlcomment>")
                continue;

            GameCatalogueEntry gce;

            gce.mName = gameNode.second.get<std::string>("<xmlattr>.name");
            gce.mLink = gameNode.second.get<std::string>("<xmlattr>.link");
            gce.mDescription = gameNode.second.get<std::string>("<xmlattr>.description");
            gce.mPictureFile = gameNode.second.get<std::string>("<xmlattr>.picture");

260 261 262
            const auto filePath = JoinPaths("cache", gce.mPictureFile);

            const auto fullfname = GetFullFileName(filePath);
263 264 265 266

            gce.mBmpPtr.reset(new GsBitmap);

            gce.mBmpPtr->loadImg(fullfname);
267

268 269
            mGameCatalogue.push_back(gce);
        }
270
    }
271 272 273 274 275
    catch(std::exception const&  ex)
    {
        gLogging << "Exception while reading game node (Legacy): " << ex.what() << "\n";
        return false;
    }
276 277
    catch(...)
    {
278
        gLogging << "Unknown Exception while reading game node\n.";
279 280 281 282 283 284
        return false;
    }

    return true;
}

285 286 287



288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
bool GameDownloader::loadCatalogue(const std::string &catalogueFile)
{
    // Create an empty property tree object
    using boost::property_tree::ptree;
    ptree pt;


    try
    {



        // Load the XML file into the property tree. If reading fails
        // (cannot open file, parse error), an exception is thrown.
        read_xml(catalogueFile, pt);

        bool ok = false;

306
        gLogging << "Reading Games from Store...\n" ;
307
        ok |= readGamesNode(pt);
308
        gLogging << "Reading Games from Store (Legacy)...\n" ;
309 310 311
        ok |= readLegacyCatalogue(pt);

        return ok;
312 313 314 315 316 317 318 319 320 321

    }
    catch(...)
    {
        return false;
    }

    return true;
}

322 323 324 325 326 327 328 329 330 331 332
bool GameDownloader::downloadCatalogue()
{
    pCancelDownload = &mCancelDownload;

    const int res = downloadFile(mCatalogFName, mProgress, "");

    if(res==0)
        return true;

    return false;
}
333

334
bool GameDownloader::checkForMissingGames( std::vector< std::string > &missingList )
335
{
336
    // Load game catalogue
337
    if( !loadCatalogue(mCatalogFName) )
338
    {
Gerhard Stein's avatar
Gerhard Stein committed
339 340
        // try with search paths
        for(searchpathlist::const_iterator p = tSearchPaths.begin(); p != tSearchPaths.end(); p++)
341
        {
Gerhard Stein's avatar
Gerhard Stein committed
342 343
            std::string newPath  = *p;
            ReplaceFileVariables(newPath);
344
            newPath = JoinPaths(newPath, mCatalogFName);
345

Gerhard Stein's avatar
Gerhard Stein committed
346
            gLogging.ftextOut("Looking at: %s<br>", newPath.c_str() );
347

348 349
            if(loadCatalogue(newPath))
            {
350
                mCataFound = true;
351 352
                break;
            }
Gerhard Stein's avatar
Gerhard Stein committed
353 354 355

        }

356
        if(!mCataFound)
Gerhard Stein's avatar
Gerhard Stein committed
357 358 359 360 361 362 363 364
        {
            // If not found search within for subdirectories
            std::set<std::string> dirs;
            FileListAdder fileListAdder;
            GetFileList(dirs, fileListAdder, ".", false, FM_DIR);

            for(std::set<std::string>::iterator i = dirs.begin(); i != dirs.end(); ++i)
            {
365
                const std::string newPath = JoinPaths(*i, mCatalogFName);
Gerhard Stein's avatar
Gerhard Stein committed
366 367 368 369 370

                gLogging.ftextOut("Looking at: %s<br>", newPath.c_str() );

                if(loadCatalogue(newPath))
                {
371
                    mCataFound = true;
Gerhard Stein's avatar
Gerhard Stein committed
372 373 374
                    break;
                }
            }
375 376 377 378 379
        }

    }
    else
    {
380
        mCataFound = true;
381 382
    }

383
    if(!mCataFound)
384
    {
Gerhard Stein's avatar
Gerhard Stein committed
385
        gLogging.ftextOut("Sorry, the catalogue file \"%s\" was not found <br>", mCatalogFName.c_str() );
386 387 388
        return -1;
    }

389 390 391 392
    // Get the first path. We assume that one is writable
    std::string searchPaths;
    GetExactFileName(GetFirstSearchPath(), searchPaths);

393 394
    const auto downloadPath = JoinPaths(searchPaths, "downloads");

Gerhard Stein's avatar
Gerhard Stein committed
395 396
    std::vector<GameCatalogueEntry> reducedCatalogue;

397
    // Need to check for a list of downloaded stuff and what we still need
398
    for( const auto &gameEntry : mGameCatalogue )
399
    {
400
        const std::string gameFile = gameEntry.mLink;
401 402 403

        const auto downloadGamePath = JoinPaths(downloadPath, gameFile);

404 405
        gLogging << "Scanning \"" << gameEntry.mName << "\"\n";

406 407
        if( !IsFileAvailable(downloadGamePath) )
        {
408
            missingList.push_back(gameEntry.mName);
Gerhard Stein's avatar
Gerhard Stein committed
409
            reducedCatalogue.push_back(gameEntry);
410 411 412 413
            continue;
        }
    }

Gerhard Stein's avatar
Gerhard Stein committed
414 415
    mGameCatalogue = reducedCatalogue;

416 417 418 419
    return true;
}


420 421
int GameDownloader::handle()
{
422 423
    int res = 0;

424 425
    pCancelDownload = &mCancelDownload;

426 427 428 429 430 431
    if(mDownloadCatalogue)
    {
        downloadCatalogue();
        return res;
    }

432 433 434 435
    // Get the first path. We assume that one is writable
    std::string searchPaths;
    GetExactFileName(GetFirstSearchPath(), searchPaths);

Gerhard Stein's avatar
Gerhard Stein committed
436
    const auto fullDownloadPath = JoinPaths(searchPaths, "downloads");
437 438 439 440
    const auto gamesPath = JoinPaths(searchPaths, "games");


    // Create Download directory if it does not exist yet
Gerhard Stein's avatar
Gerhard Stein committed
441
    CreateRecDir(fullDownloadPath);
442

443
    // Go through the missing pieces
Gerhard Stein's avatar
Gerhard Stein committed
444
    const auto &gameFileName = mGameFileName;
445
    const auto &gameName = mGameName;
446
    {
447 448
        gDlfrom = mProgress = 0;
        gDlto = 900;
449

Gerhard Stein's avatar
Gerhard Stein committed
450
        const auto downloadGamePath = JoinPaths("downloads", gameFileName);
451 452 453

        if( !IsFileAvailable(downloadGamePath) )
        {
454
            gLogging.ftextOut( FONTCOLORS::GREEN, "Downloading file \"%s\"", gameFileName.c_str());
455

456
            // TODO: We also must pass the gamepath and a downloads folder we all the file packages can be set.
Gerhard Stein's avatar
Gerhard Stein committed
457
            res = downloadFile(gameFileName, mProgress, "downloads");
458 459
        }

460
        mProgress = gDlto;
461 462 463

        // TODO: Now the downloaded stuff must be extracted to the games directory
        // At this point the file should be available
464
        const std::string destDir = JoinPaths(gamesPath, gameName);
465 466
        if( IsFileAvailable(downloadGamePath) )
        {
467 468 469
            // Create subdirectory
            CreateRecDir( destDir );

470 471 472
            const std::string fullZipPath = JoinPaths(fullDownloadPath, gameFileName);

            const int retVal = unzipFile(fullZipPath.c_str(), destDir.c_str());
Gerhard Stein's avatar
Gerhard Stein committed
473

474
            gLogging.ftextOut( FONTCOLORS::BLACK, "Extracting downloaded file \"%s\" to \"%s\".\n<br>",
475 476 477
                               fullZipPath.c_str(),
                               destDir.c_str() );

Gerhard Stein's avatar
Gerhard Stein committed
478 479 480
            // If unpacking files fails, we should delete it.
            if(retVal != 0)
            {
481
                gLogging.ftextOut( FONTCOLORS::RED, "Error: Trying to remove broken file \"%s\"", downloadGamePath.c_str());
Gerhard Stein's avatar
Gerhard Stein committed
482
                remove( GetFullFileName(downloadGamePath).c_str() );
Gerhard Stein's avatar
Gerhard Stein committed
483 484 485
            }
            else
            {
486
                gLogging.ftextOut( FONTCOLORS::GREEN, "File \"%s\" extracted successfully to \"%s\".\n<br>",
487 488
                                   downloadGamePath.c_str(),
                                   destDir.c_str());
Gerhard Stein's avatar
Gerhard Stein committed
489
            }
490 491 492
        }
        else
        {
Gerhard Stein's avatar
Gerhard Stein committed
493
            const std::string errStr = "Something went wrong with downloading \"" + gameFileName + "\"!";
494
            gLogging.ftextOut(FONTCOLORS::PURPLE, errStr.c_str() );
495
        }
496 497 498 499 500

        if(res != CURLE_OK)
        {
            remove(downloadGamePath.c_str());
        }
501 502 503
    }

    mProgress = 1000;
504 505 506

    return res;
}
Gerhard Stein's avatar
Gerhard Stein committed
507