2014-10-29 12:37:38 +00:00
|
|
|
#include <base/system.h>
|
|
|
|
#include <engine/storage.h>
|
|
|
|
#include "fetcher.h"
|
|
|
|
|
|
|
|
CFetchTask::CFetchTask()
|
|
|
|
{
|
|
|
|
m_pNext = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
CFetcher::CFetcher()
|
|
|
|
{
|
|
|
|
m_pStorage = NULL;
|
|
|
|
m_pHandle = NULL;
|
|
|
|
m_Lock = lock_create();
|
|
|
|
m_pFirst = NULL;
|
|
|
|
m_pLast = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool CFetcher::Init()
|
|
|
|
{
|
|
|
|
m_pStorage = Kernel()->RequestInterface<IStorage>();
|
|
|
|
if(!curl_global_init(CURL_GLOBAL_DEFAULT) && (m_pHandle = curl_easy_init()))
|
|
|
|
return true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
CFetcher::~CFetcher()
|
|
|
|
{
|
|
|
|
if(m_pHandle)
|
|
|
|
curl_easy_cleanup(m_pHandle);
|
|
|
|
curl_global_cleanup();
|
|
|
|
}
|
|
|
|
|
2015-01-19 22:00:08 +00:00
|
|
|
void CFetcher::QueueAdd(CFetchTask *pTask, const char *pUrl, const char *pDest, int StorageType, void *pUser, COMPFUNC pfnCompCb, PROGFUNC pfnProgCb)
|
2014-10-29 12:37:38 +00:00
|
|
|
{
|
|
|
|
str_copy(pTask->m_pUrl, pUrl, sizeof(pTask->m_pUrl));
|
|
|
|
str_copy(pTask->m_pDest, pDest, sizeof(pTask->m_pDest));
|
|
|
|
pTask->m_StorageType = StorageType;
|
|
|
|
pTask->m_pfnProgressCallback = pfnProgCb;
|
|
|
|
pTask->m_pfnCompCallback = pfnCompCb;
|
|
|
|
pTask->m_pUser = pUser;
|
|
|
|
pTask->m_Size = pTask->m_Progress = 0;
|
2015-01-28 12:13:56 +00:00
|
|
|
pTask->m_Abort = false;
|
2014-10-29 12:37:38 +00:00
|
|
|
|
|
|
|
lock_wait(m_Lock);
|
2015-02-27 21:06:19 +00:00
|
|
|
if(!m_pThHandle)
|
|
|
|
{
|
|
|
|
m_pThHandle = thread_create(&FetcherThread, this);
|
|
|
|
thread_detach(m_pThHandle);
|
|
|
|
}
|
|
|
|
|
2015-01-19 22:00:08 +00:00
|
|
|
if(!m_pFirst)
|
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
m_pFirst = pTask;
|
|
|
|
m_pLast = m_pFirst;
|
|
|
|
}
|
2015-01-19 22:00:08 +00:00
|
|
|
else
|
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
m_pLast->m_pNext = pTask;
|
|
|
|
m_pLast = pTask;
|
|
|
|
}
|
|
|
|
pTask->m_State = CFetchTask::STATE_QUEUED;
|
|
|
|
lock_release(m_Lock);
|
|
|
|
}
|
|
|
|
|
2015-02-12 17:58:54 +00:00
|
|
|
void CFetcher::Escape(char *pBuf, size_t size, const char *pStr)
|
|
|
|
{
|
|
|
|
char *pEsc = curl_easy_escape(0, pStr, 0);
|
|
|
|
str_copy(pBuf, pEsc, size);
|
|
|
|
curl_free(pEsc);
|
|
|
|
}
|
|
|
|
|
2014-10-29 12:37:38 +00:00
|
|
|
void CFetcher::FetcherThread(void *pUser)
|
|
|
|
{
|
2015-01-19 22:00:08 +00:00
|
|
|
CFetcher *pFetcher = (CFetcher *)pUser;
|
2014-10-29 12:37:38 +00:00
|
|
|
dbg_msg("fetcher", "Thread started...");
|
2015-01-19 22:00:08 +00:00
|
|
|
while(1)
|
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
lock_wait(pFetcher->m_Lock);
|
2015-01-19 22:00:08 +00:00
|
|
|
CFetchTask *pTask = pFetcher->m_pFirst;
|
|
|
|
if(pTask)
|
|
|
|
pFetcher->m_pFirst = pTask->m_pNext;
|
2014-10-29 12:37:38 +00:00
|
|
|
lock_release(pFetcher->m_Lock);
|
2015-01-19 22:00:08 +00:00
|
|
|
if(pTask)
|
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
dbg_msg("fetcher", "Task got %s:%s", pTask->m_pUrl, pTask->m_pDest);
|
|
|
|
pFetcher->FetchFile(pTask);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
thread_sleep(10);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool CFetcher::FetchFile(CFetchTask *pTask)
|
|
|
|
{
|
2015-01-19 22:00:08 +00:00
|
|
|
for(int i = 0; pTask->m_pDest[i] != '\0'; i++)
|
|
|
|
{
|
|
|
|
if(pTask->m_pDest[i] == '/')
|
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
pTask->m_pDest[i] = '\0';
|
2015-02-08 09:53:06 +00:00
|
|
|
m_pStorage->CreateFolder(pTask->m_pDest, pTask->m_StorageType);
|
2014-10-29 12:37:38 +00:00
|
|
|
pTask->m_pDest[i] = '/';
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-27 21:06:19 +00:00
|
|
|
char aPath[256];
|
|
|
|
m_pStorage->GetCompletePath(pTask->m_StorageType, pTask->m_pDest, aPath, sizeof(aPath));
|
|
|
|
IOHANDLE File = io_open(aPath, IOFLAG_WRITE);
|
|
|
|
|
2014-10-29 12:37:38 +00:00
|
|
|
|
|
|
|
char aErr[CURL_ERROR_SIZE];
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_ERRORBUFFER, aErr);
|
|
|
|
|
2015-02-05 18:41:48 +00:00
|
|
|
//curl_easy_setopt(m_pHandle, CURLOPT_VERBOSE, 1L);
|
2015-01-19 23:01:30 +00:00
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_FOLLOWLOCATION, 1L);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_MAXREDIRS, 4L);
|
2014-10-29 12:37:38 +00:00
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_FAILONERROR, 1L);
|
2015-01-19 21:19:27 +00:00
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_CAINFO, "data/ca-ddnet.pem");
|
2014-10-29 12:37:38 +00:00
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_URL, pTask->m_pUrl);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_WRITEDATA, File);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_WRITEFUNCTION, &CFetcher::WriteToFile);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_NOPROGRESS, 0);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_PROGRESSDATA, pTask);
|
|
|
|
curl_easy_setopt(m_pHandle, CURLOPT_PROGRESSFUNCTION, &CFetcher::ProgressCallback);
|
|
|
|
|
|
|
|
dbg_msg("fetcher", "Downloading %s", pTask->m_pDest);
|
|
|
|
pTask->m_State = CFetchTask::STATE_RUNNING;
|
2015-01-28 12:13:56 +00:00
|
|
|
int ret = curl_easy_perform(m_pHandle);
|
|
|
|
if(ret != CURLE_OK)
|
2015-01-19 22:00:08 +00:00
|
|
|
{
|
2014-10-29 12:37:38 +00:00
|
|
|
dbg_msg("fetcher", "Task failed. libcurl error: %s", aErr);
|
2015-01-28 12:13:56 +00:00
|
|
|
pTask->m_State = (ret == CURLE_ABORTED_BY_CALLBACK) ? CFetchTask::STATE_ABORTED : CFetchTask::STATE_ERROR;
|
2014-10-29 12:37:38 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
io_close(File);
|
2015-02-27 21:06:19 +00:00
|
|
|
dbg_msg("fetcher", "Task done %s", pTask->m_pDest);
|
|
|
|
pTask->m_State = CFetchTask::STATE_DONE;
|
2014-10-29 12:37:38 +00:00
|
|
|
if(pTask->m_pfnCompCallback)
|
|
|
|
pTask->m_pfnCompCallback(pTask, pTask->m_pUser);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void CFetcher::WriteToFile(char *pData, size_t size, size_t nmemb, void *pFile)
|
|
|
|
{
|
|
|
|
io_write((IOHANDLE)pFile, pData, size*nmemb);
|
|
|
|
}
|
|
|
|
|
|
|
|
int CFetcher::ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr)
|
|
|
|
{
|
|
|
|
CFetchTask *pTask = (CFetchTask *)pUser;
|
2015-01-19 21:19:27 +00:00
|
|
|
//dbg_msg("fetcher", "DlCurr:%f, DlTotal:%f", DlCurr, DlTotal);
|
2014-10-29 12:37:38 +00:00
|
|
|
pTask->m_Current = DlCurr;
|
2015-01-28 11:57:11 +00:00
|
|
|
pTask->m_Size = DlTotal;
|
2015-01-19 22:00:08 +00:00
|
|
|
pTask->m_Progress = (100 * DlCurr) / (DlTotal ? DlTotal : 1);
|
2014-10-29 12:37:38 +00:00
|
|
|
if(pTask->m_pfnProgressCallback)
|
|
|
|
pTask->m_pfnProgressCallback(pTask, pTask->m_pUser);
|
2015-01-28 12:13:56 +00:00
|
|
|
return pTask->m_Abort ? -1 : 0;
|
2015-01-19 21:19:27 +00:00
|
|
|
}
|