curl下载器linux版

#include <iostream> #include <assert.h> #include <algorithm> #include <curl/curl.h> #include <pthread.h> #include <unistd.h> #include "CurlDownloader.h" //下载临时文件后缀 #define DOWNLOAD_TMP_FILE_EXT ".dltmp" //检测是否支持多线程分片传输的字符串 #define RANGE_TEST_FLAG "RangeTest" //用于检测是否支持多线程下载接收大小 #define RANGE_TEST_RECV_SIZE 1024 CURLSH* CurlDownloader::sharedns_handle = NULL; CurlDownConfig CurlDownloader::g_curlDownCfg; //线程信息 typedef struct _tThreadInfo{ unsigned long ulBegPos; //下载起始位置 unsigned long ulBlockSize; //本线程负责下载的数据大小 unsigned long ulRecvSize; //本线程已经接收到的数据大小 CURL* pCurl; pthread_t thrdId; int iTryTimes; //失败已经重试次数 _tThreadInfo() // 默认构造函数 { ulBegPos = 0; ulBlockSize = 0; ulRecvSize = 0; pCurl = NULL; // thrdId.p = NULL; // thrdId.x = 0; thrdId = 0; iTryTimes = 0; } }ThreadInfo; void myprint(const std::string& strLog) { printf("%s\n", strLog.c_str()); } void FFlushEx(FILE* stream) { int duphandle; fflush(stream); duphandle = dup(fileno(stream)); close(duphandle); } void ReplaceStr(std::string &src, const std::string &target, const std::string &replacement) { if (target == replacement) { return; } std::string::size_type startpos = 0; while (startpos != std::string::npos) { startpos = src.find(target, startpos); if(startpos != std::string::npos) { src.replace(startpos, target.length(), replacement); startpos += replacement.length(); } } } void SplitStr(const std::string& s, const std::string &delim, std::vector<std::string> *ret) { if (!ret) { return; } ret->clear(); size_t last = 0; size_t index=s.find_first_of(delim,last); while (index!=std::string::npos) { ret->push_back(s.substr(last,index-last)); last=index+delim.length(); index=s.find_first_of(delim,last); } if (index-last>0) { ret->push_back(s.substr(last,index-last)); } } static bool checkConnect(const char *dst, int cnt) { FILE *stream; char recvBuf[16] = {0}; char cmdBuf[256] = {0}; if (NULL == dst || cnt <= 0) { return false; } sprintf(cmdBuf, "ping %s -c %d -i 0.2 | grep time= | wc -l", dst, cnt); stream = popen(cmdBuf, "r"); fread(recvBuf, sizeof(char), sizeof(recvBuf)-1, stream); pclose(stream); if (atoi(recvBuf) > 0) { return true; } return false; } unsigned long GetTickCount() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); } /** * 功能：拷贝文件函数 * 参数： * sourceFileNameWithPath：源文件名（带路径） * targetFileNameWithPath：目标文件名（带路径） * 返回值： * true: 拷贝成功 * false：拷贝失败 * author:wangchangshuai jlu */ static bool copyFile(const char *sourceFileNameWithPath, const char *targetFileNameWithPath) { FILE *fpR, *fpW; char buffer[BUFFER_SIZE]; int lenR, lenW; if ((fpR = fopen(sourceFileNameWithPath, "r")) == NULL) { printf("The file '%s' can not be opened! \n", sourceFileNameWithPath); return false; } if ((fpW = fopen(targetFileNameWithPath, "w")) == NULL) { printf("The file '%s' can not be opened! \n", targetFileNameWithPath); fclose(fpR); return false; } memset(buffer, 0, BUFFER_SIZE); while ((lenR = fread(buffer, 1, BUFFER_SIZE, fpR)) > 0) { if ((lenW = fwrite(buffer, 1, lenR, fpW)) != lenR) { printf("Write to file '%s' failed!\n", targetFileNameWithPath); fclose(fpR); fclose(fpW); return false; } memset(buffer, 0, BUFFER_SIZE); } fclose(fpR); fclose(fpW); return true; } std::string Log_Format(const char *lpszFormat, ...) { char szLog[1024]={0}; va_list ap; va_start(ap, lpszFormat); vsprintf(szLog, lpszFormat, ap); va_end(ap); return szLog; } class MyLog{ public: MyLog(LPLOGFUN pLogFun, const char* szFileName, const char* szFuncName, long lLine) : m_pLogFun(pLogFun) { std::string strFileName(szFileName ? szFileName : ""); int pos = strFileName.rfind('\\'); if (pos != -1) strFileName = strFileName.substr(pos + 1, strFileName.size() - pos - 1); m_szFileName = strFileName; m_szFuncName = std::string(szFuncName ? szFuncName : ""); m_lLine = lLine; }; ~MyLog(){}; void operator<<(const std::string& strLog) { if (m_pLogFun) { //SYSTEMTIME curTime; //GetLocalTime(&curTime); char ch[512] = { 0 }; //sprintf(ch, "%02d:%02d:%02d.%6d %s:%d %s(): ", // curTime.wHour, curTime.wMinute, curTime.wSecond, // curTime.wMilliseconds, m_szFileName.c_str(), m_lLine, // m_szFuncName.c_str()); sprintf(ch, "%s:%d %s(): ", m_szFileName.c_str(), m_lLine, m_szFuncName.c_str()); std::string strInfo = std::string(ch) + strLog; m_pLogFun(strInfo); } } private: LPLOGFUN m_pLogFun; std::string m_szFileName, m_szFuncName; long m_lLine; }; #define RICH_FORMAT_LOG MyLog(g_curlDownCfg.pLogFun, __FILE__, __FUNCTION__, __LINE__)<<Log_Format CurlDownloader::CurlDownloader(void) { m_pFile = NULL; m_bPause = false; m_bSupportMultiDown = false; m_bTerminate = false; m_ulFullFileSize = 0; m_ulDownFileSize = 0; m_dSpeed = 0.0; m_strUrl = m_strDownloadPath = ""; pthread_mutex_init(&m_mutexFile, NULL); m_strProxy = ""; m_iHttpCode = 200; m_bRedirected = false; m_strOriginalUrl = ""; m_curlCode = CURLE_OK; m_bInitOver = false; m_bInitInterrupt = false; } CurlDownloader::~CurlDownloader(void) { Pause(); pthread_mutex_lock(&m_mutexFile); if (m_pFile) { fclose(m_pFile); m_pFile = NULL; } pthread_mutex_unlock(&m_mutexFile); ClearThreadInfo(); //文件互质变量释放 pthread_mutex_destroy(&m_mutexFile); } void CurlDownloader::ClearThreadInfo() { std::vector<ThreadInfo*>::const_iterator ita = m_vecThrdInfo.begin(); while (ita != m_vecThrdInfo.end()) { ThreadInfo* pInfo = *ita++; if (pInfo) { if (pInfo->pCurl) { curl_easy_cleanup(pInfo->pCurl); pInfo->pCurl = NULL; } delete pInfo; pInfo = NULL; } } m_vecThrdInfo.clear(); } bool CurlDownloader::Start(const std::string &strUrl, const std::string &strDownloadPath, int *pThreadCount /* = NULL*/) { RICH_FORMAT_LOG("Start() START"); int iThreadCountX = 0; if (pThreadCount) { assert(*pThreadCount); iThreadCountX = *pThreadCount; } else { iThreadCountX = g_curlDownCfg.iMaxThreadCount; } m_bPause = false; m_bTerminate = false; m_strUrl = strUrl; //备份原始URL m_strOriginalUrl = strUrl; m_strDownloadPath = strDownloadPath; //防止重新开始，所以要重新初始化基本参数 m_ulFullFileSize = 0; m_ulDownFileSize = 0; m_bSupportMultiDown = false; m_bRedirected = false; m_curlCode = CURLE_OK; m_iHttpCode = 200; // ClearThreadInfo(); //删除本地文件 // DeleteFileA(strDownloadPath.c_str()); remove(strDownloadPath.c_str()); std::string strTmpFile = strDownloadPath + DOWNLOAD_TMP_FILE_EXT; //通过临时文件判断是否已经下载过 if (access(strTmpFile.c_str(), F_OK) == -1) // 文件不存在 { //文件不存在，新的下载 if (!DownloadInit()) { return false; } //最优级线程个数 int iCount = (int)ceil(1.0*m_ulFullFileSize/(g_curlDownCfg.iMinBlockSize)); // ceil 返回大于或者等于指定表达式的最小整数 // 线程个数不能超过iThreadCountX const int iThreadCount = (m_bSupportMultiDown ? min(iCount, iThreadCountX) : 1); m_ulDownFileSize = 0; unsigned long ulFileSize = m_ulFullFileSize; //临时信息长度