MINIO服务器基于AWS S3 SDK 文件上传及下载（C++实现类）

// // Created by yfzx on 2023/3/22. // // // Created by yfzx on 2023/3/14. // #include "MINIOGetAndPut.h" /// 判断待上传的文件是否存在 inline bool file_exists(const string& name) { struct stat buffer; return (stat(name.c_str(), &buffer) == 0); } /// 初始化S3连接 S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){ // 初始化 S3 Client Aws::Client::ClientConfiguration cfg; cfg.endpointOverride = IPPort; cfg.scheme = Aws::Http::Scheme::HTTP; cfg.verifySSL = false; Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken); S3Client client(cred, cfg, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); return client; } /// 从MINIO中下载文件或数据 bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String &region) { /// 初始化下载请求 Aws::S3::Model::GetObjectRequest object_request; object_request.SetBucket(fromBucket); object_request.SetKey(objectKey); /// 从S3服务器中下载数据 Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request); if (get_object_outcome.IsSuccess()) { /// 保存文件 Aws::OFStream local_file; local_file.open(savePath, std::ios::out | std::ios::binary); local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf(); std::cout << "Done!" << std::endl; return true; } else { auto err = get_object_outcome.GetError(); std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; return false; } } /// 往MINIO上上传文件或数据 bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String &region) { // 判断文件是否存在 if (!file_exists(file_name)) { cout << "ERROR: 找不到这个文件，这个文件不存在" << endl; return false; } /// 初始化上传请求 Aws::S3::Model::PutObjectRequest object_request; object_request.SetBucket(fromBucket); object_request.SetKey(objectKey); /// 建立文件输入串流，可为任何串流 const shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", file_name.c_str(), ios_base::in | ios_base::binary); object_request.SetBody(input_data); /// 开始上传至S3 Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request); if (!put_object_outcome.IsSuccess()) { auto error = put_object_outcome.GetError(); cout << "ERROR: " << error.GetExceptionName() << ": " << error.GetMessage() << endl; return false; }else { cout << "success" << endl; return true; } } /// 启动上传程序 bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){ // 初始化 Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); /// 设置ip、acccessKey、secretKey Aws::String IP = Ips; Aws::String accessKeyId = acccessKey; Aws::String secretKey = secretKeyS; S3Client client; client = InitS3Client(IP, accessKeyId, secretKey, securityToken); Aws::InitAPI(options); { /// 设置上传的目标桶和文件路径 const Aws::String bucket_name = bucket_names; const Aws::String object_name = object_names; if (!doPutObject(object_name, bucket_name, client, imagename, " ")) { std::string errorInfo = object_name + " 上传失败"; writeLog(L_ERROR, errorInfo.c_str()); return false; } } writeLog(L_INFO, "====> 文件上传成功"); return true; } /// 启动下载程序 bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken, bool &flags){ // 初始化Aws API Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); /// 设置ip、acccessKey、secretKey Aws::String IP = Ips; Aws::String accessKeyId = acccessKey; Aws::String secretKey = secretKeyS; /// 初始化S3服务 S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken); /// 根据get接口的桶名称设置文件所在的桶 Aws::S3::Model::Bucket temps{}; std::string stringK = bucket_name; temps.SetName(stringK); /// 查找MINIO相关数据 Aws::S3::Model::ListObjectsRequest objects_request; objects_request.WithBucket(temps.GetName()); /// 设置桶路径 objects_request.SetPrefix(downPath); /// 设置文件夹路径 auto list_objects_outcome = client.ListObjects(objects_request); /// 判断存储桶及相关连接是否有效 if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } if (list_objects_outcome.IsSuccess()) { /// 获取该目录下的前一千个文件 Aws::Vector<Aws::S3::Model::Object> object_list = list_objects_outcome.GetResult().GetContents(); if (object_list.size() == 1000){ /// 当下载数量大于一千时 循环下载文件 while(object_list.size() == 1000){ auto finally_object = object_list[0]; for (auto const &s3_object : object_list) { if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } /// 设置桶名称 const Aws::String bucket_name = stringK; /// 设置待下载的文件名称 const Aws::String object_name = s3_object.GetKey(); /// 设置保存文件路径及文件名称 std::string NewPath; size_t indexNew = object_name.find_last_of('/'); NewPath = object_name.substr(indexNew + 1, object_name.length()); const Aws::String SavePath = imagePath + NewPath; size_t indexK = downPath.find_last_of('/'); std::string newPathEs = downPath.substr(0, indexK); std::string strK; strK = imagePath + "/" + NewPath; const Aws::String SavePaths = strK; //筛选数据 std::string fileName; size_t index = object_name.find_last_of('.'); fileName = object_name.substr(index + 1, object_name.length()); if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){ /// 启动下载 if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) { std::string errorInfo = object_name + " 下载失败"; writeLog(L_ERROR, errorInfo.c_str()); continue; } } } finally_object = object_list[999]; /// 重新配置相关路径 objects_request.WithBucket(temps.GetName()); /// 设置桶路径 objects_request.SetPrefix(finally_object.GetKey()); /// 设置新文件夹路径