//
// Created by yfzx on 2023/3/22.
//
//
// Created by yfzx on 2023/3/14.
//
#include "MINIOGetAndPut.h"
/// 判断待上传的文件是否存在
inline bool file_exists(const string& name)
{
struct stat buffer;
return (stat(name.c_str(), &buffer) == 0);
}
/// 初始化S3连接
S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){
// 初始化 S3 Client
Aws::Client::ClientConfiguration cfg;
cfg.endpointOverride = IPPort;
cfg.scheme = Aws::Http::Scheme::HTTP;
cfg.verifySSL = false;
Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);
S3Client client(cred, cfg,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
return client;
}
/// 从MINIO中下载文件或数据
bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String ®ion)
{
/// 初始化下载请求
Aws::S3::Model::GetObjectRequest object_request;
object_request.SetBucket(fromBucket);
object_request.SetKey(objectKey);
/// 从S3服务器中下载数据
Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);
if (get_object_outcome.IsSuccess()) {
/// 保存文件
Aws::OFStream local_file;
local_file.open(savePath, std::ios::out | std::ios::binary);
local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();
std::cout << "Done!" << std::endl;
return true;
} else {
auto err = get_object_outcome.GetError();
std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
return false;
}
}
/// 往MINIO上上传文件或数据
bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String ®ion)
{
// 判断文件是否存在
if (!file_exists(file_name)) {
cout << "ERROR: 找不到这个文件，这个文件不存在"
<< endl;
return false;
}
/// 初始化上传请求
Aws::S3::Model::PutObjectRequest object_request;
object_request.SetBucket(fromBucket);
object_request.SetKey(objectKey);
/// 建立文件输入串流，可为任何串流
const shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::FStream>("SampleAllocationTag",
file_name.c_str(),
ios_base::in | ios_base::binary);
object_request.SetBody(input_data);
/// 开始上传至S3
Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);
if (!put_object_outcome.IsSuccess()) {
auto error = put_object_outcome.GetError();
cout << "ERROR: " << error.GetExceptionName() << ": "
<< error.GetMessage() << endl;
return false;
}else {
cout << "success" << endl;
return true;
}
}
/// 启动上传程序
bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){
// 初始化
Aws::SDKOptions options;
options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
Aws::InitAPI(options);
/// 设置ip、acccessKey、secretKey
Aws::String IP = Ips;
Aws::String accessKeyId = acccessKey;
Aws::String secretKey = secretKeyS;
S3Client client;
client = InitS3Client(IP, accessKeyId, secretKey, securityToken);
Aws::InitAPI(options);
{
/// 设置上传的目标桶和文件路径
const Aws::String bucket_name = bucket_names;
const Aws::String object_name = object_names;
if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {
std::string errorInfo = object_name + " 上传失败";
writeLog(L_ERROR, errorInfo.c_str());
return false;
}
}
writeLog(L_INFO, "====> 文件上传成功");
return true;
}
/// 启动下载程序
bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,
bool &flags){
// 初始化Aws API
Aws::SDKOptions options;
options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
Aws::InitAPI(options);
/// 设置ip、acccessKey、secretKey
Aws::String IP = Ips;
Aws::String accessKeyId = acccessKey;
Aws::String secretKey = secretKeyS;
/// 初始化S3服务
S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);
/// 根据get接口的桶名称设置文件所在的桶
Aws::S3::Model::Bucket temps{};
std::string stringK = bucket_name;
temps.SetName(stringK);
/// 查找MINIO相关数据
Aws::S3::Model::ListObjectsRequest objects_request;
objects_request.WithBucket(temps.GetName()); /// 设置桶路径
objects_request.SetPrefix(downPath); /// 设置文件夹路径
auto list_objects_outcome = client.ListObjects(objects_request); /// 判断存储桶及相关连接是否有效
if (flags){
writeLog(L_INFO, "====> 文件下载终止");
return false;
}
if (list_objects_outcome.IsSuccess())
{
/// 获取该目录下的前一千个文件
Aws::Vector<Aws::S3::Model::Object> object_list =
list_objects_outcome.GetResult().GetContents();
if (object_list.size() == 1000){
/// 当下载数量大于一千时 循环下载文件
while(object_list.size() == 1000){
auto finally_object = object_list[0];
for (auto const &s3_object : object_list)
{
if (flags){
writeLog(L_INFO, "====> 文件下载终止");
return false;
}
/// 设置桶名称
const Aws::String bucket_name = stringK;
/// 设置待下载的文件名称
const Aws::String object_name = s3_object.GetKey();
/// 设置保存文件路径及文件名称
std::string NewPath;
size_t indexNew = object_name.find_last_of('/');
NewPath = object_name.substr(indexNew + 1, object_name.length());
const Aws::String SavePath = imagePath + NewPath;
size_t indexK = downPath.find_last_of('/');
std::string newPathEs = downPath.substr(0, indexK);
std::string strK;
strK = imagePath + "/" + NewPath;
const Aws::String SavePaths = strK;
//筛选数据
std::string fileName;
size_t index = object_name.find_last_of('.');
fileName = object_name.substr(index + 1, object_name.length());
if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
/// 启动下载
if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
std::string errorInfo = object_name + " 下载失败";
writeLog(L_ERROR, errorInfo.c_str());
continue;
}
}
}
finally_object = object_list[999];
/// 重新配置相关路径
objects_request.WithBucket(temps.GetName()); /// 设置桶路径
objects_request.SetPrefix(finally_object.GetKey()); /// 设置新文件夹路径
