STORMfs 可以让你直接将云存储安装为本地的文件系统，支持 Amazon S3, Eucalyptus 等

/* * stormfs - A FUSE abstraction layer for cloud storage * Copyright (C) 2011 Ben LeMasurier <ben.lemasurier@gmail.com> * * This program can be distributed under the terms of the GNU GPL. * See the file COPYING. */ #define _GNU_SOURCE #include "config.h" #include <stdio.h> #include <stdlib.h> #include <stdint.h> #include <string.h> #include <stdbool.h> #include <errno.h> #include <ctype.h> #include <time.h> #include <sys/time.h> #include <sys/select.h> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> #include <curl/curl.h> #include <curl/easy.h> #include <pthread.h> #include <glib.h> #include "stormfs.h" #include "curl.h" #define CURL_RETRIES 3 #define SHA1_BLOCK_SIZE 64 #define SHA1_LENGTH 20 #define MAX_REQUESTS 100 #define POOL_SIZE 100 #define DEFAULT_MIME_TYPE "application/octet-stream" #define MULTIPART_MIN 20971520 /* Minimum size for multipart files */ #define MULTIPART_CHUNK 10485760 /* 10MB */ #define MULTIPART_COPY_SIZE 524288000 /* 500MB */ #define MAX_FILE_SIZE 104857600000 /* 97.65GB (10,000 * 10MB) */ static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t shared_lock = PTHREAD_MUTEX_INITIALIZER; struct stormfs_curl { int verify_ssl; const char *url; const char *bucket; const char *access_key; const char *secret_key; GList *pool; GList *marker; bool pool_full; CURLM *multi; CURLSH *share; } curl; typedef struct { int fd; int part_num; char *path; char *etag; char *upload_id; size_t size; } FILE_PART; typedef struct { CURL *c; bool in_use; } CURL_HANDLE; typedef struct { char *memory; size_t size; } HTTP_RESPONSE; typedef struct { CURL *c; char *url; char *path; bool done; HTTP_RESPONSE response; struct curl_slist *headers; } HTTP_REQUEST; struct post_data { const char *readptr; int remaining; }; uid_t get_uid(const char *s) { return (uid_t) strtoul(s, (char **) NULL, 10); } gid_t get_gid(const char *s) { return (gid_t) strtoul(s, (char **) NULL, 10); } mode_t get_mode(const char *s) { return (mode_t) strtoul(s, (char **) NULL, 10); } time_t get_ctime(const char *s) { return (time_t) strtoul(s, (char **) NULL, 10); } time_t get_mtime(const char *s) { return (time_t) strtoul(s, (char **) NULL, 10); } dev_t get_rdev(const char *s) { return (dev_t) strtoul(s, (char **) NULL, 10); } off_t get_size(const char *s) { return (off_t) strtoul(s, (char **) NULL, 10); } blkcnt_t get_blocks(off_t size) { return size / 512 + 1; } static char * gid_to_s(gid_t gid) { char s[100]; snprintf(s, 100, "%lu", (unsigned long) gid); return strdup(s); } static char * rdev_to_s(dev_t rdev) { char s[100]; snprintf(s, 100, "%lu", (unsigned long) rdev); return strdup(s); } static char * uid_to_s(uid_t uid) { char s[100]; snprintf(s, 100, "%lu", (unsigned long) uid); return strdup(s); } static char * mode_to_s(mode_t mode) { char s[100]; snprintf(s, 100, "%lu", (unsigned long) mode); return strdup(s); } static char * time_to_s(time_t t) { char s[100]; snprintf(s, 100, "%ld", (long) t); return strdup(s); } GList * add_header(GList *headers, HTTP_HEADER *h) { headers = strip_header(headers, h->key); headers = g_list_append(headers, h); return headers; } GList * stat_to_headers(GList *headers, struct stat *st) { headers = add_header(headers, gid_header(st->st_gid)); headers = add_header(headers, uid_header(st->st_uid)); headers = add_header(headers, mode_header(st->st_mode)); headers = add_header(headers, ctime_header(st->st_ctime)); headers = add_header(headers, mtime_header(st->st_mtime)); headers = add_header(headers, rdev_header(st->st_rdev)); return headers; } int headers_to_stat(GList *headers, struct stat *stbuf) { GList *head = NULL, *next = NULL; head = g_list_first(headers); while(head != NULL) { next = head->next; HTTP_HEADER *header = head->data; // TODO: clean this up. if(strcmp(header->key, "x-amz-meta-uid") == 0) stbuf->st_uid = get_uid(header->value); else if(strcmp(header->key, "x-amz-meta-gid") == 0) stbuf->st_gid = get_gid(header->value); else if(strcmp(header->key, "x-amz-meta-ctime") == 0) stbuf->st_ctime = get_ctime(header->value); else if(strcmp(header->key, "x-amz-meta-mtime") == 0) stbuf->st_mtime = get_mtime(header->value); else if(strcmp(header->key, "x-amz-meta-rdev") == 0) stbuf->st_rdev = get_rdev(header->value); else if(strcmp(header->key, "Last-Modified") == 0 && stbuf->st_mtime == 0) stbuf->st_mtime = get_mtime(header->value); else if(strcmp(header->key, "x-amz-meta-mode") == 0) stbuf->st_mode = get_mode(header->value); else if(strcmp(header->key, "Content-Length") == 0) stbuf->st_size = get_size(header->value); else if(strcmp(header->key, "Content-Type") == 0) if(strstr(header->value, "x-directory")) stbuf->st_mode |= S_IFDIR; head = next; } return 0; } char char_to_hex(char c) { static char hex[] = "0123456789abcdef"; return hex[c & 15]; } static int cmpstringp(const void *p1, const void *p2) { return strcmp(*(char **) p1, *(char **) p2); } static FILE_PART * create_part(int part_num, char *upload_id) { FILE_PART *fp = g_new0(FILE_PART, 1); fp->part_num = part_num; fp->upload_id = strdup(upload_id); fp->path = strdup("/tmp/stormfs.XXXXXX"); if((fp->fd = mkstemp(fp->path)) == -1) { perror("mkstemp"); free(fp->path); free(fp); return NULL; } return fp; } static void free_part(FILE_PART *fp) { free(fp->path); free(fp->etag); free(fp->upload_id); free(fp); } void free_parts(GList *parts) { g_list_foreach(parts, (GFunc) free_part, NULL); g_list_free(parts); } void free_header(HTTP_HEADER *h) { free(h->key); free(h->value); free(h); } void free_headers(GList *headers) { g_list_foreach(headers, (GFunc) free_header, NULL); g_list_free(headers); } char * url_encode(char *s) { char *p = s; char *buf = g_malloc((strlen(s) * 3) + 1); char *pbuf = buf; // NOTE: '/' will not be url encoded while(*p) { if(isalnum(*p) || *p == '/' || *p == '-' || *p == '_' || *p == '.' || *p == '~') *pbuf++ = *p; else if(*p == ' ') *pbuf++ = '+'; else *pbuf++ = '%', *pbuf++ = char_to_hex(*p >> 4), *pbuf++ = char_to_hex(*p & 15); p++; } *pbuf = '\0'; return buf; } static char * get_resource(const char *path) { int path_len = strlen(path); int bucket_len = strlen(curl.bucket); char *resource = g_malloc0(sizeof(char) * path_len + bucket_len + 2); strncpy(resource, "/", 1); strncat(resource, curl.bucket, bucket_len); strncat(resource, path, path_len); return resource; } HTTP_HEADER * acl_header(const char *acl) { HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("x-amz-acl"); h->value = strdup(acl); return h; } HTTP_HEADER * content_header(const char *type) { HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("Content-Type"); if(type == NULL) h->value = strdup(DEFAULT_MIME_TYPE); else h->value = strdup(type); return h; } HTTP_HEADER * copy_meta_header() { HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("x-amz-metadata-directive"); h->value = strdup("COPY"); return h; } HTTP_HEADER * copy_source_header(const char *path) { HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("x-amz-copy-source"); h->value = get_resource(path); return h; } HTTP_HEADER * copy_source_range_header(off_t first, off_t last) { HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("x-amz-copy-source-range"); if(asprintf(&h->value, "bytes=%jd-%jd", (intmax_t) first, (intmax_t) last) == -1) fprintf(stderr, "unable to allocate memory\n"); return h; } HTTP_HEADER * ctime_header(time_t t) { char *s = time_to_s(t); HTTP_HEADER *h = g_new0(HTTP_HEADER, 1); h->key = strdup("x-amz-meta-ctime"