目前我們有兩種路徑使用Ceph的塊存儲:
- 利用QEMU/KVM通過librbd與 Ceph 塊設備交互,主要爲虛擬機提供塊存儲設備,如下圖所示;
- 利用kernel module與Host kernel交互,主要爲物理機提供塊設備支持。
Librbd 是Ceph提供的塊存儲接口的抽象,它提供C/C++、Python等多種接口。對於C++,最主要的兩個類就是RBD
和 Image
。 RBD
主要負責創建、刪除、克隆映像等操作,而Image
類負責映像的讀寫等操作。
準備工作
對於任何客戶端應用,都需要首先連接到一個運行良好的Ceph集羣。
獲取集羣句柄
//聲明Rados對象,並初始化
librados::Rados rados;
ret = rados.init("admin"); // just use the client.admin keyring
if (ret < 0) { // let's handle any error that might have come back
std::cerr << "couldn't initialize rados! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "we just set up a rados cluster object" << std::endl;
}
//獲取配置文件信息: /etc/ceph/ceph.conf
// 1. 根據命令行參數
/*
ret = rados.conf_parse_argv(argc, argv);
if (ret < 0) {
// This really can't happen, but we need to check to be a good citizen.
std::cerr << "failed to parse config options! error " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "we just parsed our config options" << std::endl;
// We also want to apply the config file if the user specified
// one, and conf_parse_argv won't do that for us.
for (int i = 0; i < argc; ++i) {
if ((strcmp(argv[i], "-c") == 0) || (strcmp(argv[i], "--conf") == 0)) {
ret = rados.conf_read_file(argv[i+1]);
if (ret < 0) {
// This could fail if the config file is malformed, but it'd be hard.
std::cerr << "failed to parse config file " << argv[i+1]
<< "! error" << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
}
break;
}
}
}
*/
// 2. 程序裏面指定
ret = rados.conf_read_file("/etc/ceph/ceph.conf");
if (ret < 0) {
// This could fail if the config file is malformed, but it'd be hard.
std::cerr << "failed to parse config file! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
連接集羣
ret = rados.connect();
if (ret < 0) {
std::cerr << "couldn't connect to cluster! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "we just connected to the rados cluster" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
創建I/O上下文環境
如果沒有存儲池,需要先新建一個存儲池。
新建存儲池
const char *pool_name = "gnar";
ret = rados.pool_create(pool_name);
if (ret < 0) {
std::cerr << "couldn't create pool! error " << ret << std::endl;
ret = EXIT_FAILURE;
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "we just created a new pool named " << pool_name << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
新建I/O上下文環境
librados::IoCtx io_ctx; //I/O上下文
const char *pool_name = "gnar";
ret = rados.ioctx_create(pool_name, io_ctx);
if (ret < 0) {
std::cerr << "couldn't setup ioctx! err " << ret << std::endl;
ret = EXIT_FAILURE;
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "we just created an ioctx for our pool" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
若干RBD映像API
聲明RBD對象,創建rbd映像
librbd::RBD rbd;
const char *image_name = "rumboo";
uint64_t init_size = (uint64_t) 200 * 1024 * 1024; //映像初始化大小200MB
uint64_t features = 1; //影響feature個數
int order = 22; //默認值爲22, 即4MB (1 << 22)
ret = rbd.create2(io_ctx, image_name, init_size, features, &order);
if (ret < 0) {
std::cerr << "couldn't create rbd image! err " << ret << std::endl;
ret = EXIT_FAILURE;
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "We just created an rbd image" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
打開rbd映像
librbd::RBD rbd;
const char *image_name = "rumboo";
librbd::Image image;
ret = rbd.open(io_ctx, image, image_name);
if (ret < 0) {
std::cerr << "couldn't open rbd image! err " << ret << std::endl;
ret = EXIT_FAILURE;
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "We just opened an rbd image" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
查看映像大小
uint64_t size = 0;
ret = image.size(&size);
if (ret < 0) {
std::cerr << "couldn't get image size! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "The size of the image is " << size << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
調整映像大小
size = (uint64_t) 500 * 1024 * 1024; //調整映像大小500MB
ret = image.resize(size);
if (ret < 0) {
std::cerr << "couldn't change the size of the image! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "We just change the size of the image" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
查看映像ID
std::string id;
ret = image.get_id(&id);
if (ret < 0) {
std::cerr << "couldn't get image ID! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "The ID of the image is " << id << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
查看映像features
features = 0;
ret = image.features(&features);
if (ret < 0) {
std::cerr << "couldn't get image features! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "The features of the image are " << features << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
查看映像狀態信息
librbd::image_info_t info;
ret = image.stat(info, sizeof(info));
if (ret < 0) {
std::cerr << "couldn't get image stat_info! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "info.size is " << info.size << std::endl;
std::cout << "info.obj_size is " << info.obj_size << std::endl;
std::cout << "info.num_objs is " << info.num_objs << std::endl;
std::cout << "info.order is " << info.order << std::endl;
std::cout << "info.block_name_prefix is " << info.block_name_prefix << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
查看存儲池ID
std::cout << "data pool id is " << image.get_data_pool_id() << std::endl;
- 1
查看block_name_prefix
std::cout << "block name prefix is " << image.get_block_name_prefix() << std::endl;
- 1
查看flags
uint64_t flags = 0;
ret = image.get_flags(&flags);
if (ret < 0) {
std::cerr << "couldn't get image flags! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "image flags is " << flags << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
查看條帶化參數
std::cout << "image stripe unit is " << image.get_stripe_unit() << std::endl;
std::cout << "image stripe count is " << image.get_stripe_count() << std::endl;
- 1
- 2
RBD映像數據讀寫
數據讀寫 – synchronous
uint64_t ofs_w = (uint64_t) 0; //讀寫偏移量
uint64_t ofs_r = (uint64_t) 0;
size_t len_w = 100; //讀寫長度
size_t len_r = 100;
ceph::bufferlist bl_w; //讀寫bufferlist
ceph::bufferlist bl_r;
const char *fn_i = "input"; //讀寫文件名
const char *fn_o = "output";
std::string error;
ret = bl_r.read_file(fn_i, &error);
std::cout << "read file ret = " << ret << std::endl;
if (ret < 0) {
std::cerr << "couldn't read file! err " << ret << std::endl;
ret = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "We just read a file" << std::endl;
}
ssize_t ret_w = image.write2(ofs_w, len_w, bl_r, 0);
ssize_t ret_r = image.read2(ofs_r, len_r, bl_w, 0);
ret = bl_w.write_file(fn_o, 0644);
std::cout << "write file ret = " << ret << std::endl;
if (ret < 0) {
std::cerr << "couldn't write file! err " << ret << std::endl;
ret = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "We just wrote a file" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
數據讀寫 – asynchronous
std::string data = "foo";
uint64_t ofs_aiow = (uint64_t) 100; //讀寫偏移量
uint64_t ofs_aior = (uint64_t) 100;
size_t len_aiow = 600; //讀寫長度
size_t len_aior = 600;
ceph::bufferlist bl_aiow; //讀寫bufferlist
ceph::bufferlist bl_aior;
librbd::RBD::AioCompletion *write_completion = new librbd::RBD::AioCompletion(
NULL, (librbd::callback_t) simple_write_cb); //讀寫AioCompletion
librbd::RBD::AioCompletion *read_completion = new librbd::RBD::AioCompletion(
NULL, (librbd::callback_t) simple_read_cb);
for (int i = 0; i < 200; ++i) {
bl_aior.append(data);
}
std::cout << bl_aior.to_str() << std::endl;
ret = image.aio_write2(ofs_aiow, len_aiow, bl_aior, write_completion, 0);
if (ret < 0) {
std::cerr << "couldn't start write! error " << ret << std::endl;
ret = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
}
write_completion->wait_for_complete(); //等待寫完成
ret_w = write_completion->get_return_value();
if (ret_w < 0) {
std::cerr << "couldn't write! error " << ret << std::endl;
ret_w = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "we just write data successfully, return value is " << ret_w << std::endl;
}
ret = image.aio_read2(ofs_aior, len_aior, bl_aiow, read_completion, 0);
if (ret < 0) {
std::cerr << "couldn't start read! error " << ret << std::endl;
ret = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
}
read_completion->wait_for_complete(); //等待讀完成
ret_r = read_completion->get_return_value();
if (ret_r < 0) {
std::cerr << "couldn't read! error " << ret << std::endl;
ret_r = EXIT_FAILURE;
image.close(); //關閉rbd映像
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_FAILURE;
} else {
std::cout << "we just read data successfully, return value is " << ret_r << std::endl;
}
std::cout << bl_aiow.to_str() << std::endl;
write_completion->release();
read_completion->release();
void simple_write_cb(librbd::completion_t cb, void *arg) {
std::cout << "write completion cb called!" << std::endl;
}
//簡單的回調函數,用於librbd::RBD::AioCompletion
void simple_read_cb(librbd::completion_t cb, void *arg) {
std::cout << "read completion cb called!" << std::endl;
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
收尾工作
在最後,一定不能忘記關閉rbd映像、I/O上下文,斷開集羣連接。
ret = image.close(); //關閉rbd映像
if (ret < 0) {
std::cerr << "couldn't close rbd image! err " << ret << std::endl;
ret = EXIT_FAILURE;
return EXIT_FAILURE;
} else {
std::cout << "we just closed an rbd image" << std::endl;
}
io_ctx.close(); //關閉I/O上下文
rados.shutdown(); //斷開集羣連接
return EXIT_SUCCESS;
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
<link href="http://csdnimg.cn/release/phoenix/production/markdown_views-0bc64ada25.css" rel="stylesheet">
</div>