Ceph radosgw where to store data?

Posted on | In | |

基于ceph jewel 10.2.5版本

参考文档：http://docs.ceph.com/docs/master/radosgw/layout/

测试数据

创建一个user：ictfox
创建这个user下的一个bucket：bruins
往bucket里写一个object：hello.txt

所以这里的测试数据有：

  • user info
  • bucket info
  • object info

注：本文描述了查询radosgw数据和元数据的方法，没有深入去解析一些命令输出；若有错误，欢迎指正 ;)

RadosGW的pools

查看radosgw对应的所有pools，其所有的数据也都存在这些pools里。

1
2
3
4
5
6
7
8
9
10
11
12
# rados lspools
.rgw.root
default.rgw.control
default.rgw.data.root
default.rgw.gc
default.rgw.log
default.rgw.users.uid
default.rgw.users.keys
default.rgw.buckets.index
default.rgw.buckets.data
default.rgw.users.swift
default.rgw.users.email

很多pools的用途通过名字就能明确，若要明确pools的用途，请参考代码: rgw/rgw_rados.cc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
static string default_storage_pool_suffix = "rgw.buckets.data";
static string default_bucket_index_pool_suffix = "rgw.buckets.index";
static string default_storage_extra_pool_suffix = "rgw.buckets.non-ec";

static string zone_info_oid_prefix = "zone_info.";
static string zone_names_oid_prefix = "zone_names.";
static string region_info_oid_prefix = "region_info.";
static string zone_group_info_oid_prefix = "zonegroup_info.";
static string realm_names_oid_prefix = "realms_names.";
static string realm_info_oid_prefix = "realms.";
static string default_region_info_oid = "default.region";
static string default_zone_group_info_oid = "default.zonegroup";
static string period_info_oid_prefix = "periods.";
static string period_latest_epoch_info_oid = ".latest_epoch";
static string region_map_oid = "region_map";
static string zonegroup_map_oid = "zonegroup_map";
static string log_lock_name = "rgw_log_lock";
static string default_realm_info_oid = "default.realm";
const string default_zonegroup_name = "default";
const string default_zone_name = "default";
static string zonegroup_names_oid_prefix = "zonegroups_names.";

static string RGW_DEFAULT_ZONE_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_ZONEGROUP_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_REALM_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_PERIOD_ROOT_POOL = "rgw.root";
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
int RGWZoneParams::fix_pool_names()
{

    list<string> zones;
    int r = store->list_zones(zones);
    if (r < 0) {
        ldout(cct, 10) << "WARNING: store->list_zones() returned r=" << r << dendl;
    }

    set<string> pool_names;
    r = get_zones_pool_names_set(cct, store, zones, id, pool_names);
    if (r < 0) {
        ldout(cct, 0) << "Error: get_zones_pool_names" << r << dendl;
        return r;
    }

    domain_root = fix_zone_pool_name(pool_names, name, ".rgw.data.root", domain_root.name);
    if (!metadata_heap.name.empty()) {
        metadata_heap = fix_zone_pool_name(pool_names, name, ".rgw.meta", metadata_heap.name);
    }
    control_pool = fix_zone_pool_name(pool_names, name, ".rgw.control", control_pool.name);
    gc_pool = fix_zone_pool_name(pool_names, name,".rgw.gc", gc_pool.name);
    log_pool = fix_zone_pool_name(pool_names, name, ".rgw.log", log_pool.name);
    intent_log_pool = fix_zone_pool_name(pool_names, name, ".rgw.intent-log", intent_log_pool.name);
    usage_log_pool = fix_zone_pool_name(pool_names, name, ".rgw.usage", usage_log_pool.name);
    user_keys_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.keys", user_keys_pool.name);
    user_email_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.email", user_email_pool.name);
    user_swift_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.swift", user_swift_pool.name);
    user_uid_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.uid", user_uid_pool.name);

    for(auto& iter : placement_pools) {
        iter.second.index_pool = fix_zone_pool_name(pool_names, name, "." + default_bucket_index_pool_suffix,
                                 iter.second.index_pool);
        iter.second.data_pool = fix_zone_pool_name(pool_names, name, "." + default_storage_pool_suffix,
                                iter.second.data_pool);
        iter.second.data_extra_pool= fix_zone_pool_name(pool_names, name, "." + default_storage_extra_pool_suffix,
                                     iter.second.data_extra_pool);
    }

    return 0;
}

radosgw的元数据信息

通过命令radosgw-admin查看原数据信息，包括bucket信息和user信息。

user信息

radosgw-admin user info命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# radosgw-admin user info --uid=ictfox
{
    "user_id": "ictfox",
    "display_name": "mike",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "ictfox",
            "access_key": "IYZ800MDM7VF3EDLISC7",
            "secret_key": "05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "temp_url_keys": []
}

radosgw-admin metadata命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
# radosgw-admin metadata list user
[
    "ictfox"
]
# radosgw-admin metadata get user:ictfox
{
    "key": "user:ictfox",
    "ver": {
        "tag": "_laJXG6YTAtNd-Dzt78okwU5",
        "ver": 1
    },
    "mtime": "2017-03-14 08:49:04.735466Z",
    "data": {
        "user_id": "ictfox",
        "display_name": "mike",
        "email": "",
        "suspended": 0,
        "max_buckets": 1000,
        "auid": 0,
        "subusers": [],
        "keys": [
            {
                "user": "ictfox",
                "access_key": "IYZ800MDM7VF3EDLISC7",
                "secret_key": "05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS"
            }
        ],
        "swift_keys": [],
        "caps": [],
        "op_mask": "read, write, delete",
        "default_placement": "",
        "placement_tags": [],
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "temp_url_keys": [],
        "attrs": [
            {
                "key": "user.rgw.idtag",
                "val": ""
            },
            {
                "key": "user.rgw.manifest",
                "val": ""
            }
        ]
    }
}

bucket信息

radosgw-admin bucket stats命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# radosgw-admin bucket stats --bucket=bruins
{
    "bucket": "bruins",
    "pool": "default.rgw.buckets.data",
    "index_pool": "default.rgw.buckets.index",
    "id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "owner": "ictfox",
    "ver": "0#11",
    "master_ver": "0#0",
    "mtime": "2017-03-15 00:46:37.189715",
    "max_marker": "0#",
    "usage": {
        "rgw.main": {
            "size_kb": 1,
            "size_kb_actual": 4,
            "num_objects": 1
        }
    },
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    }
}

radosgw-admin metadata命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# radosgw-admin metadata list bucket
[
    "bruins",
]
# radosgw-admin metadata get bucket:bruins
{
    "key": "bucket:bruins",
    "ver": {
        "tag": "_osgTpZcD-jCFgcwdi3yvHmH",
        "ver": 1
    },
    "mtime": "2017-03-15 04:46:37.193584Z",
    "data": {
        "bucket": {
            "name": "bruins",
            "pool": "default.rgw.buckets.data",
            "data_extra_pool": "default.rgw.buckets.non-ec",
            "index_pool": "default.rgw.buckets.index",
            "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
            "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
            "tenant": ""
        },
        "owner": "ictfox",
        "creation_time": "0.000000",
        "linked": "true",
        "has_bucket_info": "false"
    }
}

# radosgw-admin metadata list bucket.instance
[
    "bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
]

# radosgw-admin metadata get bucket.instance:bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
{
    "key": "bucket.instance:bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "ver": {
        "tag": "_n2XczjnP1PYx3Hg88iruBbm",
        "ver": 1
    },
    "mtime": "2017-03-15 04:46:37.189715Z",
    "data": {
        "bucket_info": {
            "bucket": {
                "name": "bruins",
                "pool": "default.rgw.buckets.data",
                "data_extra_pool": "default.rgw.buckets.non-ec",
                "index_pool": "default.rgw.buckets.index",
                "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
                "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
                "tenant": ""
            },
            "creation_time": "0.000000",
            "owner": "ictfox",
            "flags": 0,
            "zonegroup": "d40f14ea-4700-43e7-a92e-fa232f28b590",
            "placement_rule": "default-placement",
            "has_instance_obj": "true",
            "quota": {
                "enabled": false,
                "max_size_kb": -1,
                "max_objects": -1
            },
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": ""
        },
        "attrs": [
            {
                "key": "user.rgw.acl",
                "val": "AgJ7AAAAAwISAAAABgAAAGljdGZveAQAAABtaWtlAwNdAAAAAQEAAAAGAAAAaWN0Zm94DwAAAAEAAAAGAAAAaWN0Zm94BAMyAAAAAgIEAAAAAAAAAAYAAABpY3Rmb3gAAAAAAAAAAAICBAAAAA8AAAAEAAAAbWlrZQAAAAAAAAAA"
            },
            {
                "key": "user.rgw.idtag",
                "val": ""
            },
            {
                "key": "user.rgw.manifest",
                "val": ""
            }
        ]
    }
}

rgw的user uid信息保存格式

每个user在该pool里都有两个objects

1
2
3
# rados ls -p default.rgw.users.uid
ictfox.buckets
ictfox

查看两个objects的omap和存储数据：

1
2
3
4
# rados -p default.rgw.users.uid listomapvals ictfox
# rados -p default.rgw.users.uid get ictfox file
# cat file
ictfox   .IYZ800MDM7VF3EDLISC7(05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHSmikeictfoxIYZ800MDM7VF3EDLISC7HIYZ800MDM7VF3EDLISC7(05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS���������������������������������
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# rados -p default.rgw.users.uid listomapkeys ictfox.buckets
bruins
# rados -p default.rgw.users.uid listomapvals ictfox.buckets
bruins
value (246 bytes) :
00000000  07 05 f0 00 00 00 00 00  00 00 0c 00 00 00 00 00  |................|
00000010  00 00 2d c7 c8 58 01 00  00 00 00 00 00 00 07 03  |..-..X..........|
00000020  c1 00 00 00 06 00 00 00  62 72 75 69 6e 73 18 00  |........bruins..|
00000030  00 00 64 65 66 61 75 6c  74 2e 72 67 77 2e 62 75  |..default.rgw.bu|
00000040  63 6b 65 74 73 2e 64 61  74 61 2c 00 00 00 30 37  |ckets.data,...07|
00000050  30 65 63 31 32 39 2d 32  65 38 34 2d 34 37 66 65  |0ec129-2e84-47fe|
00000060  2d 62 33 32 61 2d 62 30  62 66 39 39 31 34 61 35  |-b32a-b0bf9914a5|
00000070  31 66 2e 31 34 32 37 34  2e 31 2c 00 00 00 30 37  |1f.14274.1,...07|
00000080  30 65 63 31 32 39 2d 32  65 38 34 2d 34 37 66 65  |0ec129-2e84-47fe|
00000090  2d 62 33 32 61 2d 62 30  62 66 39 39 31 34 61 35  |-b32a-b0bf9914a5|
000000a0  31 66 2e 31 34 32 37 34  2e 31 19 00 00 00 64 65  |1f.14274.1....de|
000000b0  66 61 75 6c 74 2e 72 67  77 2e 62 75 63 6b 65 74  |fault.rgw.bucket|
000000c0  73 2e 69 6e 64 65 78 1a  00 00 00 64 65 66 61 75  |s.index....defau|
000000d0  6c 74 2e 72 67 77 2e 62  75 63 6b 65 74 73 2e 6e  |lt.rgw.buckets.n|
000000e0  6f 6e 2d 65 63 00 10 00  00 00 00 00 00 01 2d c7  |on-ec.........-.|
000000f0  c8 58 1a 7f b4 0b                                 |.X....|
000000f6

rgw的user keys信息保存格式

每个user在该pool里都有一个以<access_key>的值命名的object

1
2
# rados ls -p default.rgw.users.keys
IYZ800MDM7VF3EDLISC7

查看该object的omap和存储数据：

1
2
3
4
# rados -p default.rgw.users.keys listomapvals IYZ800MDM7VF3EDLISC7
# rados -p default.rgw.users.keys get IYZ800MDM7VF3EDLISC7 file
# cat file
ictfox

rgw的user bucket原数据信息保存格式

格式：.dir.<bucket id>

1
2
# rados -p default.rgw.buckets.index ls
.dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

查看该object的omap和存储数据：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# rados -p default.rgw.buckets.index get .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 file
# cat file
#
# rados -p default.rgw.buckets.index listomapkeys .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
hello.txt
# rados -p default.rgw.buckets.index listomapvals .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
hello.txt
value (196 bytes) :
00000000  08 03 be 00 00 00 09 00  00 00 68 65 6c 6c 6f 2e  |..........hello.|
00000010  74 78 74 06 00 00 00 00  00 00 00 01 04 03 53 00  |txt...........S.|
00000020  00 00 01 0c 00 00 00 00  00 00 00 21 e8 c8 58 42  |...........!..XB|
00000030  2f 40 29 20 00 00 00 65  64 30 37 36 32 38 37 35  |/@) ...ed0762875|
00000040  33 32 65 38 36 33 36 35  65 38 34 31 65 39 32 62  |32e86365e841e92b|
00000050  66 63 35 30 64 38 63 06  00 00 00 69 63 74 66 6f  |fc50d8c....ictfo|
00000060  78 04 00 00 00 6d 69 6b  65 00 00 00 00 0c 00 00  |x....mike.......|
00000070  00 00 00 00 00 00 00 00  00 00 00 00 00 01 01 02  |................|
00000080  00 00 00 09 06 0a 2c 00  00 00 30 37 30 65 63 31  |......,...070ec1|
00000090  32 39 2d 32 65 38 34 2d  34 37 66 65 2d 62 33 32  |29-2e84-47fe-b32|
000000a0  61 2d 62 30 62 66 39 39  31 34 61 35 31 66 2e 31  |a-b0bf9914a51f.1|
000000b0  34 32 34 35 2e 35 00 00  00 00 00 00 00 00 00 00  |4245.5..........|
000000c0  00 00 00 00                                       |....|
000000c4

rgw的user bucket中对象数据保存格式

每个object没有omap信息

1
2
3
4
5
6
7
# rados -p default.rgw.buckets.data ls
070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt
# rados -p default.rgw.buckets.data get 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt file
# cat file
Hello World!

# rados -p default.rgw.buckets.data listomapvals 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt

查看object的xattr信息

1
2
3
4
5
6
7
8
9
10
# rados -p default.rgw.buckets.data listxattr 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt
user.rgw.acl
user.rgw.cache_control
user.rgw.etag
user.rgw.idtag
user.rgw.manifest
user.rgw.pg_ver
user.rgw.source_zone
# rados -p default.rgw.buckets.data getxattr 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt user.rgw.acl
{ictfoxmike]ictfoxictfox2ictfoxmike

rgw的user bucket的metadata信息

1
2
3
# rados -p default.rgw.data.root ls
bruins
.bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

每个object没有omap信息

1
2
# rados -p default.rgw.data.root listomapvals bruins
# rados -p default.rgw.data.root listomapvals .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

查看object的存储数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# rados -p default.rgw.data.root get bruins file
# cat file

�bruinsdefault.rgw.buckets.data,070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1,070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1default.rgw.buckets.indexdefault.rgw.buckets.non-ecictfoxictfox

# rados -p default.rgw.data.root get .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 file
# ceph-dencoder type RGWBucketInfo import file decode dump_json
{
    "bucket": {
        "name": "bruins",
        "pool": "default.rgw.buckets.data",
        "data_extra_pool": "default.rgw.buckets.non-ec",
        "index_pool": "default.rgw.buckets.index",
        "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
        "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
        "tenant": ""
    },
    "creation_time": "0.000000",
    "owner": "ictfox",
    "flags": 0,
    "zonegroup": "d40f14ea-4700-43e7-a92e-fa232f28b590",
    "placement_rule": "default-placement",
    "has_instance_obj": "true",
    "quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "num_shards": 0,
    "bi_shard_hash_type": 0,
    "requester_pays": "false",
    "has_website": "false",
    "swift_versioning": "false",
    "swift_ver_location": ""
}

查看object的xattr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
# rados -p default.rgw.data.root listxattr bruins
ceph.objclass.version
# rados -p default.rgw.data.root getxattr bruins ceph.objclass.version
$_osgTpZcD-jCFgcwdi3yvHmH

# rados -p default.rgw.data.root listxattr .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
ceph.objclass.version
user.rgw.acl
# rados -p default.rgw.data.root getxattr .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 user.rgw.acl > file
# ceph-dencoder type RGWAccessControlPolicy import file decode dump_json
{
    "acl": {
        "acl_user_map": [
            {
                "user": "ictfox",
                "acl": 15
            }
        ],
        "acl_group_map": [],
        "grant_map": [
            {
                "id": "ictfox",
                "grant": {
                    "type": {
                        "type": 0
                    },
                    "id": "ictfox",
                    "email": "",
                    "permission": {
                        "flags": 15
                    },
                    "name": "mike",
                    "group": 0
                }
            }
        ]
    },
    "owner": {
        "id": "ictfox",
        "display_name": "mike"
    }
}
支持原创