[关闭]
@zwei 2016-04-22T10:46:13.000000Z 字数 23276 阅读 5987

openstack liberty版本使用 ceph 作为存储后端

ceph


  1. glance 使用rbd driver 作为后端存储
  2. cinder 使用rbd driver 作为后端存储
  3. nova 使用images_type 为rbd

glance driver 分析流程和总结

  1. glance image-create 分析
  2. glance image-delete 分析
  1. #相关的配置选项
  2. # RADOS images will be chunked into objects of this size (in
  3. # megabytes). For best performance, this should be a power of two.
  4. # (integer value)
  5. #rbd_store_chunk_size = 8
  6. rbd_store_chunk_size = 8
  7. #指定rbd image 的 块大小为8M
  8. #对应着 order 23 (8192 kB objects) 8M = 2 ^ 23
  9. # RADOS pool in which images are stored. (string value)
  10. #rbd_store_pool = images
  11. rbd_store_pool = rbd
  12. # 指定 pool 的名称
  13. # RADOS user to authenticate as (only applicable if using Cephx. If
  14. # <None>, a default will be chosen based on the client. section in
  15. # rbd_store_ceph_conf) (string value)
  16. #rbd_store_user = <None>
  17. rbd_store_user = admin
  18. # 指定 链接ceph 的用户
  19. # Ceph configuration file path. If <None>, librados will locate the
  20. # default config. If using cephx authentication, this file should
  21. # include a reference to the right keyring in a client.<USER> section
  22. # (string value)
  23. #rbd_store_ceph_conf = /etc/ceph/ceph.conf
  24. # 指定ceph的 配置文件信息
  25. # ceph.conf 中必须指定 keyring = /opt/ceph1/ceph.client.admin.keyring
  26. # 就是admin 用户的密码文件
  27. # Timeout value (in seconds) used when connecting to ceph cluster. If
  28. # value <= 0, no timeout is set and default librados value is used.
  29. # (integer value)
  30. #rados_connect_timeout = 0
  31. # 设置 链接 rados 的超时时间

1, glance image-create 上传一个镜像

  1. ##代码分析
  2. def add(self, image_id, image_file, image_size, context=None):
  3. """
  4. Stores an image file with supplied identifier to the backend
  5. storage system and returns a tuple containing information
  6. about the stored image.
  7. :param image_id: The opaque image identifier
  8. :param image_file: The image data to write, as a file-like object
  9. :param image_size: The size of the image data to write, in bytes
  10. :retval tuple of URL in backing store, bytes written, checksum
  11. and a dictionary with storage system specific information
  12. :raises `glance_store.exceptions.Duplicate` if the image already
  13. existed
  14. """
  15. checksum = hashlib.md5()
  16. image_name = str(image_id)
  17. with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
  18. fsid = None
  19. if hasattr(conn, 'get_fsid'):
  20. fsid = conn.get_fsid()
  21. with conn.open_ioctx(self.pool) as ioctx:
  22. order = int(math.log(self.WRITE_CHUNKSIZE, 2))
  23. LOG.debug('creating image %s with order %d and size %d',
  24. image_name, order, image_size)
  25. if image_size == 0:
  26. LOG.warning(_("since image size is zero we will be doing "
  27. "resize-before-write for each chunk which "
  28. "will be considerably slower than normal"))
  29. try:
  30. loc = self._create_image(fsid, conn, ioctx, image_name,
  31. image_size, order)
  32. except rbd.ImageExists:
  33. msg = _('RBD image %s already exists') % image_id
  34. raise exceptions.Duplicate(message=msg)
  35. try:
  36. with rbd.Image(ioctx, image_name) as image:
  37. bytes_written = 0
  38. offset = 0
  39. chunks = utils.chunkreadable(image_file,
  40. self.WRITE_CHUNKSIZE)
  41. for chunk in chunks:
  42. # If the image size provided is zero we need to do
  43. # a resize for the amount we are writing. This will
  44. # be slower so setting a higher chunk size may
  45. # speed things up a bit.
  46. if image_size == 0:
  47. chunk_length = len(chunk)
  48. length = offset + chunk_length
  49. bytes_written += chunk_length
  50. LOG.debug(_("resizing image to %s KiB") %
  51. (length / units.Ki))
  52. image.resize(length)
  53. LOG.debug(_("writing chunk at offset %s") %
  54. (offset))
  55. offset += image.write(chunk, offset)
  56. checksum.update(chunk)
  57. if loc.snapshot:
  58. image.create_snap(loc.snapshot)
  59. image.protect_snap(loc.snapshot)
  60. except Exception as exc:
  61. log_msg = (_LE("Failed to store image %(img_name)s "
  62. "Store Exception %(store_exc)s") %
  63. {'img_name': image_name,
  64. 'store_exc': exc})
  65. LOG.error(log_msg)
  66. # Delete image if one was created
  67. try:
  68. target_pool = loc.pool or self.pool
  69. self._delete_image(target_pool, loc.image,
  70. loc.snapshot)
  71. except exceptions.NotFound:
  72. pass
  73. raise exc
  74. # Make sure we send back the image size whether provided or inferred.
  75. if image_size == 0:
  76. image_size = bytes_written
  77. return (loc.get_uri(), image_size, checksum.hexdigest(), {})
  78. 功能总结:
  79. 使用 glance v2 craete image 的时候在rbd 中创建的流程
  80. 1. loc = self._create_image() 先创建一个rbd image 指定order size image_name
  81. 2. offset += image.write(chunk, offset) 在把数据写入rbd image
  82. 3. image.create_snap(loc.snapshot) 然后在 给这个image 创建一个 snap名称的快照
  83. 4. 最后返回的location 一个image 快照的 rbd 地址

glance image-delete 分析

  1. ## 代码分析
  2. @capabilities.check
  3. def delete(self, location, context=None):
  4. """
  5. Takes a `glance_store.location.Location` object that indicates
  6. where to find the image file to delete.
  7. :location `glance_store.location.Location` object, supplied
  8. from glance_store.location.get_location_from_uri()
  9. :raises NotFound if image does not exist;
  10. InUseByStore if image is in use or snapshot unprotect failed
  11. """
  12. loc = location.store_location
  13. target_pool = loc.pool or self.pool
  14. self._delete_image(target_pool, loc.image, loc.snapshot)
  15. def _delete_image(self, target_pool, image_name,
  16. snapshot_name=None, context=None):
  17. """
  18. Delete RBD image and snapshot.
  19. :param image_name Image's name
  20. :param snapshot_name Image snapshot's name
  21. :raises NotFound if image does not exist;
  22. InUseByStore if image is in use or snapshot unprotect failed
  23. """
  24. with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn:
  25. with conn.open_ioctx(target_pool) as ioctx:
  26. try:
  27. # First remove snapshot.
  28. if snapshot_name is not None:
  29. with rbd.Image(ioctx, image_name) as image:
  30. try:
  31. image.unprotect_snap(snapshot_name)
  32. except rbd.ImageBusy:
  33. log_msg = _("snapshot %(image)s@%(snap)s "
  34. "could not be unprotected because "
  35. "it is in use")
  36. LOG.debug(log_msg %
  37. {'image': image_name,
  38. 'snap': snapshot_name})
  39. raise exceptions.InUseByStore()
  40. image.remove_snap(snapshot_name)
  41. # Then delete image.
  42. rbd.RBD().remove(ioctx, image_name)
  43. except rbd.ImageNotFound:
  44. msg = _("RBD image %s does not exist") % image_name
  45. raise exceptions.NotFound(message=msg)
  46. except rbd.ImageBusy:
  47. log_msg = _("image %s could not be removed "
  48. "because it is in use")
  49. LOG.debug(log_msg % image_name)
  50. raise exceptions.InUseByStore()
  51. 总结:
  52. image-delete 命令具体如下步骤:
  53. 1 先对snap 接触保护
  54. 2 在对snap 进行删除
  55. 3 在对image 进行删除
  56. 4 如果这个snap在使用中则删除不掉eg 基于这个image 创建vmvolume

cinder driver 的分析流程和总结

  1. cinder-volume 服务使用rbd
  2. cinder-backup 服务使用rbd

cinder-volume 分析和总结

  1. #### ceph 配置选项
  2. # The name of ceph cluster (string value)
  3. #rbd_cluster_name = ceph
  4. rbd_cluster_name = ceph
  5. # The RADOS pool where rbd volumes are stored (string value)
  6. #rbd_pool = rbd
  7. rbd_pool = cinder
  8. # The RADOS client name for accessing rbd volumes - only set when using cephx
  9. # authentication (string value)
  10. #rbd_user = <None>
  11. rbd_user = admin
  12. # Path to the ceph configuration file (string value)
  13. #rbd_ceph_conf =
  14. rbd_ceph_conf = /etc/ceph/ceph.conf
  15. # Flatten volumes created from snapshots to remove dependency from volume to
  16. # snapshot (boolean value)
  17. #rbd_flatten_volume_from_snapshot = false
  18. rbd_flatten_volume_from_snapshot = true
  19. # 在创建基于 snapshot 的卷时候 是否flatten(扶平)新的 volume 独立出来
  20. # 就是让 新的volume 和 snapshot 没有关系。
  21. # The libvirt uuid of the secret for the rbd_user volumes (string value)
  22. #rbd_secret_uuid = <None>
  23. #设置 libvirt 链接 ceph 的认证密腰id
  24. #在计算节点上使用 virsh secret-list 查看
  25. #(.venv)[root@cinder cinder-7.0.0]# virsh secret-list
  26. #UUID Usage
  27. #-----------------------------------------------------------
  28. #7535b2cb-87e7-4d0a-b458-f0ab6adda149 ceph client.admin ceph2 secret
  29. #ec43b2f7-44c3-466b-a9b1-d32fbc3b04d5 ceph client.admin secret
  30. # Directory where temporary image files are stored when the volume driver does
  31. # not write them directly to the volume. Warning: this option is now
  32. # deprecated, please use image_conversion_dir instead. (string value)
  33. #volume_tmp_dir = <None>
  34. #设置下载 glance 镜像的临时目录
  35. # Maximum number of nested volume clones that are taken before a flatten
  36. # occurs. Set to 0 to disable cloning. (integer value)
  37. #rbd_max_clone_depth = 5
  38. rbd_max_clone_depth = 5
  39. # 设置 基于source volume 创建一个卷的时候 检测 源卷的 clone 深度是否为5
  40. # 如果 source volume 的 depth 为5 则 对 source volume 进行 flatten (扶平)
  41. # 也就是 让 source volume 独立出来, 和 source volume 的perent volume 没有关系
  42. # 注意:这里是对 所创建卷的 父卷也就是 source volume 进行 flatten 而不是 对所
  43. # 要创建的卷
  44. # 注意 值 > 0 才会有效果, < 0 则会一直做full copy , 效率低下但安全。
  45. # Volumes will be chunked into objects of this size (in megabytes). (integer
  46. # value)
  47. #rbd_store_chunk_size = 4
  48. rbd_store_chunk_size = 4
  49. # 设置 rbd image 的块的大小
  50. # Timeout value (in seconds) used when connecting to ceph cluster. If value <
  51. # 0, no timeout is set and default librados value is used. (integer value)
  52. #rados_connect_timeout = -1
  53. #rados_connect_timeout = -1
  54. # 设置链接 ceph 的超时时间 大于 等于 0 有效
  55. # Number of retries if connection to ceph cluster failed. (integer value)
  56. #rados_connection_retries = 3
  57. rados_connection_retries = 3
  58. # 设置重新链接的次数
  59. # Interval value (in seconds) between connection retries to ceph cluster.
  60. # (integer value)
  61. #rados_connection_interval = 5
  62. rados_connection_interval = 5
  63. # 设置重新 链接ceph 的间隔时间

分析 create api接口

  1. cinder help create
  2. Creates a volume.
  3. Positional arguments:
  4. <size> Size of volume, in GBs. (Required unless snapshot-id
  5. /source-volid is specified).
  6. Optional arguments:
  7. --consisgroup-id <consistencygroup-id>
  8. ID of a consistency group where the new volume belongs
  9. to. Default=None.
  10. --snapshot-id <snapshot-id>
  11. Creates volume from snapshot ID. Default=None.
  12. --source-volid <source-volid>
  13. Creates volume from volume ID. Default=None.
  14. --source-replica <source-replica>
  15. Creates volume from replicated volume ID.
  16. Default=None.
  17. --image-id <image-id>
  18. Creates volume from image ID. Default=None.
  19. --image <image> Creates a volume from image (ID or name).
  20. Default=None.
  21. --name <name> Volume name. Default=None.
  22. --description <description>
  23. Volume description. Default=None.
  24. --volume-type <volume-type>
  25. Volume type. Default=None.
  26. --availability-zone <availability-zone>
  27. Availability zone for volume. Default=None.
  28. --metadata [<key=value> [<key=value> ...]]
  29. Metadata key and value pairs. Default=None.
  30. --hint <key=value> Scheduler hint, like in nova.
  31. --allow-multiattach Allow volume to be attached more than once.
  32. Default=False
  33. ### 基于 snapshot 创建一个卷
  34. ### 基于 image 创建一个卷
  35. ### 基于 source volume 创建一个卷
  36. ### 创建一个 裸卷
  37. #### cinder driver rbd 代码分析
  38. if create_type == 'raw':
  39. model_update = self._create_raw_volume(volume_ref=volume_ref,
  40. **volume_spec)
  41. elif create_type == 'snap':
  42. model_update = self._create_from_snapshot(context,
  43. volume_ref=volume_ref,
  44. **volume_spec)
  45. elif create_type == 'source_vol':
  46. model_update = self._create_from_source_volume(
  47. context, volume_ref=volume_ref, **volume_spec)
  48. elif create_type == 'source_replica':
  49. model_update = self._create_from_source_replica(
  50. context, volume_ref=volume_ref, **volume_spec)
  51. elif create_type == 'image':
  52. model_update = self._create_from_image(context,
  53. volume_ref=volume_ref,
  54. **volume_spec)
  55. ##### 1, 创建一个 raw 的卷
  56. def create_volume(self, volume):
  57. """Creates a logical volume."""
  58. size = int(volume['size']) * units.Gi
  59. LOG.debug("creating volume '%s'", volume['name'])
  60. chunk_size = self.configuration.rbd_store_chunk_size * units.Mi
  61. order = int(math.log(chunk_size, 2))
  62. with RADOSClient(self) as client:
  63. self.RBDProxy().create(client.ioctx,
  64. utils.convert_str(volume['name']),
  65. size,
  66. order,
  67. old_format=False,
  68. features=client.features)
  69. 总结:创建一个 rbd image
  70. #####2, 创建一个 snap 的卷
  71. def create_volume_from_snapshot(self, volume, snapshot):
  72. """Creates a volume from a snapshot."""
  73. self._clone(volume, self.configuration.rbd_pool,
  74. snapshot['volume_name'], snapshot['name'])
  75. if self.configuration.rbd_flatten_volume_from_snapshot:
  76. self._flatten(self.configuration.rbd_pool, volume['name'])
  77. if int(volume['size']):
  78. self._resize(volume)
  79. def _clone(self, volume, src_pool, src_image, src_snap):
  80. LOG.debug('cloning %(pool)s/%(img)s@%(snap)s to %(dst)s',
  81. dict(pool=src_pool, img=src_image, snap=src_snap,
  82. dst=volume['name']))
  83. with RADOSClient(self, src_pool) as src_client:
  84. with RADOSClient(self) as dest_client:
  85. self.RBDProxy().clone(src_client.ioctx,
  86. utils.convert_str(src_image),
  87. utils.convert_str(src_snap),
  88. dest_client.ioctx,
  89. utils.convert_str(volume['name']),
  90. features=src_client.features)
  91. 总结:
  92. ## 类似 rbd clone --order 23 252656cd-b791-4334-a0e6-547fd4a410e3@snap c5098738-8039-49df-975c-aea802fe0c7e 命令
  93. ## rbd_flatten_volume_from_snapshot 如果设置为true 则会 扶平 新创建的卷就是
  94. ## 让新卷独立出来。
  95. #####3, 创建一个基于 source volume 的卷
  96. def create_cloned_volume(self, volume, src_vref):
  97. """Create a cloned volume from another volume.
  98. Since we are cloning from a volume and not a snapshot, we must first
  99. create a snapshot of the source volume.
  100. The user has the option to limit how long a volume's clone chain can be
  101. by setting rbd_max_clone_depth. If a clone is made of another clone
  102. and that clone has rbd_max_clone_depth clones behind it, the source
  103. volume will be flattened.
  104. """
  105. src_name = utils.convert_str(src_vref['name'])
  106. dest_name = utils.convert_str(volume['name'])
  107. flatten_parent = False
  108. # Do full copy if requested
  109. if self.configuration.rbd_max_clone_depth <= 0:
  110. with RBDVolumeProxy(self, src_name, read_only=True) as vol:
  111. vol.copy(vol.ioctx, dest_name)
  112. return
  113. # Otherwise do COW clone.
  114. with RADOSClient(self) as client:
  115. depth = self._get_clone_depth(client, src_name)
  116. # If source volume is a clone and rbd_max_clone_depth reached,
  117. # flatten the source before cloning. Zero rbd_max_clone_depth means
  118. # infinite is allowed.
  119. if depth == self.configuration.rbd_max_clone_depth:
  120. LOG.debug("maximum clone depth (%d) has been reached - "
  121. "flattening source volume",
  122. self.configuration.rbd_max_clone_depth)
  123. flatten_parent = True
  124. src_volume = self.rbd.Image(client.ioctx, src_name)
  125. try:
  126. # First flatten source volume if required.
  127. if flatten_parent:
  128. _pool, parent, snap = self._get_clone_info(src_volume,
  129. src_name)
  130. # Flatten source volume
  131. LOG.debug("flattening source volume %s", src_name)
  132. src_volume.flatten()
  133. # Delete parent clone snap
  134. parent_volume = self.rbd.Image(client.ioctx, parent)
  135. try:
  136. parent_volume.unprotect_snap(snap)
  137. parent_volume.remove_snap(snap)
  138. finally:
  139. parent_volume.close()
  140. # Create new snapshot of source volume
  141. clone_snap = "%s.clone_snap" % dest_name
  142. LOG.debug("creating snapshot='%s'", clone_snap)
  143. src_volume.create_snap(clone_snap)
  144. src_volume.protect_snap(clone_snap)
  145. except Exception:
  146. # Only close if exception since we still need it.
  147. src_volume.close()
  148. raise
  149. # Now clone source volume snapshot
  150. try:
  151. LOG.debug("cloning '%(src_vol)s@%(src_snap)s' to "
  152. "'%(dest)s'",
  153. {'src_vol': src_name, 'src_snap': clone_snap,
  154. 'dest': dest_name})
  155. self.RBDProxy().clone(client.ioctx, src_name, clone_snap,
  156. client.ioctx, dest_name,
  157. features=client.features)
  158. except Exception:
  159. src_volume.unprotect_snap(clone_snap)
  160. src_volume.remove_snap(clone_snap)
  161. raise
  162. finally:
  163. src_volume.close()
  164. if volume['size'] != src_vref['size']:
  165. LOG.debug("resize volume '%(dst_vol)s' from %(src_size)d to "
  166. "%(dst_size)d",
  167. {'dst_vol': volume['name'], 'src_size': src_vref['size'],
  168. 'dst_size': volume['size']})
  169. self._resize(volume)
  170. LOG.debug("clone created successfully")
  171. 总结:
  172. configuration.rbd_max_clone_depth 的值 <= 0 的时候
  173. 创建卷会 基于 source volume copy 一个新的卷
  174. 新卷 源卷 数据一样的 但是没有父子关系
  175. 类似: rbd -p rbd copy test_ld_01 test_ld_01_copy 命令
  176. configuration.rbd_max_clone_depth 的值 >= 0 的时候, source volume 的卷的depth
  177. 值小于 configuration.rbd_max_clone_depth 的值
  178. 则会在 source volume 上创建一个为名为 新卷id + clone_snap 的快照
  179. 例如: [root@osd0 ~]# rbd -p glance-02 snap ls volume-5996e0fb-fbc9-4219-b09b-04980737dd29
  180. SNAPID NAME SIZE
  181. 43 volume-0aa9e66d-a032-4db3-bde9-6a68a7cf4436.clone_snap 1024 MB
  182. configuration.rbd_max_clone_depth 的值 >= 0 的时候, source volume 的卷的depth
  183. 值大于等于 configuration.rbd_max_clone_depth 的值
  184. eg: configuration.rbd_max_clone_depth = 5 source volume 刚好是 莫一个volume 的第5 子孙, 就是source volume 的爷爷的爷爷
  185. 这时候会把 source volume 在独立出来 就是 faltten 出来,和以前的父辈没有关系。
  186. 然后在给这个 source volume snap 然后基于这个 snap clone 一个新的 卷。
  187. 总结: 就是一个 volume 只能有5代子孙。
  188. #####6, 创建一个基于 image 的卷 这个有点复杂根据image 的location有关
  189. ######具体看代码
  190. def _create_from_image(self, context, volume_ref,
  191. image_location, image_id, image_meta,
  192. image_service, **kwargs):
  193. LOG.debug("Cloning %(volume_id)s from image %(image_id)s "
  194. " at location %(image_location)s.",
  195. {'volume_id': volume_ref['id'],
  196. 'image_location': image_location, 'image_id': image_id})
  197. # Create the volume from an image.
  198. #
  199. # First see if the driver can clone the image directly.
  200. #
  201. # NOTE (singn): two params need to be returned
  202. # dict containing provider_location for cloned volume
  203. # and clone status.
  204. model_update, cloned = self.driver.clone_image(context,
  205. volume_ref,
  206. image_location,
  207. image_meta,
  208. image_service)
  209. # Try and clone the image if we have it set as a glance location.
  210. if not cloned and 'cinder' in CONF.allowed_direct_url_schemes:
  211. model_update, cloned = self._clone_image_volume(context,
  212. volume_ref,
  213. image_location,
  214. image_meta)
  215. #####调用底层的rbd driver
  216. def clone_image(self, context, volume,
  217. image_location, image_meta,
  218. image_service):
  219. if image_location:
  220. # Note: image_location[0] is glance image direct_url.
  221. # image_location[1] contains the list of all locations (including
  222. # direct_url) or None if show_multiple_locations is False in
  223. # glance configuration.
  224. if image_location[1]:
  225. url_locations = [location['url'] for
  226. location in image_location[1]]
  227. else:
  228. url_locations = [image_location[0]]
  229. # iterate all locations to look for a cloneable one.
  230. for url_location in url_locations:
  231. if url_location and self._is_cloneable(
  232. url_location, image_meta):
  233. _prefix, pool, image, snapshot = \
  234. self._parse_location(url_location)
  235. self._clone(volume, pool, image, snapshot)
  236. self._resize(volume)
  237. return {'provider_location': None}, True
  238. return ({}, False)
  239. def _is_cloneable(self, image_location, image_meta):
  240. try:
  241. fsid, pool, image, snapshot = self._parse_location(image_location)
  242. except exception.ImageUnacceptable as e:
  243. LOG.debug('not cloneable: %s.', e)
  244. return False
  245. if self._get_fsid() != fsid:
  246. LOG.debug('%s is in a different ceph cluster.', image_location)
  247. return False
  248. if image_meta['disk_format'] != 'raw':
  249. LOG.debug("rbd image clone requires image format to be "
  250. "'raw' but image %(image)s is '%(format)s'",
  251. {"image", image_location,
  252. "format", image_meta['disk_format']})
  253. return False
  254. # check that we can read the image
  255. try:
  256. with RBDVolumeProxy(self, image,
  257. pool=pool,
  258. snapshot=snapshot,
  259. read_only=True):
  260. return True
  261. except self.rbd.Error as e:
  262. LOG.debug('Unable to open image %(loc)s: %(err)s.',
  263. dict(loc=image_location, err=e))
  264. return False
  265. ##总结:
  266. 1,查看locationrbd:// 格式的
  267. 2,判断location 是否是统一ceph 集群
  268. 3,判断image 的格式是否是raw
  269. 4,判断指定的 snap 是否存在
  270. 总结 location rbd 的格式的:
  271. 如果locationrbd 的并且是统一集群的则可以使用 rbd clone的方式创建一个volume
  272. 查看结果:
  273. (.venv)[root@cinder cinder-7.0.0]# rbd2 -p glance-02 info volume-17272cee-314f-4e27-9807-f183e603a991
  274. rbd image 'volume-17272cee-314f-4e27-9807-f183e603a991':
  275. size 1024 MB in 128 objects
  276. order 23 (8192 kB objects)
  277. block_name_prefix: rbd_data.d53d43ceeb80
  278. format: 2
  279. features: layering, striping
  280. parent: glance-02/8deb9c62-d194-4147-8f26-8f7082a1cbf8@snap
  281. overlap: 9532 kB
  282. stripe unit: 4096 kB
  283. stripe count: 1
  284. ### 当 image 中的location 不是rbd 格式或者 不是同一个ceph 集群则使用下列代码
  285. def _create_from_image_download(self, context, volume_ref, image_location,
  286. image_id, image_service):
  287. # TODO(harlowja): what needs to be rolled back in the clone if this
  288. # volume create fails?? Likely this should be a subflow or broken
  289. # out task in the future. That will bring up the question of how
  290. # do we make said subflow/task which is only triggered in the
  291. # clone image 'path' resumable and revertable in the correct
  292. # manner.
  293. model_update = self.driver.create_volume(volume_ref)
  294. updates = dict(model_update or dict(), status='downloading')
  295. try:
  296. volume_ref = self.db.volume_update(context,
  297. volume_ref['id'], updates)
  298. except exception.CinderException:
  299. LOG.exception(_LE("Failed updating volume %(volume_id)s with "
  300. "%(updates)s"),
  301. {'volume_id': volume_ref['id'],
  302. 'updates': updates})
  303. self._copy_image_to_volume(context, volume_ref,
  304. image_id, image_location, image_service)
  305. return model_update
  306. 总结: 基于 image 创建 是先下载 image 镜像到本地临时目录然后载copy rbd image 中。
  307. def copy_image_to_volume(self, context, volume, image_service, image_id):
  308. tmp_dir = self._image_conversion_dir()
  309. with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp:
  310. image_utils.fetch_to_raw(context, image_service, image_id,
  311. tmp.name,
  312. self.configuration.volume_dd_blocksize,
  313. size=volume['size'])
  314. self.delete_volume(volume)
  315. chunk_size = self.configuration.rbd_store_chunk_size * units.Mi
  316. order = int(math.log(chunk_size, 2))
  317. # keep using the command line import instead of librbd since it
  318. # detects zeroes to preserve sparseness in the image
  319. args = ['rbd', 'import',
  320. '--pool', self.configuration.rbd_pool,
  321. '--order', order,
  322. tmp.name, volume['name'],
  323. '--new-format']
  324. args.extend(self._ceph_args())
  325. self._try_execute(*args)
  326. self._resize(volume)
  327. 总结: 把先前创建rbd image 的镜像删除掉,然后在通过 rbd import 命令把数据导入到指定的image 中。
  328. 注意: rbd import --pool glance-01 --order 22 /var/lib/cinder/conversion/tmpKJBgmg volume-6cccbace-00b6-4c17-b5b7-fcf773728bbe --new-format --id admin --conf /opt/ceph1/ceph.conf --cluster ceph 会自建一个rbd iamge 镜像。
  329. 最后删除掉本地临时的 image 镜像。

分析 delete api 接口

  1. (.venv)[root@cinder ~]# cinder help delete
  2. usage: cinder delete <volume> [<volume> ...]
  3. Removes one or more volumes.
  4. Positional arguments:
  5. <volume> Name or ID of volume or volumes to delete.
  6. ### 代码分析
  7. self._notify_about_volume_usage(context, volume_ref, "delete.start")
  8. try:
  9. # NOTE(flaper87): Verify the driver is enabled
  10. # before going forward. The exception will be caught
  11. # and the volume status updated.
  12. utils.require_driver_initialized(self.driver)
  13. self.driver.remove_export(context, volume_ref)
  14. if unmanage_only:
  15. self.driver.unmanage(volume_ref)
  16. else:
  17. self.driver.delete_volume(volume_ref)
  18. except exception.VolumeIsBusy:
  19. LOG.error(_LE("Unable to delete busy volume."),
  20. resource=volume_ref)
  21. # If this is a destination volume, we have to clear the database
  22. # record to avoid user confusion.
  23. self._clear_db(context, is_migrating_dest, volume_ref,
  24. 'available')
  25. return True
  26. except Exception:
  27. with excutils.save_and_reraise_exception():
  28. # If this is a destination volume, we have to clear the
  29. # database record to avoid user confusion.
  30. self._clear_db(context, is_migrating_dest, volume_ref,
  31. 'error_deleting')
  32. ### manager 调用的 是 driver.remove_export 和 delete_volume
  33. def delete_volume(self, volume):
  34. """Deletes a logical volume."""
  35. # NOTE(dosaboy): this was broken by commit cbe1d5f. Ensure names are
  36. # utf-8 otherwise librbd will barf.
  37. volume_name = utils.convert_str(volume['name'])
  38. with RADOSClient(self) as client:
  39. try:
  40. rbd_image = self.rbd.Image(client.ioctx, volume_name)
  41. except self.rbd.ImageNotFound:
  42. LOG.info(_LI("volume %s no longer exists in backend"),
  43. volume_name)
  44. return
  45. clone_snap = None
  46. parent = None
  47. # Ensure any backup snapshots are deleted
  48. self._delete_backup_snaps(rbd_image)
  49. # If the volume has non-clone snapshots this delete is expected to
  50. # raise VolumeIsBusy so do so straight away.
  51. try:
  52. snaps = rbd_image.list_snaps()
  53. for snap in snaps:
  54. if snap['name'].endswith('.clone_snap'):
  55. LOG.debug("volume has clone snapshot(s)")
  56. # We grab one of these and use it when fetching parent
  57. # info in case the volume has been flattened.
  58. clone_snap = snap['name']
  59. break
  60. raise exception.VolumeIsBusy(volume_name=volume_name)
  61. # Determine if this volume is itself a clone
  62. _pool, parent, parent_snap = self._get_clone_info(rbd_image,
  63. volume_name,
  64. clone_snap)
  65. finally:
  66. rbd_image.close()
  67. @utils.retry(self.rbd.ImageBusy, retries=3)
  68. def _try_remove_volume(client, volume_name):
  69. self.RBDProxy().remove(client.ioctx, volume_name)
  70. if clone_snap is None:
  71. LOG.debug("deleting rbd volume %s", volume_name)
  72. try:
  73. _try_remove_volume(client, volume_name)
  74. except self.rbd.ImageBusy:
  75. msg = (_("ImageBusy error raised while deleting rbd "
  76. "volume. This may have been caused by a "
  77. "connection from a client that has crashed and, "
  78. "if so, may be resolved by retrying the delete "
  79. "after 30 seconds has elapsed."))
  80. LOG.warning(msg)
  81. # Now raise this so that volume stays available so that we
  82. # delete can be retried.
  83. raise exception.VolumeIsBusy(msg, volume_name=volume_name)
  84. except self.rbd.ImageNotFound:
  85. LOG.info(_LI("RBD volume %s not found, allowing delete "
  86. "operation to proceed."), volume_name)
  87. return
  88. # If it is a clone, walk back up the parent chain deleting
  89. # references.
  90. if parent:
  91. LOG.debug("volume is a clone so cleaning references")
  92. self._delete_clone_parent_refs(client, parent, parent_snap)
  93. else:
  94. # If the volume has copy-on-write clones we will not be able to
  95. # delete it. Instead we will keep it as a silent volume which
  96. # will be deleted when it's snapshot and clones are deleted.
  97. new_name = "%s.deleted" % (volume_name)
  98. self.RBDProxy().rename(client.ioctx, volume_name, new_name)
  99. 总结:
  100. volume rbd 中不存在的时候, cinder delete 命令还是会正确执行,并且更新数据库
  101. ### self._delete_backup_snaps(rbd_image) 分析
  102. def _delete_backup_snaps(self, rbd_image):
  103. backup_snaps = self._get_backup_snaps(rbd_image)
  104. if backup_snaps:
  105. for snap in backup_snaps:
  106. rbd_image.remove_snap(snap['name'])
  107. else:
  108. LOG.debug("volume has no backup snaps")
  109. def _get_backup_snaps(self, rbd_image):
  110. """Get list of any backup snapshots that exist on this volume.
  111. There should only ever be one but accept all since they need to be
  112. deleted before the volume can be.
  113. """
  114. # NOTE(dosaboy): we do the import here otherwise we get import conflict
  115. # issues between the rbd driver and the ceph backup driver. These
  116. # issues only seem to occur when NOT using them together and are
  117. # triggered when the ceph backup driver imports the rbd volume driver.
  118. from cinder.backup.drivers import ceph
  119. return ceph.CephBackupDriver.get_backup_snaps(rbd_image)
  120. ###cinder/backup/drivers/ceph.py
  121. @classmethod
  122. def get_backup_snaps(cls, rbd_image, sort=False):
  123. """Get all backup snapshots for the given rbd image.
  124. NOTE: this call is made public since these snapshots must be deleted
  125. before the base volume can be deleted.
  126. """
  127. snaps = rbd_image.list_snaps()
  128. backup_snaps = []
  129. for snap in snaps:
  130. search_key = cls.backup_snapshot_name_pattern()
  131. result = re.search(search_key, snap['name'])
  132. if result:
  133. backup_snaps.append({'name': result.group(0),
  134. 'backup_id': result.group(1),
  135. 'timestamp': result.group(2)})
  136. if sort:
  137. # Sort into ascending order of timestamp
  138. backup_snaps.sort(key=lambda x: x['timestamp'], reverse=True)
  139. return backup_snaps
  140. @staticmethod
  141. def backup_snapshot_name_pattern():
  142. """Returns the pattern used to match backup snapshots.
  143. It is essential that snapshots created for purposes other than backups
  144. do not have this name format.
  145. """
  146. return r"^backup\.([a-z0-9\-]+?)\.snap\.(.+)$"
  147. 总结:
  148. 获取快照中是否有 backup 开头的 snap 如果有则删除这些快照。
  149. 接下来接着分析:
  150. 获取所有的 clone_snap 结尾的 snap
  151. 如果还有 其他的snap 则会报错 "is busy" 不能删除
  152. 如果这个image 没有 clone_snap 的快照则 使用remove | rm 删除这个image
  153. 如果这个image clone_snap 的快照则 调用 rename 修改名称 加上 .delete
  154. 总结所有的delete 功能特性:
  155. 1, rbd中没有image 则更新数据库 信息
  156. 2image backup 相关的snap 则删除 backup 相关快照
  157. 3image clone_snap 相关的snap rename 卷名称,不真正删除
  158. 4image 没有clone_snap 相关的snap remove 真正删除卷

nova 调用ceph 存储


添加新批注
在作者公开此批注前,只有你和作者可见。
回复批注