Merge branch 'develop' into librbd

Author: GowthamShanmugam

README.rst

@@ -1,6 +1,6 @@
 ===============================
 ceph-integration
-================
+===============================
 
 Tendrl integration for Ceph Storage
 
@@ -20,13 +20,13 @@ Builds
 ------
 
 .. image:: https://travis-ci.org/Tendrl/ceph_integration.svg?branch=master
-    :target: https://travis-ci.org/Tendrl/ceph_integration
+  :target: https://travis-ci.org/Tendrl/ceph-integration
 
 Code Coverage
 -------------
 
 .. image:: https://coveralls.io/repos/github/Tendrl/ceph_integration/badge.svg?branch=master
-    :target: https://coveralls.io/github/Tendrl/ceph_integration?branch=master
+  :target: https://coveralls.io/github/Tendrl/ceph_integration?branch=master
 
 Developer/Install documentation
 -----------------------

etc/tendrl/ceph-integration/ceph-integration.conf.yaml.sample

@@ -4,3 +4,4 @@ tendrl_ansible_exec_file: $HOME/.tendrl/node-agent/ansible_exec
 log_cfg_path: /etc/tendrl/ceph-integration/ceph-integration_logging.yaml
 log_level: DEBUG
 logging_socket_path: /var/run/tendrl/message.sock
+with_internal_profiling: False

tendrl-ceph-integration.spec

@@ -1,5 +1,5 @@
 Name: tendrl-ceph-integration
-Version: 1.4.1
+Version: 1.4.2
 Release: 1%{?dist}
 BuildArch: noarch
 Summary: Tendrl bridge for Ceph Storage
@@ -60,6 +60,9 @@ py.test -v tendrl/ceph_integration/tests || :
 %{_sysconfdir}/tendrl/ceph-integration/ceph-integration.conf.yaml
 
 %changelog
+* Mon Jun 19 2017 Rohan Kanade <[email protected]> - 1.4.2-1
+- Release tendrl-ceph-integration v1.4.2
+
 * Thu Jun 08 2017 Rohan Kanade <[email protected]> - 1.4.1-1
 - Release tendrl-ceph-integration v1.4.1
 

tendrl/ceph_integration/manager/__init__.py

@@ -80,6 +80,10 @@ def main():
     NS.tendrl_context.save()
     NS.ceph.definitions.save()
     NS.ceph.config.save()
+    
+    if NS.config.data.get("with_internal_profiling", False):
+        from tendrl.commons import profiler
+        profiler.start()
 
     m = CephIntegrationManager()
     m.start()

tendrl/ceph_integration/objects/definition/__init__.py

@@ -17,6 +17,9 @@ def __init__(self, *args, **kwargs):
         self.value = 'clusters/{0}/_NS/definitions'
 
     def get_parsed_defs(self):
+        if self._parsed_defs:
+            return self._parsed_defs
+        
         self._parsed_defs = yaml.safe_load(self.data)
         return self._parsed_defs
 

tendrl/ceph_integration/objects/definition/ceph.yaml

@@ -21,6 +21,8 @@ namespace.ceph:
           - Pool.quota_max_bytes
       pre_run:
         - ceph.objects.Pool.atoms.NamedPoolNotExists
+      post_run:
+        - ceph.objects.Pool.atoms.CheckPoolAvailable
       run: ceph.flows.CreatePool
       type: Create
       uuid: faeab231-69e9-4c9d-b5ef-a67ed057f98b
@@ -41,6 +43,8 @@ namespace.ceph:
           - ECProfile.directory
           - ECProfile.ruleset_failure_domain
       run: ceph.flows.CreateECProfile
+      post_run:
+        - ceph.objects.ECProfile.atoms.CheckECProfileAvailable
       type: Create
       uuid: faeab231-69e9-4c9d-b5ef-a67ed057f98d
   objects:
@@ -102,6 +106,16 @@ namespace.ceph:
           run: ceph.objects.ECProfile.atoms.Delete
           type: Delete
           uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3740
+        CheckECProfileAvailable:
+          enabled: true
+          help: Check if ECProfile available
+          inputs:
+            mandatory:
+              - ECProfile.name
+          name: ECProfile available
+          run: ceph.objects.ECProfile.atoms.CheckECProfileAvailable
+          type: Check
+          uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3759
       flows:
         DeleteECProfile:
           tags:
@@ -212,6 +226,17 @@ namespace.ceph:
           run: ceph.objects.Rbd.atoms.RbdExists
           type: Get
           uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3922
+        CheckRbdAvailable:
+          enabled: true
+          help: Check if rbd is available
+          inputs:
+            mandatory:
+              - Rbd.pool_id
+              - Rbd.name
+          name: Rbd available
+          run: ceph.objects.Rbd.atoms.CheckRbdAvailable
+          type: Check
+          uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3929
       flows:
         CreateRbd:
           tags:
@@ -239,6 +264,8 @@ namespace.ceph:
               - Rbd.pool_quota_max_bytes
           pre_run:
             - ceph.objects.Rbd.atoms.RbdNotExists
+          post_run:
+            - ceph.objects.Rbd.atoms.CheckRbdAvailable
           run: ceph.objects.Rbd.flows.CreateRbd
           type: Create
           uuid: 9bc41d8f-a0cf-420a-b2fe-18761e07f3d2
@@ -442,6 +469,16 @@ namespace.ceph:
           run: ceph.objects.Pool.atoms.PoolNotExists
           type: Get
           uuid: 9a2df258-9b24-4fd3-a66f-ee346e2e3832
+        CheckPoolAvailable:
+          enabled: true
+          help: check if pool available
+          inputs:
+            mandatory:
+              - Pool.poolname
+          name: pool is available
+          run: ceph.objects.Pool.atoms.CheckPoolAvailable
+          type: Check
+          uuid: 9a2df258-9b24-4fd3-a66f-ee346e2e3839
       flows:
         DeletePool:
           tags:
@@ -565,12 +602,18 @@ namespace.ceph:
         uuid:
           help: UUID
           type: String
+        hostname:
+          help: Name of the host
+          type: String
         public_addr:
           help: Public Address
           type: String
         cluster_addr:
           help: Cluster Address
           type: String
+        device_path:
+          help: device path
+          type: String
         heartbeat_front_addr:
           help: Heartbeat front address
           type: String
@@ -610,8 +653,17 @@ namespace.ceph:
         last_clean_end:
           help: Last clean end
           type: int
+        total:
+          help: total size of osd
+          type: int
+        used:
+          help: used size
+          type: int
+        used_pcnt:
+          help: used percent
+          type: String
       help: "Osd"
       enabled: true
-      value: clusters/$TendrlContext.integration_id/Osds/$Osd.uuid
+      value: clusters/$TendrlContext.integration_id/Osds/$Osd.id
       list: clusters/$TendrlContext.integration_id/Osds
 tendrl_schema_version: 0.3

tendrl/ceph_integration/objects/ecprofile/atoms/check_ec_profile_available/__init__.py

@@ -0,0 +1,42 @@
+import etcd
+import gevent
+
+from tendrl.commons.event import Event
+from tendrl.commons.message import Message
+from tendrl.commons import objects
+from tendrl.commons.objects import AtomExecutionFailedError
+
+
+class CheckECProfileAvailable(objects.BaseAtom):
+    def __init__(self, *args, **kwargs):
+        super(CheckECProfileAvailable, self).__init__(*args, **kwargs)
+
+    def run(self):
+        retry_count = 0
+        while True:
+            try:
+                NS.ceph.objects.ECProfile(
+                    name=self.parameters['ECProfile.name']
+                ).load()
+                return True
+            except etcd.EtcdKeyNotFound:
+                retry_count += 1
+                gevent.sleep(1)
+                if retry_count == 600:
+                    Event(
+                        Message(
+                            priority="error",
+                            publisher=NS.publisher_id,
+                            payload={
+                                "message": "ECProfile %s not reflected in tendrl yet. Timing out" %
+                                self.parameters['ECProfile.name']
+                            },
+                            job_id=self.parameters['job_id'],
+                            flow_id=self.parameters['flow_id'],
+                            cluster_id=NS.tendrl_context.integration_id,
+                       )
+                    )
+                    raise AtomExecutionFailedError(
+                        "ECProfile %s not reflected in tendrl yet. Timing out" %
+                        self.parameters['ECProfile.name']
+                    )

tendrl/ceph_integration/objects/osd/__init__.py

@@ -1,39 +1,39 @@
-from tendrl.commons import objects
+from tendrl.integrations.ceph.objects import osd
 
 
-class Osd(objects.BaseObject):
+class Osd(osd.Osd):
     def __init__(self, id=None,
-                 uuid=None, public_addr=None, cluster_addr=None,
-                 heartbeat_front_addr=None, heartbeat_back_addr=None,
+                 uuid=None, hostname=None, public_addr=None, cluster_addr=None,
+                 device_path=None, heartbeat_front_addr=None, heartbeat_back_addr=None,
                  down_at=None, up_from=None, lost_at=None,
                  osd_up=None, osd_in=None, up_thru=None,
                  weight=None, primary_affinity=None,
                  state=None, last_clean_begin=None,
-                 last_clean_end=None, *args, **kwargs):
-        super(Osd, self).__init__(*args, **kwargs)
-
-        self.id = id
-        self.uuid = uuid
-        self.public_addr = public_addr
-        self.cluster_addr = cluster_addr
-        self.heartbeat_front_addr = heartbeat_front_addr
-        self.heartbeat_back_addr = heartbeat_back_addr
-        self.down_at = down_at
-        self.up_from = up_from
-        self.lost_at = lost_at
-        self.osd_up = osd_up
-        self.osd_in = osd_in
-        self.up_thru = up_thru
-        self.weight = weight
-        self.primary_affinity = primary_affinity
-        self.state = state
-        self.last_clean_begin = last_clean_begin
-        self.last_clean_end = last_clean_end
-        self.value = 'clusters/{0}/Osds/{1}'
-
-    def render(self):
-        self.value = self.value.format(
-            NS.tendrl_context.integration_id,
-            self.uuid
+                 last_clean_end=None, total=None, used=None, used_pcnt=None,
+                 *args, **kwargs):
+        super(Osd, self).__init__(
+            id=id,
+            uuid=uuid,
+            hostname=hostname,
+            public_addr=public_addr,
+            cluster_addr=cluster_addr,
+            device_path=device_path,
+            heartbeat_front_addr=heartbeat_front_addr,
+            heartbeat_back_addr=heartbeat_back_addr,
+            down_at=down_at,
+            up_from=up_from,
+            lost_at=lost_at,
+            osd_up=osd_up,
+            osd_in=osd_in,
+            up_thru=up_thru,
+            weight=weight,
+            primary_affinity=primary_affinity,
+            state=state,
+            last_clean_begin=last_clean_begin,
+            last_clean_end=last_clean_end,
+            total=total,
+            used=used,
+            used_pcnt=used_pcnt,
+            *args,
+            **kwargs
         )
-        return super(Osd, self).render()

tendrl/ceph_integration/objects/pool/atoms/check_pool_available/__init__.py

@@ -0,0 +1,54 @@
+import etcd
+import gevent
+
+from tendrl.ceph_integration.objects.pool import Pool
+from tendrl.commons.event import Event
+from tendrl.commons.message import Message
+from tendrl.commons import objects
+from tendrl.commons.objects import AtomExecutionFailedError
+
+
+class CheckPoolAvailable(objects.BaseAtom):
+    def __init__(self, *args, **kwargs):
+        super(CheckPoolAvailable, self).__init__(*args, **kwargs)
+
+    def run(self):
+        retry_count = 0
+        while True:
+            pools = None
+            try:
+                pools = NS._int.client.read(
+                    "clusters/%s/Pools" % NS.tendrl_context.integration_id
+                )
+            except etcd.EtcdKeyNotFound:
+                pass
+            
+            if pools:
+                for entry in pools.leaves:
+                    try:
+                        pool = Pool(pool_id=entry.key.split("Pools/")[-1]).load()
+                        if pool.pool_name == self.parameters['Pool.poolname']:
+                            return True
+                    except etcd.EtcdKeyNotFound:
+                        continue
+            
+            retry_count += 1
+            gevent.sleep(1)
+            if retry_count == 600:
+                Event(
+                    Message(
+                        priority="error",
+                        publisher=NS.publisher_id,
+                        payload={
+                            "message": "Pool %s not reflected in tendrl yet. Timing out" %
+                            self.parameters['Pool.pool_name']
+                        },
+                        job_id=self.parameters['job_id'],
+                        flow_id=self.parameters['flow_id'],
+                        cluster_id=NS.tendrl_context.integration_id,
+                   )
+                )
+                raise AtomExecutionFailedError(
+                    "Pool %s not reflected in tendrl yet. Timing out" %
+                    self.parameters['Pool.pool_name']
+                )

tendrl/ceph_integration/objects/rbd/atoms/check_rbd_available/__init__.py

@@ -0,0 +1,43 @@
+import etcd
+import gevent
+
+from tendrl.commons.event import Event
+from tendrl.commons.message import Message
+from tendrl.commons import objects
+from tendrl.commons.objects import AtomExecutionFailedError
+
+
+class CheckRbdAvailable(objects.BaseAtom):
+    def __init__(self, *args, **kwargs):
+        super(CheckRbdAvailable, self).__init__(*args, **kwargs)
+
+    def run(self):
+        retry_count = 0
+        while True:
+            try:
+                NS.ceph.objects.Rbd(
+                    pool_id=self.parameters['Rbd.pool_id'],
+                    name=self.parameters['Rbd.name']
+                ).load()
+                return True
+            except etcd.EtcdKeyNotFound:
+                retry_count += 1
+                gevent.sleep(1)
+                if retry_count == 600:
+                    Event(
+                        Message(
+                            priority="error",
+                            publisher=NS.publisher_id,
+                            payload={
+                                "message": "Rbd %s not reflected in tendrl yet. Timing out" %
+                                self.parameters['Rbd.name']
+                            },
+                            job_id=self.parameters['job_id'],
+                            flow_id=self.parameters['flow_id'],
+                            cluster_id=NS.tendrl_context.integration_id,
+                       )
+                    )
+                    raise AtomExecutionFailedError(
+                        "Rbd %s not reflected in tendrl yet. Timing out" %
+                        self.parameters['Rbd.name']
+                    )