Skip to content

Commit

Permalink
Merge branch 'develop' into librbd
Browse files Browse the repository at this point in the history
  • Loading branch information
GowthamShanmugam authored Jun 21, 2017
2 parents ff9d753 + 071b28f commit 1cb19ac
Show file tree
Hide file tree
Showing 12 changed files with 274 additions and 37 deletions.
6 changes: 3 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
===============================
ceph-integration
================
===============================

Tendrl integration for Ceph Storage

Expand All @@ -20,13 +20,13 @@ Builds
------

.. image:: https://travis-ci.org/Tendrl/ceph_integration.svg?branch=master
:target: https://travis-ci.org/Tendrl/ceph_integration
:target: https://travis-ci.org/Tendrl/ceph-integration

Code Coverage
-------------

.. image:: https://coveralls.io/repos/github/Tendrl/ceph_integration/badge.svg?branch=master
:target: https://coveralls.io/github/Tendrl/ceph_integration?branch=master
:target: https://coveralls.io/github/Tendrl/ceph_integration?branch=master

Developer/Install documentation
-----------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ tendrl_ansible_exec_file: $HOME/.tendrl/node-agent/ansible_exec
log_cfg_path: /etc/tendrl/ceph-integration/ceph-integration_logging.yaml
log_level: DEBUG
logging_socket_path: /var/run/tendrl/message.sock
with_internal_profiling: False
5 changes: 4 additions & 1 deletion tendrl-ceph-integration.spec
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Name: tendrl-ceph-integration
Version: 1.4.1
Version: 1.4.2
Release: 1%{?dist}
BuildArch: noarch
Summary: Tendrl bridge for Ceph Storage
Expand Down Expand Up @@ -60,6 +60,9 @@ py.test -v tendrl/ceph_integration/tests || :
%{_sysconfdir}/tendrl/ceph-integration/ceph-integration.conf.yaml

%changelog
* Mon Jun 19 2017 Rohan Kanade <[email protected]> - 1.4.2-1
- Release tendrl-ceph-integration v1.4.2

* Thu Jun 08 2017 Rohan Kanade <[email protected]> - 1.4.1-1
- Release tendrl-ceph-integration v1.4.1

Expand Down
4 changes: 4 additions & 0 deletions tendrl/ceph_integration/manager/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ def main():
NS.tendrl_context.save()
NS.ceph.definitions.save()
NS.ceph.config.save()

if NS.config.data.get("with_internal_profiling", False):
from tendrl.commons import profiler
profiler.start()

m = CephIntegrationManager()
m.start()
Expand Down
3 changes: 3 additions & 0 deletions tendrl/ceph_integration/objects/definition/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ def __init__(self, *args, **kwargs):
self.value = 'clusters/{0}/_NS/definitions'

def get_parsed_defs(self):
if self._parsed_defs:
return self._parsed_defs

self._parsed_defs = yaml.safe_load(self.data)
return self._parsed_defs

Expand Down
54 changes: 53 additions & 1 deletion tendrl/ceph_integration/objects/definition/ceph.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ namespace.ceph:
- Pool.quota_max_bytes
pre_run:
- ceph.objects.Pool.atoms.NamedPoolNotExists
post_run:
- ceph.objects.Pool.atoms.CheckPoolAvailable
run: ceph.flows.CreatePool
type: Create
uuid: faeab231-69e9-4c9d-b5ef-a67ed057f98b
Expand All @@ -41,6 +43,8 @@ namespace.ceph:
- ECProfile.directory
- ECProfile.ruleset_failure_domain
run: ceph.flows.CreateECProfile
post_run:
- ceph.objects.ECProfile.atoms.CheckECProfileAvailable
type: Create
uuid: faeab231-69e9-4c9d-b5ef-a67ed057f98d
objects:
Expand Down Expand Up @@ -102,6 +106,16 @@ namespace.ceph:
run: ceph.objects.ECProfile.atoms.Delete
type: Delete
uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3740
CheckECProfileAvailable:
enabled: true
help: Check if ECProfile available
inputs:
mandatory:
- ECProfile.name
name: ECProfile available
run: ceph.objects.ECProfile.atoms.CheckECProfileAvailable
type: Check
uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3759
flows:
DeleteECProfile:
tags:
Expand Down Expand Up @@ -212,6 +226,17 @@ namespace.ceph:
run: ceph.objects.Rbd.atoms.RbdExists
type: Get
uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3922
CheckRbdAvailable:
enabled: true
help: Check if rbd is available
inputs:
mandatory:
- Rbd.pool_id
- Rbd.name
name: Rbd available
run: ceph.objects.Rbd.atoms.CheckRbdAvailable
type: Check
uuid: 7a2df258-9b24-4fd3-a66f-ee346e2e3929
flows:
CreateRbd:
tags:
Expand Down Expand Up @@ -239,6 +264,8 @@ namespace.ceph:
- Rbd.pool_quota_max_bytes
pre_run:
- ceph.objects.Rbd.atoms.RbdNotExists
post_run:
- ceph.objects.Rbd.atoms.CheckRbdAvailable
run: ceph.objects.Rbd.flows.CreateRbd
type: Create
uuid: 9bc41d8f-a0cf-420a-b2fe-18761e07f3d2
Expand Down Expand Up @@ -442,6 +469,16 @@ namespace.ceph:
run: ceph.objects.Pool.atoms.PoolNotExists
type: Get
uuid: 9a2df258-9b24-4fd3-a66f-ee346e2e3832
CheckPoolAvailable:
enabled: true
help: check if pool available
inputs:
mandatory:
- Pool.poolname
name: pool is available
run: ceph.objects.Pool.atoms.CheckPoolAvailable
type: Check
uuid: 9a2df258-9b24-4fd3-a66f-ee346e2e3839
flows:
DeletePool:
tags:
Expand Down Expand Up @@ -565,12 +602,18 @@ namespace.ceph:
uuid:
help: UUID
type: String
hostname:
help: Name of the host
type: String
public_addr:
help: Public Address
type: String
cluster_addr:
help: Cluster Address
type: String
device_path:
help: device path
type: String
heartbeat_front_addr:
help: Heartbeat front address
type: String
Expand Down Expand Up @@ -610,8 +653,17 @@ namespace.ceph:
last_clean_end:
help: Last clean end
type: int
total:
help: total size of osd
type: int
used:
help: used size
type: int
used_pcnt:
help: used percent
type: String
help: "Osd"
enabled: true
value: clusters/$TendrlContext.integration_id/Osds/$Osd.uuid
value: clusters/$TendrlContext.integration_id/Osds/$Osd.id
list: clusters/$TendrlContext.integration_id/Osds
tendrl_schema_version: 0.3
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import etcd
import gevent

from tendrl.commons.event import Event
from tendrl.commons.message import Message
from tendrl.commons import objects
from tendrl.commons.objects import AtomExecutionFailedError


class CheckECProfileAvailable(objects.BaseAtom):
def __init__(self, *args, **kwargs):
super(CheckECProfileAvailable, self).__init__(*args, **kwargs)

def run(self):
retry_count = 0
while True:
try:
NS.ceph.objects.ECProfile(
name=self.parameters['ECProfile.name']
).load()
return True
except etcd.EtcdKeyNotFound:
retry_count += 1
gevent.sleep(1)
if retry_count == 600:
Event(
Message(
priority="error",
publisher=NS.publisher_id,
payload={
"message": "ECProfile %s not reflected in tendrl yet. Timing out" %
self.parameters['ECProfile.name']
},
job_id=self.parameters['job_id'],
flow_id=self.parameters['flow_id'],
cluster_id=NS.tendrl_context.integration_id,
)
)
raise AtomExecutionFailedError(
"ECProfile %s not reflected in tendrl yet. Timing out" %
self.parameters['ECProfile.name']
)
62 changes: 31 additions & 31 deletions tendrl/ceph_integration/objects/osd/__init__.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,39 @@
from tendrl.commons import objects
from tendrl.integrations.ceph.objects import osd


class Osd(objects.BaseObject):
class Osd(osd.Osd):
def __init__(self, id=None,
uuid=None, public_addr=None, cluster_addr=None,
heartbeat_front_addr=None, heartbeat_back_addr=None,
uuid=None, hostname=None, public_addr=None, cluster_addr=None,
device_path=None, heartbeat_front_addr=None, heartbeat_back_addr=None,
down_at=None, up_from=None, lost_at=None,
osd_up=None, osd_in=None, up_thru=None,
weight=None, primary_affinity=None,
state=None, last_clean_begin=None,
last_clean_end=None, *args, **kwargs):
super(Osd, self).__init__(*args, **kwargs)

self.id = id
self.uuid = uuid
self.public_addr = public_addr
self.cluster_addr = cluster_addr
self.heartbeat_front_addr = heartbeat_front_addr
self.heartbeat_back_addr = heartbeat_back_addr
self.down_at = down_at
self.up_from = up_from
self.lost_at = lost_at
self.osd_up = osd_up
self.osd_in = osd_in
self.up_thru = up_thru
self.weight = weight
self.primary_affinity = primary_affinity
self.state = state
self.last_clean_begin = last_clean_begin
self.last_clean_end = last_clean_end
self.value = 'clusters/{0}/Osds/{1}'

def render(self):
self.value = self.value.format(
NS.tendrl_context.integration_id,
self.uuid
last_clean_end=None, total=None, used=None, used_pcnt=None,
*args, **kwargs):
super(Osd, self).__init__(
id=id,
uuid=uuid,
hostname=hostname,
public_addr=public_addr,
cluster_addr=cluster_addr,
device_path=device_path,
heartbeat_front_addr=heartbeat_front_addr,
heartbeat_back_addr=heartbeat_back_addr,
down_at=down_at,
up_from=up_from,
lost_at=lost_at,
osd_up=osd_up,
osd_in=osd_in,
up_thru=up_thru,
weight=weight,
primary_affinity=primary_affinity,
state=state,
last_clean_begin=last_clean_begin,
last_clean_end=last_clean_end,
total=total,
used=used,
used_pcnt=used_pcnt,
*args,
**kwargs
)
return super(Osd, self).render()
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import etcd
import gevent

from tendrl.ceph_integration.objects.pool import Pool
from tendrl.commons.event import Event
from tendrl.commons.message import Message
from tendrl.commons import objects
from tendrl.commons.objects import AtomExecutionFailedError


class CheckPoolAvailable(objects.BaseAtom):
def __init__(self, *args, **kwargs):
super(CheckPoolAvailable, self).__init__(*args, **kwargs)

def run(self):
retry_count = 0
while True:
pools = None
try:
pools = NS._int.client.read(
"clusters/%s/Pools" % NS.tendrl_context.integration_id
)
except etcd.EtcdKeyNotFound:
pass

if pools:
for entry in pools.leaves:
try:
pool = Pool(pool_id=entry.key.split("Pools/")[-1]).load()
if pool.pool_name == self.parameters['Pool.poolname']:
return True
except etcd.EtcdKeyNotFound:
continue

retry_count += 1
gevent.sleep(1)
if retry_count == 600:
Event(
Message(
priority="error",
publisher=NS.publisher_id,
payload={
"message": "Pool %s not reflected in tendrl yet. Timing out" %
self.parameters['Pool.pool_name']
},
job_id=self.parameters['job_id'],
flow_id=self.parameters['flow_id'],
cluster_id=NS.tendrl_context.integration_id,
)
)
raise AtomExecutionFailedError(
"Pool %s not reflected in tendrl yet. Timing out" %
self.parameters['Pool.pool_name']
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import etcd
import gevent

from tendrl.commons.event import Event
from tendrl.commons.message import Message
from tendrl.commons import objects
from tendrl.commons.objects import AtomExecutionFailedError


class CheckRbdAvailable(objects.BaseAtom):
def __init__(self, *args, **kwargs):
super(CheckRbdAvailable, self).__init__(*args, **kwargs)

def run(self):
retry_count = 0
while True:
try:
NS.ceph.objects.Rbd(
pool_id=self.parameters['Rbd.pool_id'],
name=self.parameters['Rbd.name']
).load()
return True
except etcd.EtcdKeyNotFound:
retry_count += 1
gevent.sleep(1)
if retry_count == 600:
Event(
Message(
priority="error",
publisher=NS.publisher_id,
payload={
"message": "Rbd %s not reflected in tendrl yet. Timing out" %
self.parameters['Rbd.name']
},
job_id=self.parameters['job_id'],
flow_id=self.parameters['flow_id'],
cluster_id=NS.tendrl_context.integration_id,
)
)
raise AtomExecutionFailedError(
"Rbd %s not reflected in tendrl yet. Timing out" %
self.parameters['Rbd.name']
)
Loading

0 comments on commit 1cb19ac

Please sign in to comment.