initial (re)import

Author: Ron Gorodetzky

.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+*~
+build
+dist
+src/clusto.egg-info
+doc/.build

README

@@ -0,0 +1,139 @@
+*****
+THIS IS OUT OF DATE AND NEEDS TO BE UPDATED
+*****
+
+=== A lot of this stuff has changed I'll have to update it ===
+
+Things seem to fall into 3 general categories:
+
+1. Equipment - Things like servers, switches, racks, routers, etc.
+
+2. Parts - Things like NICS, ports, U (rack space), etc.
+
+3. Resources - IP ranges/network blocks, space (U, datacenter space), etc.
+
+These are loose categories 
+
+
+Kinds of matching:
+
+1. the given key value pairs exist in Thing
+2. for the given keys the values match exactly in the Thing
+3. the given key/values exactly match attributes of Thing
+
+
+Questions I'd like to be able to answer questions like:
+
+* get 4 servers in datacenter Z with status == unallocated 
+  each in a different rack
+
+servers = []
+d = clusto.getByName('datacenterZ')
+racks = d.getConnectedMatching(AttributeDict(Racks.allMetaAttrs()))
+for r in racks:
+    s = r.getConnectedMatching(Servers.allMetaAttrs() + [('status', 'unallocated')])
+    servers.append(s.pop())
+
+
+* get console for server X
+* add service W on server X to loadbalancer vip Y
+
+clusto expressions (maybe)
+-----------------------
+
+driver:Console connected to name:server2
+ - return the console connected to 'server2'
+
+driver:Server
+ - return all serves
+
+driver:Server connected to name:rack002
+ - return servers in rack002
+
+attr:foo,12 and driver:Server
+ - return servers with attr foo,12
+
+pattern
+-------
+alphanumeric
+* glob supported ? 
+* regex supported ?
+
+pattern types
+----------
+name:<somename> 
+driver:<driver>
+attr:<key,val>
+
+operators
+---------
+not, or, and, 
+
+functions
+---------
+haskey
+connected to
+
+
+clusto categorization ideas
+===========================
+
+Class, Role, Service
+
+Class is similar to the class concept from fai.  The classes of a server, for
+example, might include information about what software is installed on that
+machine or how much ram the machine has or how many disks.
+
+Role is a description of the job of a Thing.  So a server could have the roles
+of Production, LiveWebserver, and Mail.  These roles are generally more
+abstract and don't refer to specific software or hardware properties
+themselves.  The can however require that the server be a member of certain
+Classes.  So the Blog role might require that the server be connected to the
+Webserver class while the LiveSite role would require the server be connected
+to the Apache and BigMem classes.
+
+A Service is an instance of a role. So you can have dbslave14... (not sure of
+the value of this)
+
+A Pool is just a grouping of Things.  For example you could have many Servers
+with a 'dbslave' role but with some in a 'production' pool and others in a
+'development' pool.  
+
+(how do you put two db roles on one machine?)
+  ( up to sysadmins.  might use VMs, might use vservers (like openvz), might
+    put things on separate ports)
+
+
+clusto command line ideas
+=========================
+
+clusto connect foo1 foo2
+ - connect foo1 and foo2
+
+clusto power off server1
+ 
+clusto locate server1
+ - should return:
+ server1 -> connected to port 4 on switch2a
+ server1 -> connected to RU13 in rack101
+ 
+clusto console server1
+ - connect to console on server1
+
+clusto attr del <key> [<value>]
+clusto attr add <key> <value> [<value>] [<value>] ...
+clusto attr get <key>
+clusto attr set <key> <value> [<value>] [<value>] ...
+ - modify Thing attributes
+
+clusto network setvlan server1 eth1 vlan203
+  - put eth1 on server1 onto vlan203
+
+clusto rebuild server1
+ - reinstall operating system onto server1
+
+Reserverd Attrs
+===============
+
+_ip
+_in

contrib/README

@@ -0,0 +1,16 @@
+dhcpgen.py -- Generate a list of host sections for ISC dhcpd
+discovery.py -- Module for discovering information about hosts, not directly
+				callable
+etchosts.py -- Attempt to discover and create clusto server objects based on
+			   the content of a hosts file. May need modification in main() to
+			   achieve the desired effect.
+parsearp.py -- Imported by etchosts.py, allows discovery and creation of objects
+			   based on the output of scapy's arpping command
+sysinfo.py -- Uses paramiko to connect to servers and gather information about
+			  disks, processors, and memory to store in clusto attributes
+watcher.py -- If run with the "snmp" option, it watches for SNMP traps and
+			  creates new clusto objects bound to the correct rack and switch
+			  port. If run with the "dhcp" option, it will watch for DHCP
+			  requests containing vendor IDs matching PXE clients and IPMI
+			  cards. Associates the MAC address with the proper interface on
+			  the server object.

contrib/clustohttp.py

@@ -0,0 +1,138 @@
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+from urllib import urlencode, quote
+from urlparse import urlsplit, urljoin
+import httplib
+import logging
+
+from pprint import pprint
+
+def request(method, url, body='', headers={}):
+    logging.debug('%s %s' % (method, url))
+    if type(body) != type(''):
+        body = urlencode(body)
+    url = urlsplit(url, 'http')
+
+    conn = httplib.HTTPConnection(url.hostname, url.port)
+    if url.query:
+        query = '%s?%s' % (url.path, url.query)
+    else:
+        query = url.path
+    conn.request(method, query, body, headers)
+    response = conn.getresponse()
+    length = response.getheader('Content-length', None)
+    if length:
+        data = response.read(int(length))
+    else:
+        data = response.read()
+    conn.close()
+    if response.status >= 400:
+        logging.debug('Server error %s: %s' % (response.status, data))
+    return (response.status, response.getheaders(), data)
+
+class ClustoProxy(object):
+    def __init__(self, url):
+        self.url = url
+
+    def get_entities(self, **kwargs):
+        for k, v in kwargs.items():
+            if k == 'attrs':
+                kwargs[k] = json.dumps(v)
+        status, headers, response = request('POST', self.url + '/query/get_entities?%s' % urlencode(kwargs))
+        if status != 200:
+            raise Exception(response)
+        return [EntityProxy(self.url + x) for x in json.loads(response)]
+
+    def get_by_name(self, name):
+        status, headers, response = request('GET', self.url + '/query/get_by_name?name=%s' % quote(name))
+        if status != 200:
+            raise Exception(response)
+        obj = json.loads(response)
+        return EntityProxy(self.url + obj['object'])
+
+    def get_from_pools(self, pools, clusto_types=None):
+        url = self.url + '/query/get_from_pools?pools=%s' % ','.join(pools)
+        if clusto_types:
+            url += '&types=' + ','.join(clusto_types)
+        status, headers, response = request('GET', url)
+        if status != 200:
+            raise Exception(response)
+        return [EntityProxy(self.url + x) for x in json.loads(response)]
+
+    def get_ip_manager(self, ip):
+        status, headers, response = request('GET', self.url + '/query/get_ip_manager?ip=%s' % ip)
+        if status != 200:
+            raise Exception(response)
+        return EntityProxy(self.url + json.loads(response))
+
+class EntityProxy(object):
+    def __init__(self, url):
+        self.url = url
+        self.name = self.url.rsplit('/', 1)[1]
+
+    def __getattr__(self, action):
+        def method(**kwargs):
+            data = {}
+            for k, v in kwargs.items():
+                if isinstance(v, bool):
+                    v = int(v)
+                if not type(v) in (int, str, unicode):
+                    v = json.dumps(v)
+                data[k] = v
+            if data:
+                status, headers, response = request('GET', '%s/%s?%s' % (self.url, action, urlencode(data)))
+            else:
+                status, headers, response = request('GET', '%s/%s' % (self.url, action))
+            if status != 200:
+                raise Exception(response)
+            if response:
+                return json.loads(response)
+            else:
+                return None
+        return method
+
+    def contents(self):
+        return [EntityProxy(urljoin(self.url, x)) for x in self.show()['contents']]
+
+    def parents(self):
+        return [EntityProxy(urljoin(self.url, x)) for x in self.show()['parents']]
+
+    def attrs(self, **kwargs):
+        return self.__getattr__('attrs')(**kwargs)['attrs']
+    
+    def set_port_attr(self, porttype, portnum, key, value):
+        return self.__getattr__('set_port_attr')(porttype=porttype, portnum=portnum, key=key, value=value)
+
+    def get_port_attr(self, porttype, portnum, key):
+        return self.__getattr__('get_port_attr')(porttype=porttype, portnum=portnum, key=key)
+
+    def __str__(self):
+        return urlsplit(self.url).path
+
+    def __repr__(self):
+        return 'EntityProxy(%s)' % repr(self.url)
+
+def test():
+    clusto = ClustoProxy('http://127.0.0.1:9996')
+    server = clusto.get_entities(attrs=[{'subkey': 'mac', 'value': '00:a0:d1:e9:3d:dc'}])
+    server = server[0]
+    print server
+    assert server.name == 's0104'
+    attr = server.get_port_attr('nic-eth', 1, 'mac')
+    server.set_port_attr('nic-eth', 1, 'mac', attr)
+    newattr = server.get_port_attr('nic-eth', 1, 'mac')
+    print repr((attr, newattr))
+    assert newattr == attr
+    #print server.parents()
+    #obj = clusto.get_by_name('s1100')
+    #pprint(obj.ports())
+    #pprint(obj.attrs(key='dhcp', merge_container_attrs=True))
+    #webservers = clusto.get_from_pools(['webservers-lolcat', 'production'])
+    #pprint(webservers)
+    #pprint(webservers[0].contents())
+
+if __name__ == '__main__':
+    test()

contrib/dhcpgen.py

@@ -0,0 +1,52 @@
+from clusto.scripthelpers import init_script
+from clusto.drivers import PenguinServer, IPManager
+import clusto
+import sys
+
+from traceback import format_exc
+
+ATTR_MAP = {
+    'tftp-server': 'next-server',
+    'tftp-filename': 'filename',
+    'nfsroot': 'option root-path',
+}
+
+def main():
+    #for server in clusto.get_entities(clusto_drivers=[PenguinServer]):
+    for server in clusto.get_by_name('fai').contents():
+        out = 'host %s { ' % server.name
+
+        try:
+            mac = server.get_port_attr('nic-eth', 1, 'mac')
+            if not mac:
+                sys.stderr.write('No nic-eth:1 mac for %s\n' % server.name)
+                continue
+            out += 'hardware ethernet %s; ' % mac
+        except:
+            sys.stderr.write(format_exc() + '\n')
+            continue
+
+        ip = IPManager.get_ips(server)
+        if ip:
+            ip = ip[0]
+            out += 'fixed-address %s; ' % ip
+
+        options = {}
+        for attr in server.attrs(key='dhcp', merge_container_attrs=True):
+            if attr.subkey in options: continue
+            if not attr.subkey in ATTR_MAP:
+                sys.stderr.write('Unknown subkey: %s\n' % attr.subkey)
+                continue
+            options[ATTR_MAP[attr.subkey]] = attr.value
+
+        for key, value in options.items():
+            out += '%s %s; ' % (key, value)
+
+        out += '}\n'
+
+        sys.stdout.write(out)
+        sys.stdout.flush()
+
+if __name__ == '__main__':
+    init_script()
+    main()