validate_yaml

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#########+#########+#########+#########+#########+#########+#########+#########+#########+#########+#########+#########+
# Copyright (C) 2009  Joe Blaylock <jrbl@jrbl.org>
#
#This program is free software: you can redistribute it and/or modify it under 
#the terms of the GNU General Public License as published by the Free Software 
#Foundation, either version 3 of the License, or (at your option) any later 
#version.
#
#This program is distributed in the hope that it will be useful, but WITHOUT 
#ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
#FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
#details.
#
#You should have received a copy of the GNU General Public License along with 
#this program.  If not, see <http://www.gnu.org/licenses/>.
"""Validates that the YAML file provided conforms to our expected format.
"""

# Imports
import os, sys

from DictDB import DictDB
from EasyIO import ewriteln


EXPECTED_FORMAT_EN = """
id string:
   'real name': some text
   'email': [ "some text", ... ]
   'irc': [ "some_text", ... ]
   'wiki': [ "some text", ... ]"""
HELP_USAGE_EN = """usage: %prog <opts> [file.yaml]

Default behaviour is to read in the yaml file, sanity-check its contents,
and write out a version with record-internal redundancies removed."""

def validateText(thing):
    """This had better be a string or unicode object."""
    assert isinstance(thing, basestring)
    if isinstance(thing, unicode):
        return thing.encode("utf-8")
    else: return thing

def validateListOfText(thing):
    """Validate the contents of a list of strings, filter unnecessary null strings, return"""
    assert isinstance(thing, list)
    for item in thing:
        validateText(item)
    if len(thing) > 1:
        thing = filter(None, list(set(thing)))
    return thing

def NEWvalidateListOfText(thing):
    """Validate the contents of a list of strings, filter unnecessary null strings, return"""
    assert isinstance(thing, list)
    if len(thing) > 1:
        thing = filter(None, list(set(thing)))
    for i in range(len(thing)):
        thing[i] = validateText(thing[i])
    return thing

# Test Harness
if __name__ == "__main__":
    import optparse    
    usage = HELP_USAGE_EN
    parser = optparse.OptionParser(usage = usage)
    parser.add_option('-w', '--write', dest="write_out", action="store_true", default=True,
                      help="Clean up the file on disk after validation (default).")
    parser.add_option('-W', '--no-write', dest="write_out", action="store_false", default=True,
                      help="Don't modify the file on disk - simply validate its contents.")
    parser.add_option('-f', '--format', dest="show_format", action="store_true", default=False,
                      help="Display the expected YAML file format to the screen.")
    parser.add_option('-o', '--outfile', dest="outfile", action="store", metavar="FILE", 
                      help="When writing cleaned YAML, filename to which we should write")

    options, args = parser.parse_args()
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit()

    if options.show_format:
        print EXPECTED_FORMAT_EN
        sys.exit()

    if len(args) != 1: parser.error("Exactly one YAML input file must be specified.")
    else:
        filename = args[0]
        data = DictDB(filename, format='yaml')

        for key in data.keys():
            assert isinstance(key, int)
            record = data[key]

            try:
                name = record['real name']
            except KeyError, msg:
                # missing keys should be an error, but we can fix it.
                ewriteln("Warning: no real name field for key %s; correcting..." % key)
                name = ''
            validateText(name)
            data[key]['real name'] = name

            try:
                email = record['email']
            except KeyError, msg:
                # missing keys should be an error, but we can fix it.
                ewriteln("Warning: no email field for key %s; correcting..." % key)
                email = ['']
            email = validateListOfText(email)
            data[key]['email'] = email

            try:
                irc = record['irc']
            except KeyError, msg:
                # missing keys should be an error, but we can fix it.
                ewriteln("Warning: no irc nicks field for key %s; correcting..." % key)
                irc = ['']
            irc = validateListOfText(irc)
            data[key]['irc'] = irc

            try:
                wiki = record['wiki']
            except KeyError, msg:
                # missing keys should be an error, but we can fix it.
                ewriteln("Warning: no wiki data field for key %s; correcting..." % key)
                wiki = ['']
            wiki = validateListOfText(wiki)
            data[key]['wiki'] = wiki

        # If we haven't thrown an exception by now, we're fine.
        sys.stdout.write("Everything checks out OK.\n")

    # Since we did some filtering, serialize back to disk
    if options.write_out:
        ofile = args[0]
        if options.outfile != None:
            ofile = options.outfile
        data.sync(ofile)
        #writeYAML(data, ofile)