autopep8, starting refactoring

Author: gustavo

fextractor

@@ -25,21 +25,19 @@ import csv
 import sys
 import random
 
-from vdiscover.Detection  import GetArgs, GetFiles, GetCmd
+from vdiscover.Detection import GetArgs, GetFiles, GetCmd
 
 # static feature extraction
 
 from vdiscover.RandomWalk import RandomWalkElf
 
 # dynamic feature extraction
 
-from vdiscover.Process    import Process
-from vdiscover.Mutation   import NullMutator, RandomByteMutator, RandomExpanderMutator, RandomInputMutator
-from vdiscover.Printer    import TypePrinter
-from vdiscover.Misc       import readmodfile
-from vdiscover.Input      import prepare_inputs
-
-
+from vdiscover.Process import Process
+from vdiscover.Mutation import NullMutator, RandomByteMutator, RandomExpanderMutator, RandomInputMutator
+from vdiscover.Printer import TypePrinter
+from vdiscover.Misc import readmodfile
+from vdiscover.Input import prepare_inputs
 
 
 if __name__ == "__main__":
@@ -48,16 +46,19 @@ if __name__ == "__main__":
     random.seed()
 
     # To help argparse to detect the number of columns correctly
-    #os.environ['COLUMNS'] = str(os.popen('stty size', 'r').read().split()[1]) #str(shutil.get_terminal_size().columns)
+    # os.environ['COLUMNS'] = str(os.popen('stty size',
+    # 'r').read().split()[1]) #str(shutil.get_terminal_size().columns)
 
     if open("/proc/sys/kernel/randomize_va_space").read().strip() != "0":
         print("Address space layout randomization (ASLR) is enabled, disable it before continue to use the cache")
         print("Hint: # echo 0 > /proc/sys/kernel/randomize_va_space")
         sys.exit(-1)
 
     # Arguments
-    parser = argparse.ArgumentParser(description='Feature extraction of VDiscover')
-    parser.add_argument("testcase", help="Testcase to analyze", type=str, default=None)
+    parser = argparse.ArgumentParser(
+        description='Feature extraction of VDiscover')
+    parser.add_argument(
+        "testcase", help="Testcase to analyze", type=str, default=None)
 
     parser.add_argument("--static",
                         help="Extract only static features from an executable",
@@ -67,54 +68,80 @@ if __name__ == "__main__":
                         help="Extract only dynamic features from a testcase",
                         action="store_true", default=False)
 
-    parser.add_argument("--mclass", type=str,
-                        help="Include class column, to use later in training mode",
-                        action="store", default=None)
+    parser.add_argument(
+        "--mclass",
+        type=str,
+        help="Include class column, to use later in training mode",
+        action="store",
+        default=None)
 
     parser.add_argument("--out-file",
                         help="File to output the extracted features",
                         type=str, default="/dev/stdout")
 
-    parser.add_argument("--max-subtraces-collected", type=int,
-                        help="Maximum number of subtraces collected (static features only)", default=100)
-
-    parser.add_argument("--max-subtraces-explored", type=int,
-                        help="Maximum number of subtraces explored (static features only)", default=10000)
-
-    parser.add_argument("--min-subtrace-size", type=int,
-                        help="Minumum number of events in each subtrace collected (static features only)", default=3)
-
-    parser.add_argument("--show-stdout",
-                        help="Don't use /dev/null as stdout/stderr (dynamic features only)",
-                        action="store_true", default=False)
-
-    parser.add_argument("--inc-mods",
-                        help="Only extract features from the libraries matching the strings inside this file (dynamic features only)",
-                        type=str, default=None)
-
-    parser.add_argument("--ign-mods",
-                        help="Ignore extracted features from the libraries matching the string inside this file (dynamic features only)",
-                        type=str, default=None)
-
-    parser.add_argument("--timeout", dest="timeout", type=int,
-                        help="Timeout in seconds (dynamic features only)", default=3)
-
-    parser.add_argument("--max-mutations", type=int,
-                        help="Maximum number of mutations to the original testcase (dynamic features only)", default=0)
+    parser.add_argument(
+        "--max-subtraces-collected",
+        type=int,
+        help="Maximum number of subtraces collected (static features only)",
+        default=100)
+
+    parser.add_argument(
+        "--max-subtraces-explored",
+        type=int,
+        help="Maximum number of subtraces explored (static features only)",
+        default=10000)
+
+    parser.add_argument(
+        "--min-subtrace-size",
+        type=int,
+        help="Minumum number of events in each subtrace collected (static features only)",
+        default=3)
+
+    parser.add_argument(
+        "--show-stdout",
+        help="Don't use /dev/null as stdout/stderr (dynamic features only)",
+        action="store_true",
+        default=False)
+
+    parser.add_argument(
+        "--inc-mods",
+        help="Only extract features from the libraries matching the strings inside this file (dynamic features only)",
+        type=str,
+        default=None)
+
+    parser.add_argument(
+        "--ign-mods",
+        help="Ignore extracted features from the libraries matching the string inside this file (dynamic features only)",
+        type=str,
+        default=None)
+
+    parser.add_argument(
+        "--timeout",
+        dest="timeout",
+        type=int,
+        help="Timeout in seconds (dynamic features only)",
+        default=3)
+
+    parser.add_argument(
+        "--max-mutations",
+        type=int,
+        help="Maximum number of mutations to the original testcase (dynamic features only)",
+        default=0)
 
     options = parser.parse_args()
     testcase = options.testcase
 
     static_only = options.static
     dynamic_only = options.dynamic
 
-    if (not static_only and not dynamic_only) or (static_only and dynamic_only):
-      print "The feature extraction requires to select either static of dynamic features exclusively"
-      exit(-1)
+    if (not static_only and not dynamic_only) or (
+            static_only and dynamic_only):
+        print "The feature extraction requires to select either static of dynamic features exclusively"
+        exit(-1)
 
     max_subtraces_collected = options.max_subtraces_collected
-    max_subtraces_explored  = options.max_subtraces_explored
-    min_subtrace_size       = options.min_subtrace_size
+    max_subtraces_explored = options.max_subtraces_explored
+    min_subtrace_size = options.min_subtrace_size
 
     incmodfile = options.inc_mods
     ignmodfile = options.ign_mods
@@ -135,39 +162,42 @@ if __name__ == "__main__":
 
     if static_only:
 
-      RandomWalkElf(program, csvfile, mclass, max_subtraces_collected, max_subtraces_explored, min_subtrace_size)
+        RandomWalkElf(program, csvfile, mclass, max_subtraces_collected,
+                      max_subtraces_explored, min_subtrace_size)
 
     elif dynamic_only:
 
-      os.chdir("inputs")
+        os.chdir("inputs")
 
-      envs = dict()
-      args = GetArgs()
-      files = GetFiles()
+        envs = dict()
+        args = GetArgs()
+        files = GetFiles()
 
-      original_inputs = RandomInputMutator(args + files, NullMutator)
-      #expanded_input_generator = RandomInputMutator(args + files, RandomExpanderMutator)
-      mutated_input_generator = RandomInputMutator(args + files, RandomByteMutator)
-      if included_mods == []:
-        included_mods = [program]
+        original_inputs = RandomInputMutator(args + files, NullMutator)
+        #expanded_input_generator = RandomInputMutator(args + files, RandomExpanderMutator)
+        mutated_input_generator = RandomInputMutator(
+            args + files, RandomByteMutator)
+        if included_mods == []:
+            included_mods = [program]
 
-      app = Process(program, envs, timeout, included_mods, ignored_mods, no_stdout = not show_stdout )
-      prt = TypePrinter(csvfile, testcase, mclass)
+        app = Process(program, envs, timeout, included_mods,
+                      ignored_mods, no_stdout=not show_stdout)
+        prt = TypePrinter(csvfile, testcase, mclass)
 
-      # unchanged input
-      null_mutt, original_input = original_inputs.next()
-      original_events = app.getData(prepare_inputs(original_input))
+        # unchanged input
+        null_mutt, original_input = original_inputs.next()
+        original_events = app.getData(prepare_inputs(original_input))
 
-      if original_events is None:
-        print "Execution of",program,"failed!"
-        exit(-1)
+        if original_events is None:
+            print "Execution of", program, "failed!"
+            exit(-1)
 
-      prt.print_events(program,original_events) 
+        prt.print_events(program, original_events)
 
-      for (i, (d, mutated)) in enumerate(mutated_input_generator):
+        for (i, (d, mutated)) in enumerate(mutated_input_generator):
 
-        if i >= max_mut:
-          break
+            if i >= max_mut:
+                break
 
-        events = app.getData(prepare_inputs(mutated))
-        prt.print_events(program,events)
+            events = app.getData(prepare_inputs(mutated))
+            prt.print_events(program, events)

setup.py

@@ -12,10 +12,13 @@
     url='http://vdiscover.org/',
     author='G.Grieco',
     author_email='[email protected]',
-    scripts=['fextractor', 'vpredictor', 'tcreator', 'tseeder', 'vd'],
+    scripts=[
+        'fextractor',
+        'vpredictor',
+        'tcreator',
+        'tseeder',
+        'vd'],
     install_requires=[
         "python-ptrace",
-        "scikit-learn"
-    ],
+        "scikit-learn"],
 )
-

tcreator

@@ -25,61 +25,73 @@ import sys
 import csv
 
 from vdiscover.Detection import WriteTestcase
-concatenate = lambda *lists: reduce((lambda a,b: a.extend(b) or a),lists,[])
+from functools import reduce
+concatenate = lambda *lists: reduce((lambda a, b: a.extend(b) or a), lists, [])
 
 if __name__ == "__main__":
 
     # Arguments
-    parser = argparse.ArgumentParser(description='A small utility to create new test cases using a name and a command line')
-    parser.add_argument("--name", help="The name of the ", type=str, default=None)
-    parser.add_argument("--cmd", help="Command-line to execute", type=str, default=None)
-    parser.add_argument("--batch", help="A csv with the command lines", type=str, default=None)
-
-    parser.add_argument("--copy", help="Force the copy of the files in command lines instead of symbolic linking", action='store_true', default=False)
-
-    parser.add_argument("outdir", help="Output directory to write testcases", type=str, default=None)
+    parser = argparse.ArgumentParser(
+        description='A small utility to create new test cases using a name and a command line')
+    parser.add_argument("--name", help="The name of the ",
+                        type=str, default=None)
+    parser.add_argument(
+        "--cmd", help="Command-line to execute", type=str, default=None)
+    parser.add_argument(
+        "--batch", help="A csv with the command lines", type=str, default=None)
+
+    parser.add_argument(
+        "--copy",
+        help="Force the copy of the files in command lines instead of symbolic linking",
+        action='store_true',
+        default=False)
+
+    parser.add_argument(
+        "outdir",
+        help="Output directory to write testcases",
+        type=str,
+        default=None)
 
     options = parser.parse_args()
     name = options.name
     cmd = options.cmd
     in_file = options.batch
     copy = options.copy
-    out_dir= options.outdir
+    out_dir = options.outdir
 
     if (name is not None and cmd is not None) ^ (in_file is not None):
-      pass
+        pass
     else:
-      #or (name not is None and cmd is not None) and in_file is None:
-      print "Either name and command should be used or an input file"
-      exit(-1)
+        # or (name not is None and cmd is not None) and in_file is None:
+        print "Either name and command should be used or an input file"
+        exit(-1)
 
     try:
-      os.makedirs(out_dir)
+        os.makedirs(out_dir)
     except:
-      pass
+        pass
 
     if in_file is not None:
-      infile = open(in_file,"r")
-      csvreader = csv.reader(infile, delimiter='\t')
-      os.chdir(out_dir)
+        infile = open(in_file, "r")
+        csvreader = csv.reader(infile, delimiter='\t')
+        os.chdir(out_dir)
 
-      for i,row in enumerate(csvreader):
-        args = filter(lambda x: x is not '', row[0].split(" "))
-        name = args[0].replace("/","_")+":"+str(i)
-        WriteTestcase(name,args[0],args[1:], copy)
+        for i, row in enumerate(csvreader):
+            args = filter(lambda x: x is not '', row[0].split(" "))
+            name = args[0].replace("/", "_") + ":" + str(i)
+            WriteTestcase(name, args[0], args[1:], copy)
 
     else:
 
-      os.chdir(out_dir)
-      args = cmd.split("'")
-      args = map(lambda x: x.split(" "), args)
-      pargs = []
-
-      for arg in args:
-        if arg <> '':
-          pargs = pargs + arg
-      #args = concatenate(args)
-      print "Procesing '" + " ".join(pargs) + "'"
-      #args = filter(lambda x: x is not '', cmd.split(" "))
-      WriteTestcase(name,pargs[0],pargs[1:], copy)
-
+        os.chdir(out_dir)
+        args = cmd.split("'")
+        args = map(lambda x: x.split(" "), args)
+        pargs = []
+
+        for arg in args:
+            if arg != '':
+                pargs = pargs + arg
+        #args = concatenate(args)
+        print "Procesing '" + " ".join(pargs) + "'"
+        #args = filter(lambda x: x is not '', cmd.split(" "))
+        WriteTestcase(name, pargs[0], pargs[1:], copy)