Skip to content

Commit

Permalink
autopep8, starting refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
gustavo committed Jan 19, 2017
1 parent 8c947b7 commit 2f1cfb3
Show file tree
Hide file tree
Showing 30 changed files with 2,740 additions and 2,449 deletions.
168 changes: 99 additions & 69 deletions fextractor
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,19 @@ import csv
import sys
import random

from vdiscover.Detection import GetArgs, GetFiles, GetCmd
from vdiscover.Detection import GetArgs, GetFiles, GetCmd

# static feature extraction

from vdiscover.RandomWalk import RandomWalkElf

# dynamic feature extraction

from vdiscover.Process import Process
from vdiscover.Mutation import NullMutator, RandomByteMutator, RandomExpanderMutator, RandomInputMutator
from vdiscover.Printer import TypePrinter
from vdiscover.Misc import readmodfile
from vdiscover.Input import prepare_inputs


from vdiscover.Process import Process
from vdiscover.Mutation import NullMutator, RandomByteMutator, RandomExpanderMutator, RandomInputMutator
from vdiscover.Printer import TypePrinter
from vdiscover.Misc import readmodfile
from vdiscover.Input import prepare_inputs


if __name__ == "__main__":
Expand All @@ -48,16 +46,19 @@ if __name__ == "__main__":
random.seed()

# To help argparse to detect the number of columns correctly
#os.environ['COLUMNS'] = str(os.popen('stty size', 'r').read().split()[1]) #str(shutil.get_terminal_size().columns)
# os.environ['COLUMNS'] = str(os.popen('stty size',
# 'r').read().split()[1]) #str(shutil.get_terminal_size().columns)

if open("/proc/sys/kernel/randomize_va_space").read().strip() != "0":
print("Address space layout randomization (ASLR) is enabled, disable it before continue to use the cache")
print("Hint: # echo 0 > /proc/sys/kernel/randomize_va_space")
sys.exit(-1)

# Arguments
parser = argparse.ArgumentParser(description='Feature extraction of VDiscover')
parser.add_argument("testcase", help="Testcase to analyze", type=str, default=None)
parser = argparse.ArgumentParser(
description='Feature extraction of VDiscover')
parser.add_argument(
"testcase", help="Testcase to analyze", type=str, default=None)

parser.add_argument("--static",
help="Extract only static features from an executable",
Expand All @@ -67,54 +68,80 @@ if __name__ == "__main__":
help="Extract only dynamic features from a testcase",
action="store_true", default=False)

parser.add_argument("--mclass", type=str,
help="Include class column, to use later in training mode",
action="store", default=None)
parser.add_argument(
"--mclass",
type=str,
help="Include class column, to use later in training mode",
action="store",
default=None)

parser.add_argument("--out-file",
help="File to output the extracted features",
type=str, default="/dev/stdout")

parser.add_argument("--max-subtraces-collected", type=int,
help="Maximum number of subtraces collected (static features only)", default=100)

parser.add_argument("--max-subtraces-explored", type=int,
help="Maximum number of subtraces explored (static features only)", default=10000)

parser.add_argument("--min-subtrace-size", type=int,
help="Minumum number of events in each subtrace collected (static features only)", default=3)

parser.add_argument("--show-stdout",
help="Don't use /dev/null as stdout/stderr (dynamic features only)",
action="store_true", default=False)

parser.add_argument("--inc-mods",
help="Only extract features from the libraries matching the strings inside this file (dynamic features only)",
type=str, default=None)

parser.add_argument("--ign-mods",
help="Ignore extracted features from the libraries matching the string inside this file (dynamic features only)",
type=str, default=None)

parser.add_argument("--timeout", dest="timeout", type=int,
help="Timeout in seconds (dynamic features only)", default=3)

parser.add_argument("--max-mutations", type=int,
help="Maximum number of mutations to the original testcase (dynamic features only)", default=0)
parser.add_argument(
"--max-subtraces-collected",
type=int,
help="Maximum number of subtraces collected (static features only)",
default=100)

parser.add_argument(
"--max-subtraces-explored",
type=int,
help="Maximum number of subtraces explored (static features only)",
default=10000)

parser.add_argument(
"--min-subtrace-size",
type=int,
help="Minumum number of events in each subtrace collected (static features only)",
default=3)

parser.add_argument(
"--show-stdout",
help="Don't use /dev/null as stdout/stderr (dynamic features only)",
action="store_true",
default=False)

parser.add_argument(
"--inc-mods",
help="Only extract features from the libraries matching the strings inside this file (dynamic features only)",
type=str,
default=None)

parser.add_argument(
"--ign-mods",
help="Ignore extracted features from the libraries matching the string inside this file (dynamic features only)",
type=str,
default=None)

parser.add_argument(
"--timeout",
dest="timeout",
type=int,
help="Timeout in seconds (dynamic features only)",
default=3)

parser.add_argument(
"--max-mutations",
type=int,
help="Maximum number of mutations to the original testcase (dynamic features only)",
default=0)

options = parser.parse_args()
testcase = options.testcase

static_only = options.static
dynamic_only = options.dynamic

if (not static_only and not dynamic_only) or (static_only and dynamic_only):
print "The feature extraction requires to select either static of dynamic features exclusively"
exit(-1)
if (not static_only and not dynamic_only) or (
static_only and dynamic_only):
print "The feature extraction requires to select either static of dynamic features exclusively"
exit(-1)

max_subtraces_collected = options.max_subtraces_collected
max_subtraces_explored = options.max_subtraces_explored
min_subtrace_size = options.min_subtrace_size
max_subtraces_explored = options.max_subtraces_explored
min_subtrace_size = options.min_subtrace_size

incmodfile = options.inc_mods
ignmodfile = options.ign_mods
Expand All @@ -135,39 +162,42 @@ if __name__ == "__main__":

if static_only:

RandomWalkElf(program, csvfile, mclass, max_subtraces_collected, max_subtraces_explored, min_subtrace_size)
RandomWalkElf(program, csvfile, mclass, max_subtraces_collected,
max_subtraces_explored, min_subtrace_size)

elif dynamic_only:

os.chdir("inputs")
os.chdir("inputs")

envs = dict()
args = GetArgs()
files = GetFiles()
envs = dict()
args = GetArgs()
files = GetFiles()

original_inputs = RandomInputMutator(args + files, NullMutator)
#expanded_input_generator = RandomInputMutator(args + files, RandomExpanderMutator)
mutated_input_generator = RandomInputMutator(args + files, RandomByteMutator)
if included_mods == []:
included_mods = [program]
original_inputs = RandomInputMutator(args + files, NullMutator)
#expanded_input_generator = RandomInputMutator(args + files, RandomExpanderMutator)
mutated_input_generator = RandomInputMutator(
args + files, RandomByteMutator)
if included_mods == []:
included_mods = [program]

app = Process(program, envs, timeout, included_mods, ignored_mods, no_stdout = not show_stdout )
prt = TypePrinter(csvfile, testcase, mclass)
app = Process(program, envs, timeout, included_mods,
ignored_mods, no_stdout=not show_stdout)
prt = TypePrinter(csvfile, testcase, mclass)

# unchanged input
null_mutt, original_input = original_inputs.next()
original_events = app.getData(prepare_inputs(original_input))
# unchanged input
null_mutt, original_input = original_inputs.next()
original_events = app.getData(prepare_inputs(original_input))

if original_events is None:
print "Execution of",program,"failed!"
exit(-1)
if original_events is None:
print "Execution of", program, "failed!"
exit(-1)

prt.print_events(program,original_events)
prt.print_events(program, original_events)

for (i, (d, mutated)) in enumerate(mutated_input_generator):
for (i, (d, mutated)) in enumerate(mutated_input_generator):

if i >= max_mut:
break
if i >= max_mut:
break

events = app.getData(prepare_inputs(mutated))
prt.print_events(program,events)
events = app.getData(prepare_inputs(mutated))
prt.print_events(program, events)
11 changes: 7 additions & 4 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@
url='http://vdiscover.org/',
author='G.Grieco',
author_email='[email protected]',
scripts=['fextractor', 'vpredictor', 'tcreator', 'tseeder', 'vd'],
scripts=[
'fextractor',
'vpredictor',
'tcreator',
'tseeder',
'vd'],
install_requires=[
"python-ptrace",
"scikit-learn"
],
"scikit-learn"],
)

84 changes: 48 additions & 36 deletions tcreator
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,61 +25,73 @@ import sys
import csv

from vdiscover.Detection import WriteTestcase
concatenate = lambda *lists: reduce((lambda a,b: a.extend(b) or a),lists,[])
from functools import reduce
concatenate = lambda *lists: reduce((lambda a, b: a.extend(b) or a), lists, [])

if __name__ == "__main__":

# Arguments
parser = argparse.ArgumentParser(description='A small utility to create new test cases using a name and a command line')
parser.add_argument("--name", help="The name of the ", type=str, default=None)
parser.add_argument("--cmd", help="Command-line to execute", type=str, default=None)
parser.add_argument("--batch", help="A csv with the command lines", type=str, default=None)

parser.add_argument("--copy", help="Force the copy of the files in command lines instead of symbolic linking", action='store_true', default=False)

parser.add_argument("outdir", help="Output directory to write testcases", type=str, default=None)
parser = argparse.ArgumentParser(
description='A small utility to create new test cases using a name and a command line')
parser.add_argument("--name", help="The name of the ",
type=str, default=None)
parser.add_argument(
"--cmd", help="Command-line to execute", type=str, default=None)
parser.add_argument(
"--batch", help="A csv with the command lines", type=str, default=None)

parser.add_argument(
"--copy",
help="Force the copy of the files in command lines instead of symbolic linking",
action='store_true',
default=False)

parser.add_argument(
"outdir",
help="Output directory to write testcases",
type=str,
default=None)

options = parser.parse_args()
name = options.name
cmd = options.cmd
in_file = options.batch
copy = options.copy
out_dir= options.outdir
out_dir = options.outdir

if (name is not None and cmd is not None) ^ (in_file is not None):
pass
pass
else:
#or (name not is None and cmd is not None) and in_file is None:
print "Either name and command should be used or an input file"
exit(-1)
# or (name not is None and cmd is not None) and in_file is None:
print "Either name and command should be used or an input file"
exit(-1)

try:
os.makedirs(out_dir)
os.makedirs(out_dir)
except:
pass
pass

if in_file is not None:
infile = open(in_file,"r")
csvreader = csv.reader(infile, delimiter='\t')
os.chdir(out_dir)
infile = open(in_file, "r")
csvreader = csv.reader(infile, delimiter='\t')
os.chdir(out_dir)

for i,row in enumerate(csvreader):
args = filter(lambda x: x is not '', row[0].split(" "))
name = args[0].replace("/","_")+":"+str(i)
WriteTestcase(name,args[0],args[1:], copy)
for i, row in enumerate(csvreader):
args = filter(lambda x: x is not '', row[0].split(" "))
name = args[0].replace("/", "_") + ":" + str(i)
WriteTestcase(name, args[0], args[1:], copy)

else:

os.chdir(out_dir)
args = cmd.split("'")
args = map(lambda x: x.split(" "), args)
pargs = []

for arg in args:
if arg <> '':
pargs = pargs + arg
#args = concatenate(args)
print "Procesing '" + " ".join(pargs) + "'"
#args = filter(lambda x: x is not '', cmd.split(" "))
WriteTestcase(name,pargs[0],pargs[1:], copy)

os.chdir(out_dir)
args = cmd.split("'")
args = map(lambda x: x.split(" "), args)
pargs = []

for arg in args:
if arg != '':
pargs = pargs + arg
#args = concatenate(args)
print "Procesing '" + " ".join(pargs) + "'"
#args = filter(lambda x: x is not '', cmd.split(" "))
WriteTestcase(name, pargs[0], pargs[1:], copy)
Loading

0 comments on commit 2f1cfb3

Please sign in to comment.