Remove OS assumption (#15)

* removed kali specific installs - test 1

* removed kali specific installs - test 2

* removed kali specific installs - test 2

* removed kali specific installs - test 4

* removed kali specific installs - test 5

* removed kali specific installs - test 6

* removed kali specific installs - test 7

* removed kali specific installs - test 8

* removed kali specific installs - test 9

* removed kali specific installs - test 10

* removed kali specific installs - test 11

* removed kali specific installs - test 12

* removed kali specific installs - test 13

* removed kali specific installs - test 14

* all works locally, fixing up tests et al

* trying pipeline again; round 2

* looks good locally; testing pipeline again

* test 18?

* test 19

* 20

* 21

.github/workflows/pythonapp.yml

@@ -47,7 +47,7 @@ jobs:
         pipenv install -d
     - name: Test with pytest
       run: |
-        pipenv install pytest
+        pipenv install pytest cmd2 luigi
         pipenv run python -m pytest tests/test_install
 
   test-recon:
@@ -67,7 +67,7 @@ jobs:
         pipenv install -d
     - name: Test with pytest
       run: |
-        pipenv install pytest
+        pipenv install pytest cmd2 luigi
         pipenv run python -m pytest tests/test_recon
 
   test-web:
@@ -87,5 +87,5 @@ jobs:
         pipenv install -d
     - name: Test with pytest
       run: |
-        pipenv install pytest
+        pipenv install pytest cmd2 luigi
         pipenv run python -m pytest tests/test_web

.gitignore

@@ -103,4 +103,6 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-.idea
+.idea
+Pipfile
+Pipfile.lock

docs/conf.py

@@ -38,12 +38,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx.ext.coverage",
-    "sphinx.ext.napoleon",
-    "sphinxarg.ext",
-]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.napoleon", "sphinxarg.ext"]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]

recon-pipeline.py

@@ -16,12 +16,16 @@
 # suppress "You should consider upgrading via the 'pip install --upgrade pip' command." warning
 os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
 
+# in case we need pipenv, add its default --user installed directory to the PATH
+sys.path.append(str(Path.home() / ".local" / "bin"))
+
 # third party imports
 import cmd2  # noqa: E402
 from cmd2.ansi import style  # noqa: E402
 
 # project's module imports
 from recon import get_scans, tools, scan_parser, install_parser, status_parser  # noqa: F401,E402
+from recon.config import defaults  # noqa: F401,E402
 
 # select loop, handles async stdout/stderr processing of subprocesses
 selector = selectors.DefaultSelector()
@@ -58,6 +62,9 @@ def __init__(self, *args, **kwargs):
         self.sentry = False
         self.prompt = "recon-pipeline> "
         self.selectorloop = None
+        self.continue_install = True
+
+        Path(defaults.get("tools-dir")).mkdir(parents=True, exist_ok=True)
 
         # register hooks to handle selector loop start and cleanup
         self.register_preloop_hook(self._preloop_hook)
@@ -215,7 +222,7 @@ def do_install(self, args):
                     continue
 
                 self.async_alert(
-                    style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True,)
+                    style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True)
                 )
 
                 # install the dependency before continuing with installation
@@ -224,13 +231,17 @@ def do_install(self, args):
         if tools.get(args.tool).get("installed"):
             return self.async_alert(style(f"[!] {args.tool} is already installed.", fg="yellow"))
         else:
-
             # list of return values from commands run during each tool installation
             # used to determine whether the tool installed correctly or not
             retvals = list()
 
             self.async_alert(style(f"[*] Installing {args.tool}...", fg="bright_yellow"))
 
+            addl_env_vars = tools.get(args.tool).get("environ")
+
+            if addl_env_vars is not None:
+                addl_env_vars.update(dict(os.environ))
+
             for command in tools.get(args.tool).get("commands"):
                 # run all commands required to install the tool
 
@@ -240,11 +251,15 @@ def do_install(self, args):
                 if tools.get(args.tool).get("shell"):
 
                     # go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True
-                    proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
+                    proc = subprocess.Popen(
+                        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars
+                    )
                 else:
 
                     # "normal" command, split up the string as usual and run it
-                    proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
+                    proc = subprocess.Popen(
+                        shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars
+                    )
 
                 out, err = proc.communicate()