Merge pull request #923 from pgrenaud/add-pipx-and-virtual-env-support

Add pipx and virtual env support

Author: six2dez

install.sh

@@ -83,31 +83,39 @@ declare -A gotools=(
 	["urlfinder"]="go install -v github.com/projectdiscovery/urlfinder/cmd/urlfinder@latest"
 )
 
-# Declare repositories and their paths
-declare -A repos=(
-	["dorks_hunter"]="six2dez/dorks_hunter"
+# Declare pipx tools and their paths
+declare -A pipxtools=(
 	["dnsvalidator"]="vortexau/dnsvalidator"
 	["interlace"]="codingo/Interlace"
 	["wafw00f"]="EnableSecurity/wafw00f"
+	["commix"]="commixproject/commix"
+	["urless"]="xnl-h4ck3r/urless"
+	["ghauri"]="r0oth3x49/ghauri"
+	["xnLinkFinder"]="xnl-h4ck3r/xnLinkFinder"
+	["porch-pirate"]="MandConsultingGroup/porch-pirate"
+	["MetaFinder"]="Josue87/MetaFinder"
+	["EmailFinder"]="Josue87/EmailFinder"
+)
+
+# Declare repositories and their paths
+declare -A repos=(
+	["dorks_hunter"]="six2dez/dorks_hunter"
 	["gf"]="tomnomnom/gf"
 	["Gf-Patterns"]="1ndianl33t/Gf-Patterns"
 	["Corsy"]="s0md3v/Corsy"
 	["CMSeeK"]="Tuhinshubhra/CMSeeK"
 	["fav-up"]="pielco11/fav-up"
 	["massdns"]="blechschmidt/massdns"
 	["Oralyzer"]="r0075h3ll/Oralyzer"
-	["testssl.sh"]="drwetter/testssl.sh"
-	["commix"]="commixproject/commix"
+	["testssl"]="drwetter/testssl.sh"
 	["JSA"]="w9w/JSA"
 	["CloudHunter"]="belane/CloudHunter"
 	["ultimate-nmap-parser"]="shifty0g/ultimate-nmap-parser"
 	["pydictor"]="LandGrey/pydictor"
 	["gitdorks_go"]="damit5/gitdorks_go"
-	["urless"]="xnl-h4ck3r/urless"
 	["smuggler"]="defparam/smuggler"
 	["Web-Cache-Vulnerability-Scanner"]="Hackmanit/Web-Cache-Vulnerability-Scanner"
 	["regulator"]="cramppet/regulator"
-	["ghauri"]="r0oth3x49/ghauri"
 	["gitleaks"]="gitleaks/gitleaks"
 	["trufflehog"]="trufflesecurity/trufflehog"
 	["nomore403"]="devploit/nomore403"
@@ -116,10 +124,6 @@ declare -A repos=(
 	["ffufPostprocessing"]="Damian89/ffufPostprocessing"
 	["misconfig-mapper"]="intigriti/misconfig-mapper"
 	["Spoofy"]="MattKeeley/Spoofy"
-	["xnLinkFinder"]="xnl-h4ck3r/xnLinkFinder"
-	["porch-pirate"]="MandConsultingGroup/porch-pirate"
-	["MetaFinder"]="Josue87/MetaFinder"
-	["EmailFinder"]="Josue87/EmailFinder"
 )
 
 # Function to display the banner
@@ -169,6 +173,43 @@ function install_tools() {
 		fi
 	done
 
+	echo -e "\n${bblue}Running: Installing pipx tools (${#repos[@]})${reset}\n"
+
+	local pipx_step=0
+	local failed_pipx_tools=()
+
+	for pipxtool in "${!pipxtools[@]}"; do
+		((pipx_step++))
+		if [[ $upgrade_tools == "false" ]]; then
+			if command -v "$pipxtool" &>/dev/null; then
+				echo -e "[${yellow}SKIPPING${reset}] $pipxtool already installed at $(command -v "$pipxtool")"
+				continue
+			fi
+		fi
+
+		# Install the pipx tool
+		eval pipx install "git+https://github.com/${pipxtools[$pipxtool]}" &>/dev/null
+		exit_status=$?
+		if [[ $exit_status -ne 0 ]]; then
+			echo -e "${red}Failed to install $pipxtool, try manually (${pipx_step}/${#pipxtools[@]})${reset}"
+			failed_pipx_tools+=("$pipxtool")
+			double_check=true
+			continue
+		fi
+
+		# Upgrade the pipx tool
+		eval pipx upgrade "${pipxtool}" &>/dev/null
+		exit_status=$?
+		if [[ $exit_status -ne 0 ]]; then
+			echo -e "${red}Failed to upgrade $pipxtool, try manually (${pipx_step}/${#pipxtools[@]})${reset}"
+			failed_pipx_tools+=("$pipxtool")
+			double_check=true
+			continue
+		fi
+
+		echo -e "${yellow}$pipxtool installed (${pipx_step}/${#pipxtools[@]})${reset}"
+	done
+
 	echo -e "\n${bblue}Running: Installing repositories (${#repos[@]})${reset}\n"
 
 	local repos_step=0
@@ -212,9 +253,14 @@ function install_tools() {
 			continue
 		fi
 
-		# Install dependencies if setup.py exists
-		if [[ -f "setup.py" ]]; then
-			eval "$SUDO pipx install . $DEBUG_STD" &>/dev/null
+		# Install requirements inside a virtual environment
+		if [[ -s "requirements.txt" ]]; then
+			if [[ ! -f "venv/bin/activate" ]]; then
+				python3 -m venv venv &>/dev/null
+			fi
+			source venv/bin/activate
+			eval "pip3 install --upgrade -r requirements.txt $DEBUG_STD" &>/dev/null
+			deactivate
 		fi
 
 		# Special handling for certain repositories
@@ -273,6 +319,10 @@ function install_tools() {
 		echo -e "\n${red}Failed to install the following Go tools: ${failed_tools[*]}${reset}"
 	fi
 
+	if [[ ${#failed_pipx_tools[@]} -ne 0 ]]; then
+		echo -e "\n${red}Failed to install the following pipx tools: ${failed_pipx_tools[*]}${reset}"
+	fi
+
 	if [[ ${#failed_repos[@]} -ne 0 ]]; then
 		echo -e "\n${red}Failed to clone or update the following repositories:\n${failed_repos[*]}${reset}"
 	fi
@@ -448,7 +498,7 @@ function install_yum() {
 
 # Function to install required packages for Arch-based systems
 function install_pacman() {
-	"$SUDO" pacman -Sy --noconfirm python python-pip base-devel gcc cmake ruby git curl libpcap pipx whois wget zip pv bind openssl libffi libxml2 libxslt zlib nmap jq lynx medusa xorg-server-xvfb &>/dev/null
+	"$SUDO" pacman -Sy --noconfirm python python-pip base-devel gcc cmake ruby git curl libpcap python-pipx whois wget zip pv bind openssl libffi libxml2 libxslt zlib nmap jq lynx medusa xorg-server-xvfb &>/dev/null
 	curl https://sh.rustup.rs -sSf | sh -s -- -y >/dev/null 2>&1
 	source "${HOME}/.cargo/env"
 	cargo install ripgen &>/dev/null
@@ -475,12 +525,8 @@ function initial_setup() {
 	touch "${dir}/.github_tokens"
 	touch "${dir}/.gitlab_tokens"
 
-	wget -N -c https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py &>/dev/null
-	python3 /tmp/get-pip.py &>/dev/null
-	rm -f /tmp/get-pip.py
-
-	pipx install reconftw &>/dev/null
-	pipx inject mkdocs -r requirements.txt &>/dev/null
+	eval pipx ensurepath $DEBUG_STD
+	source "${HOME}/${profile_shell}"
 
 	install_tools
 
@@ -522,26 +568,6 @@ function initial_setup() {
 		eval git -C "${dir}/massdns" pull $DEBUG_STD
 	fi
 
-	# Interlace
-	if [[ ! -d "${dir}/interlace" ]]; then
-		#printf "${yellow}Cloning Interlace...${reset}\n"
-		eval git clone https://github.com/codingo/Interlace.git "${dir}/interlace" $DEBUG_STD
-		eval cd "${dir}/interlace" && eval $SUDO python3 setup.py install $DEBUG_STD
-	else
-		#printf "${yellow}Updating Interlace...${reset}\n"
-		eval git -C "${dir}/interlace" pull $DEBUG_STD
-	fi
-
-	# wafw00f
-	if [[ ! -d "${dir}/wafw00f" ]]; then
-		#printf "${yellow}Cloning wafw00f...${reset}\n"
-		eval git clone https://github.com/EnableSecurity/wafw00f.git "${dir}/wafw00f" $DEBUG_STD
-		eval cd "${dir}/wafw00f" && eval $SUDO python3 setup.py install $DEBUG_STD
-	else
-		#printf "${yellow}Updating wafw00f...${reset}\n"
-		eval git -C "${dir}/wafw00f" pull $DEBUG_STD
-	fi
-
 	# gf patterns
 	if [[ ! -d "$HOME/.gf" ]]; then
 		#printf "${yellow}Installing gf patterns...${reset}\n"

reconftw.sh

@@ -120,32 +120,42 @@ function tools_installed() {
 	# Define tools and their paths/commands
 	declare -A tools_files=(
 		["dorks_hunter"]="${tools}/dorks_hunter/dorks_hunter.py"
+		["dorks_hunter_python"]="${tools}/dorks_hunter/venv/bin/python3"
 		["fav-up"]="${tools}/fav-up/favUp.py"
+		["fav-up_python"]="${tools}/fav-up/venv/bin/python3"
 		["Corsy"]="${tools}/Corsy/corsy.py"
-		["testssl.sh"]="${tools}/testssl.sh/testssl.sh"
+		["Corsy_python"]="${tools}/Corsy/venv/bin/python3"
+		["testssl"]="${tools}/testssl.sh/testssl.sh"
 		["CMSeeK"]="${tools}/CMSeeK/cmseek.py"
+		["CMSeeK_python"]="${tools}/CMSeeK/venv/bin/python3"
 		["OneListForAll"]="$fuzz_wordlist"
 		["lfi_wordlist"]="$lfi_wordlist"
 		["ssti_wordlist"]="$ssti_wordlist"
 		["subs_wordlist"]="$subs_wordlist"
 		["subs_wordlist_big"]="$subs_wordlist_big"
 		["resolvers"]="$resolvers"
 		["resolvers_trusted"]="$resolvers_trusted"
-		["commix"]="${tools}/commix/commix.py"
 		["getjswords"]="${tools}/getjswords.py"
 		["JSA"]="${tools}/JSA/jsa.py"
+		["JSA_python"]="${tools}/JSA/venv/bin/python3"
 		["CloudHunter"]="${tools}/CloudHunter/cloudhunter.py"
+		["CloudHunter_python"]="${tools}/CloudHunter/venv/bin/python3"
 		["nmap-parse-output"]="${tools}/ultimate-nmap-parser/ultimate-nmap-parser.sh"
 		["pydictor"]="${tools}/pydictor/pydictor.py"
-		["urless"]="${tools}/urless/urless/urless.py"
 		["smuggler"]="${tools}/smuggler/smuggler.py"
 		["regulator"]="${tools}/regulator/main.py"
+		["regulator_python"]="${tools}/regulator/venv/bin/python3"
 		["nomore403"]="${tools}/nomore403/nomore403"
 		["ffufPostprocessing"]="${tools}/ffufPostprocessing/ffufPostprocessing"
 		["misconfig-mapper"]="${tools}/misconfig-mapper/misconfig-mapper"
 		["spoofy"]="${tools}/Spoofy/spoofy.py"
+		["spoofy_python"]="${tools}/Spoofy/venv/bin/python3"
 		["swaggerspy"]="${tools}/SwaggerSpy/swaggerspy.py"
+		["swaggerspy_python"]="${tools}/SwaggerSpy/venv/bin/python3"
 		["LeakSearch"]="${tools}/LeakSearch/LeakSearch.py"
+		["LeakSearch_python"]="${tools}/LeakSearch/venv/bin/python3"
+		["Oralyzer"]="${tools}/Oralyzer/oralyzer.py"
+		["Oralyzer_python"]="${tools}/Oralyzer/venv/bin/python3"
 	)
 
 	declare -A tools_folders=(
@@ -219,6 +229,8 @@ function tools_installed() {
 		["sns"]="sns"
 		["sourcemapper"]="sourcemapper"
 		["jsluice"]="jsluice"
+		["commix"]="commix"
+		["urless"]="urless"
 		["dnstake"]="dnstake"
 	)
 
@@ -280,7 +292,7 @@ function google_dorks() {
 	if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GOOGLE_DORKS == true ]] && [[ $OSINT == true ]]; then
 		start_func "${FUNCNAME[0]}" "Running: Google Dorks in process"
 
-		python3 "${tools}/dorks_hunter/dorks_hunter.py" -d "$domain" -o "osint/dorks.txt"
+		"${tools}/dorks_hunter/venv/bin/python3" "${tools}/dorks_hunter/dorks_hunter.py" -d "$domain" -o "osint/dorks.txt"
 		end_func "Results are saved in $domain/osint/dorks.txt" "${FUNCNAME[0]}"
 	else
 		if [[ $GOOGLE_DORKS == false ]] || [[ $OSINT == false ]]; then
@@ -464,7 +476,7 @@ function apileaks() {
 		fi
 
 		# Run swaggerspy.py and handle errors
-		python3 swaggerspy.py "$domain" 2>>"$LOGFILE" | grep -i "[*]\|URL" >"${dir}/osint/swagger_leaks.txt"
+		"${tools}/SwaggerSpy/venv/bin/python3" swaggerspy.py "$domain" 2>>"$LOGFILE" | grep -i "[*]\|URL" >"${dir}/osint/swagger_leaks.txt"
 
 		# Return to the previous directory
 		if ! popd >/dev/null; then
@@ -518,7 +530,7 @@ function emails() {
 		fi
 
 		# Run LeakSearch.py and handle errors
-		python3 LeakSearch.py -k "$domain" -o "${dir}/.tmp/passwords.txt" 1>>"$LOGFILE"
+		"${tools}/LeakSearch/venv/bin/python3" LeakSearch.py -k "$domain" -o "${dir}/.tmp/passwords.txt" 1>>"$LOGFILE"
 
 		# Return to the previous directory
 		if ! popd >/dev/null; then
@@ -642,7 +654,7 @@ function spoof() {
 		fi
 
 		# Run spoofy.py and handle errors
-		./spoofy.py -d "$domain" >"${dir}/osint/spoof.txt"
+		"${tools}/Spoofy/venv/bin/python3" spoofy.py -d "$domain" >"${dir}/osint/spoof.txt"
 
 		# Return to the previous directory
 		if ! popd >/dev/null; then
@@ -1805,7 +1817,7 @@ function sub_regex_permut() {
 		fi
 
 		# Run the main.py script
-		python3 main.py -t "$domain" -f "${dir}/subdomains/subdomains.txt" -o "${dir}/.tmp/${domain}.brute" \
+		"${tools}/regulator/venv/bin/python3" main.py -t "$domain" -f "${dir}/subdomains/subdomains.txt" -o "${dir}/.tmp/${domain}.brute" \
 			2>>"$LOGFILE" >/dev/null
 
 		# Return to the previous directory
@@ -2389,7 +2401,7 @@ function s3buckets() {
 		esac
 
 		# Debug: Print the full CloudHunter command
-		printf "CloudHunter command: python3 %s/cloudhunter.py %s -r %s/resolvers.txt -t 50 [URL]\n" "$tools/CloudHunter" "$PERMUTATION_FLAG" "$tools/CloudHunter" >>"$LOGFILE"
+		printf "CloudHunter command: %s/venv/bin/python3 %s/cloudhunter.py %s -r %s/resolvers.txt -t 50 [URL]\n" "$tools/CloudHunter" "$tools/CloudHunter" "$PERMUTATION_FLAG" "$tools/CloudHunter" >>"$LOGFILE"
 
 		# Debug: Check if files exist
 		if [[ -f "$tools/CloudHunter/cloudhunter.py" ]]; then
@@ -2421,7 +2433,7 @@ function s3buckets() {
 					printf "%b[!] Failed to cd to %s.%b\n" "$bred" "$tools/CloudHunter" "$reset"
 					return 1
 				fi
-				if ! python3 ./cloudhunter.py ${PERMUTATION_FLAG#-p } -r ./resolvers.txt -t 50 "$url"; then
+				if ! "${tools}/CloudHunter/venv/bin/python3" ./cloudhunter.py ${PERMUTATION_FLAG#-p } -r ./resolvers.txt -t 50 "$url"; then
 					printf "%b[!] CloudHunter command failed for URL %s.%b\n" "$bred" "$url" "$reset"
 				fi
 			) >>"$dir/subdomains/cloudhunter_open_buckets.txt" 2>>"$LOGFILE"
@@ -2920,7 +2932,7 @@ function favicon() {
 		fi
 
 		# Run the favicon IP lookup tool
-		python3 favUp.py -w "$domain" -sc -o favicontest.json 2>>"$LOGFILE" >/dev/null
+		"${tools}/fav-up/venv/bin/python3" "${tools}/fav-up/favUp.py" -w "$domain" -sc -o favicontest.json 2>>"$LOGFILE" >/dev/null
 
 		# Process the results if favicontest.json exists and is not empty
 		if [[ -s "favicontest.json" ]]; then
@@ -3456,7 +3468,7 @@ function cms_scanner() {
 		fi
 
 		# Run CMSeeK with timeout
-		if ! timeout -k 1m "${CMSSCAN_TIMEOUT}s" python3 "${tools}/CMSeeK/cmseek.py" -l .tmp/cms.txt --batch -r &>>"$LOGFILE"; then
+		if ! timeout -k 1m "${CMSSCAN_TIMEOUT}s" "${tools}/CMSeeK/venv/bin/python3" "${tools}/CMSeeK/cmseek.py" -l .tmp/cms.txt --batch -r &>>"$LOGFILE"; then
 			exit_status=$?
 			if [[ ${exit_status} -eq 124 || ${exit_status} -eq 137 ]]; then
 				echo "TIMEOUT cmseek.py - investigate manually for $dir" >>"$LOGFILE"
@@ -3560,13 +3572,13 @@ function urlchecks() {
 				grep "$domain" .tmp/url_extract_tmp.txt | grep -E '^((http|https):\/\/)?([a-zA-Z0-9\-\.]+\.)+[a-zA-Z]{1,}(\/.*)?$' | grep -aEi "\.js$" | anew -q .tmp/url_extract_js.txt
 				grep "$domain" .tmp/url_extract_tmp.txt | grep -E '^((http|https):\/\/)?([a-zA-Z0-9\-\.]+\.)+[a-zA-Z]{1,}(\/.*)?$' | grep -aEi "\.js\.map$" | anew -q .tmp/url_extract_jsmap.txt
 				if [[ $DEEP == true ]] && [[ -s ".tmp/url_extract_js.txt" ]]; then
-					interlace -tL .tmp/url_extract_js.txt -threads 10 -c "python3 ${tools}/JSA/jsa.py -f _target_ | anew -q .tmp/url_extract_tmp.txt" &>/dev/null
+					interlace -tL .tmp/url_extract_js.txt -threads 10 -c "${tools}/JSA/venv/bin/python3 ${tools}/JSA/jsa.py -f _target_ | anew -q .tmp/url_extract_tmp.txt" &>/dev/null
 				fi
 
 				grep "$domain" .tmp/url_extract_tmp.txt | grep -E '^((http|https):\/\/)?([a-zA-Z0-9\-\.]+\.)+[a-zA-Z]{1,}(\/.*)?$' | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" | anew -q .tmp/url_extract_tmp2.txt
 
 				if [[ -s ".tmp/url_extract_tmp2.txt" ]]; then
-					python3 "${tools}/urless/urless/urless.py" <.tmp/url_extract_tmp2.txt | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/null
+					urless <.tmp/url_extract_tmp2.txt | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/null
 				fi
 
 				if [[ -s ".tmp/url_extract_uddup.txt" ]]; then
@@ -3764,7 +3776,7 @@ function jschecks() {
 				grep -iE "\.js($|\?)" .tmp/subjslinks.txt | anew -q .tmp/url_extract_js.txt
 			fi
 
-			python3 "${tools}/urless/urless/urless.py" <.tmp/url_extract_js.txt |
+			urless <.tmp/url_extract_js.txt |
 				anew -q js/url_extract_js.txt 2>>"$LOGFILE" >/dev/null
 
 			printf "%bRunning: Resolving JS URLs 2/6%b\n" "$yellow" "$reset"
@@ -4180,7 +4192,7 @@ function cors() {
 		# Proceed only if webs_all.txt exists and is non-empty
 		if [[ -s "webs/webs_all.txt" ]]; then
 			printf "${yellow}\n[$(date +'%Y-%m-%d %H:%M:%S')] Running: Corsy for CORS Scan${reset}\n\n"
-			python3 "${tools}/Corsy/corsy.py" -i "webs/webs_all.txt" -o "vulns/cors.txt" 2>>"$LOGFILE" >/dev/null
+			"${tools}/Corsy/venv/bin/python3" "${tools}/Corsy/corsy.py" -i "webs/webs_all.txt" -o "vulns/cors.txt" 2>>"$LOGFILE" >/dev/null
 		else
 			end_func "No webs/webs_all.txt file found, CORS Scan skipped." "${FUNCNAME[0]}"
 			return
@@ -4225,7 +4237,7 @@ function open_redirect() {
 			qsreplace FUZZ <"gf/redirect.txt" | sed '/FUZZ/!d' | anew -q ".tmp/tmp_redirect.txt"
 
 			# Run Oralyzer with the generated payloads
-			python3 "${tools}/Oralyzer/oralyzer.py" -l ".tmp/tmp_redirect.txt" -p "${tools}/Oralyzer/payloads.txt" >"vulns/redirect.txt" 2>>"$LOGFILE" >/dev/null
+			"${tools}/Oralyzer/venv/bin/python3" "${tools}/Oralyzer/oralyzer.py" -l ".tmp/tmp_redirect.txt" -p "${tools}/Oralyzer/payloads.txt" >"vulns/redirect.txt" 2>>"$LOGFILE" >/dev/null
 
 			# Remove ANSI color codes from the output
 			sed -r -i "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g" "vulns/redirect.txt"
@@ -4650,7 +4662,7 @@ function command_injection() {
 				# Run Commix if enabled
 				if [[ $SQLMAP == true ]]; then
 					printf "${yellow}\n[$(date +'%Y-%m-%d %H:%M:%S')] Running: Commix for Command Injection Checks${reset}\n\n"
-					python3 "${tools}/commix/commix.py" --batch -m ".tmp/tmp_rce.txt" --output-dir "vulns/command_injection" 2>>"$LOGFILE" >/dev/null
+					commix --batch -m ".tmp/tmp_rce.txt" --output-dir "vulns/command_injection" 2>>"$LOGFILE" >/dev/null
 				fi
 
 				# Additional tools can be integrated here (e.g., Ghauri, sqlmap)