From af7e55543329308195de27bcc4fa49b7dd018520 Mon Sep 17 00:00:00 2001 From: iilyak Date: Wed, 22 Nov 2023 06:23:37 -0800 Subject: [PATCH] Setup asdf and direnv managed development environment (#44) * Setup asdf and direnv managed development environment Now `cd` into the project directory would activate correct versions of build dependencies. --- .envrc | 41 +++++++ .gitignore | 6 ++ .tool-versions | 4 + Makefile | 286 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 128 +++++++++++++++++++++- 5 files changed, 461 insertions(+), 4 deletions(-) create mode 100644 .envrc create mode 100644 .tool-versions create mode 100644 Makefile diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..9f22f3cb --- /dev/null +++ b/.envrc @@ -0,0 +1,41 @@ +tools::has() { + type "$1" &>/dev/null +} + +if ! tools::has tput; then + echo "'tput' tool is missing" +fi + +console::bold() { + tput bold +} + +console::red() { + tput setaf 1 +} + +console::reset() { + tput sgr0 +} + +die() { + echo -e "$(console::red)$(console::bold)FATAL: ${*}$(console::reset)" >&2 + exit 1 +} + + +asdf::has() { + asdf current $1 >>/dev/null 2>&1 +} + +use asdf + +while read asdf_tool; do + if ! asdf::has ${asdf_tool}; then asdf direnv local ${asdf_tool}; fi +done < <(cat .tool-versions | grep -v '\#') + +watch_file ".asdf" +dotenv_if_exists .envrc.local + +export JAVA_HOME=$(dirname $(dirname $(expand_path $(asdf which java) "/"))) +export ASDF_DIRENV_CLOUSEAU=true diff --git a/.gitignore b/.gitignore index 522466d9..b561cfda 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,9 @@ build.log build.log .cache clouseau.iml +dependency-reduced-pom.xml + + +# direnv & asdf +.asdf +.envrc.local diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..aec3a948 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,4 @@ +direnv 2.32.3 +maven 3.8.2 +java zulu-7.48.0.11 +scala 2.9.1.final diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e32013ec --- /dev/null +++ b/Makefile @@ -0,0 +1,286 @@ +instances = clouseau1 clouseau2 clouseau3 +ifneq ($(WITH_COOKIE),) +with-cookie := -Dlauncher=with-cookie -Dcookie=$(WITH_COOKIE) +endif + +.PHONY: help +# target: help - Print this help +help: + @egrep "^# target: " Makefile \ + | sed -e 's/^# target: //g' \ + | sort \ + | awk '{printf(" %-20s", $$1); $$1=$$2=""; print "-" $$0}' + +.PHONY: build +# target: build - Build package, run tests and create distribution +build: .asdf check-epmd + @mvn + +# target: clouseau1 - Start local inistance of clouseau1 node +# target: clouseau2 - Start local inistance of clouseau2 node +# target: clouseau3 - Start local inistance of clouseau3 node +$(instances): .asdf check-epmd + @mvn scala:run -Dname=$@ $(with-cookie) + +.PHONY: clean +# target: clean - Remove build artifacts +clean: + @mvn clean + +check-epmd: + @if ! pgrep epmd >>/dev/null 2>&1 ; then echo "Error: Please start 'epmd' first"; exit 1; fi + +# ========================== +# Setting up the environment +# -------------------------- + +define CURRENT_SHELL +"$(shell basename $${SHELL})" +endef + +define CURRENT_OS +"$(shell uname -s | tr '[:upper:]' '[:lower:]')" +endef + +# Testing from fish +# set SHELL bash && make .asdf +# set SHELL fish && make .asdf +# set SHELL zsh && make .asdf +# set SHELL unsupported && make .asdf +# Testing from other shells +# SHELL=bash && make .asdf +# SHELL=fish && make .asdf +# SHELL=zsh && make .asdf +# SHELL=unsupported && make .asdf + +ifndef ASDF_DIRENV_BIN +.asdf: + @case $(CURRENT_SHELL)-$(CURRENT_OS) in \ + fish-darwin) echo "$$FISH_DARWIN_HELP" ;; \ + bash-darwin) echo "$$BASH_DARWIN_HELP" ;; \ + zsh-darwin) echo "$$ZSH_DARWIN_HELP" ;; \ + *) echo "$$CONTRIBUTE_SHELL_SETUP" ;; \ + esac +else +.asdf: + @touch $@ + @touch .envrc +endif + +define FISH_DARWIN_HELP +There are two steps which need to be done to set up environment. + + - The asdf tool manager would need to be installed + - The direnv plugin for asdf manager would need to be installed + + += Setting up asdf tool manager + + You can install asdf from brew by following command. + + ``` + brew install asdf + ``` + + Once you finish installation add following line to ~/.config/fish/config.fish + + ``` + source (brew --prefix asdf)/libexec/asdf.fish + ``` + + You can do it by copy/paste of the following line + + ``` + echo -e "\\nsource (brew --prefix asdf)/libexec/asdf.fish" >> ~/.config/fish/config.fish + ``` + += Setting up direnv + + Run following commands to install direnv plugin + + ``` + asdf plugin-add direnv + asdf install direnv latest + asdf global direnv latest + ``` + + Once you have done with above commands add following to the ~/.config/fish/config.fish + + ``` + # Hook direnv into your shell. + asdf exec direnv hook fish | source + + # A shortcut for asdf managed direnv. + function direnv + asdf exec direnv "$$argv" + end + ``` + + You can do it by copy/paste of the following + + ``` + printf "\ + # Hook direnv into your shell. + asdf exec direnv hook fish | source + + # A shortcut for asdf managed direnv. + function direnv + asdf exec direnv \\"\$$argv\\" + end + " >> ~/.config/fish/config.fish + ``` + += Enabling direnv integration + +- After finishing the above two steps restart your shell. +- Run following `asdf direnv setup --shell fish --version latest` +- Restart your shell once again + +endef +export FISH_DARWIN_HELP + +define BASH_DARWIN_HELP + +There are two steps which need to be done to set up environment. + + - The asdf tool manager would need to be installed + - The direnv plugin for asdf manager would need to be installed + + += Setting up asdf tool manager + + You can install asdf from brew by following command. + + ``` + brew install asdf + ``` + + Once you finish installation add following line to ~/.bash_profile + + ``` + source $$(brew --prefix asdf)/libexec/asdf.sh + ``` + + You can do it by copy/paste of the following line + ``` + echo -e "\n. $$(brew --prefix asdf)/libexec/asdf.sh" >> ~/.bash_profile + ``` + += Setting up direnv + + Run following commands to install direnv plugin + + ``` + asdf plugin-add direnv + asdf install direnv latest + asdf global direnv latest + ``` + + Once you have done with above commands add following to the ~/.bash_profile + + ``` + # Hook direnv into your shell. + eval "$$(asdf exec direnv hook bash)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$$@"; } + ``` + + You can do it by copy/paste of the following + + ``` + cat << EOF >> ~/.bashrc + # Hook direnv into your shell. + eval "$$(asdf exec direnv hook bash)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$$@"; } + EOF + ``` + += Enabling direnv integration + +- After finishing the above two steps restart your shell. +- Run following `asdf direnv setup --shell bash --version latest` +- Restart your shell once again + +endef +export BASH_DARWIN_HELP + +define ZSH_DARWIN_HELP + +There are two steps which need to be done to set up environment. + + - The asdf tool manager would need to be installed + - The direnv plugin for asdf manager would need to be installed + + += Setting up asdf tool manager + + You can install asdf from brew by following command. + + ``` + brew install asdf + ``` + + Once you finish installation add following line to ~/.zshrc + (or ${ZDOTDIR}/.zshrc if you use custom location). + + ``` + source $$(brew --prefix asdf)/libexec/asdf.sh + ``` + + You can do it by copy/paste of the following line + ``` + echo -e "\n. $$(brew --prefix asdf)/libexec/asdf.sh" >> $${ZDOTDIR:-~}/.zshrc + ``` + += Setting up direnv + + Run following commands to install direnv plugin + + ``` + asdf plugin-add direnv + asdf install direnv latest + asdf global direnv latest + ``` + + Once you have done with above commands add following to the ~/.zshrc + (or ${ZDOTDIR}/.zshrc if you use custom location). + + + ``` + # Hook direnv into your shell. + eval "$$(asdf exec direnv hook zsh)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$$@"; } + ``` + + You can do it by copy/paste of the following + + ``` + cat << EOF >> ~/.zshrc + # Hook direnv into your shell. + eval "$$(asdf exec direnv hook zsh)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$$@"; } + EOF + ``` + += Enabling direnv integration + +- After finishing the above two steps restart your shell. +- Run following `asdf direnv setup --shell zsh --version latest` +- Restart your shell once again + +endef +export ZSH_DARWIN_HELP + + +define CONTRIBUTE_SHELL_SETUP + We detected $(CURRENT_SHELL) running on $(CURRENT_OS). Unfortunately + documentation for this combination is not available. Contributions are welcome. +endef +export CONTRIBUTE_SHELL_SETUP diff --git a/README.md b/README.md index f458dd2c..d486c53f 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,128 @@ Expose Lucene features to erlang RPC. +## Setting up the development environment + +We use a combination of [`direnv`](https://github.com/direnv/direnv) and [`asdf`](https://github.com/asdf-vm/asdf) to manage development environment. +The minimal set of tools you need to install manually is: + +- xcode +- brew +- asdf +- git +- coreutils + +1. install asdf itself +``` +brew install asdf +``` +2. integrate asdf with your shell + - fish + ``` + echo -e "\nsource $(brew --prefix asdf)/libexec/asdf.fish" >> ~/.config/fish/config.fish + ``` + - bash + ``` + echo -e "\n. $(brew --prefix asdf)/libexec/asdf.sh" >> ~/.bash_profile + ``` + - zsh + ``` + echo -e "\n. $(brew --prefix asdf)/libexec/asdf.sh" >> ${ZDOTDIR:-~}/.zshrc + ``` +3. install direnv plugin +``` +asdf plugin-add direnv +asdf install direnv latest +asdf global direnv latest +``` +4. integrate direnv plugin with your shell + - fish + ``` + printf "\ + # Hook direnv into your shell. + asdf exec direnv hook fish | source + + # A shortcut for asdf managed direnv. + function direnv + asdf exec direnv \"\$argv\" + end + " >> ~/.config/fish/config.fish + ``` + - bash + ``` + cat << EOF >> ~/.bashrc + # Hook direnv into your shell. + eval "$(asdf exec direnv hook bash)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$@"; } + EOF + ``` + - zsh + ``` + cat << EOF >> ~/.zshrc + # Hook direnv into your shell. + eval "$(asdf exec direnv hook zsh)" + + # A shortcut for asdf managed direnv. + direnv() { asdf exec direnv "$@"; } + EOF + ``` +5. enable `use asdf` feature +``` +echo 'source "$(asdf direnv hook asdf)"' >> ~/.config/direnv/direnvrc +``` +6. exit from the current shell or start a new one +7. `cd` into the directory where you checked out `clouseau` and do `direnv allow` +8. do `make tools` (we need it only when we add new tools into `.tool-versions`) + +Now every time you `cd` into the project directory the correct versions of tools +will be activated. + +### Custom Java + +In some cases version of Java is not available in asdf-java. +There is a trick which makes globally available version recognized by +asdf. + +``` +ln -s /Library/Java/JavaVirtualMachines/jdk1.7.0_80.jdk/Contents/Home ~/.asdf/installs/java/oracle-1.7.0 +asdf reshim java +``` + +## Building distribution + +Just run `make` to get `target/clouseau-{version}-SNAPSHOT.zip` file. + +## Running locally + +To run single node instance use `make clouseau1`. + +Usually for testing all three nodes might be needed. In such case run following +from separate terminals: + +``` +make clouseau1 +make clouseau2 +make clouseau3 +``` + +## Specifying Erlang cookie + +The cookie for a node can be specified using following + +``` +make clouseau2 WITH_COOKIE=2123 +``` + +## Changing tools version + +The list of tools are configured in a `.tool-versions` file. Make sure you +run `make tools` every time you add new tools. You don't need to do it when +there is just a version change. + ## Configuration options -This guide explains the various clouseau configuration options available, and how to use them to tune clouseau performance and scalability. There are two categories of clouseau options, first category is about tuning the JVM (ex: Xmx) and other category of options that go into clouseau.ini. +This guide explains the various clouseau configuration options available, and how to use them to tune clouseau performance and scalability. There are two categories of clouseau options, first category is about tuning the JVM (ex: Xmx) and other category of options that go into clouseau.ini. Clouseau configuration options (as determined by the relevant role in chef-repo) are stored in `/opt/clouseau/etc/clouseau.ini` and some options (about JVM tuning) go into the command used to start and stop clouseau. @@ -52,7 +172,7 @@ This option allows to specify the maximum number indices opened at a given point These options allows us to close the search indices if there is no activity within a specified interval. As mentioned in the above section about the `max_indexes_open`, when number of indices opened reaches the `max_indexes_open` limit then clouseau will close the index that was opened first even if there is an activity on that index. This was leading to errors and hence added this option to close the idle indices first. Basically we will close the idle indices so that we can avoid reaching the limit specified in `max_indexes_open`. So that we will close the idle indices first and if we still reach the limit then clouseau will close the oldest index (based on the open time). -If `close_if_idle` is set to true, then it will start monitoring the activity of the indices and will close the index if there is no activity in two consecutive idle check intervals. By default the `idle_check_interval_secs` is 300 seconds (5 minutes) and can be overridden by specifying the `idle_check_interval_secs`. In the example configuration specified in the above section, this was set to 600 seconds, which will close the index if there is no activity for anywhere between 601 to 1200 seconds. +If `close_if_idle` is set to true, then it will start monitoring the activity of the indices and will close the index if there is no activity in two consecutive idle check intervals. By default the `idle_check_interval_secs` is 300 seconds (5 minutes) and can be overridden by specifying the `idle_check_interval_secs`. In the example configuration specified in the above section, this was set to 600 seconds, which will close the index if there is no activity for anywhere between 601 to 1200 seconds. This variation (instead of fixed) in the index closing due to inactivity is a result of the implementation approach, where during the first index idle check interval we mark the index as idle and close that index if the state is still idle during the next check. And if there is any activity between the two `index idle check` intervals then it will not be closed. @@ -64,7 +184,7 @@ This variation (instead of fixed) in the index closing due to inactivity is a re s:maintenance_mode("heap dump of clouseau"). ``` 2. Find the clouseau pid (process: "java [...] com.cloudant.clouseau.Main /opt/cloudant/etc/clouseau.ini" ): - + ``` ps aux | grep clouseau ``` @@ -84,5 +204,5 @@ This variation (instead of fixed) in the index closing due to inactivity is a re ``` rsync -avz db..cloudant.com:/srv/heap.bin . ``` - + 7. Analyze the heap dump using `visualVM` tool or Eclipse memory analyzer tool(http://www.eclipse.org/mat/downloads.php).