Skip to content

Commit

Permalink
Add --no-conda flag (#107)
Browse files Browse the repository at this point in the history
  • Loading branch information
@epwalsh
epwalsh authored Jan 9, 2025
1 parent 950c644 commit 35895e8
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 55 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## Unreleased

### Added

- Added `--no-conda` flag to avoid creating Python environments with conda.

## [v1.10.1](https://github.com/allenai/beaker-gantry/releases/tag/v1.10.1) - 2025-01-09

### Fixed
Expand Down
60 changes: 36 additions & 24 deletions gantry/commands/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@
is_flag=True,
help="""If set, gantry will skip setting up a Python environment altogether.""",
)
@click.option(
"--no-conda",
is_flag=True,
help="""If set, gantry will not use conda to construct a Python environment,
and instead will potentially use the default Python environment on the image.""",
)
@click.option(
"--replicas",
type=int,
Expand Down Expand Up @@ -301,6 +307,7 @@ def run(
priority: Optional[str] = None,
install: Optional[str] = None,
no_python: bool = False,
no_conda: bool = False,
replicas: Optional[int] = None,
leader_selection: bool = False,
host_networking: bool = False,
Expand Down Expand Up @@ -469,6 +476,7 @@ def run(
priority=priority,
install=install,
no_python=no_python,
no_conda=no_conda,
replicas=replicas,
leader_selection=leader_selection,
host_networking=host_networking or (bool(replicas) and leader_selection),
Expand Down Expand Up @@ -605,6 +613,7 @@ def build_experiment_spec(
priority: Optional[Union[str, Priority]] = None,
install: Optional[str] = None,
no_python: bool = False,
no_conda: bool = False,
replicas: Optional[int] = None,
leader_selection: bool = False,
host_networking: bool = False,
Expand Down Expand Up @@ -662,38 +671,41 @@ def build_experiment_spec(
if no_python:
task_spec = task_spec.with_env_var(name="NO_PYTHON", value="1")
else:
if conda is not None:
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=str(conda),
)
elif Path(constants.CONDA_ENV_FILE).is_file():
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=constants.CONDA_ENV_FILE,
)
elif Path(constants.CONDA_ENV_FILE_ALTERNATE).is_file():
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=constants.CONDA_ENV_FILE_ALTERNATE,
)
if not no_conda:
if conda is not None:
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=str(conda),
)
elif Path(constants.CONDA_ENV_FILE).is_file():
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=constants.CONDA_ENV_FILE,
)
elif Path(constants.CONDA_ENV_FILE_ALTERNATE).is_file():
task_spec = task_spec.with_env_var(
name="CONDA_ENV_FILE",
value=constants.CONDA_ENV_FILE_ALTERNATE,
)
else:
task_spec = task_spec.with_env_var(
name="PYTHON_VERSION", value=".".join(platform.python_version_tuple()[:-1])
)

if venv is not None:
task_spec = task_spec.with_env_var(
name="VENV_NAME",
value=venv,
)
else:
task_spec = task_spec.with_env_var(
name="PYTHON_VERSION", value=".".join(platform.python_version_tuple()[:-1])
)
task_spec = task_spec.with_env_var(name="NO_CONDA", value="1")

if pip is not None:
task_spec = task_spec.with_env_var(
name="PIP_REQUIREMENTS_FILE",
value=str(pip),
)

if venv is not None:
task_spec = task_spec.with_env_var(
name="VENV_NAME",
value=venv,
)

if install is not None:
task_spec = task_spec.with_env_var(name="INSTALL_CMD", value=install)

Expand Down
68 changes: 37 additions & 31 deletions gantry/entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ done
# Function to check for conda, install it if needed.
function ensure_conda {
if ! command -v conda &> /dev/null; then
echo "installing conda..."
echo "[GANTRY] Installing conda..."
curl -fsSL -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
chmod +x ~/miniconda.sh
~/miniconda.sh -b -p /opt/conda
Expand All @@ -33,7 +33,11 @@ if [[ -n "$GITHUB_TOKEN" ]]; then
########################################
"
if ! command -v gh &> /dev/null; then
ensure_conda
if [[ -z "$NO_CONDA" ]]; then
ensure_conda
else
echo >&2 "error: you specified '--no-conda' but conda is needed to install the GitHub CLI. To avoid this error please ensure the GitHub CLI is already installed on your image."
fi

# Install GitHub CLI.
conda install -y gh --channel conda-forge
Expand Down Expand Up @@ -94,39 +98,41 @@ if [[ -z "$NO_PYTHON" ]]; then
PIP_REQUIREMENTS_FILE="${{ PIP_REQUIREMENTS_FILE }}"
fi

# Check if VENV_NAME is a path. If so, it should exist.
if [[ "$VENV_NAME" == */* ]]; then
if [[ ! -d "$VENV_NAME" ]]; then
echo >&2 "error: venv '$VENV_NAME' looks like a path but it doesn't exist"
exit 1
fi
fi
if [[ -z "$NO_CONDA" ]]; then
ensure_conda

ensure_conda

if conda activate "$VENV_NAME" &> /dev/null; then
echo "[GANTRY] Using existing conda environment '$VENV_NAME'"
# The virtual environment already exists. Possibly update it based on an environment file.
if [[ -f "$CONDA_ENV_FILE" ]]; then
echo "[GANTRY] Updating environment from conda env file '$CONDA_ENV_FILE'..."
conda env update -f "$CONDA_ENV_FILE"
# Check if VENV_NAME is a path. If so, it should exist.
if [[ "$VENV_NAME" == */* ]]; then
if [[ ! -d "$VENV_NAME" ]]; then
echo >&2 "error: venv '$VENV_NAME' looks like a path but it doesn't exist"
exit 1
fi
fi
else
# The virtual environment doesn't exist yet. Create it.
if [[ -f "$CONDA_ENV_FILE" ]]; then
# Create from the environment file.
echo "[GANTRY] Initializing environment from conda env file '$CONDA_ENV_FILE'..."
conda env create -n "$VENV_NAME" -f "$CONDA_ENV_FILE"
elif [[ -z "$PYTHON_VERSION" ]]; then
# Create a new empty environment with the whatever the default Python version is.
echo "[GANTRY] Initializing environment with default Python version..."
conda create -y -n "$VENV_NAME" pip

if conda activate "$VENV_NAME" &> /dev/null; then
echo "[GANTRY] Using existing conda environment '$VENV_NAME'"
# The virtual environment already exists. Possibly update it based on an environment file.
if [[ -f "$CONDA_ENV_FILE" ]]; then
echo "[GANTRY] Updating environment from conda env file '$CONDA_ENV_FILE'..."
conda env update -f "$CONDA_ENV_FILE"
fi
else
# Create a new empty environment with the specific Python version.
echo "[GANTRY] Initializing environment with Python $PYTHON_VERSION..."
conda create -y -n "$VENV_NAME" "python=$PYTHON_VERSION" pip
# The virtual environment doesn't exist yet. Create it.
if [[ -f "$CONDA_ENV_FILE" ]]; then
# Create from the environment file.
echo "[GANTRY] Initializing environment from conda env file '$CONDA_ENV_FILE'..."
conda env create -n "$VENV_NAME" -f "$CONDA_ENV_FILE"
elif [[ -z "$PYTHON_VERSION" ]]; then
# Create a new empty environment with the whatever the default Python version is.
echo "[GANTRY] Initializing environment with default Python version..."
conda create -y -n "$VENV_NAME" pip
else
# Create a new empty environment with the specific Python version.
echo "[GANTRY] Initializing environment with Python $PYTHON_VERSION..."
conda create -y -n "$VENV_NAME" "python=$PYTHON_VERSION" pip
fi
conda activate "$VENV_NAME"
fi
conda activate "$VENV_NAME"
fi

if [[ -z "$INSTALL_CMD" ]]; then
Expand Down

0 comments on commit 35895e8

Please sign in to comment.