From 489cf448df4962b5af5ac53ec37ed6e6c4012b1f Mon Sep 17 00:00:00 2001 From: maiyetum95 Date: Sun, 29 Sep 2024 00:32:43 +0200 Subject: [PATCH 1/5] add test_2d_owsc_python for drl --- README.md | 28 ++ .../test_2d_owsc_python/CMakeLists.txt | 41 ++ .../custom_io_environment.cpp | 26 ++ .../custom_io_environment.h | 16 + .../custom_io_observation.h | 23 + .../test_2d_owsc_python/custom_io_simbody.h | 36 ++ .../drl_gym_environments/README.md | 20 + .../gym_env_owsc/__init__.py | 9 + .../gym_env_owsc/envs/__init__.py | 1 + .../gym_env_owsc/envs/owsc.py | 138 ++++++ .../drl_gym_environments/setup.py | 7 + .../drl_tianshou_training/sac.py | 150 +++++++ .../test_2d_owsc_python/owsc_python.cpp | 408 ++++++++++++++++++ .../test_2d_owsc_python/owsc_python.h | 375 ++++++++++++++++ .../pybind_tool/pybind_test.py | 44 ++ ...otalViscousForceFromFluid_Run_0_result.xml | 9 + ...talViscousForceFromFluid_Run_10_result.xml | 9 + ...talViscousForceFromFluid_Run_20_result.xml | 9 + ...TotalViscousForceFromFluid_dtwdistance.xml | 4 + ...ap_TotalViscousForceFromFluid_runtimes.dat | 3 + .../regression_test_tool.py | 37 ++ 21 files changed, 1393 insertions(+) create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/CMakeLists.txt create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.cpp create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.h create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/custom_io_observation.h create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/custom_io_simbody.h create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/__init__.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/__init__.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/owsc.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/setup.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.h create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/pybind_tool/pybind_test.py create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_0_result.xml create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_10_result.xml create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_20_result.xml create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_dtwdistance.xml create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_runtimes.dat create mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/regression_test_tool.py diff --git a/README.md b/README.md index 29146f0496..cff3a73b14 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,34 @@ Please check the source code of [2D Dambreak case with python interface](https://github.com/Xiangyu-Hu/SPHinXsys/tree/master/tests/2d_examples/test_2d_dambreak_python) for the usage. +## Deep reinforcement learning in SPHinXsys + +By leveraging the Python interface of SPHinXsys, one can rapidly and modularly construct SPHinXsys simulation environments required for deep reinforcement learning (DRL). Effective training can be conducted using various algorithms from the mainstream DRL platform Tianshou. Please refer to the source code of +[2D owsc case with python interface](https://github.com/Xiangyu-Hu/SPHinXsys/tree/master/tests/extra_source_and_tests/test_2d_owsc_python). + +Step-by-Step instructions: +1. **Use conda or Python to create a virtual Python environment.** + Python 3.10 is recommended for compatibility. + + ```bash + conda create -n drl_env python=3.10 +2. **Activate the python environment and install required libraries.** + + ```bash + conda activate drl_env + pip install tianshou +3. **Set up the SPHinXsys-based reinforcement learning environment in the `drl_gym_environments` directory.** + Detailed environment setup instructions can be found in the `README.md` file. + + ```bash + cd path_to_deep_reinforcement_learning_tool/drl_gym_environments + pip install -e . +6. **Navigate to the `drl_tianshou_training` directory and start the training process.** + + ```bash + cd path_to_deep_reinforcement_learning_tool/drl_tianshou_training + python sac.py + ## Publications Main publication on the library: diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/CMakeLists.txt b/tests/extra_source_and_tests/test_2d_owsc_python/CMakeLists.txt new file mode 100644 index 0000000000..8371e08a4d --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/CMakeLists.txt @@ -0,0 +1,41 @@ +### Python, for ${Python_EXECUTABLE} +find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +### Pybind11 +find_package(pybind11 CONFIG REQUIRED) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) # main (top) cmake dir + +set(CMAKE_VERBOSE_MAKEFILE on) + +STRING(REGEX REPLACE ".*/(.*)" "\\1" CURRENT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}) +PROJECT("${CURRENT_FOLDER}") + +SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) +SET(EXECUTABLE_OUTPUT_PATH "${PROJECT_BINARY_DIR}/bin/") +SET(BUILD_INPUT_PATH "${EXECUTABLE_OUTPUT_PATH}/input") +SET(BUILD_RELOAD_PATH "${EXECUTABLE_OUTPUT_PATH}/reload") +SET(BUILD_BIND_PATH "${EXECUTABLE_OUTPUT_PATH}/bind") +SET(BUILD_DRL_PATH "${EXECUTABLE_OUTPUT_PATH}/drl") + +file(MAKE_DIRECTORY ${BUILD_INPUT_PATH}) +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_INPUT_PATH}) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/regression_test_tool/ DESTINATION ${BUILD_INPUT_PATH}) + + +file(MAKE_DIRECTORY ${BUILD_BIND_PATH}) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/pybind_tool/ + DESTINATION ${BUILD_BIND_PATH}) + +file(MAKE_DIRECTORY ${BUILD_DRL_PATH}) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/deep_reinforcement_learning_tool/ + DESTINATION ${BUILD_DRL_PATH}) + +aux_source_directory(. DIR_SRCS) +pybind11_add_module(${PROJECT_NAME} ${DIR_SRCS}) +set_target_properties(${PROJECT_NAME} PROPERTIES VS_DEBUGGER_WORKING_DIRECTORY "${EXECUTABLE_OUTPUT_PATH}") +target_link_libraries(${PROJECT_NAME} PRIVATE sphinxsys_2d) + +add_test(NAME ${PROJECT_NAME} COMMAND ${Python3_EXECUTABLE} "${EXECUTABLE_OUTPUT_PATH}/bind/pybind_test.py") +set_tests_properties(${PROJECT_NAME} PROPERTIES WORKING_DIRECTORY "${EXECUTABLE_OUTPUT_PATH}" + PASS_REGULAR_EXPRESSION "The result of TotalViscousForceFromFluid is correct based on the dynamic time warping regression test!") + diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.cpp b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.cpp new file mode 100644 index 0000000000..03532a292e --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.cpp @@ -0,0 +1,26 @@ +#include "custom_io_environment.h" +#include "sph_system.h" +namespace fs = std::filesystem; + +namespace SPH +{ +//=================================================================================================// +CustomIOEnvironment::CustomIOEnvironment(SPHSystem &sph_system, bool delete_output, int parallel_env_number, int episode_number) + : IOEnvironment(sph_system, delete_output) +{ + // Append environment_number to the output_folder_ + output_folder_ += "_env_" + std::to_string(parallel_env_number) + "_episode_" + std::to_string(episode_number); + + // Check and create the output folder with the modified path + if (!fs::exists(output_folder_)) { + fs::create_directory(output_folder_); + } + + // Handle deletion of contents in the output folder if required + if (delete_output) { + fs::remove_all(output_folder_); + fs::create_directory(output_folder_); + } +} +//=================================================================================================// +} // namespace SPH \ No newline at end of file diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.h b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.h new file mode 100644 index 0000000000..0b1c9695ac --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_environment.h @@ -0,0 +1,16 @@ +#ifndef CUSTOM_IO_ENVIRONMENT_H +#define CUSTOM_IO_ENVIRONMENT_H + +#include "io_environment.h" +#include "sph_system.h" + +namespace SPH +{ +class CustomIOEnvironment : public IOEnvironment +{ + public: + // Constructor with an additional environment_number parameter + CustomIOEnvironment(SPHSystem &sph_system, bool delete_output, int parallel_env_number, int episode_number); +}; +} // namespace SPH +#endif // CUSTOM_IO_ENVIRONMENT_H diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_observation.h b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_observation.h new file mode 100644 index 0000000000..ba00a444a8 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_observation.h @@ -0,0 +1,23 @@ +#ifndef CUSTOM_IO_OBSERVATION_H +#define CUSTOM_IO_OBSERVATION_H + +#include "io_observation.h" + +namespace SPH +{ +template +class ExtendedReducedQuantityRecording : public ReducedQuantityRecording +{ +public: + // Inherit constructors from the base class + using ReducedQuantityRecording::ReducedQuantityRecording; + + // Function to directly return the result of reduce_method_.exec() + typename LocalReduceMethodType::ReturnType getReducedQuantity() + { + return this->reduce_method_.exec(); + } +}; + +} // namespace SPH +#endif // CUSTOM_IO_OBSERVATION_H diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_simbody.h b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_simbody.h new file mode 100644 index 0000000000..e8046b04f0 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/custom_io_simbody.h @@ -0,0 +1,36 @@ +#ifndef CUSTOM_IO_SIMBODY_H +#define CUSTOM_IO_SIMBODY_H + +#include "io_simbody.h" +#include + +namespace SPH { +/** + * @class WriteSimBodyPinDataExtended + * @brief Extended class to write total force acting on a solid body and get angles to Python. + */ +class WriteSimBodyPinDataExtended : public WriteSimBodyPinData +{ +public: + WriteSimBodyPinDataExtended(SPHSystem &sph_system, SimTK::RungeKuttaMersonIntegrator &integ, + SimTK::MobilizedBody::Pin &pinbody) + : WriteSimBodyPinData(sph_system, integ, pinbody){}; + + // Function to get angle + Real getAngleToPython(size_t iteration_step = 0) + { + const SimTK::State& state = integ_.getState(); + Real angle = mobody_.getAngle(state); + return angle; + } + + // Function to get angle rate + Real getAngleRateToPython(size_t iteration_step = 0) + { + const SimTK::State& state = integ_.getState(); + Real angle_rate = mobody_.getRate(state); + return angle_rate; + } +}; +} // namespace SPH +#endif // CUSTOM_IO_SIMBODY_H diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md new file mode 100644 index 0000000000..cf8ecf98f2 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md @@ -0,0 +1,20 @@ +- **`drl_gym_environments/`**: + The main package directory that includes the environment registration logic and all the custom environment implementations. + +- **`gym_env_owsc/__init__.py`**: + This file contains the registration logic for all custom environments. + Each environment is registered with a unique ID, which can be used with Gymnasium’s `gym.make()` function. + +- **`envs/`**: + This sub-directory contains the actual implementation of all the custom environments. + + - **`envs/__init__.py`**: + Imports all custom environments so they can be properly registered. + + - **`envs/owsc.py`**: + Implements the OWSC environment, following the standard Gymnasium `Env` interface. + The environment defines unique observation and action spaces, and includes specific environment dynamics in the `reset()`, `step()`, and `render()` methods. + +- **`setup.py`**: + Provides the configuration for installing the package. + To install the environments in editable mode, run the following command from the project’s root directory. diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/__init__.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/__init__.py new file mode 100644 index 0000000000..6bad1ecbaf --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/__init__.py @@ -0,0 +1,9 @@ +from gymnasium.envs.registration import register + +register( + id="OWSC-v0", + entry_point="gym_env_owsc.envs:OWSCEnv", + kwargs={'parallel_envs': 0}, + max_episode_steps=500, + reward_threshold=500.0, +) diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/__init__.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/__init__.py new file mode 100644 index 0000000000..ba87f6f63a --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/__init__.py @@ -0,0 +1 @@ +from gym_env_owsc.envs.owsc import OWSCEnv diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/owsc.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/owsc.py new file mode 100644 index 0000000000..1b6c3305fa --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/gym_env_owsc/envs/owsc.py @@ -0,0 +1,138 @@ +import sys +import math +import numpy as np +import gymnasium as gym +from gymnasium import spaces +# add dynamic link library or shared object to python env +sys.path.append('/path/to/SPHinXsys/case/lib/dynamic link library or shared object') +import test_2d_owsc_python as test_2d + + +class OWSCEnv(gym.Env): + """Custom Environment without rendering.""" + # metadata = {"render_modes": ["human", "rgb_array"], "render_fps": 30} + + def __init__(self, render_mode=None, parallel_envs=0): + # Initialize environment parameters + self.parallel_envs = parallel_envs # Identifier for parallel simulation environments + self.episode = 1 # Current episode number + self.time_per_action = 0.1 # Time interval per action step + self.low_action = -1.0 # Minimum action value + self.max_action = 1.0 # Maximum action value + self.update_per_action = 10 # The action's effect is applied in smaller iterations within one action time step + self.low_obs = -10.0 # Minimum observation value + self.high_obs = 10.0 # Maximum observation value + self.obs_numbers = 16 # Number of observation variables + + # Define action and observation spaces for Gym + low_action = np.array([self.low_action]).astype(np.float32) + high_action = np.array([self.max_action]).astype(np.float32) + low_obs = np.full(self.obs_numbers, self.low_obs).astype(np.float32) + high_obs = np.full(self.obs_numbers, self.high_obs).astype(np.float32) + + self.action_space = spaces.Box(low_action, high_action) # Continuous action space + self.observation_space = spaces.Box(low_obs, high_obs) # Continuous observation space + + # Reset the environment at the beginning of each episode + def reset(self, seed=None, options=None): + super().reset(seed=seed) + + # Initialize the OWSC simulation with the given episode and environment setup + self.owsc = test_2d.owsc_from_sph_cpp(self.parallel_envs, self.episode) + self.action_time_steps = 0 # Track the number of action steps + self.action_time = 0.5 # Initialize action time + self.damping_coefficient = 50 # Set damping coefficient for the environment + self.total_reward_per_episode = 0.0 # Track total reward in each episode + + # Start the simulation with the given action time and damping coefficient + self.owsc.run_case(self.action_time, self.damping_coefficient) + + # Initialize observation array with zero values + self.observation = np.zeros(self.obs_numbers) + # Fill the observation array with values from the OWSC simulation + for i in range(0, 2): + self.observation[i] = self.owsc.get_wave_height(i) + self.observation[i + 2] = self.owsc.get_wave_velocity(i, 0) + self.observation[i + 4] = self.owsc.get_wave_velocity(i, 1) + self.observation[i + 6] = self.owsc.get_wave_velocity_on_flap(i, 0) + self.observation[i + 8] = self.owsc.get_wave_velocity_on_flap(i, 1) + self.observation[i + 10] = self.owsc.get_flap_position(i, 0) + self.observation[i + 12] = self.owsc.get_flap_position(i, 1) + self.observation[14] = self.owsc.get_flap_angle() + self.observation[15] = self.owsc.get_flap_angle_rate() + + self._get_obs = self.observation.astype(np.float32) + + return self._get_obs, {} + + def step(self, action): + self.action_time_steps += 1 + # Apply the action to change the damping coefficient + self.damping_change = 5.0 * action[0] + # Penalty for invalid actions + penality_0 = 0.0 + # Ensure the damping coefficient stays within valid bounds + if self.damping_coefficient + self.damping_change < 0.01: + self.damping_change = 0.01 - self.damping_coefficient + penality_0 = - 1.0 + if self.damping_coefficient + self.damping_change > 100: + self.damping_change = 100 - self.damping_coefficient + penality_0 = - 1.0 + + reward_0 = 0.0 + for i in range(self.update_per_action): + self.flap_angle_rate_previous = self.owsc.get_flap_angle_rate() + self.damping_coefficient += self.damping_change / self.update_per_action + self.action_time += self.time_per_action / self.update_per_action + self.owsc.run_case(self.action_time, self.damping_coefficient) + self.flap_angle_rate_now = self.owsc.get_flap_angle_rate() + # Calculate reward based on energy (flap angle rate) + reward_0 += self.damping_coefficient * math.pow(0.5 * (self.flap_angle_rate_now + self.flap_angle_rate_previous), 2) * self.time_per_action / self.update_per_action + # Add any penalties to the reward + reward = reward_0 + penality_0 + self.total_reward_per_episode += reward + + # Update observations from the OWSC simulation + for i in range(0, 2): + self.observation[i] = self.owsc.get_wave_height(i) + self.observation[i + 2] = self.owsc.get_wave_velocity(i, 0) + self.observation[i + 4] = self.owsc.get_wave_velocity(i, 1) + self.observation[i + 6] = self.owsc.get_wave_velocity_on_flap(i, 0) + self.observation[i + 8] = self.owsc.get_wave_velocity_on_flap(i, 1) + self.observation[i + 10] = self.owsc.get_flap_position(i, 0) + self.observation[i + 12] = self.owsc.get_flap_position(i, 1) + self.observation[14] = self.owsc.get_flap_angle() + self.observation[15] = self.owsc.get_flap_angle_rate() + + self._get_obs = self.observation.astype(np.float32) + + # Log action and reward information to files + with open(f'action_env{self.parallel_envs}_epi{self.episode}.txt', 'a') as file: + file.write(f'action_time: {self.action_time} action: {self.damping_coefficient}\n') + + with open(f'reward_env{self.parallel_envs}_epi{self.episode}.txt', 'a') as file: + file.write(f'action_time: {self.action_time} reward: {reward}\n') + + # Check if the episode is done after 200 steps + if self.action_time_steps > 99: + done = True + with open(f'reward_env{self.parallel_envs}.txt', 'a') as file: + file.write(f'episode: {self.episode} total_reward: {self.total_reward_per_episode}\n') + self.episode += 1 + else: + done = False + + # Return the updated observation, reward, done flag, and additional info + return self._get_obs, reward, done, False, {} + + # Render method (optional, no rendering in this case) + def render(self): + return 0 + + # Additional render frame logic (not implemented) + def _render_frame(self): + return 0 + + # Close the environment and cleanup (optional) + def close(self): + return 0 \ No newline at end of file diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/setup.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/setup.py new file mode 100644 index 0000000000..2596b90e68 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name="gym_env_owsc", + version="1.0", + install_requires=["gymnasium>=0.27.1", "pygame>=2.3.0"], +) diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py new file mode 100644 index 0000000000..2c9e955407 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +import gymnasium as gym +# Custom OWSC environment +import gym_env_owsc + +import argparse +import datetime +import os +import pprint + +import numpy as np +import torch + +from torch.utils.tensorboard import SummaryWriter + +from tianshou.data import Collector, ReplayBuffer, VectorReplayBuffer +from tianshou.policy import SACPolicy +from tianshou.trainer import OffpolicyTrainer +from tianshou.env import SubprocVectorEnv +from tianshou.utils import TensorboardLogger, WandbLogger +from tianshou.utils.net.common import Net +from tianshou.utils.net.continuous import ActorProb, Critic + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--task", type=str, default="OWSC-v0") # Environment ID + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--buffer-size", type=int, default=10100) + parser.add_argument("--hidden-sizes", type=int, nargs="*", default=[64, 64]) + parser.add_argument("--actor-lr", type=float, default=1e-4) + parser.add_argument("--critic-lr", type=float, default=1e-4) + parser.add_argument("--gamma", type=float, default=0.95) + parser.add_argument("--tau", type=float, default=0.005) + parser.add_argument("--alpha", type=float, default=0.25) + parser.add_argument("--auto-alpha", default=False, action="store_true") + parser.add_argument("--alpha-lr", type=float, default=5e-4) + parser.add_argument("--start-timesteps", type=int, default=100) + parser.add_argument("--epoch", type=int, default=100) + parser.add_argument("--step-per-epoch", type=int, default=100) + parser.add_argument("--step-per-collect", type=int, default=1) + parser.add_argument("--update-per-step", type=int, default=1) + parser.add_argument("--n-step", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--training-num", type=int, default=1) + parser.add_argument('--test-num', type=int, default=0) + parser.add_argument("--logdir", type=str, default="log") + parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("--resume-path", type=str, default=None) + parser.add_argument("--logger", type=str, default="tensorboard", choices=["tensorboard", "wandb"]) + parser.add_argument("--wandb-project", type=str, default="mujoco.benchmark") + parser.add_argument("--watch", default=False, action="store_true", help="Watch the play of pre-trained policy only") + return parser.parse_args() + + +def training_sac(args=get_args()): + """Main function for setting up and training a SAC agent in the OWSC environment.""" + + envs = gym.make(args.task) + # Create vectorized environments for parallel training + # envs = SubprocVectorEnv([ + # lambda i=i: gym.make(args.task, parallel_envs=i) # Create environments with different parallel_envs IDs + # for i in range(args.training_num)]) + + # Retrieve environment observation and action space details + args.state_shape = envs.observation_space.shape or envs.observation_space.n + args.action_shape = envs.action_space.shape or envs.action_space.n + args.max_action = envs.action_space.high[0] + + # seed + np.random.seed(args.seed) + torch.manual_seed(args.seed) + + # Define the actor and critic neural networks for SAC + net_a = Net(args.state_shape, hidden_sizes=args.hidden_sizes, device=args.device) + actor = ActorProb(net_a, args.action_shape, device=args.device, max_action=args.max_action, conditioned_sigma=True).to(args.device) + actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr) + + net_c1 = Net(args.state_shape, args.action_shape, hidden_sizes=args.hidden_sizes, concat=True, device=args.device) + critic1 = Critic(net_c1, device=args.device).to(args.device) + critic1_optim = torch.optim.Adam(critic1.parameters(), lr=args.critic_lr) + + net_c2 = Net(args.state_shape, args.action_shape, hidden_sizes=args.hidden_sizes, concat=True, device=args.device) + critic2 = Critic(net_c2, device=args.device).to(args.device) + critic2_optim = torch.optim.Adam(critic2.parameters(), lr=args.critic_lr) + + # Optionally, use automatic tuning of the temperature parameter (alpha) + if args.auto_alpha: + target_entropy = -np.prod(envs.action_space.shape) + log_alpha = torch.zeros(1, requires_grad=True, device=args.device) + alpha_optim = torch.optim.Adam([log_alpha], lr=args.alpha_lr) + args.alpha = (target_entropy, log_alpha, alpha_optim) + + # Initialize the SAC policy + policy = SACPolicy( + actor=actor, + actor_optim=actor_optim, + critic1=critic1, + critic1_optim=critic1_optim, + critic2=critic2, + critic2_optim=critic2_optim, + tau=args.tau, + gamma=args.gamma, + alpha=args.alpha, + estimation_step=args.n_step, + action_space=envs.action_space, + ) + + # load a previous policy + if args.resume_path: + policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) + print("Loaded agent from: ", args.resume_path) + + # Setup replay buffer and collector for training + buffer = VectorReplayBuffer(args.buffer_size, len(envs)) if args.training_num > 1 else ReplayBuffer(args.buffer_size) + train_collector = Collector(policy, envs, buffer, exploration_noise=True) + test_collector=None + train_collector.collect(n_step=args.start_timesteps, random=True) + + # Setup logging + now = datetime.datetime.now().strftime("%y%m%d-%H%M%S") + log_path = os.path.join(args.logdir, os.path.join(args.task, "sac", str(args.seed), now)) + writer = SummaryWriter(log_path) + writer.add_text("args", str(args)) + logger = TensorboardLogger(writer) if args.logger == "tensorboard" else None # Only Tensorboard logging is used + + # Save functions for best policy and checkpoints + def save_best_fn(policy): + torch.save(policy.state_dict(), os.path.join(log_path, "policy.pth")) + + # Training loop using OffpolicyTrainer + if not args.watch: + result = OffpolicyTrainer( + policy=policy, + train_collector=train_collector, + test_collector=test_collector, + episode_per_test=args.test_num, + max_epoch=args.epoch, + step_per_epoch=args.step_per_epoch, + step_per_collect=args.step_per_collect, + batch_size=args.batch_size, + save_best_fn=save_best_fn, + logger=logger, + update_per_step=args.update_per_step, + ).run() + pprint.pprint(result) + + +if __name__ == "__main__": + training_sac() diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp new file mode 100644 index 0000000000..4e8c627637 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp @@ -0,0 +1,408 @@ +/** + * @file owsc_python.cpp + * @brief This is the test of wave interaction with Oscillating Wave Surge Converter (OWSC), for DRL. + * @author Mai Ye, Chi Zhang and Xiangyu Hu + */ +#include +#include "owsc_python.h" +#include "custom_io_environment.h" +#include "custom_io_observation.h" +#include "custom_io_simbody.h" +#include "sphinxsys.h" + +using namespace SPH; +namespace py = pybind11; + +class SphBasicSystemSetting : public SphBasicGeometrySetting +{ + protected: + BoundingBox system_domain_bounds; + SPHSystem sph_system; + CustomIOEnvironment custom_io_environment; + + public: + SphBasicSystemSetting(int parallel_env, int episode_env) + : system_domain_bounds(Vec2d(-DL_Extra - BW, -BW), Vec2d(DL + BW, DH + BW)), + sph_system(system_domain_bounds, particle_spacing_ref), + custom_io_environment(sph_system, true, parallel_env, episode_env){} +}; + +class SphFlapReloadEnvironment : public SphBasicSystemSetting +{ + protected: + FluidBody water_block; + SolidBody wall_boundary, flap; + ObserverBody flap_observer, wave_velocity_observer; + + public: + SphFlapReloadEnvironment(int parallel_env, int episode_env) + : SphBasicSystemSetting(parallel_env, episode_env), + water_block(sph_system, makeShared("WaterBody")), + wall_boundary(sph_system, makeShared("Wall")), + flap(sph_system, makeShared("Flap")), + flap_observer(sph_system, "FlapObserver"), + wave_velocity_observer(sph_system, "WaveVelocityObserver") + { + water_block.defineMaterial(rho0_f, c_f, mu_f); + water_block.generateParticles(); + + wall_boundary.defineMaterial(); + wall_boundary.generateParticles(); + + flap.defineMaterial(rho0_s); + flap.generateParticles(); + + flap_observer.generateParticles(createFlapObserver()); + wave_velocity_observer.generateParticles(createWaveVelocityObserver()); + } +}; + +class SimbodyEnvironment : public SphFlapReloadEnvironment +{ + protected: + /** set up the multi body system. */ + SimTK::MultibodySystem MBsystem; + /** the bodies or matter of the system. */ + SimTK::SimbodyMatterSubsystem matter; + /** the forces of the system. */ + SimTK::GeneralForceSubsystem forces; + /** mass properties of the fixed spot. */ + FlapSystemForSimbody flap_multibody; + SimTK::Body::Rigid pin_spot_info; + SimTK::MobilizedBody::Pin pin_spot; + SimTK::Force::UniformGravity sim_gravity; + /** discrete forces acting on the bodies. */ + SimTK::Force::DiscreteForces force_on_bodies; + SimTK::Force::MobilityLinearDamper linear_damper; + SimTK::State state; + SimTK::RungeKuttaMersonIntegrator integ; + + public: + SimbodyEnvironment(int parallel_env, int episode_env) + : SphFlapReloadEnvironment(parallel_env, episode_env), + matter(MBsystem), + forces(MBsystem), + flap_multibody(flap, makeShared(createFlapSimbodyConstrainShape(), "FlapMultiBody")), + pin_spot_info(*flap_multibody.body_part_mass_properties_), + pin_spot(matter.Ground(), SimTK::Transform(SimTK::Vec3(7.92, 0.315, 0.0)), pin_spot_info, SimTK::Transform(SimTK::Vec3(0.0, 0.0, 0.0))), + sim_gravity(forces, matter, SimTK::Vec3(0.0, - gravity_g, 0.0), 0.0), + force_on_bodies(forces, matter), + linear_damper(forces, pin_spot, SimTK::MobilizerUIndex(0), 20), integ(MBsystem) + { + pin_spot.setDefaultAngle(0); + state = MBsystem.realizeTopology(); + integ.setAccuracy(1e-3); + integ.setAllowInterpolation(false); + integ.initialize(state); + } +}; + +class SphOWSC : public SimbodyEnvironment +{ + protected: + InnerRelation water_block_inner, flap_inner; + ContactRelation water_block_contact, flap_contact, flap_observer_contact_with_water, flap_observer_contact_with_flap, + wave_velocity_observer_contact_with_water; + ComplexRelation water_block_complex; + //---------------------------------------------------------------------- + // Define all numerical methods which are used in this case. + //---------------------------------------------------------------------- + Gravity gravity; + SimpleDynamics> constant_gravity; + SimpleDynamics flap_offset_position; + SimpleDynamics wall_boundary_normal_direction, flap_normal_direction; + + Dynamics1Level pressure_relaxation; + Dynamics1Level density_relaxation; + InteractionWithUpdate update_density_by_summation; + InteractionWithUpdate viscous_force; + + ReduceDynamics get_fluid_advection_time_step_size; + ReduceDynamics get_fluid_time_step_size; + BodyRegionByCell damping_buffer; + SimpleDynamics damping_wave; + BodyRegionByParticle wave_maker; + SimpleDynamics wave_making; + + InteractionWithUpdate viscous_force_from_fluid; + InteractionWithUpdate> pressure_force_from_fluid; + + ParticleSorting particle_sorting; + //---------------------------------------------------------------------- + // Coupling between SimBody and SPH + //---------------------------------------------------------------------- + ReduceDynamics force_on_spot_flap; + SimpleDynamics constraint_spot_flap; + //---------------------------------------------------------------------- + // Define the methods for I/O operations and observations of the simulation. + //---------------------------------------------------------------------- + BodyStatesRecordingToVtp write_real_body_states; + RegressionTestDynamicTimeWarping>> write_total_viscous_force_from_fluid; + /** Velocity probe. */ + ObservedQuantityRecording wave_velocity_probe; + ObservedQuantityRecording wave_velocity_on_flap_probe; + /** Observer for flap position. */ + ObservedQuantityRecording flap_position_probe; + // Interpolate the particle position in flap to move the observer accordingly. + // Seems not used? TODO: observe displacement more accurate. + InteractionDynamics> interpolation_flap_velocity_observer_position; + WriteSimBodyPinDataExtended write_flap_pin_data; + /** WaveProbes. */ + BodyRegionByCell wave_probe_buffer_no_0, wave_probe_buffer_no_1; + ExtendedReducedQuantityRecording> wave_probe_0, wave_probe_1; + //---------------------------------------------------------------------- + // Basic control parameters for time stepping. + //---------------------------------------------------------------------- + Real &physical_time = *sph_system.getSystemVariableDataByName("PhysicalTime"); + int number_of_iterations = 0; + int screen_output_interval = 100; + Real dt = 0.0; + Real total_time = 0.0; + Real relax_time = 1.0; + Real output_interval = 0.1; + /** statistics for computing time. */ + TickCount t1 = TickCount::now(); + TimeInterval interval; + + public: + explicit SphOWSC(int parallel_env, int episode_env) + : SimbodyEnvironment(parallel_env, episode_env), + water_block_inner(water_block), + flap_inner(flap), + water_block_contact(water_block, {&wall_boundary, &flap}), + flap_contact(flap, {&water_block}), + flap_observer_contact_with_water(flap_observer, {&water_block}), + flap_observer_contact_with_flap(flap_observer, {&flap}), + wave_velocity_observer_contact_with_water(wave_velocity_observer, {&water_block}), + water_block_complex(water_block_inner, water_block_contact), + + gravity(Vecd(0.0, -gravity_g)), + constant_gravity(water_block, gravity), + flap_offset_position(flap, offset), + wall_boundary_normal_direction(wall_boundary), + flap_normal_direction(flap), + + pressure_relaxation(water_block_inner, water_block_contact), + density_relaxation(water_block_inner, water_block_contact), + update_density_by_summation(water_block_inner, water_block_contact), + viscous_force(water_block_inner, water_block_contact), + + get_fluid_advection_time_step_size(water_block, U_f), + get_fluid_time_step_size(water_block), + damping_buffer(water_block, makeShared(createDampingBufferShape())), + damping_wave(damping_buffer), + wave_maker(wall_boundary, makeShared(createWaveMakerShape())), + wave_making(wave_maker), + + viscous_force_from_fluid(flap_contact), + pressure_force_from_fluid(flap_contact), + + particle_sorting(water_block), + + force_on_spot_flap(flap_multibody, MBsystem, pin_spot, integ), + constraint_spot_flap(flap_multibody, MBsystem, pin_spot, integ), + write_real_body_states(sph_system), + write_total_viscous_force_from_fluid(flap, "ViscousForceFromFluid"), + wave_velocity_probe("Velocity", wave_velocity_observer_contact_with_water), + wave_velocity_on_flap_probe("Velocity", flap_observer_contact_with_water), + flap_position_probe("Position", flap_observer_contact_with_flap), + interpolation_flap_velocity_observer_position(flap_observer_contact_with_flap, "Position", "Position"), + write_flap_pin_data(sph_system, integ, pin_spot), + wave_probe_buffer_no_0(water_block, makeShared(createWaveProbeShape(3.0), "WaveProbe_03")), + wave_probe_buffer_no_1(water_block, makeShared(createWaveProbeShape(5.0), "WaveProbe_05")), + wave_probe_0(wave_probe_buffer_no_0, "FreeSurfaceHeight"), + wave_probe_1(wave_probe_buffer_no_1, "FreeSurfaceHeight") + { + physical_time = 0.0; + //---------------------------------------------------------------------- + // Prepare the simulation with cell linked list, configuration + // and case specified initial condition if necessary. + //---------------------------------------------------------------------- + flap_offset_position.exec(); + sph_system.initializeSystemCellLinkedLists(); + sph_system.initializeSystemConfigurations(); + wall_boundary_normal_direction.exec(); + flap_normal_direction.exec(); + constant_gravity.exec(); + //---------------------------------------------------------------------- + // First output before the main loop. + //---------------------------------------------------------------------- + write_real_body_states.writeToFile(0); + write_total_viscous_force_from_fluid.writeToFile(0); + wave_velocity_probe.writeToFile(0); + wave_velocity_on_flap_probe.writeToFile(0); + flap_position_probe.writeToFile(0); + write_flap_pin_data.writeToFile(0); + wave_probe_0.writeToFile(0); + wave_probe_1.writeToFile(0); + } + + virtual ~SphOWSC(){}; + //---------------------------------------------------------------------- + // For ctest. + //---------------------------------------------------------------------- + int cmakeTest() + { + return 1; + } + //---------------------------------------------------------------------- + // Get flap angle and angle rate. + //---------------------------------------------------------------------- + Real getFlapAngle() + { + return write_flap_pin_data.getAngleToPython(number_of_iterations); + }; + + Real getFlapAngleRate() + { + return write_flap_pin_data.getAngleRateToPython(number_of_iterations); + }; + //---------------------------------------------------------------------- + // Get wave height. + //---------------------------------------------------------------------- + Real getWaveHeight(size_t number) + { + if (number == 0) + { + return wave_probe_0.getReducedQuantity(); + } + else if (number == 1) + { + return wave_probe_1.getReducedQuantity(); + } + return -1.0; + } + //---------------------------------------------------------------------- + // Get wave velocity in front of the flap. + //---------------------------------------------------------------------- + Real getWaveVelocity(int number, int direction) + { + return wave_velocity_probe.getObservedQuantity()[number][direction]; + }; + //---------------------------------------------------------------------- + // Get wave velocity on the flap. + //---------------------------------------------------------------------- + Real getWaveVelocityOnFlap(int number, int direction) + { + return wave_velocity_on_flap_probe.getObservedQuantity()[number][direction]; + }; + //---------------------------------------------------------------------- + // Get flap position. + //---------------------------------------------------------------------- + Real getFlapPositon(int number, int direction) + { + return flap_position_probe.getObservedQuantity()[number][direction]; + }; + //---------------------------------------------------------------------- + // Main loop of time stepping starts here. && For changing damping coefficient. + //---------------------------------------------------------------------- + void runCase(Real pause_time_from_python, Real dampling_coefficient_from_python) + { + while (physical_time < pause_time_from_python) + { + Real integral_time = 0.0; + while (integral_time < output_interval) + { + Real Dt = get_fluid_advection_time_step_size.exec(); + update_density_by_summation.exec(); + viscous_force.exec(); + /** Viscous force exerting on flap. */ + viscous_force_from_fluid.exec(); + + Real relaxation_time = 0.0; + while (relaxation_time < Dt) + { + pressure_relaxation.exec(dt); + pressure_force_from_fluid.exec(); + density_relaxation.exec(dt); + /** coupled rigid body dynamics. */ + if (total_time >= relax_time) + { + SimTK::State &state_for_update = integ.updAdvancedState(); + linear_damper.setDamping(state_for_update, dampling_coefficient_from_python); + Real angle = pin_spot.getAngle(state_for_update); + force_on_bodies.clearAllBodyForces(state_for_update); + force_on_bodies.setOneBodyForce(state_for_update, pin_spot, force_on_spot_flap.exec(angle)); + integ.stepBy(dt); + constraint_spot_flap.exec(); + wave_making.exec(dt); + } + interpolation_flap_velocity_observer_position.exec(); + + dt = get_fluid_time_step_size.exec(); + relaxation_time += dt; + integral_time += dt; + total_time += dt; + if (total_time >= relax_time) + physical_time += dt; + } + + if (number_of_iterations % screen_output_interval == 0) + { + std::cout << std::fixed << std::setprecision(9) << "N=" << number_of_iterations + << " Total Time = " << total_time + << " Physical Time = " << physical_time + << " Dt = " << Dt << " dt = " << dt << "\n"; + } + number_of_iterations++; + damping_wave.exec(Dt); + if (number_of_iterations % 100 == 0 && number_of_iterations != 1) + { + particle_sorting.exec(); + } + water_block.updateCellLinkedList(); + wall_boundary.updateCellLinkedList(); + flap.updateCellLinkedList(); + water_block_complex.updateConfiguration(); + flap_contact.updateConfiguration(); + flap_observer_contact_with_water.updateConfiguration(); + wave_velocity_observer_contact_with_water.updateConfiguration(); + if (total_time >= relax_time) + { + write_total_viscous_force_from_fluid.writeToFile(number_of_iterations); + wave_velocity_probe.writeToFile(number_of_iterations); + wave_velocity_on_flap_probe.writeToFile(number_of_iterations); + flap_position_probe.writeToFile(number_of_iterations); + write_flap_pin_data.writeToFile(physical_time); + wave_probe_0.writeToFile(number_of_iterations); + wave_probe_1.writeToFile(number_of_iterations); + } + } + + TickCount t2 = TickCount::now(); + if (total_time >= relax_time) + write_real_body_states.writeToFile(); + TickCount t3 = TickCount::now(); + interval += t3 - t2; + } + TickCount t4 = TickCount::now(); + + TimeInterval tt; + tt = t4 - t1 - interval; + + // This section is used for CMake testing (cmake test). + // During reinforcement learning training, this part can be commented out. + if (sph_system.GenerateRegressionData()) + { + write_total_viscous_force_from_fluid.generateDataBase(1.0e-3); + } + else + { + write_total_viscous_force_from_fluid.testResult(); + } + }; +}; + +PYBIND11_MODULE(test_2d_owsc_python, m) +{ + py::class_(m, "owsc_from_sph_cpp") + .def(py::init()) + .def("cmake_test", &SphOWSC::cmakeTest) + .def("get_flap_angle", &SphOWSC::getFlapAngle) + .def("get_flap_angle_rate", &SphOWSC::getFlapAngleRate) + .def("get_wave_height", &SphOWSC::getWaveHeight) + .def("get_wave_velocity", &SphOWSC::getWaveVelocity) + .def("get_wave_velocity_on_flap", &SphOWSC::getWaveVelocityOnFlap) + .def("get_flap_position", &SphOWSC::getFlapPositon) + .def("run_case", &SphOWSC::runCase); +} diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.h b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.h new file mode 100644 index 0000000000..e2c3c44d38 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.h @@ -0,0 +1,375 @@ +#include "sphinxsys.h" + +using namespace SPH; + +class SphBasicGeometrySetting +{ + protected: + //------------------------------------------------------------------------------ + // global parameters for the case + //------------------------------------------------------------------------------ + Real DL = 18.42; // tank length + Real DH = 1.0; // tank height + Real DL_Extra = 1.0; // for wave maker + Real Water_H = 0.691; /**< Water height. */ + + Real Flap_width = 0.12; + Real Flap_x = 7.92; + Real Flap_H = 0.48; + + Real Base_bottom_position = 0.155; + Real Base_height = 0.1; + Real particle_spacing_ref = Flap_width / 4.0; // particle spacing + Real BW = particle_spacing_ref * 4.0; // boundary width + // the offset that the rubber flap shifted above the tank + // Real flap_off = Flap_x - 0.5 * Flap_width + DL_Extra + BW; + // Real off_set = particle_spacing_ref + floor(flap_off / particle_spacing_ref) * particle_spacing_ref - flap_off; + Vec2d offset = Vec2d::Zero(); + + // water block parameters + Vec2d Water_lb = Vec2d(0.0, 0.0); // left bottom + Vec2d Water_lt = Vec2d(0.0, Water_H); // left top + Vec2d Water_rt = Vec2d(DL, Water_H); // right top + Vec2d Water_rb = Vec2d(DL, 0.356); // right bottom + Vec2d Water_slope_1 = Vec2d(DL - 6.2, 0.356); + Vec2d Water_slope_2 = Vec2d(DL - 6.2 - 3.7, 0.155); + Vec2d Water_slope_3 = Vec2d(DL - 6.2 - 3.7 - 2.4, 0.155); + Vec2d Water_slope_4 = Vec2d(DL - 6.2 - 3.7 - 2.4 - 1.3, 0.0); + + // flap constrain region parameter + Vec2d Base_lb = Vec2d(Flap_x - 0.5 * Flap_width, Base_bottom_position); // left bottom + Vec2d Base_lt = Vec2d(Flap_x - 0.5 * Flap_width, Base_bottom_position + Base_height); // left top + Vec2d Base_rt = Vec2d(Flap_x + 0.5 * Flap_width, Base_bottom_position + Base_height); // right top + Vec2d Base_rb = Vec2d(Flap_x + 0.5 * Flap_width, Base_bottom_position); + + // flap geometric parameters + Vec2d Flap_lb = Vec2d(Flap_x - 0.5 * Flap_width, Base_bottom_position + Base_height + 0.5 * Flap_width); // left bottom + Vec2d Flap_lt = Vec2d(Flap_x - 0.5 * Flap_width, Base_bottom_position + Base_height + 0.5 * Flap_width + Flap_H); // left top + Vec2d Flap_rt = Vec2d(Flap_x + 0.5 * Flap_width, Base_bottom_position + Base_height + 0.5 * Flap_width + Flap_H); // right top + Vec2d Flap_rb = Vec2d(Flap_x + 0.5 * Flap_width, Base_bottom_position + Base_height + 0.5 * Flap_width); // right bottom + + // gravity + Real gravity_g = 9.81; + + // for material properties of the fluid + Real rho0_f = 1000.0; + Real U_f = 2.0 * sqrt(0.79 * gravity_g); + Real c_f = 10.0 * U_f; + Real mu_f = 1.0e-6; + + // for material properties of the solid + Real flap_mass = 33.04; + Real flap_volume = 0.0579; + Real rho0_s = flap_mass / flap_volume; + + //------------------------------------------------------------------------------ + // geometric shape elements used in the case + //------------------------------------------------------------------------------ + std::vector createWaterBlockShape() + { + std::vector pnts; + pnts.push_back(Water_lb); + pnts.push_back(Water_lt); + pnts.push_back(Water_rt); + pnts.push_back(Water_rb); + pnts.push_back(Water_slope_1); + pnts.push_back(Water_slope_2); + pnts.push_back(Water_slope_3); + pnts.push_back(Water_slope_4); + pnts.push_back(Water_lb); + + return pnts; + } + + std::vector createFlapConstrainShape() + { + std::vector pnts2; + pnts2.push_back(Base_lb); + pnts2.push_back(Base_lt); + pnts2.push_back(Base_rt); + pnts2.push_back(Base_rb); + pnts2.push_back(Base_lb); + + return pnts2; + } + + std::vector createFlapShape() + { + std::vector pnts3; + pnts3.push_back(Flap_lb); + pnts3.push_back(Flap_lt); + pnts3.push_back(Flap_rt); + pnts3.push_back(Flap_rb); + for (int i = 1; i <= 10; i++) + { + Real angle = Real(i) * Pi / (11.0); + Real x = Flap_rb[0] - 0.5 * Flap_width * (1.0 - cos(angle)); + Real y = Flap_rb[1] - 0.5 * Flap_width * sin(angle) - 0.5 * particle_spacing_ref; + pnts3.push_back(Vecd(x, y)); + } + pnts3.push_back(Flap_lb); + + return pnts3; + } + + MultiPolygon createDampingBufferShape() + { + std::vector pnts; + pnts.push_back(Vecd(DL - 5.0, 0.356 - BW)); + pnts.push_back(Vecd(DL - 5.0, DH)); + pnts.push_back(Vecd(DL + BW, DH)); + pnts.push_back(Vecd(DL + BW, 0.356 - BW)); + pnts.push_back(Vecd(DL - 5.0, 0.356 - BW)); + + MultiPolygon multi_polygon; + multi_polygon.addAPolygon(pnts, ShapeBooleanOps::add); + return multi_polygon; + } + + std::vector createOuterWallShape() + { + std::vector pnts1; + pnts1.push_back(Vecd(-DL_Extra - BW, -BW)); + pnts1.push_back(Vecd(-DL_Extra - BW, DH + BW)); + pnts1.push_back(Vecd(DL + BW, DH + BW)); + pnts1.push_back(Vecd(DL + BW, 0.35 - BW)); + pnts1.push_back(Water_slope_1 + Vec2d(0.0, -BW)); + pnts1.push_back(Water_slope_2 + Vec2d(0.0, -BW)); + pnts1.push_back(Water_slope_3 + Vec2d(0.0, -BW)); + pnts1.push_back(Water_slope_4 + Vec2d(0.0, -BW)); + pnts1.push_back(Vecd(-DL_Extra - BW, -BW)); + + return pnts1; + } + + std::vector createInnerWallShape01() + { + std::vector pnts2; + pnts2.push_back(Water_lb); + pnts2.push_back(Vecd(0.0, DH + BW)); + pnts2.push_back(Vecd(DL, DH + BW)); + pnts2.push_back(Water_rb); + pnts2.push_back(Water_slope_1); + pnts2.push_back(Water_slope_2); + pnts2.push_back(Base_rb); + pnts2.push_back(Base_rt); + pnts2.push_back(Base_lt); + pnts2.push_back(Base_lb); + pnts2.push_back(Water_slope_3); + pnts2.push_back(Water_slope_4); + pnts2.push_back(Water_lb); + + return pnts2; + } + + std::vector createInnerWallShape02() + { + std::vector pnts3; + pnts3.push_back(Vecd(-DL_Extra, 0.0)); + pnts3.push_back(Vecd(-DL_Extra, DH + BW)); + pnts3.push_back(Vecd(-BW, DH + BW)); + pnts3.push_back(Vecd(-BW, 0.0)); + pnts3.push_back(Vecd(-DL_Extra, 0.0)); + + return pnts3; + } + + MultiPolygon createWaveMakerShape() + { + std::vector wave_make_shape; + wave_make_shape.push_back(Vecd(-BW, 0.0)); + wave_make_shape.push_back(Vecd(-BW, DH + BW)); + wave_make_shape.push_back(Vecd(0.0, DH + BW)); + wave_make_shape.push_back(Vecd(0.0, 0.0)); + wave_make_shape.push_back(Vecd(-BW, 0.0)); + + MultiPolygon multi_polygon; + multi_polygon.addAPolygon(wave_make_shape, ShapeBooleanOps::add); + return multi_polygon; + } + + //------------------------------------------------------------------------------ + // Body parts used in the case + //------------------------------------------------------------------------------ + Real h = 1.3 * particle_spacing_ref; + + MultiPolygon createFlapSimbodyConstrainShape() + { + MultiPolygon multi_polygon; + multi_polygon.addAPolygon(createFlapShape(), ShapeBooleanOps::add); + return multi_polygon; + } + + MultiPolygon createWaveProbeShape(Real x) + { + std::vector pnts; + pnts.push_back(Vecd(x - h, 0.0)); + pnts.push_back(Vecd(x - h, 1.0)); + pnts.push_back(Vecd(x + h, 1.0)); + pnts.push_back(Vecd(x + h, 0.0)); + pnts.push_back(Vecd(x - h, 0.0)); + + MultiPolygon multi_polygon; + multi_polygon.addAPolygon(pnts, ShapeBooleanOps::add); + return multi_polygon; + } + + StdVec createFlapObserver() + { + StdVec observer_positions; + observer_positions.push_back(Vecd(7.862, 0.3)); + observer_positions.push_back(Vecd(7.862, 0.5)); + + return observer_positions; + } + + StdVec createWaveVelocityObserver() + { + StdVec observer_positions; + observer_positions.push_back(Vecd(3.0, 0.55)); + observer_positions.push_back(Vecd(5.0, 0.55)); + + return observer_positions; + } +}; +//------------------------------------------------------------------------------ +// geometric shapes for the bodies used in the case +//------------------------------------------------------------------------------ +class WaterBlock : public MultiPolygonShape, public SphBasicGeometrySetting +{ + public: + explicit WaterBlock(const std::string& shape_name) : MultiPolygonShape(shape_name) + { + multi_polygon_.addAPolygon(createWaterBlockShape(), ShapeBooleanOps::add); + multi_polygon_.addAPolygon(createFlapShape(), ShapeBooleanOps::sub); + multi_polygon_.addAPolygon(createFlapConstrainShape(), ShapeBooleanOps::sub); + } +}; + +class WallBoundary : public MultiPolygonShape, public SphBasicGeometrySetting +{ + public: + explicit WallBoundary(const std::string& shape_name) : MultiPolygonShape(shape_name) + { + multi_polygon_.addAPolygon(createOuterWallShape(), ShapeBooleanOps::add); + multi_polygon_.addAPolygon(createFlapConstrainShape(), ShapeBooleanOps::add); + multi_polygon_.addAPolygon(createInnerWallShape01(), ShapeBooleanOps::sub); + multi_polygon_.addAPolygon(createInnerWallShape02(), ShapeBooleanOps::sub); + } +}; + +class Flap : public MultiPolygonShape, public SphBasicGeometrySetting +{ + public: + explicit Flap(const std::string& shape_name) : MultiPolygonShape(shape_name) + { + multi_polygon_.addAPolygon(createFlapShape(), ShapeBooleanOps::add); + } +}; + +class FlapSystemForSimbody : public SolidBodyPartForSimbody +{ + public: + FlapSystemForSimbody(SPHBody& sph_body, SharedPtr shape_ptr) + : SolidBodyPartForSimbody(sph_body, shape_ptr) + { + // Vecd mass_center = Vecd(7.92, 0.355); // 0.3355 + // initial_mass_center_ = SimTK::Vec3(mass_center[0], mass_center[1], 0.0); + /** UnitInertia_ (const RealP &xx, const RealP &yy, const RealP &zz) + * Create a principal unit inertia matrix (only non-zero on diagonal). + */ + Real Iz = 1.84 / 33.04; + body_part_mass_properties_ = + mass_properties_ptr_keeper_ + .createPtr(33.04, SimTK::Vec3(0.0), SimTK::UnitInertia(0.0, 0.0, Iz)); + } +}; + +class WaveMaking : public BodyPartMotionConstraint, public SphBasicGeometrySetting +{ + Real model_scale_; + Real gravity_; + Real water_depth_; + Real wave_height_; + Real wave_period_; + Real wave_freq_; + Real wave_stroke_; + + Vecd getDisplacement(const Real &time) + { + Vecd displacement{Vecd::Zero()}; + displacement[0] = 0.5 * wave_stroke_ * sin(wave_freq_ * time); + return displacement; + } + + Vec2d getVelocity(const Real &time) + { + Vec2d velocity{Vecd::Zero()}; + velocity[0] = 0.5 * wave_stroke_ * wave_freq_ * cos(wave_freq_ * time); + return velocity; + } + + Vec2d getAcceleration(const Real &time) + { + Vec2d acceleration{Vecd::Zero()}; + acceleration[0] = -0.5 * wave_stroke_ * wave_freq_ * wave_freq_ * sin(wave_freq_ * time); + return acceleration; + } + + void computeWaveStrokeAndFrequency() + { + Real scaled_wave_height = wave_height_ / model_scale_; + Real scaled_wave_period = wave_period_ / sqrt(model_scale_); + Real scaled_wave_freq = 2.0 * Pi / scaled_wave_period; + Real scaled_wave_amp = 0.5 * scaled_wave_height; + + int iterator = 20; + Real Tol = 1.0e-6; + Real scaled_wave_number = 1.0; + for (int i = 1; i < iterator; i++) + { + Real term1 = tanh(scaled_wave_number * water_depth_); + Real term2 = scaled_wave_freq * scaled_wave_freq / gravity_; + Real term3 = scaled_wave_number * term1 - term2; + Real term4 = term1 + scaled_wave_number * water_depth_ * (1.0 - term1 * term1); + Real wave_number_old = scaled_wave_number; + scaled_wave_number = wave_number_old - term3 / term4; + Real error = abs(scaled_wave_number - wave_number_old) / abs(scaled_wave_number); + if (error <= Tol) + break; + } + + Real term_1 = gravity_ / scaled_wave_freq / scaled_wave_freq; + Real term_2 = 2.0 * scaled_wave_number * water_depth_; + Real term_3 = scaled_wave_number * water_depth_; + Real scaled_wave_stroke = 0.5 * scaled_wave_amp * scaled_wave_number * term_1 * + (term_2 + sinh(term_2)) / (cosh(term_3) * sinh(term_3)); + + wave_stroke_ = scaled_wave_stroke; + wave_freq_ = scaled_wave_freq; + std::cout << "scaled_wave_number: " << scaled_wave_number << " Wave frequency: " << wave_freq_ << std::endl; + } + + public: + WaveMaking(BodyPartByParticle &body_part) + : BodyPartMotionConstraint(body_part), + model_scale_(25.0), gravity_(gravity_g), water_depth_(Water_H), wave_height_(5.0), + wave_period_(10.0), + acc_(particles_->registerStateVariable("Acceleration")), + physical_time_(sph_system_.getSystemVariableDataByName("PhysicalTime")) + { + computeWaveStrokeAndFrequency(); + } + + void update(size_t index_i, Real dt = 0.0) + { + Real time = *physical_time_; + pos_[index_i] = pos0_[index_i] + getDisplacement(time); + vel_[index_i] = getVelocity(time); + acc_[index_i] = getAcceleration(time); + }; + + protected: + Vecd *acc_; + Real *physical_time_; +}; diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/pybind_tool/pybind_test.py b/tests/extra_source_and_tests/test_2d_owsc_python/pybind_tool/pybind_test.py new file mode 100644 index 0000000000..e148cefa34 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/pybind_tool/pybind_test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import os +import sys +import platform +import argparse +# add dynamic link library or shared object to python env +# attention: match current python version with the version exposing the cpp code +sys_str = platform.system() +# If this doesn't works, try path_1 = os.path.abspath(os.path.join(os.getcwd(), '../..')) +path_1 = os.path.abspath(os.path.join(os.getcwd(), '..')) +if sys_str == 'Windows': + # Append 'RelWithDebInfo' or 'Debug' depending on the configuration + # For example, path_2 = 'lib/RelWithDebInfo' + path_2 = 'lib' +elif sys_str == 'Linux': + path_2 = 'lib' +else: + # depend on the system + path_2 = 'lib' +path = os.path.join(path_1, path_2) +sys.path.append(path) +# change import depending on the project name +import test_2d_owsc_python as test_2d + + +def run_case(): + parser = argparse.ArgumentParser() + # set case parameters + parser.add_argument("--parallel_env", default=0, type=int) + parser.add_argument("--episode_env", default=0, type=int) + parser.add_argument("--damping_coefficient", default=20.0, type=float) + parser.add_argument("--end_time", default=12.0, type=float) + case = parser.parse_args() + + # set project from class, which is set in cpp pybind module + project = test_2d.owsc_from_sph_cpp(case.parallel_env, case.episode_env) + if project.cmake_test() == 1: + project.run_case(case.end_time, case.damping_coefficient) + else: + print("check path: ", path) + + +if __name__ == "__main__": + run_case() diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_0_result.xml b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_0_result.xml new file mode 100644 index 0000000000..c4faad078d --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_0_result.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_10_result.xml b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_10_result.xml new file mode 100644 index 0000000000..272d814277 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_10_result.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_20_result.xml b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_20_result.xml new file mode 100644 index 0000000000..99cfc89fca --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_Run_20_result.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_dtwdistance.xml b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_dtwdistance.xml new file mode 100644 index 0000000000..2ca0472222 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_dtwdistance.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_runtimes.dat b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_runtimes.dat new file mode 100644 index 0000000000..5aa48673a0 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/Flap_TotalViscousForceFromFluid_runtimes.dat @@ -0,0 +1,3 @@ +true +21 +4 \ No newline at end of file diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/regression_test_tool.py b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/regression_test_tool.py new file mode 100644 index 0000000000..1ea47ef859 --- /dev/null +++ b/tests/extra_source_and_tests/test_2d_owsc_python/regression_test_tool/regression_test_tool.py @@ -0,0 +1,37 @@ +# !/usr/bin/env python3 +import os +import sys + +path = os.path.abspath('../../../../../PythonScriptStore/RegressionTest') +sys.path.append(path) +from regression_test_base_tool import SphinxsysRegressionTest + +""" +case name: test_2d_owsc_python +""" + +case_name = "test_2d_owsc_python" +body_name = "Flap" +parameter_name = "TotalViscousForceFromFluid" + +number_of_run_times = 0 +converged = 0 +sphinxsys = SphinxsysRegressionTest(case_name, body_name, parameter_name) + + +while True: + print("Now start a new run......") + sphinxsys.run_case() + number_of_run_times += 1 + converged = sphinxsys.read_dat_file() + print("Please note: This is the", number_of_run_times, "run!") + if number_of_run_times <= 200: + if (converged == "true"): + print("The tested parameters of all variables are converged, and the run will stop here!") + break + elif converged != "true": + print("The tested parameters of", sphinxsys.sphinxsys_parameter_name, "are not converged!") + continue + else: + print("It's too many runs but still not converged, please try again!") + break From 60010a981d2c09177c00191732345f43be9e64e5 Mon Sep 17 00:00:00 2001 From: maiyetum95 <108422104+maiyetum95@users.noreply.github.com> Date: Sun, 29 Sep 2024 00:35:05 +0200 Subject: [PATCH 2/5] Modify the format in owsc_python.cpp --- .../test_2d_owsc_python/owsc_python.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp index 4e8c627637..38bdc39809 100644 --- a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp +++ b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp @@ -89,11 +89,11 @@ class SimbodyEnvironment : public SphFlapReloadEnvironment force_on_bodies(forces, matter), linear_damper(forces, pin_spot, SimTK::MobilizerUIndex(0), 20), integ(MBsystem) { - pin_spot.setDefaultAngle(0); - state = MBsystem.realizeTopology(); - integ.setAccuracy(1e-3); - integ.setAllowInterpolation(false); - integ.initialize(state); + pin_spot.setDefaultAngle(0); + state = MBsystem.realizeTopology(); + integ.setAccuracy(1e-3); + integ.setAllowInterpolation(false); + integ.initialize(state); } }; From cff5b9e8927605b191ecb592e843f243379dae6f Mon Sep 17 00:00:00 2001 From: maiyetum95 <108422104+maiyetum95@users.noreply.github.com> Date: Sun, 29 Sep 2024 00:49:12 +0200 Subject: [PATCH 3/5] clean the code in owsc_python.cpp --- .../extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp index 38bdc39809..0ef3584f25 100644 --- a/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp +++ b/tests/extra_source_and_tests/test_2d_owsc_python/owsc_python.cpp @@ -100,6 +100,7 @@ class SimbodyEnvironment : public SphFlapReloadEnvironment class SphOWSC : public SimbodyEnvironment { protected: + SPHSystem &sph_system_; InnerRelation water_block_inner, flap_inner; ContactRelation water_block_contact, flap_contact, flap_observer_contact_with_water, flap_observer_contact_with_flap, wave_velocity_observer_contact_with_water; @@ -167,6 +168,7 @@ class SphOWSC : public SimbodyEnvironment public: explicit SphOWSC(int parallel_env, int episode_env) : SimbodyEnvironment(parallel_env, episode_env), + sph_system_(sph_system), water_block_inner(water_block), flap_inner(flap), water_block_contact(water_block, {&wall_boundary, &flap}), From ff2a90504db4a56913a741c97f90f8c5e0847107 Mon Sep 17 00:00:00 2001 From: maiyetum95 <108422104+maiyetum95@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:31:16 +0200 Subject: [PATCH 4/5] Update README.md, reorganize the framework --- .../drl_gym_environments/README.md | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md index cf8ecf98f2..686e0f1a93 100644 --- a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md @@ -1,20 +1,29 @@ +### Detailed Explanation + - **`drl_gym_environments/`**: The main package directory that includes the environment registration logic and all the custom environment implementations. -- **`gym_env_owsc/__init__.py`**: - This file contains the registration logic for all custom environments. - Each environment is registered with a unique ID, which can be used with Gymnasium’s `gym.make()` function. + - **`drl_gym_environments/setup.py`**: + Provides the configuration for installing the package. + To install the environments in editable mode, run the following command from the project’s root directory: + + ```bash + pip install -e . + ``` + + - **`drl_gym_environments/gym_env_owsc/`**: + This folder contains the OWSC custom environment. -- **`envs/`**: - This sub-directory contains the actual implementation of all the custom environments. + - **`gym_env_owsc/__init__.py`**: + This file contains the registration logic for all custom environments. + Each environment is registered with a unique ID, which can be used with Gymnasium’s `gym.make()` function. - - **`envs/__init__.py`**: - Imports all custom environments so they can be properly registered. + - **`gym_env_owsc/envs/`**: + This sub-directory contains the actual implementation of all the custom environments. - - **`envs/owsc.py`**: - Implements the OWSC environment, following the standard Gymnasium `Env` interface. - The environment defines unique observation and action spaces, and includes specific environment dynamics in the `reset()`, `step()`, and `render()` methods. + - **`envs/__init__.py`**: + Imports all custom environments so they can be properly registered. -- **`setup.py`**: - Provides the configuration for installing the package. - To install the environments in editable mode, run the following command from the project’s root directory. + - **`envs/owsc.py`**: + Implements the OWSC environment, following the standard Gymnasium `Env` interface. + The environment defines unique observation and action spaces and includes specific environment dynamics in the `reset()`, `step()`, and `render()` methods. From 46c8cc55425e71e27a3f50a2f5af11c5620479e4 Mon Sep 17 00:00:00 2001 From: maiyetum95 Date: Mon, 30 Sep 2024 19:19:03 +0200 Subject: [PATCH 5/5] 1. add code in the sac.py about how to used the trained DRL model 2. the overall guide for DRL will be moved to the SPHinXsys/tutorials/sphinx/python.rst --- README.md | 28 ------ .../drl_gym_environments/README.md | 29 ------- .../drl_tianshou_training/sac.py | 12 ++- tutorials/sphinx/python.rst | 87 +++++++++++++++++++ 4 files changed, 98 insertions(+), 58 deletions(-) delete mode 100644 tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md create mode 100644 tutorials/sphinx/python.rst diff --git a/README.md b/README.md index cff3a73b14..29146f0496 100644 --- a/README.md +++ b/README.md @@ -93,34 +93,6 @@ Please check the source code of [2D Dambreak case with python interface](https://github.com/Xiangyu-Hu/SPHinXsys/tree/master/tests/2d_examples/test_2d_dambreak_python) for the usage. -## Deep reinforcement learning in SPHinXsys - -By leveraging the Python interface of SPHinXsys, one can rapidly and modularly construct SPHinXsys simulation environments required for deep reinforcement learning (DRL). Effective training can be conducted using various algorithms from the mainstream DRL platform Tianshou. Please refer to the source code of -[2D owsc case with python interface](https://github.com/Xiangyu-Hu/SPHinXsys/tree/master/tests/extra_source_and_tests/test_2d_owsc_python). - -Step-by-Step instructions: -1. **Use conda or Python to create a virtual Python environment.** - Python 3.10 is recommended for compatibility. - - ```bash - conda create -n drl_env python=3.10 -2. **Activate the python environment and install required libraries.** - - ```bash - conda activate drl_env - pip install tianshou -3. **Set up the SPHinXsys-based reinforcement learning environment in the `drl_gym_environments` directory.** - Detailed environment setup instructions can be found in the `README.md` file. - - ```bash - cd path_to_deep_reinforcement_learning_tool/drl_gym_environments - pip install -e . -6. **Navigate to the `drl_tianshou_training` directory and start the training process.** - - ```bash - cd path_to_deep_reinforcement_learning_tool/drl_tianshou_training - python sac.py - ## Publications Main publication on the library: diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md deleted file mode 100644 index 686e0f1a93..0000000000 --- a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_gym_environments/README.md +++ /dev/null @@ -1,29 +0,0 @@ -### Detailed Explanation - -- **`drl_gym_environments/`**: - The main package directory that includes the environment registration logic and all the custom environment implementations. - - - **`drl_gym_environments/setup.py`**: - Provides the configuration for installing the package. - To install the environments in editable mode, run the following command from the project’s root directory: - - ```bash - pip install -e . - ``` - - - **`drl_gym_environments/gym_env_owsc/`**: - This folder contains the OWSC custom environment. - - - **`gym_env_owsc/__init__.py`**: - This file contains the registration logic for all custom environments. - Each environment is registered with a unique ID, which can be used with Gymnasium’s `gym.make()` function. - - - **`gym_env_owsc/envs/`**: - This sub-directory contains the actual implementation of all the custom environments. - - - **`envs/__init__.py`**: - Imports all custom environments so they can be properly registered. - - - **`envs/owsc.py`**: - Implements the OWSC environment, following the standard Gymnasium `Env` interface. - The environment defines unique observation and action spaces and includes specific environment dynamics in the `reset()`, `step()`, and `render()` methods. diff --git a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py index 2c9e955407..161b0a0253 100644 --- a/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py +++ b/tests/extra_source_and_tests/test_2d_owsc_python/deep_reinforcement_learning_tool/drl_tianshou_training/sac.py @@ -43,6 +43,7 @@ def get_args(): parser.add_argument("--n-step", type=int, default=1) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--training-num", type=int, default=1) + parser.add_argument("--test-num", type=int, default=1) parser.add_argument('--test-num', type=int, default=0) parser.add_argument("--logdir", type=str, default="log") parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") @@ -106,10 +107,19 @@ def training_sac(args=get_args()): action_space=envs.action_space, ) - # load a previous policy + # load a previous or trained policy if args.resume_path: policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) print("Loaded agent from: ", args.resume_path) + + if args.watch: + # Watch the play of the trained policy + policy.eval() + test_collector = Collector(policy, envs) + test_collector.reset() + result = test_collector.collect(n_episode=args.test_num) + print(f"Final reward: {result['rews'].mean()}, length: {result['lens'].mean()}") + return # Setup replay buffer and collector for training buffer = VectorReplayBuffer(args.buffer_size, len(envs)) if args.training_num > 1 else ReplayBuffer(args.buffer_size) diff --git a/tutorials/sphinx/python.rst b/tutorials/sphinx/python.rst new file mode 100644 index 0000000000..84d3164f37 --- /dev/null +++ b/tutorials/sphinx/python.rst @@ -0,0 +1,87 @@ +======================== +How to train deep reinforcement learning model in SPHinXsys +======================== + +By leveraging the Python interface of SPHinXsys, one can rapidly and modularly construct SPHinXsys simulation environments required for deep reinforcement learning (DRL). Effective training can be conducted using various algorithms from the mainstream DRL platform Tianshou. Please refer to the source code of +[2D owsc case with python interface](https://github.com/Xiangyu-Hu/SPHinXsys/tree/master/tests/extra_source_and_tests/test_2d_owsc_python). + +Step-by-Step instructions +--------------------------------------- + +Use conda or Python to create a virtual Python environment. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Python 3.10 is recommended for compatibility. + +.. code-block:: bash + + conda create -n drl_env python=3.10 + +Activate the python environment and install required libraries. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + conda activate drl_env + pip install tianshou + +Set up the SPHinXsys-based reinforcement learning environment in the "drl_gym_environments" directory. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + cd path_to_deep_reinforcement_learning_tool/drl_gym_environments + pip install -e . + +Navigate to the `drl_tianshou_training` directory and start the training process. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + cd path_to_deep_reinforcement_learning_tool/drl_tianshou_training + python sac.py + +Test the trained model. +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Modify three arguments: + +'resume-path': the path where your policy was saved, + +'test-number': the number of test episodes you want to run, + +'watch': True means testing, False means training. + +.. code-block:: bash + + python sac.py + + +Detailed explanation of "drl_gym_environments" +-------------------------------------------------- + +The main package directory that includes the environment registration logic and all the custom environment implementations. + +**drl_gym_environments/setup.py**: + +Provides the configuration for installing the package. + +**drl_gym_environments/gym_env_owsc**: + +This folder contains the OWSC custom environment. + +**gym_env_owsc/__init__.py**: + +This file contains the registration logic for all custom environments. Each environment is registered with a unique ID, which can be used with Gymnasium’s `gym.make()` function. + +**gym_env_owsc/envs**: + +This sub-directory contains the actual implementation of all the custom environments. + +**envs/__init__.py**: + +Imports all custom environments so they can be properly registered. + +**envs/owsc.py**: + +Implements the OWSC environment, following the standard Gymnasium `Env` interface. The environment defines unique observation and action spaces and includes specific environment dynamics in the `reset()`, `step()`, and `render()` methods. \ No newline at end of file