Github actions pipeline

.github/workflows/minetester.yml

@@ -0,0 +1,51 @@
+name: Build and test Minetester
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    branches: [ "develop" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Linux dependencies
+      run: make linux_deps
+    - name: Install build dependencies
+      run: make python_build_deps
+    - name: Init submodules
+      run: make repos
+    - name: Build SDL2
+      run: make sdl2
+    - name: Build zmqpp
+      run: make zmqpp
+    - name: Create Protobuf files
+      run: make proto
+    - name: Build irrlicht
+      run: bash util/minetester/build_irrlicht.sh
+    - name: Build minetest binary
+      run: make minetest
+    - name: Build minetester wheel
+      run: make minetester
+    - name:  Install minetester along with dependencies
+      run: |
+        python -m pip install --upgrade pip
+        make install
+    - name: Run pre-commit hooks
+      run: |
+        pip install pre-commit
+        pre-commit install
+        pre-commit run --all-files

.gitmodules

@@ -1,14 +1,14 @@
-[submodule "lib/irrlichtmt"]
-	path = lib/irrlichtmt
-	url = https://github.com/EleutherAI/irrlicht.git 
-	branch = headless-rendering
-
-[submodule "lib/zmqpp"]
-	path = lib/zmqpp
-	url = https://github.com/zeromq/zmqpp.git
 [submodule "games/minetest_game"]
 	path = games/minetest_game
 	url = https://github.com/minetest/minetest_game.git
 [submodule "lib/SDL"]
 	path = lib/SDL
 	url = https://github.com/libsdl-org/SDL.git
+	branch = SDL2
+[submodule "lib/zmqpp"]
+	path = lib/zmqpp
+	url = https://github.com/zeromq/zmqpp.git
+[submodule "lib/irrlicht"]
+	path = lib/irrlichtmt
+	url = https://github.com/EleutherAI/irrlicht.git
+	branch = headless-rendering

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+# Linting
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-ast
+    files: ^minetester/.*$
+  - id: trailing-whitespace
+    files: ^minetester/.*$
+  - id: end-of-file-fixer
+    exclude_types: [jupyter]
+    files: ^minetester/.*$
+  - id: check-added-large-files
+    files: ^minetester/.*$
+- repo: https://github.com/psf/black
+  rev: 22.6.0
+  hooks:
+  - id: black
+    files: ^minetester/.*$
+  - id: black-jupyter
+    files: ^minetester/.*$
+- repo: https://github.com/PyCQA/isort
+  rev: 5.12.0
+  hooks:
+  - id: isort
+    args: ["--profile", "black", "--filter-files"]
+    files: ^minetester/.*$
+# Python static analysis
+- repo: https://github.com/pycqa/flake8
+  rev: '5.0.4'
+  hooks:
+  - id: flake8
+    additional_dependencies:
+      - darglint~=1.8.1
+      - flake8-blind-except==0.2.1
+      - flake8-builtins~=1.5.3
+      - flake8-commas~=2.1.0
+      - flake8-debugger~=4.1.2
+      - flake8-docstrings~=1.6.0
+    files: ^minetester/.*$

Makefile

@@ -1,8 +1,9 @@
-.PHONY: all deps repos sdl2 package zmqpp minetester minetest install demo proto clean
+.PHONY: all deps repos sdl2 package zmqpp irrlicht minetester minetest install demo proto clean
 
 MINETESTER_VERSION := 0.0.1
 SDL2_CMAKE_FILE := lib/SDL/build/lib/cmake/SDL2/sdl2-config.cmake
 ZMQPP_LIB_FILE := lib/zmqpp/build/max-g++/libzmqpp.a
+IRRLICHTMT_LIB_FILE := lib/irrlichtmt/lib/Linux/libIrrlichtMt.a
 MINETEST_BINARY := bin/minetest
 DEBUG_BINARY := bin/debug
 MINETESTER_WHEEL := build/package/wheel/minetester-$(MINETESTER_VERSION)-py3-none-manylinux_2_35_x86_64.whl
@@ -38,6 +39,10 @@ $(ZMQPP_LIB_FILE):
 
 zmqpp: $(ZMQPP_LIB_FILE)
 
+$(IRRLICHTMT_LIB_FILE):
+	util/minetester/build_irrlicht.sh
+
+irrlicht: $(IRRLICHTMT_LIB_FILE)
 
 $(MINETEST_BINARY):
 	#build minetest binary

README.md

@@ -15,13 +15,18 @@ Minetester
 ==========
 Minetester is the Python package that exposes Minetest environments via the `gym(nasium)` interface.
 After [building the minetest executable](https://github.com/EleutherAI/minetest/blob/develop/build_instructions.txt) you can install it with:
-```
+``` bash
 pip install -e .
 ```
 To verify the installation run
-```
+``` bash
 python -m minetester.scripts.test_loop
 ```
+To build the documentation in `docs-minetester` please run
+``` bash
+pip install -e .[docs]
+cd docs-minetester && make livehtml
+```
 
 Quick Build Instructions for Linux
 ==================================
@@ -45,7 +50,7 @@ make clean #clean up build artifacts
 
 Additionally the makefile supports a utility to clean only the minetester install
 
-```
+```bash
 make clean_minetester #remove existing minetester install
 make minetester #build minetester python library
 make install #install python library into local environment along with nessesary dependencies

build_instructions.txt

@@ -1,46 +1,20 @@
-1. Install prereqs
-  1. sudo apt-get install xvfb g++ make libzmq3-dev libtool pkg-config build-essential autoconf automake libc6-dev cmake libpng-dev libjpeg-dev libxi-dev libgl1-mesa-dev libsqlite3-dev libogg-dev libvorbis-dev libopenal-dev libcurl4-gnutls-dev libfreetype6-dev zlib1g-dev libgmp-dev libjsoncpp-dev libzstd-dev libluajit-5.1-dev protobuf-compiler
-
-2. Build SDL2
-  1. clone the SDL2 repo https://github.com/libsdl-org/SDL
-  2. checkout release-2.26.2 https://github.com/libsdl-org/SDL/tree/release-2.26.2
-  3. create a build directory inside the SDL repo
-  4. cd into it and run ../configure --prefix=/path/to/SDL/dir/build && make && make install
-
-3. Build zmqpp
-  1. clone https://github.com/zeromq/zmqpp
-  2. checkout the develop branch
-  3. run make
-
-4. Clone EAI alignment minetest repos
-  1. clone https://github.com/EleutherAI/minetest
-  2. checkout the develop branch
-  3. clone https://github.com/EleutherAI/irrlicht
-  4. checkout headless-renderer
-
-5. Clone minetest game spec
-  1. clone https://github.com/minetest/minetest_game
-  2. checkout master branch
-
-5. Establish symlinks
-  1. cd into minetest/lib
-  2. rm -r zmqpp irrlichtmt
-  3. ln -s ../../zmqpp/ zmqpp
-  4. ln -s ../../irrlicht/ irrlichtmt
-  5. cd into minetest/games
-  6. ln -s ../../minetest_game/ minetest_game
-
-6. Build minetest
-  1. cd into minetest
-  2. either run
-     cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=1 -DSDL2_DIR=<path to SDL repo>/SDL/build/lib/cmake/SDL2/
-     or
-     cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=0 -DSDL2_DIR=
-  3. run make -j$(nproc)
-
-7. setup python
-  1. create and activate a new python conda env or venv (tested with python3.9)
-  2. pip install gym matplotlib protobuf==3.20.1 psutil zmq -e ./minetest
-  3. cd into the scripts directory and run compile_proto.sh
-  4. run python -m minetester.scripts.test_loop
-
+# 1. Install debian dependencies, equivalent commands are nessesary for other distros
+make deb_deps
+# 2. Install build dependencies into the local python environment (we reccomend using a venv)
+make python_build_deps
+# 3. Init submodules
+make repos
+# 4. Build sdl2
+make sdl2
+# 5. Build zmqpp
+make zmqpp
+# 6. Create c++ and python protobuf files
+make proto
+# 7. Build minetest binary
+make minetest
+# 8. Build minetester python library
+make minetester
+# 9. Install minetester into local environment along with necessary dependencies
+make install
+# 10. Run the demo script
+make demo

clientmods/mods.conf

@@ -1,7 +1,7 @@
-load_mod_random_v0 = true
 load_mod_rewards = true
 load_mod_treechop_v1 = false
 load_mod_treechop_shaped_v0 = false
 load_mod_random_v1 = false
-load_mod_preview = false
 load_mod_treechop_v0 = false
+load_mod_random_v0 = false
+load_mod_preview = false

docs-minetester/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+livehtml:
+	sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) --open-browser

docs-minetester/build.sh

@@ -0,0 +1 @@
+sphinx-apidoc -o ./source  ../minetester ../minetester/proto

docs-minetester/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs-minetester/source/.gitignore

@@ -0,0 +1 @@
+_api

docs-minetester/source/_templates/module.rst

@@ -0,0 +1,30 @@
+{{ fullname | escape | underline }}
+
+Description
+-----------
+
+.. automodule:: {{ fullname | escape }}
+
+{% if classes %}
+Classes
+-------
+.. autosummary:
+    :toctree: _autosummary
+
+    {% for class in classes %}
+        {{ class }}
+    {% endfor %}
+
+{% endif %}
+
+{% if functions %}
+Functions
+---------
+.. autosummary:
+    :toctree: _autosummary
+
+    {% for function in functions %}
+        {{ function }}
+    {% endfor %}
+
+{% endif %}

docs-minetester/source/advanced/client_api.rst

@@ -0,0 +1,37 @@
+Client API
+==========
+
+The API provided for controlling and reading out the Minetest client is based on two libraries:
+
+- `Protocol Buffers <https://protobuf.dev/>`_
+- `ZeroMQ <https://zeromq.org/>`_
+
+Protobuf objects are used for serializing and deserializing the messages, that are being sent using ZeroMQ.
+
+Protobuf objects
+----------------
+
+The objects are defined in `/proto/objects.proto <https://github.com/EleutherAI/minetest/blob/develop/proto/objects.proto>`_.
+The Minetest client receives ``Action`` messages and sends out ``Observation`` messages.
+
+By default the Protobuf objects are compiled for C++ and Python using the
+`/scripts/compile_proto.sh <https://github.com/EleutherAI/minetest/blob/develop/scripts/compile_proto.sh>`_ script.
+This can be easily adjusted to compile for other languages.
+
+ZeroMQ message patterns
+-----------------------
+
+There are two differnt ZeroMQ messaging patterns in use depending on the type of the client:
+
+- dumb Minetest client <-> controller client: ``REQ/REP``
+
+  In each step the dumb client sends a blocking request containing an observation to the controller
+  which awaits the request and replies with an action.
+- recording Minetest client <-> data gathering client : ``PUB/SUB``
+
+  In each step the recording client publishes an observation and the data gathering client
+  subscribes to the topic in order to read out the observations.
+  See `this script <https://github.com/EleutherAI/minetest/blob/develop/scripts/data_recorder.py>`_
+  for an example of how to implement a data gathering client.
+
+In both cases the ZMQ socket address is passed to the Minetest client via the ``--client-address`` command line argument.