Reduce requirements (#594)

# Description
Closes #517 

Added new requirements file `requirements_full.txt` with all optional
requirements.

Getting rust, cargo and poetry properly installed in the docker images
for building the wheels is followed up in issue #601.

Follow up with issue #602 - Avoid skipping tests when build Python
wheels.

**Remaining things to be done:**
- [x] Fix failing tests
- [x] Make it possible to install dlite with `pip install
DLite-Python[full]` to include all optional dependencies.
- [x] Document how to install DLite with all dependencies included.
- [x] Ensure that tests run if yaml is missing (avoid false CI failures
on this PR before we get everything correctly installed in the docker
containers used for building the wheels. This should not be used to skip
yaml-dependent tests when testing the wheels).
- [ ] Agree on whether to accept this PR or the solution suggested in
issue #603.

## Type of change
- [x] Bug fix & code cleanup
- [ ] New feature
- [ ] Documentation update
- [ ] Test update

## Checklist for the reviewer
This checklist should be used as a help for the reviewer.

- [ ] Is the change limited to one issue?
- [ ] Does this PR close the issue?
- [ ] Is the code easy to read and understand?
- [ ] Do all new feature have an accompanying new test?
- [ ] Has the documentation been updated as necessary?

.github/docker/Dockerfile-manylinux.template

@@ -24,9 +24,10 @@
 # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="7 8 9 10"
+ARG PY_MINORS="7 8 9 10 11"
 
-COPY requirements.txt /tmp/requirements.txt
+#COPY requirements.txt /tmp/requirements.txt
+COPY requirements_full.txt requirements_dev.txt /tmp/
 
 {{ EXTRA_PRE }}
 
@@ -49,13 +50,16 @@ RUN yum update -y && \
 # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+# Change required version of pydantic to be <2
+  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \
+# Install required Python packages
   mkdir -p /ci/pip_cache && \
   if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
     python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
-    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \
+    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
   done
 
 {{ EXTRA_POST }}

.github/docker/Dockerfile-mingw

@@ -46,6 +46,8 @@ RUN dnf install -y \
         cppcheck \
         doxygen \
         ccache \
+        rust \
+        cargo \
         wine-core \
         wine-filesystem \
         mingw-filesystem-base \

.github/docker/Dockerfile-musllinux.template

@@ -23,7 +23,10 @@
 #
 FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
-COPY requirements.txt /tmp/requirements.txt
+ARG PY_MINORS="7 8 9 10 11"
+
+#COPY requirements.txt /tmp/requirements.txt
+COPY requirements_full.txt requirements_dev.txt /tmp/
 
 # Do not use distutils distributed with setuptools
 # This is due to base changes in the distutils API, removing msvccompiler,
@@ -40,10 +43,13 @@ RUN apk add -u \
 # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+# Change required version of pydantic to be <2
+  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \
+# Install required Python packages
   mkdir -p /ci/pip_cache && \
-  for minor in 7 8 9 10; do \
+  for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
     python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
-    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \
+    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
   done

.github/workflows/cd_docs.yml

@@ -25,7 +25,7 @@ env:
   PUBLISH_DIR: pages
 
 jobs:
-  build:
+  compile:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -87,7 +87,7 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
 
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
 
     steps:
       - name: Deploy to GitHub Pages

.github/workflows/ci_dependabot.yml

@@ -20,7 +20,7 @@ jobs:
       extra_to_dos: "- [ ] Make sure that the PR is **squash** merged, with a sensible commit message."
       update_pre-commit: true
       python_version: "3.8"
-      install_extras: "[dev,docs]"
+      install_extras: "[full,dev,docs]"
       skip_pre-commit_hooks: pylint
     secrets:
       PAT: ${{ secrets.RELEASE_PAT }}

.github/workflows/ci_tests.yml

@@ -29,6 +29,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python3 -m pip install --upgrade pip -r requirements.txt
+        python3 -m pip install -r requirements_full.txt
         python3 -m pip install -r requirements_dev.txt
         python3 -m pip install -r requirements_doc.txt
 

README.md

@@ -37,9 +37,15 @@ Installation
 DLite is available on PyPI and can be installed with pip
 
 ```shell
-pip install dlite-python
+pip install dlite-python[full]
 ```
 
+The bracket `[full]` is optional, but ensures that you install all optional
+dependencies together with DLite.
+Without `[full]` you get a minimal DLite installation that only depends on
+NumPy.
+This would disable most storage plugins, except for the built-in
+"json", "bson" and "rdf" (when compiled against Redland librdf).
 For alternative installation methods, see the [installation instructions].
 
 

bindings/python/tests/test_storage.py

@@ -11,6 +11,12 @@
 except ModuleNotFoundError:
     HAVE_PYTEST = False
 
+try:
+    import yaml
+    HAVE_YAML = True
+except ModuleNotFoundError:
+    HAVE_YAML = False
+
 
 thisdir = os.path.abspath(os.path.dirname(__file__))
 
@@ -48,13 +54,9 @@
 
 
 # Test yaml
-try:
-    import yaml
-except ImportError:
-    pass
-else:
-    print("--- testing yaml")
-    inst.save("yaml://inst.yaml?mode=w")
+if HAVE_YAML:
+    print('--- testing yaml')
+    inst.save('yaml://inst.yaml?mode=w')
     del inst
     inst = dlite.Instance.from_url("yaml://inst.yaml#my-data")
 
@@ -106,8 +108,9 @@
 
 
 # Tests for issue #587
-bytearr = inst.to_bytes("yaml")
-# print(bytes(bytearr).decode())
+if HAVE_YAML:
+    bytearr = inst.to_bytes("yaml")
+    #print(bytes(bytearr).decode())
 if HAVE_PYTEST:
     with pytest.raises(dlite.DLiteError):
         inst.to_bytes("json")

doc/contributors_guide/tips_and_tricks.md

@@ -9,7 +9,7 @@ See the [Build and install to a virtual Python environment] in the installation
 Debugging tests failing inside docker on GitHub
 -----------------------------------------------
 1. Set up a local virtual environment with the same version of Python as in the failing test.
-   See [Build and install to a virtual Python environment].
+   See [Build against Python environment].
 2. Enter your virtual environment with dlite and install cibuildwheel.
 
         workon dlite
@@ -89,4 +89,4 @@ Debugging tests failing inside docker on GitHub
 
 
 [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/
-[Build and install to a virtual Python environment]: https://sintef.github.io/dlite/getting_started/installation.html#build-and-install-to-a-virtual-python-environment
+[Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment

doc/getting_started/build/runtime_dependencies.md

@@ -8,24 +8,45 @@ On Ubuntu, they can be installed with
     sudo apt install python3 gfortran librdf libhdf5
 
 When DLite is compiled with Python bindings, additional runtime features may be enabled by installing one of more of the following optional Python packages
-- [tripper], optional, (used for property mappings)
-- [PyYAML], optional (used for generic YAML storage plugin)
-- [psycopg2], optional (used for generic PostgreSQL storage plugin)
+- [tripper], used for property mappings
+- [pint], used for units conversion in property mappings
+- [pydantic], used for testing support for pydantic models
+- [typing_extensions], needed by pydantic v2
+- [rdflib], used by `rdf` storage plugin
+- [PyYAML], used by `yaml` storage plugin
+- [psycopg2], used by `postgresql` storage plugin
     Note that in some cases a GSSAPI error is raised when using psycopg2
     by pip installing psycopg2-binary.
     This is solved by installing from source as described in their documentation.
-- [pandas], optional (used for csv storage plugin)
-- [pymongo], optional, (used for mongodb storage plugin)
-- [mongomock], optional, used for testing mongodb storage plugin.
+- [pandas], used by the `csv` storage plugin
+- [requests], used by `http` storage plugin
+- [jinja2], used by `template` storage plugin
+- [pymongo], used by the `mongodb` storage plugin
 
 These optional dependencies can be installed with
 
+    pip install -r requirements_full.txt
+
+Separate requirements can also be installed for development
+
     pip install -r requirements_dev.txt
 
+or for building the documentation
+
+    pip install -r requirements_doc.txt
+
+
 
 [tripper]: https://pypi.org/project/tripper/
+[pint]: https://pint.readthedocs.io/en/stable/
+[pydantic]: https://docs.pydantic.dev/
+[typing_extensions]: https://github.com/python/typing_extensions
+[rdflib]: https://rdflib.readthedocs.io/
 [PyYAML]: https://pypi.org/project/PyYAML/
 [psycopg2]: https://pypi.org/project/psycopg2/
 [pandas]: https://pandas.pydata.org/
 [pymongo]: https://github.com/mongodb/mongo-python-driver
 [mongomock]: https://github.com/mongomock/mongomock
+[openpyxl]: https://openpyxl.readthedocs.io/
+[requests]: https://requests.readthedocs.io/
+[jinja2]: https://jinja.palletsprojects.com/

doc/getting_started/installation.md

@@ -10,21 +10,34 @@ Installing with pip
 If you are using Python, the easiest way to install DLite is with pip:
 
 ```shell
-pip install DLite-Python
+pip install DLite-Python[full]
 ```
 
-This will give you DLite together with all optional Python dependencies (see [runtime dependencies]).
+The bracket `[full]` is optional, but ensures that you install DLite
+together with all optional dependencies needed for additional features
+and storage plugins. (see also [runtime dependencies]).
 
 
 Development installation
 ------------------------
 If you to contribute or develop DLite, you should [build from source].
 
+Python dependencies for development can be installed with
+
+```shell
+pip install DLite-Python[dev]
+```
+
+Install additional Python packages for building documentation with
+
+```shell
+pip install DLite-Python[doc]
+```
 
 
 [Python]: https://www.python.org/
 [Fortran]: https://en.wikipedia.org/wiki/Fortran
 [HDF5]: https://support.hdfgroup.org/HDF5/
 [librdf]: https://librdf.org/
-[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime-dependencies.html
+[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime_dependencies.html
 [build from source]: https://sintef.github.io/dlite/getting_started/build/build.html

python/setup.py

@@ -156,19 +156,26 @@ def build_extension(self, ext: CMakeExtension) -> None:
             str(cmake_bdist_dir / ext.name), str(Path(output_dir) / ext.name)
         )
 
-extra_requirements = [
-    "fortran-language-server",
-    "PyYAML",
-    "psycopg2-binary==2.9.5",
-    "pandas",
-    "pymongo",
-    "rdflib",
-    "tripper",
-    "pint",
-]
-
 requirements = ["numpy"]
 
+# Populate extra_requirements from requirements_*.txt
+extra_requirements = {}
+for name in "full", "dev", "doc":
+    with open(SOURCE_DIR / f"requirements_{name}.txt", "r") as f:
+        extra_requirements[name] = [
+            line.strip() for line in f.readlines() if not line.startswith("#")
+        ]
+
+# Temporary workaround!
+# Require pydantic <2. Needed before we have managed to install rust in
+# the docker image for wheels
+extras = extra_requirements["full"]
+for i, pkg in enumerate(extras):
+    match = re.match("^(pydantic>.*<)", pkg)
+    if match:
+        extras[i] = f"{match.groups()[0]}2"
+
+
 version = re.search(
     r"project\([^)]*VERSION\s+([0-9.]+)",
     (SOURCE_DIR / "CMakeLists.txt").read_text(),
@@ -204,10 +211,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         "Programming Language :: Python :: 3.11",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    install_requires=requirements + extra_requirements,
-    # For now, the extra requirements are hard requirements.
-    # See issue #222: https://github.com/SINTEF/dlite/issues/222
-    # extras_require={"all": extra_requirements},
+    install_requires=requirements,
+    extras_require=extra_requirements,
     packages=["dlite"],
     scripts=[
         str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"),

requirements.txt

@@ -1,14 +1 @@
-fortran-language-server>=1.12.0,<1.13
-numpy>=1.20.3,<1.25.0
-PyYAML>=5.4.1,<7
-# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5
-psycopg2-binary==2.9.5
-pandas>=1.2,<2.1
-rdflib>=4.2.1,<7
-pint>=0.15,<0.23
-openpyxl>=3.0.9,<3.2
-pymongo>=4.4.0,<5
-tripper>=0.2.5,<0.3
-pydantic>=1.10.0,<2
-typing_extensions>=4.1,<5
-requests>=2.10,<3
+numpy>=1.14.5,<1.25.0

requirements_dev.txt

@@ -3,4 +3,4 @@ ipython>=7.34.0,<9
 ipykernel>=6.0.1,<7
 ipython_genutils~=0.2.0
 mongomock>=4.1.2,<5
-pydantic>=1.9.0,<2
+openpyxl>=3.0.9,<3.2

requirements_full.txt

@@ -0,0 +1,14 @@
+# Optional requirements - used by various plugins or additional features like mappings
+fortran-language-server>=1.12.0,<1.13
+PyYAML>=5.4.1,<7
+# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5
+psycopg2-binary==2.9.5
+pandas>=1.2,<2.1
+rdflib>=4.2.1,<7
+pint>=0.15,<1
+pymongo>=4.4.0,<5
+tripper>=0.2.5,<0.3
+requests>=2.10,<3
+jinja2>=3.0,<4
+pydantic>=1.10.0,<3
+typing_extensions>=4.1,<5