diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 0e68ed675..a36856481 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -24,9 +24,10 @@ # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest -ARG PY_MINORS="7 8 9 10" +ARG PY_MINORS="7 8 9 10 11" -COPY requirements.txt /tmp/requirements.txt +#COPY requirements.txt /tmp/requirements.txt +COPY requirements_full.txt requirements_dev.txt /tmp/ {{ EXTRA_PRE }} @@ -49,13 +50,16 @@ RUN yum update -y && \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ +# Change required version of pydantic to be <2 + sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \ +# Install required Python packages mkdir -p /ci/pip_cache && \ if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \ for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \ done {{ EXTRA_POST }} diff --git a/.github/docker/Dockerfile-mingw b/.github/docker/Dockerfile-mingw index ac8f11337..c904325af 100644 --- a/.github/docker/Dockerfile-mingw +++ b/.github/docker/Dockerfile-mingw @@ -46,6 +46,8 @@ RUN dnf install -y \ cppcheck \ doxygen \ ccache \ + rust \ + cargo \ wine-core \ wine-filesystem \ mingw-filesystem-base \ diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index 22a3f26fb..002919dcf 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -23,7 +23,10 @@ # FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest -COPY requirements.txt /tmp/requirements.txt +ARG PY_MINORS="7 8 9 10 11" + +#COPY requirements.txt /tmp/requirements.txt +COPY requirements_full.txt requirements_dev.txt /tmp/ # Do not use distutils distributed with setuptools # This is due to base changes in the distutils API, removing msvccompiler, @@ -40,10 +43,13 @@ RUN apk add -u \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ +# Change required version of pydantic to be <2 + sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \ +# Install required Python packages mkdir -p /ci/pip_cache && \ - for minor in 7 8 9 10; do \ + for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \ done diff --git a/.github/workflows/cd_docs.yml b/.github/workflows/cd_docs.yml index 3a955e91e..e1207fe8d 100644 --- a/.github/workflows/cd_docs.yml +++ b/.github/workflows/cd_docs.yml @@ -25,7 +25,7 @@ env: PUBLISH_DIR: pages jobs: - build: + compile: runs-on: ubuntu-latest steps: - name: Checkout @@ -87,7 +87,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest - needs: build + needs: compile steps: - name: Deploy to GitHub Pages diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index 47437becc..92fb37f1d 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -20,7 +20,7 @@ jobs: extra_to_dos: "- [ ] Make sure that the PR is **squash** merged, with a sensible commit message." update_pre-commit: true python_version: "3.8" - install_extras: "[dev,docs]" + install_extras: "[full,dev,docs]" skip_pre-commit_hooks: pylint secrets: PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index f8610f9eb..173c3fd22 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -29,6 +29,7 @@ jobs: - name: Install Python dependencies run: | python3 -m pip install --upgrade pip -r requirements.txt + python3 -m pip install -r requirements_full.txt python3 -m pip install -r requirements_dev.txt python3 -m pip install -r requirements_doc.txt diff --git a/README.md b/README.md index 3079800a8..64693110e 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,15 @@ Installation DLite is available on PyPI and can be installed with pip ```shell -pip install dlite-python +pip install dlite-python[full] ``` +The bracket `[full]` is optional, but ensures that you install all optional +dependencies together with DLite. +Without `[full]` you get a minimal DLite installation that only depends on +NumPy. +This would disable most storage plugins, except for the built-in +"json", "bson" and "rdf" (when compiled against Redland librdf). For alternative installation methods, see the [installation instructions]. diff --git a/bindings/python/tests/test_storage.py b/bindings/python/tests/test_storage.py index 6c28082dd..4963e0706 100755 --- a/bindings/python/tests/test_storage.py +++ b/bindings/python/tests/test_storage.py @@ -11,6 +11,12 @@ except ModuleNotFoundError: HAVE_PYTEST = False +try: + import yaml + HAVE_YAML = True +except ModuleNotFoundError: + HAVE_YAML = False + thisdir = os.path.abspath(os.path.dirname(__file__)) @@ -48,13 +54,9 @@ # Test yaml -try: - import yaml -except ImportError: - pass -else: - print("--- testing yaml") - inst.save("yaml://inst.yaml?mode=w") +if HAVE_YAML: + print('--- testing yaml') + inst.save('yaml://inst.yaml?mode=w') del inst inst = dlite.Instance.from_url("yaml://inst.yaml#my-data") @@ -106,8 +108,9 @@ # Tests for issue #587 -bytearr = inst.to_bytes("yaml") -# print(bytes(bytearr).decode()) +if HAVE_YAML: + bytearr = inst.to_bytes("yaml") + #print(bytes(bytearr).decode()) if HAVE_PYTEST: with pytest.raises(dlite.DLiteError): inst.to_bytes("json") diff --git a/doc/contributors_guide/tips_and_tricks.md b/doc/contributors_guide/tips_and_tricks.md index 8e1bcb1ff..ef4af3429 100644 --- a/doc/contributors_guide/tips_and_tricks.md +++ b/doc/contributors_guide/tips_and_tricks.md @@ -9,7 +9,7 @@ See the [Build and install to a virtual Python environment] in the installation Debugging tests failing inside docker on GitHub ----------------------------------------------- 1. Set up a local virtual environment with the same version of Python as in the failing test. - See [Build and install to a virtual Python environment]. + See [Build against Python environment]. 2. Enter your virtual environment with dlite and install cibuildwheel. workon dlite @@ -89,4 +89,4 @@ Debugging tests failing inside docker on GitHub [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/ -[Build and install to a virtual Python environment]: https://sintef.github.io/dlite/getting_started/installation.html#build-and-install-to-a-virtual-python-environment +[Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment diff --git a/doc/getting_started/build/runtime_dependencies.md b/doc/getting_started/build/runtime_dependencies.md index a11464e97..c681e6ac4 100644 --- a/doc/getting_started/build/runtime_dependencies.md +++ b/doc/getting_started/build/runtime_dependencies.md @@ -8,24 +8,45 @@ On Ubuntu, they can be installed with sudo apt install python3 gfortran librdf libhdf5 When DLite is compiled with Python bindings, additional runtime features may be enabled by installing one of more of the following optional Python packages -- [tripper], optional, (used for property mappings) -- [PyYAML], optional (used for generic YAML storage plugin) -- [psycopg2], optional (used for generic PostgreSQL storage plugin) +- [tripper], used for property mappings +- [pint], used for units conversion in property mappings +- [pydantic], used for testing support for pydantic models +- [typing_extensions], needed by pydantic v2 +- [rdflib], used by `rdf` storage plugin +- [PyYAML], used by `yaml` storage plugin +- [psycopg2], used by `postgresql` storage plugin Note that in some cases a GSSAPI error is raised when using psycopg2 by pip installing psycopg2-binary. This is solved by installing from source as described in their documentation. -- [pandas], optional (used for csv storage plugin) -- [pymongo], optional, (used for mongodb storage plugin) -- [mongomock], optional, used for testing mongodb storage plugin. +- [pandas], used by the `csv` storage plugin +- [requests], used by `http` storage plugin +- [jinja2], used by `template` storage plugin +- [pymongo], used by the `mongodb` storage plugin These optional dependencies can be installed with + pip install -r requirements_full.txt + +Separate requirements can also be installed for development + pip install -r requirements_dev.txt +or for building the documentation + + pip install -r requirements_doc.txt + + [tripper]: https://pypi.org/project/tripper/ +[pint]: https://pint.readthedocs.io/en/stable/ +[pydantic]: https://docs.pydantic.dev/ +[typing_extensions]: https://github.com/python/typing_extensions +[rdflib]: https://rdflib.readthedocs.io/ [PyYAML]: https://pypi.org/project/PyYAML/ [psycopg2]: https://pypi.org/project/psycopg2/ [pandas]: https://pandas.pydata.org/ [pymongo]: https://github.com/mongodb/mongo-python-driver [mongomock]: https://github.com/mongomock/mongomock +[openpyxl]: https://openpyxl.readthedocs.io/ +[requests]: https://requests.readthedocs.io/ +[jinja2]: https://jinja.palletsprojects.com/ diff --git a/doc/getting_started/installation.md b/doc/getting_started/installation.md index a64c8ee62..9d098d506 100644 --- a/doc/getting_started/installation.md +++ b/doc/getting_started/installation.md @@ -10,21 +10,34 @@ Installing with pip If you are using Python, the easiest way to install DLite is with pip: ```shell -pip install DLite-Python +pip install DLite-Python[full] ``` -This will give you DLite together with all optional Python dependencies (see [runtime dependencies]). +The bracket `[full]` is optional, but ensures that you install DLite +together with all optional dependencies needed for additional features +and storage plugins. (see also [runtime dependencies]). Development installation ------------------------ If you to contribute or develop DLite, you should [build from source]. +Python dependencies for development can be installed with + +```shell +pip install DLite-Python[dev] +``` + +Install additional Python packages for building documentation with + +```shell +pip install DLite-Python[doc] +``` [Python]: https://www.python.org/ [Fortran]: https://en.wikipedia.org/wiki/Fortran [HDF5]: https://support.hdfgroup.org/HDF5/ [librdf]: https://librdf.org/ -[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime-dependencies.html +[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime_dependencies.html [build from source]: https://sintef.github.io/dlite/getting_started/build/build.html diff --git a/python/setup.py b/python/setup.py index 0d87bc633..e371efebb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -156,19 +156,26 @@ def build_extension(self, ext: CMakeExtension) -> None: str(cmake_bdist_dir / ext.name), str(Path(output_dir) / ext.name) ) -extra_requirements = [ - "fortran-language-server", - "PyYAML", - "psycopg2-binary==2.9.5", - "pandas", - "pymongo", - "rdflib", - "tripper", - "pint", -] - requirements = ["numpy"] +# Populate extra_requirements from requirements_*.txt +extra_requirements = {} +for name in "full", "dev", "doc": + with open(SOURCE_DIR / f"requirements_{name}.txt", "r") as f: + extra_requirements[name] = [ + line.strip() for line in f.readlines() if not line.startswith("#") + ] + +# Temporary workaround! +# Require pydantic <2. Needed before we have managed to install rust in +# the docker image for wheels +extras = extra_requirements["full"] +for i, pkg in enumerate(extras): + match = re.match("^(pydantic>.*<)", pkg) + if match: + extras[i] = f"{match.groups()[0]}2" + + version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", (SOURCE_DIR / "CMakeLists.txt").read_text(), @@ -204,10 +211,8 @@ def build_extension(self, ext: CMakeExtension) -> None: "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", ], - install_requires=requirements + extra_requirements, - # For now, the extra requirements are hard requirements. - # See issue #222: https://github.com/SINTEF/dlite/issues/222 - # extras_require={"all": extra_requirements}, + install_requires=requirements, + extras_require=extra_requirements, packages=["dlite"], scripts=[ str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"), diff --git a/requirements.txt b/requirements.txt index a0fa0e6d0..a5da0575c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1 @@ -fortran-language-server>=1.12.0,<1.13 -numpy>=1.20.3,<1.25.0 -PyYAML>=5.4.1,<7 -# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 -psycopg2-binary==2.9.5 -pandas>=1.2,<2.1 -rdflib>=4.2.1,<7 -pint>=0.15,<0.23 -openpyxl>=3.0.9,<3.2 -pymongo>=4.4.0,<5 -tripper>=0.2.5,<0.3 -pydantic>=1.10.0,<2 -typing_extensions>=4.1,<5 -requests>=2.10,<3 +numpy>=1.14.5,<1.25.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index b1560ce6b..772ed35a7 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,4 +3,4 @@ ipython>=7.34.0,<9 ipykernel>=6.0.1,<7 ipython_genutils~=0.2.0 mongomock>=4.1.2,<5 -pydantic>=1.9.0,<2 +openpyxl>=3.0.9,<3.2 diff --git a/requirements_full.txt b/requirements_full.txt new file mode 100644 index 000000000..6ddb93b7d --- /dev/null +++ b/requirements_full.txt @@ -0,0 +1,14 @@ +# Optional requirements - used by various plugins or additional features like mappings +fortran-language-server>=1.12.0,<1.13 +PyYAML>=5.4.1,<7 +# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 +psycopg2-binary==2.9.5 +pandas>=1.2,<2.1 +rdflib>=4.2.1,<7 +pint>=0.15,<1 +pymongo>=4.4.0,<5 +tripper>=0.2.5,<0.3 +requests>=2.10,<3 +jinja2>=3.0,<4 +pydantic>=1.10.0,<3 +typing_extensions>=4.1,<5