From 7aa38a10b07d713e88841238af6efdb507ca5dbe Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 14 Aug 2023 11:45:22 +0200 Subject: [PATCH 01/49] Initial cleanup of requirements --- .github/workflows/ci_tests.yml | 1 + python/setup.py | 16 +++++----------- requirements.txt | 12 ------------ 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 031fe6098..8dfe33758 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -46,6 +46,7 @@ jobs: - name: Install Python dependencies run: | pip install --upgrade pip -r requirements.txt + pip install -r requirements_full.txt pip install -r requirements_dev.txt pip install -r requirements_doc.txt - name: check numpy diff --git a/python/setup.py b/python/setup.py index 0d87bc633..67b3dfbee 100644 --- a/python/setup.py +++ b/python/setup.py @@ -156,19 +156,13 @@ def build_extension(self, ext: CMakeExtension) -> None: str(cmake_bdist_dir / ext.name), str(Path(output_dir) / ext.name) ) -extra_requirements = [ - "fortran-language-server", - "PyYAML", - "psycopg2-binary==2.9.5", - "pandas", - "pymongo", - "rdflib", - "tripper", - "pint", -] - requirements = ["numpy"] +with open(SOURCE_DIR / "requirements_full.txt", "r") as f: + extra_requirements = [ + line for line in f.readlines() if line not startswith("#") + ] + version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", (SOURCE_DIR / "CMakeLists.txt").read_text(), diff --git a/requirements.txt b/requirements.txt index 4c9bd4dec..f08732da3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1 @@ -fortran-language-server>=1.12.0,<1.13 numpy>=1.20.3,<1.25.0 -PyYAML>=5.4.1,<7 -# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 -psycopg2-binary==2.9.5 -pandas>=1.2,<2.1 -rdflib>=4.2.1,<7 -pint>=0.15,<0.23 -openpyxl>=3.0.9,<3.2 -pymongo>=4.4.0,<5 -tripper>=0.2.5,<0.3 -pydantic>=1.10.0,<2 - From 186819ea7912ebe38d68305d1a8574e9e1d32e2e Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 14 Aug 2023 11:55:50 +0200 Subject: [PATCH 02/49] Fixed typo --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 67b3dfbee..0229f297c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -160,7 +160,7 @@ def build_extension(self, ext: CMakeExtension) -> None: with open(SOURCE_DIR / "requirements_full.txt", "r") as f: extra_requirements = [ - line for line in f.readlines() if line not startswith("#") + line for line in f.readlines() if not line.startswith("#") ] version = re.search( From 20a9a8ff7f0d57ad8375dd69f356db59cc8facf2 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 14 Aug 2023 13:50:39 +0200 Subject: [PATCH 03/49] Entered extra_requirements explicit to setup.py for now... --- python/setup.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 0229f297c..cc18ef3be 100644 --- a/python/setup.py +++ b/python/setup.py @@ -158,10 +158,25 @@ def build_extension(self, ext: CMakeExtension) -> None: requirements = ["numpy"] -with open(SOURCE_DIR / "requirements_full.txt", "r") as f: - extra_requirements = [ - line for line in f.readlines() if not line.startswith("#") - ] +# Uncomment when requirements_full.txt have entered the container +#with open(SOURCE_DIR / "requirements_full.txt", "r") as f: +# extra_requirements = [ +# line.strip() for line in f.readlines() if not line.startswith("#") +# ] +extra_requirements = [ + "fortran-language-server>=1.12.0,<1.13", + "PyYAML>=5.4.1,<7", + "psycopg2-binary==2.9.5", + "pandas>=1.2,<2.1", + "rdflib>=4.2.1,<7", + "pint>=0.15,<1", + "openpyxl>=3.0.9,<3.2", + "pymongo>=4.4.0,<5", + "tripper>=0.2.5,<0.3", + "pydantic>=1.10.0,<3", + "requests>=2.10,<3", +] + version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", From 34c06ba6e3ca76765aa640448bc87071da32a9e3 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 14 Aug 2023 17:58:14 +0200 Subject: [PATCH 04/49] Updated README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8bbd3ef58..f4871cdac 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,11 @@ Installation DLite is available on PyPI and can be installed with pip ```shell -pip install dlite-python +pip install dlite-python[full] ``` +The bracket `[full]` is optional, but ensures that you install all optional +dependencies together with DLite. For alternative installation methods, see the [installation instructions]. From f8daee88c9b7eb773117396beafde99ff466916b Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 01:07:10 +0200 Subject: [PATCH 05/49] Added missing requirements_full.txt --- requirements_full.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 requirements_full.txt diff --git a/requirements_full.txt b/requirements_full.txt new file mode 100644 index 000000000..fe345bdd7 --- /dev/null +++ b/requirements_full.txt @@ -0,0 +1,12 @@ +fortran-language-server>=1.12.0,<1.13 +PyYAML>=5.4.1,<7 +# psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 +psycopg2-binary==2.9.5 +pandas>=1.2,<2.1 +rdflib>=4.2.1,<7 +pint>=0.15,<1 +openpyxl>=3.0.9,<3.2 +pymongo>=4.4.0,<5 +tripper>=0.2.5,<0.3 +pydantic>=1.10.0,<3 +requests>=2.10,<3 From bbce3292664a418e63d45a3d274887b4f8248e71 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 01:18:35 +0200 Subject: [PATCH 06/49] Require Pydantic v1 until PR #592 has been merged --- requirements_full.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_full.txt b/requirements_full.txt index fe345bdd7..51add208b 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -8,5 +8,5 @@ pint>=0.15,<1 openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 -pydantic>=1.10.0,<3 +pydantic>=1.10.0,<2 requests>=2.10,<3 From 850a2a246e0791a2c7fd3e1c29a6917b2376c564 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 12:04:23 +0200 Subject: [PATCH 07/49] Added typing_extensions to requirements --- requirements_full.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_full.txt b/requirements_full.txt index fe345bdd7..f9bb127da 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -9,4 +9,5 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 pydantic>=1.10.0,<3 +typing_extensions>=4.1,<5 requests>=2.10,<3 From 51e15c94ff3b73eb33f7bccbefb7e1f719f12f70 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 12:08:23 +0200 Subject: [PATCH 08/49] Added typing_extensions to extra_requirements in setup.py --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index cc18ef3be..28436325f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -174,6 +174,7 @@ def build_extension(self, ext: CMakeExtension) -> None: "pymongo>=4.4.0,<5", "tripper>=0.2.5,<0.3", "pydantic>=1.10.0,<3", + "typing_extensions>=4.1,<5", "requests>=2.10,<3", ] From 399cc22e36a72fa14f044888d31fd6105c2e2809 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 13:08:03 +0200 Subject: [PATCH 09/49] Added jsonschema<4.18 to avoid rust dependency. Lets see if it works... --- python/setup.py | 1 + requirements_full.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/python/setup.py b/python/setup.py index 28436325f..a73c391b5 100644 --- a/python/setup.py +++ b/python/setup.py @@ -176,6 +176,7 @@ def build_extension(self, ext: CMakeExtension) -> None: "pydantic>=1.10.0,<3", "typing_extensions>=4.1,<5", "requests>=2.10,<3", + "jsonschema>=4.0,<4.18", ] diff --git a/requirements_full.txt b/requirements_full.txt index f9bb127da..181bcb418 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -11,3 +11,4 @@ tripper>=0.2.5,<0.3 pydantic>=1.10.0,<3 typing_extensions>=4.1,<5 requests>=2.10,<3 +jsonschema>=4.0,<4.18 From 8e8cc18d4e63855d6224be61668f5f21d771d4b4 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 13:37:56 +0200 Subject: [PATCH 10/49] Install rust, cargo and poetry since Pydantic v2 seems to require them when building wheels on some platforms... --- .github/workflows/ci_build_wheels.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index bf75839f8..e8d15a4fe 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -83,11 +83,16 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Install rust and cargo (needed by Pydantic v2) + run: | + apt install rust cargo + - name: Install cibuildwheel run: | python -m pip install -U pip python -m pip install -U setuptools wheel python -m pip install -U cibuildwheel + python -m pip install -U poetry # See pyproject.toml (under python/) for cibuildwheel configuration - name: Build wheels From 466741bc2e3eeb5a45ea366b20f34a758821bfe6 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 13:43:11 +0200 Subject: [PATCH 11/49] Cleanded up ci_build_wheel --- .github/workflows/ci_build_wheels.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index e8d15a4fe..461ed932d 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -83,16 +83,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Install rust and cargo (needed by Pydantic v2) + - name: Update pip + run: | + python -m pip install -U pip + + - name: Install rust, cargo and poetry (needed by Pydantic v2) run: | apt install rust cargo + python -m pip install -U poetry - name: Install cibuildwheel run: | - python -m pip install -U pip python -m pip install -U setuptools wheel python -m pip install -U cibuildwheel - python -m pip install -U poetry # See pyproject.toml (under python/) for cibuildwheel configuration - name: Build wheels From af332b48df30c19e5820e47bb9f697da2b1b2f53 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 13:46:54 +0200 Subject: [PATCH 12/49] Install with apt-get --- .github/workflows/ci_build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 461ed932d..5e6881d16 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -89,7 +89,7 @@ jobs: - name: Install rust, cargo and poetry (needed by Pydantic v2) run: | - apt install rust cargo + sudo apt-get install rust cargo python -m pip install -U poetry - name: Install cibuildwheel From 67dacbf5d3e9ced76501811496ec1af6d94a24b0 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 13:56:18 +0200 Subject: [PATCH 13/49] Only run apt-get if the command is available --- .github/workflows/ci_build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 5e6881d16..6129f902a 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -89,7 +89,7 @@ jobs: - name: Install rust, cargo and poetry (needed by Pydantic v2) run: | - sudo apt-get install rust cargo + python -c 'import subprocess; subprocess.check_call(["apt-get", "--help"])' && sudo apt-get install rust cargo python -m pip install -U poetry - name: Install cibuildwheel From 1159649c5eef81e9237515a7f5ac369d8fbd4b39 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 14:03:36 +0200 Subject: [PATCH 14/49] Install rust the recommended way --- .github/workflows/ci_build_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 6129f902a..6433ff162 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -89,8 +89,7 @@ jobs: - name: Install rust, cargo and poetry (needed by Pydantic v2) run: | - python -c 'import subprocess; subprocess.check_call(["apt-get", "--help"])' && sudo apt-get install rust cargo - python -m pip install -U poetry + python -c 'import sys; assert sys.platform.startswith("linux")' && sudo apt-get install curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh && source $HOME/.cargo/env && python -m pip install -U poetry - name: Install cibuildwheel run: | From 9d32e07d76e048176cb7582cc62f1f6bc9bb3867 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 14:11:17 +0200 Subject: [PATCH 15/49] Install rust non-interactively --- .github/workflows/ci_build_wheels.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 6433ff162..a35149aa8 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -88,8 +88,12 @@ jobs: python -m pip install -U pip - name: Install rust, cargo and poetry (needed by Pydantic v2) - run: | - python -c 'import sys; assert sys.platform.startswith("linux")' && sudo apt-get install curl && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh && source $HOME/.cargo/env && python -m pip install -U poetry + run: > + python -c 'import sys; assert sys.platform.startswith("linux")' && + sudo apt-get install curl && + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && + source $HOME/.cargo/env && + python -m pip install -U poetry - name: Install cibuildwheel run: | From 95801ed7b4b9cb91f65bd831aba0cac5f9b8fbe3 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 14:29:52 +0200 Subject: [PATCH 16/49] Try to install rust via apt --- .github/workflows/ci_build_wheels.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index a35149aa8..7fd737335 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -88,13 +88,22 @@ jobs: python -m pip install -U pip - name: Install rust, cargo and poetry (needed by Pydantic v2) + #run: > + # python -c 'import sys, subprocess; if sys.platform.startswith("linux"): + # subprocess.call(" + # sudo apt-get install curl && + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && + # source $HOME/.cargo/env && + # python -m pip install -U poetry + # ", check=True, shell=True)' run: > - python -c 'import sys; assert sys.platform.startswith("linux")' && - sudo apt-get install curl && - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && - source $HOME/.cargo/env && - python -m pip install -U poetry - + python -c ' + import sys, subprocess; + if sys.platform.startswith("linux"): + subprocess.call(" + sudo apt install rustc && + python -m pip install -U poetry + ", check=True, shell=True)' - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From 3ff5739f1f09a8427910728695a42e2cad68585e Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 14:31:47 +0200 Subject: [PATCH 17/49] Fixed Python indentations --- .github/workflows/ci_build_wheels.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 7fd737335..82d9acd58 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -97,13 +97,11 @@ jobs: # python -m pip install -U poetry # ", check=True, shell=True)' run: > - python -c ' - import sys, subprocess; - if sys.platform.startswith("linux"): - subprocess.call(" - sudo apt install rustc && - python -m pip install -U poetry - ", check=True, shell=True)' + python -c 'import sys, subprocess; if sys.platform.startswith("linux"): + subprocess.call(" + sudo apt install rustc && + python -m pip install -U poetry + ", check=True, shell=True)' - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From 8f502d58559410c30172c1bb5c04cc3ebe7c94a1 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 15:01:30 +0200 Subject: [PATCH 18/49] Formatting --- .github/workflows/ci_build_wheels.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 82d9acd58..f271ad477 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -98,10 +98,11 @@ jobs: # ", check=True, shell=True)' run: > python -c 'import sys, subprocess; if sys.platform.startswith("linux"): - subprocess.call(" - sudo apt install rustc && - python -m pip install -U poetry - ", check=True, shell=True)' + subprocess.call( + "sudo apt install rustc && python -m pip install -U poetry", + check=True, + shell=True, + )' - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From c965bb3566420abb4946d70c3056c4243a215831 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 15:34:39 +0200 Subject: [PATCH 19/49] Replace if with and in inline python expression --- .github/workflows/ci_build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index f271ad477..153d0685a 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -97,7 +97,7 @@ jobs: # python -m pip install -U poetry # ", check=True, shell=True)' run: > - python -c 'import sys, subprocess; if sys.platform.startswith("linux"): + python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.call( "sudo apt install rustc && python -m pip install -U poetry", check=True, From 2b4b97f0b922b8b6d4a010fd8e327a47b84cd356 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 15:37:43 +0200 Subject: [PATCH 20/49] more formatting --- .github/workflows/ci_build_wheels.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 153d0685a..ee9d212a5 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -97,12 +97,9 @@ jobs: # python -m pip install -U poetry # ", check=True, shell=True)' run: > - python -c 'import sys, subprocess; sys.platform.startswith("linux") and - subprocess.call( - "sudo apt install rustc && python -m pip install -U poetry", - check=True, - shell=True, - )' + python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.call( + "sudo apt install rustc && python -m pip install -U poetry", + check=True, shell=True)' - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From f221c22577b750613b4a40ed59d6c0a5d03c5a5e Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 15:39:38 +0200 Subject: [PATCH 21/49] Use subprocess.run() --- .github/workflows/ci_build_wheels.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index ee9d212a5..9df2dd92b 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -89,15 +89,14 @@ jobs: - name: Install rust, cargo and poetry (needed by Pydantic v2) #run: > - # python -c 'import sys, subprocess; if sys.platform.startswith("linux"): - # subprocess.call(" + # python -c 'import sys, subprocess; if sys.platform.startswith("linux"): subprocess.run(" # sudo apt-get install curl && # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && # source $HOME/.cargo/env && # python -m pip install -U poetry # ", check=True, shell=True)' run: > - python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.call( + python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.run( "sudo apt install rustc && python -m pip install -U poetry", check=True, shell=True)' - name: Install cibuildwheel From 67ed009d75ee360c3565081929fc8795451f84d6 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 16:18:55 +0200 Subject: [PATCH 22/49] Remove syntax error --- .github/workflows/ci_build_wheels.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 9df2dd92b..b00c0f27a 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -95,10 +95,8 @@ jobs: # source $HOME/.cargo/env && # python -m pip install -U poetry # ", check=True, shell=True)' - run: > - python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.run( - "sudo apt install rustc && python -m pip install -U poetry", - check=True, shell=True)' + run: | + python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.run("sudo apt install -y rustc && python -m pip install -U poetry", check=True, shell=True)' - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From 6437a6348a563580ec901f4ecafae8cafac882ca Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 17:47:37 +0200 Subject: [PATCH 23/49] Also install cargo and change ' to " to also support windows --- .github/workflows/ci_build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index b00c0f27a..1214d1f3e 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -96,7 +96,7 @@ jobs: # python -m pip install -U poetry # ", check=True, shell=True)' run: | - python -c 'import sys, subprocess; sys.platform.startswith("linux") and subprocess.run("sudo apt install -y rustc && python -m pip install -U poetry", check=True, shell=True)' + python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo llvm-15 lld-15 clang-15 && python -m pip install -U poetry\", check=True, shell=True)" - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From 33d31b406c2d06e1682ee366f30f5932a152c15d Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 17:52:41 +0200 Subject: [PATCH 24/49] Remove recommended but unrecognised ubuntu packages --- .github/workflows/ci_build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 1214d1f3e..7adf64cfa 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -96,7 +96,7 @@ jobs: # python -m pip install -U poetry # ", check=True, shell=True)' run: | - python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo llvm-15 lld-15 clang-15 && python -m pip install -U poetry\", check=True, shell=True)" + python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo && python -m pip install -U poetry\", check=True, shell=True)" - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From a7a50fc833ed997afb44f1c9cfb50abc05b8f51f Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 18:07:56 +0200 Subject: [PATCH 25/49] Moved installing of rust to a separte python script --- .github/install_rust.py | 10 ++++++++++ .github/workflows/ci_build_wheels.yml | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .github/install_rust.py diff --git a/.github/install_rust.py b/.github/install_rust.py new file mode 100644 index 000000000..4242ee049 --- /dev/null +++ b/.github/install_rust.py @@ -0,0 +1,10 @@ +# A python script for installing rust in a Linux environment +import subprocess +import sys + + +if sys.platform.startswith("linux"): + subprocess.run( + "sudo apt install -y rustc cargo && python -m pip install -U poetry", + check=True, shell=True, + ) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 7adf64cfa..3d2c7bc4a 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -95,8 +95,12 @@ jobs: # source $HOME/.cargo/env && # python -m pip install -U poetry # ", check=True, shell=True)' + #run: | + # python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo && python -m pip install -U poetry\", check=True, shell=True)" run: | - python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo && python -m pip install -U poetry\", check=True, shell=True)" + pwd + ls + python .github/install_rust.py - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From b5fa54405a28c29af91621dbeb2dae3dea8c0789 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 18:10:54 +0200 Subject: [PATCH 26/49] removed commented out code --- .github/workflows/ci_build_wheels.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 3d2c7bc4a..82e365844 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -88,15 +88,6 @@ jobs: python -m pip install -U pip - name: Install rust, cargo and poetry (needed by Pydantic v2) - #run: > - # python -c 'import sys, subprocess; if sys.platform.startswith("linux"): subprocess.run(" - # sudo apt-get install curl && - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && - # source $HOME/.cargo/env && - # python -m pip install -U poetry - # ", check=True, shell=True)' - #run: | - # python -c "import sys, subprocess; sys.platform.startswith(\"linux\") and subprocess.run(\"sudo apt install -y rustc cargo && python -m pip install -U poetry\", check=True, shell=True)" run: | pwd ls From e0a45449981145ea8022f7d69678d5fc5a37701d Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 19:32:45 +0200 Subject: [PATCH 27/49] Try with pydantic v1 --- .github/workflows/ci_build_wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 82e365844..7b74c2c04 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -89,8 +89,6 @@ jobs: - name: Install rust, cargo and poetry (needed by Pydantic v2) run: | - pwd - ls python .github/install_rust.py - name: Install cibuildwheel run: | From 969f101390b99624b3c64f00a29cdb69ee96f20f Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 19:57:13 +0200 Subject: [PATCH 28/49] Renamed github workflows --- .github/workflows/cd_docs.yml | 2 +- .github/workflows/ci_automerge_dependabot.yml | 2 +- .github/workflows/{ci_tests.yml => ci_build_and_test.yml} | 4 ++-- .github/workflows/ci_build_wheels.yml | 2 +- .../workflows/{ci_dependabot.yml => ci_dependabot_pr.yml} | 2 +- .../{static_analysis.yml => ci_static_analysis.yml} | 4 ++-- .github/workflows/container_builds_weekly.yml | 2 +- requirements_full.txt | 7 ++++--- 8 files changed, 13 insertions(+), 12 deletions(-) rename .github/workflows/{ci_tests.yml => ci_build_and_test.yml} (98%) rename .github/workflows/{ci_dependabot.yml => ci_dependabot_pr.yml} (97%) rename .github/workflows/{static_analysis.yml => ci_static_analysis.yml} (92%) diff --git a/.github/workflows/cd_docs.yml b/.github/workflows/cd_docs.yml index 3a955e91e..7c0eb05eb 100644 --- a/.github/workflows/cd_docs.yml +++ b/.github/workflows/cd_docs.yml @@ -25,7 +25,7 @@ env: PUBLISH_DIR: pages jobs: - build: + cd_docs: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/ci_automerge_dependabot.yml b/.github/workflows/ci_automerge_dependabot.yml index 914a94e30..28d5c2799 100644 --- a/.github/workflows/ci_automerge_dependabot.yml +++ b/.github/workflows/ci_automerge_dependabot.yml @@ -5,7 +5,7 @@ on: branches: [ci/dependabot-updates] jobs: - update-dependabot-branch: + ci_automerge_dependabot: name: External uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v2.4.0 if: github.repository_owner == 'sintef' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_build_and_test.yml similarity index 98% rename from .github/workflows/ci_tests.yml rename to .github/workflows/ci_build_and_test.yml index 8dfe33758..18848a423 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_build_and_test.yml @@ -1,9 +1,9 @@ -name: CI tests +name: CI build and test on: [push] jobs: - build: + ci_build_and_test: runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 3d2c7bc4a..c5666b36a 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -10,7 +10,7 @@ env: container_registry: ghcr.io jobs: - build_wheels: + ci_build_wheels: name: Build wheels runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot_pr.yml similarity index 97% rename from .github/workflows/ci_dependabot.yml rename to .github/workflows/ci_dependabot_pr.yml index 47437becc..2fb36263d 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot_pr.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: - create-collected-pr: + ci_dependabot_pr: name: External uses: SINTEF/ci-cd/.github/workflows/ci_update_dependencies.yml@v2.4.0 if: github.repository_owner == 'sintef' diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/ci_static_analysis.yml similarity index 92% rename from .github/workflows/static_analysis.yml rename to .github/workflows/ci_static_analysis.yml index 4881bb837..b3ccd648e 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/ci_static_analysis.yml @@ -1,9 +1,9 @@ -name: Static code analysis +name: CI Static code analysis on: [push] jobs: - static-analysis: + ci_static_analysis: runs-on: ubuntu-latest diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml index 0174c8659..584860a1d 100644 --- a/.github/workflows/container_builds_weekly.yml +++ b/.github/workflows/container_builds_weekly.yml @@ -15,7 +15,7 @@ env: CONTAINER_REGISTRY: ghcr.io jobs: - build_wheel_linux_dockers: + container_builds: name: Build wheel containers runs-on: ubuntu-20.04 strategy: diff --git a/requirements_full.txt b/requirements_full.txt index db0994170..95b628e08 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -8,7 +8,8 @@ pint>=0.15,<1 openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 -pydantic>=1.10.0,<3 -typing_extensions>=4.1,<5 requests>=2.10,<3 -#jsonschema>=4.0,<4.18 +pydantic>=1.10.0,<2 +#pydantic>=1.10.0,<3 +#typing_extensions>=4.1,<5 +jsonschema>=4.0,<4.18 From f509b11e61dda5908e2b6742adf665689bc7674c Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 20:03:37 +0200 Subject: [PATCH 29/49] Added requirements_full to ci docker file --- .github/docker/Dockerfile-manylinux.template | 4 +++- .github/docker/Dockerfile-musllinux.template | 4 +++- requirements_full.txt | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 0e68ed675..128b45cb1 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -27,6 +27,7 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="7 8 9 10" COPY requirements.txt /tmp/requirements.txt +COPY requirements_full.txt /tmp/requirements_full.txt {{ EXTRA_PRE }} @@ -55,7 +56,8 @@ RUN yum update -y && \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done {{ EXTRA_POST }} diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index 22a3f26fb..fe2888a17 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -24,6 +24,7 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest COPY requirements.txt /tmp/requirements.txt +COPY requirements_full.txt /tmp/requirements_full.txt # Do not use distutils distributed with setuptools # This is due to base changes in the distutils API, removing msvccompiler, @@ -45,5 +46,6 @@ RUN apk add -u \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/requirements_full.txt b/requirements_full.txt index 95b628e08..6d46e7bcb 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -9,7 +9,7 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 -pydantic>=1.10.0,<2 +#pydantic>=1.10.0,<2 #pydantic>=1.10.0,<3 #typing_extensions>=4.1,<5 -jsonschema>=4.0,<4.18 +#jsonschema>=4.0,<4.18 From db423669407b7222859ec9a7d5eecba4458be447 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 20:05:46 +0200 Subject: [PATCH 30/49] Added Python 11 to manylinux docker template --- .github/docker/Dockerfile-manylinux.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 128b45cb1..2bf33b270 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -24,7 +24,7 @@ # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest -ARG PY_MINORS="7 8 9 10" +ARG PY_MINORS="7 8 9 10 11" COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt From 0fc2fe8ad8b49512499abe1fbe4c84b4c2577f94 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 20:21:50 +0200 Subject: [PATCH 31/49] Moved installation of rust cargo and poetry to docker templates --- .github/docker/Dockerfile-manylinux.template | 6 ++++-- .github/docker/Dockerfile-musllinux.template | 6 ++++-- .github/workflows/ci_build_wheels.yml | 7 ++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 2bf33b270..3a3270d3c 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -42,7 +42,9 @@ RUN yum update -y && \ swig \ libcurl-devel \ libxslt-devel \ - libxml2-devel && \ + libxml2-devel \ + rust \ + cargo && \ if [ "{{ TYPE }}{{ ARCH }}" != "2014i686" ]; then yum install -y hdf5-devel; fi && \ if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then yum install -y postgresql91-devel; fi && \ # Unpack static libraries @@ -55,7 +57,7 @@ RUN yum update -y && \ for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ + python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index fe2888a17..a8430de50 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -35,7 +35,9 @@ RUN apk add -u \ redland \ rasqal \ hdf5 \ - swig && \ + swig \ + rust\ + cargo && \ # Unpack static libraries # It's necessary to be in /opt/_internal because the internal libraries # exist here. @@ -45,7 +47,7 @@ RUN apk add -u \ for minor in 7 8 9 10; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ + python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 02fe9f0e2..928b35ed6 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -87,9 +87,10 @@ jobs: run: | python -m pip install -U pip - - name: Install rust, cargo and poetry (needed by Pydantic v2) - run: | - python .github/install_rust.py + #- name: Install rust, cargo and poetry (needed by Pydantic v2) + # run: | + # python .github/install_rust.py + - name: Install cibuildwheel run: | python -m pip install -U setuptools wheel From 469fa372d6a4731f027ec914a37a8751e2095178 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 20:54:54 +0200 Subject: [PATCH 32/49] Reverted back to old action names --- .github/workflows/ci_automerge_dependabot.yml | 2 +- .github/workflows/ci_build_wheels.yml | 9 +++++---- .../{ci_dependabot_pr.yml => ci_dependabot.yml} | 2 +- .../workflows/{ci_build_and_test.yml => ci_tests.yml} | 4 ++-- .github/workflows/container_builds_weekly.yml | 2 +- .../{ci_static_analysis.yml => static_analysis.yml} | 4 ++-- python/setup.py | 7 ++++--- 7 files changed, 16 insertions(+), 14 deletions(-) rename .github/workflows/{ci_dependabot_pr.yml => ci_dependabot.yml} (97%) rename .github/workflows/{ci_build_and_test.yml => ci_tests.yml} (98%) rename .github/workflows/{ci_static_analysis.yml => static_analysis.yml} (92%) diff --git a/.github/workflows/ci_automerge_dependabot.yml b/.github/workflows/ci_automerge_dependabot.yml index 28d5c2799..914a94e30 100644 --- a/.github/workflows/ci_automerge_dependabot.yml +++ b/.github/workflows/ci_automerge_dependabot.yml @@ -5,7 +5,7 @@ on: branches: [ci/dependabot-updates] jobs: - ci_automerge_dependabot: + update-dependabot-branch: name: External uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v2.4.0 if: github.repository_owner == 'sintef' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 928b35ed6..69adaf82f 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -10,7 +10,7 @@ env: container_registry: ghcr.io jobs: - ci_build_wheels: + build_wheels: name: Build wheels runs-on: ${{ matrix.os }} strategy: @@ -83,9 +83,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Update pip - run: | - python -m pip install -U pip + #- name: Update pip + # run: | + # python -m pip install -U pip #- name: Install rust, cargo and poetry (needed by Pydantic v2) # run: | @@ -93,6 +93,7 @@ jobs: - name: Install cibuildwheel run: | + python -m pip install -U pip python -m pip install -U setuptools wheel python -m pip install -U cibuildwheel diff --git a/.github/workflows/ci_dependabot_pr.yml b/.github/workflows/ci_dependabot.yml similarity index 97% rename from .github/workflows/ci_dependabot_pr.yml rename to .github/workflows/ci_dependabot.yml index 2fb36263d..47437becc 100644 --- a/.github/workflows/ci_dependabot_pr.yml +++ b/.github/workflows/ci_dependabot.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: - ci_dependabot_pr: + create-collected-pr: name: External uses: SINTEF/ci-cd/.github/workflows/ci_update_dependencies.yml@v2.4.0 if: github.repository_owner == 'sintef' diff --git a/.github/workflows/ci_build_and_test.yml b/.github/workflows/ci_tests.yml similarity index 98% rename from .github/workflows/ci_build_and_test.yml rename to .github/workflows/ci_tests.yml index 18848a423..8dfe33758 100644 --- a/.github/workflows/ci_build_and_test.yml +++ b/.github/workflows/ci_tests.yml @@ -1,9 +1,9 @@ -name: CI build and test +name: CI tests on: [push] jobs: - ci_build_and_test: + build: runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml index 584860a1d..0174c8659 100644 --- a/.github/workflows/container_builds_weekly.yml +++ b/.github/workflows/container_builds_weekly.yml @@ -15,7 +15,7 @@ env: CONTAINER_REGISTRY: ghcr.io jobs: - container_builds: + build_wheel_linux_dockers: name: Build wheel containers runs-on: ubuntu-20.04 strategy: diff --git a/.github/workflows/ci_static_analysis.yml b/.github/workflows/static_analysis.yml similarity index 92% rename from .github/workflows/ci_static_analysis.yml rename to .github/workflows/static_analysis.yml index b3ccd648e..4881bb837 100644 --- a/.github/workflows/ci_static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -1,9 +1,9 @@ -name: CI Static code analysis +name: Static code analysis on: [push] jobs: - ci_static_analysis: + static-analysis: runs-on: ubuntu-latest diff --git a/python/setup.py b/python/setup.py index a73c391b5..ebd64ea14 100644 --- a/python/setup.py +++ b/python/setup.py @@ -173,10 +173,11 @@ def build_extension(self, ext: CMakeExtension) -> None: "openpyxl>=3.0.9,<3.2", "pymongo>=4.4.0,<5", "tripper>=0.2.5,<0.3", - "pydantic>=1.10.0,<3", - "typing_extensions>=4.1,<5", "requests>=2.10,<3", - "jsonschema>=4.0,<4.18", + #"pydantic>=1.10.0,<2", + #"pydantic>=1.10.0,<3", + #"typing_extensions>=4.1,<5", + #"jsonschema>=4.0,<4.18", ] From 36d67087a8c37f6bcdeddca092357e2a94a338fd Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 21:42:02 +0200 Subject: [PATCH 33/49] Remove unused stuff --- .github/install_rust.py | 10 ---------- .github/workflows/ci_build_wheels.yml | 8 -------- 2 files changed, 18 deletions(-) delete mode 100644 .github/install_rust.py diff --git a/.github/install_rust.py b/.github/install_rust.py deleted file mode 100644 index 4242ee049..000000000 --- a/.github/install_rust.py +++ /dev/null @@ -1,10 +0,0 @@ -# A python script for installing rust in a Linux environment -import subprocess -import sys - - -if sys.platform.startswith("linux"): - subprocess.run( - "sudo apt install -y rustc cargo && python -m pip install -U poetry", - check=True, shell=True, - ) diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 69adaf82f..bf75839f8 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -83,14 +83,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - #- name: Update pip - # run: | - # python -m pip install -U pip - - #- name: Install rust, cargo and poetry (needed by Pydantic v2) - # run: | - # python .github/install_rust.py - - name: Install cibuildwheel run: | python -m pip install -U pip From 4bbb805ce780698ece498c8d695829f99a482334 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 22:17:23 +0200 Subject: [PATCH 34/49] Added "full" to extras --- .github/workflows/ci_dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index 47437becc..92fb37f1d 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -20,7 +20,7 @@ jobs: extra_to_dos: "- [ ] Make sure that the PR is **squash** merged, with a sensible commit message." update_pre-commit: true python_version: "3.8" - install_extras: "[dev,docs]" + install_extras: "[full,dev,docs]" skip_pre-commit_hooks: pylint secrets: PAT: ${{ secrets.RELEASE_PAT }} From c00714b1b833499a6e220f07f9c8eba494d16771 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 22:46:21 +0200 Subject: [PATCH 35/49] Do not install default version of numpy when building docker container --- .github/docker/Dockerfile-manylinux.template | 3 +-- .github/docker/Dockerfile-mingw | 2 ++ .github/docker/Dockerfile-musllinux.template | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 3a3270d3c..ef4093549 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -26,7 +26,7 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="7 8 9 10 11" -COPY requirements.txt /tmp/requirements.txt +#COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt {{ EXTRA_PRE }} @@ -58,7 +58,6 @@ RUN yum update -y && \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/.github/docker/Dockerfile-mingw b/.github/docker/Dockerfile-mingw index ac8f11337..c904325af 100644 --- a/.github/docker/Dockerfile-mingw +++ b/.github/docker/Dockerfile-mingw @@ -46,6 +46,8 @@ RUN dnf install -y \ cppcheck \ doxygen \ ccache \ + rust \ + cargo \ wine-core \ wine-filesystem \ mingw-filesystem-base \ diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index a8430de50..b6aa1524c 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -23,7 +23,7 @@ # FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest -COPY requirements.txt /tmp/requirements.txt +#COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt # Do not use distutils distributed with setuptools @@ -48,6 +48,5 @@ RUN apk add -u \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements.txt && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done From b600b8a3c2311a92358d8c025f38f8d1b7cd036a Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Wed, 16 Aug 2023 08:07:01 +0200 Subject: [PATCH 36/49] Reduced lowest version of numpy to oldest-required-numpy Added pydantic<2 to requirements --- python/setup.py | 40 ++++++++++++++++++++-------------------- requirements.txt | 2 +- requirements_full.txt | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/python/setup.py b/python/setup.py index ebd64ea14..a4270ae0e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -159,26 +159,26 @@ def build_extension(self, ext: CMakeExtension) -> None: requirements = ["numpy"] # Uncomment when requirements_full.txt have entered the container -#with open(SOURCE_DIR / "requirements_full.txt", "r") as f: -# extra_requirements = [ -# line.strip() for line in f.readlines() if not line.startswith("#") -# ] -extra_requirements = [ - "fortran-language-server>=1.12.0,<1.13", - "PyYAML>=5.4.1,<7", - "psycopg2-binary==2.9.5", - "pandas>=1.2,<2.1", - "rdflib>=4.2.1,<7", - "pint>=0.15,<1", - "openpyxl>=3.0.9,<3.2", - "pymongo>=4.4.0,<5", - "tripper>=0.2.5,<0.3", - "requests>=2.10,<3", - #"pydantic>=1.10.0,<2", - #"pydantic>=1.10.0,<3", - #"typing_extensions>=4.1,<5", - #"jsonschema>=4.0,<4.18", -] +with open(SOURCE_DIR / "requirements_full.txt", "r") as f: + extra_requirements = [ + line.strip() for line in f.readlines() if not line.startswith("#") + ] +#extra_requirements = [ +# "fortran-language-server>=1.12.0,<1.13", +# "PyYAML>=5.4.1,<7", +# "psycopg2-binary==2.9.5", +# "pandas>=1.2,<2.1", +# "rdflib>=4.2.1,<7", +# "pint>=0.15,<1", +# "openpyxl>=3.0.9,<3.2", +# "pymongo>=4.4.0,<5", +# "tripper>=0.2.5,<0.3", +# "requests>=2.10,<3", +# "pydantic>=1.10.0,<2", +# #"pydantic>=1.10.0,<3", +# #"typing_extensions>=4.1,<5", +# #"jsonschema>=4.0,<4.18", +#] version = re.search( diff --git a/requirements.txt b/requirements.txt index f08732da3..a5da0575c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -numpy>=1.20.3,<1.25.0 +numpy>=1.14.5,<1.25.0 diff --git a/requirements_full.txt b/requirements_full.txt index 6d46e7bcb..dea85c3e6 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -9,7 +9,7 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 -#pydantic>=1.10.0,<2 +pydantic>=1.10.0,<2 #pydantic>=1.10.0,<3 #typing_extensions>=4.1,<5 #jsonschema>=4.0,<4.18 From 9ae97587ae112bac68897646ed973ae83bd29ce8 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Wed, 16 Aug 2023 09:54:22 +0200 Subject: [PATCH 37/49] Try to allow pydantic v2 --- python/setup.py | 19 +------------------ requirements_full.txt | 8 ++++---- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/python/setup.py b/python/setup.py index a4270ae0e..d9a7346d3 100644 --- a/python/setup.py +++ b/python/setup.py @@ -158,28 +158,11 @@ def build_extension(self, ext: CMakeExtension) -> None: requirements = ["numpy"] -# Uncomment when requirements_full.txt have entered the container +# Read extra_requirements from requirements_full.txt with open(SOURCE_DIR / "requirements_full.txt", "r") as f: extra_requirements = [ line.strip() for line in f.readlines() if not line.startswith("#") ] -#extra_requirements = [ -# "fortran-language-server>=1.12.0,<1.13", -# "PyYAML>=5.4.1,<7", -# "psycopg2-binary==2.9.5", -# "pandas>=1.2,<2.1", -# "rdflib>=4.2.1,<7", -# "pint>=0.15,<1", -# "openpyxl>=3.0.9,<3.2", -# "pymongo>=4.4.0,<5", -# "tripper>=0.2.5,<0.3", -# "requests>=2.10,<3", -# "pydantic>=1.10.0,<2", -# #"pydantic>=1.10.0,<3", -# #"typing_extensions>=4.1,<5", -# #"jsonschema>=4.0,<4.18", -#] - version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", diff --git a/requirements_full.txt b/requirements_full.txt index dea85c3e6..98a5a9103 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -9,7 +9,7 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 -pydantic>=1.10.0,<2 -#pydantic>=1.10.0,<3 -#typing_extensions>=4.1,<5 -#jsonschema>=4.0,<4.18 +#pydantic>=1.10.0,<2 +pydantic>=1.10.0,<3 +typing_extensions>=4.1,<5 +jsonschema>=4.0,<4.18 From aa848a02dded227f87233aec3196f264ba3d2b4f Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Wed, 16 Aug 2023 12:38:25 +0200 Subject: [PATCH 38/49] Temporarily require Pydantic v1 in setup.py --- python/setup.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index d9a7346d3..514b65e73 100644 --- a/python/setup.py +++ b/python/setup.py @@ -159,10 +159,26 @@ def build_extension(self, ext: CMakeExtension) -> None: requirements = ["numpy"] # Read extra_requirements from requirements_full.txt -with open(SOURCE_DIR / "requirements_full.txt", "r") as f: - extra_requirements = [ - line.strip() for line in f.readlines() if not line.startswith("#") - ] +#with open(SOURCE_DIR / "requirements_full.txt", "r") as f: +# extra_requirements = [ +# line.strip() for line in f.readlines() if not line.startswith("#") +# ] +extra_requirements = [ # TO BE REPLACED WITH THE ABOVE + "fortran-language-server>=1.12.0,<1.13", + "PyYAML>=5.4.1,<7", + "psycopg2-binary==2.9.5", + "pandas>=1.2,<2.1", + "rdflib>=4.2.1,<7", + "pint>=0.15,<1", + "openpyxl>=3.0.9,<3.2", + "pymongo>=4.4.0,<5", + "tripper>=0.2.5,<0.3", + "requests>=2.10,<3", + "pydantic>=1.10.0,<2", + #"pydantic>=1.10.0,<3", + #"typing_extensions>=4.1,<5", + #"jsonschema>=4.0,<4.18", +] version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", From 7013153ea1259c6c11749ba8ac3edc6209b2a75f Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Wed, 16 Aug 2023 20:06:00 +0200 Subject: [PATCH 39/49] Removed attempted installation of rust in manylinux and musllinux. Cleanup. --- .github/docker/Dockerfile-manylinux.template | 6 ++---- .github/docker/Dockerfile-musllinux.template | 6 ++---- python/setup.py | 2 +- requirements_full.txt | 1 - 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index ef4093549..bce0b8786 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -42,9 +42,7 @@ RUN yum update -y && \ swig \ libcurl-devel \ libxslt-devel \ - libxml2-devel \ - rust \ - cargo && \ + libxml2-devel && \ if [ "{{ TYPE }}{{ ARCH }}" != "2014i686" ]; then yum install -y hdf5-devel; fi && \ if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then yum install -y postgresql91-devel; fi && \ # Unpack static libraries @@ -57,7 +55,7 @@ RUN yum update -y && \ for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ + python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index b6aa1524c..fe392b84c 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -35,9 +35,7 @@ RUN apk add -u \ redland \ rasqal \ hdf5 \ - swig \ - rust\ - cargo && \ + swig && # Unpack static libraries # It's necessary to be in /opt/_internal because the internal libraries # exist here. @@ -47,6 +45,6 @@ RUN apk add -u \ for minor in 7 8 9 10; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy poetry && \ + python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ done diff --git a/python/setup.py b/python/setup.py index 514b65e73..e125d1c9e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -163,7 +163,7 @@ def build_extension(self, ext: CMakeExtension) -> None: # extra_requirements = [ # line.strip() for line in f.readlines() if not line.startswith("#") # ] -extra_requirements = [ # TO BE REPLACED WITH THE ABOVE +extra_requirements = [ # TO BE REPLACED WITH THE ABOVE, SEE ISSUE #601 "fortran-language-server>=1.12.0,<1.13", "PyYAML>=5.4.1,<7", "psycopg2-binary==2.9.5", diff --git a/requirements_full.txt b/requirements_full.txt index 98a5a9103..2e869a101 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -12,4 +12,3 @@ requests>=2.10,<3 #pydantic>=1.10.0,<2 pydantic>=1.10.0,<3 typing_extensions>=4.1,<5 -jsonschema>=4.0,<4.18 From 1db91490fc6f432f609fe33ee64413e98d15d639 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Thu, 17 Aug 2023 22:54:49 +0200 Subject: [PATCH 40/49] Installed extras --- .github/workflows/cd_docs.yml | 4 ++-- python/setup.py | 43 +++++++++++++++-------------------- requirements_full.txt | 8 ++++--- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cd_docs.yml b/.github/workflows/cd_docs.yml index 7c0eb05eb..e1207fe8d 100644 --- a/.github/workflows/cd_docs.yml +++ b/.github/workflows/cd_docs.yml @@ -25,7 +25,7 @@ env: PUBLISH_DIR: pages jobs: - cd_docs: + compile: runs-on: ubuntu-latest steps: - name: Checkout @@ -87,7 +87,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest - needs: build + needs: compile steps: - name: Deploy to GitHub Pages diff --git a/python/setup.py b/python/setup.py index a4270ae0e..e371efebb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -158,27 +158,22 @@ def build_extension(self, ext: CMakeExtension) -> None: requirements = ["numpy"] -# Uncomment when requirements_full.txt have entered the container -with open(SOURCE_DIR / "requirements_full.txt", "r") as f: - extra_requirements = [ - line.strip() for line in f.readlines() if not line.startswith("#") - ] -#extra_requirements = [ -# "fortran-language-server>=1.12.0,<1.13", -# "PyYAML>=5.4.1,<7", -# "psycopg2-binary==2.9.5", -# "pandas>=1.2,<2.1", -# "rdflib>=4.2.1,<7", -# "pint>=0.15,<1", -# "openpyxl>=3.0.9,<3.2", -# "pymongo>=4.4.0,<5", -# "tripper>=0.2.5,<0.3", -# "requests>=2.10,<3", -# "pydantic>=1.10.0,<2", -# #"pydantic>=1.10.0,<3", -# #"typing_extensions>=4.1,<5", -# #"jsonschema>=4.0,<4.18", -#] +# Populate extra_requirements from requirements_*.txt +extra_requirements = {} +for name in "full", "dev", "doc": + with open(SOURCE_DIR / f"requirements_{name}.txt", "r") as f: + extra_requirements[name] = [ + line.strip() for line in f.readlines() if not line.startswith("#") + ] + +# Temporary workaround! +# Require pydantic <2. Needed before we have managed to install rust in +# the docker image for wheels +extras = extra_requirements["full"] +for i, pkg in enumerate(extras): + match = re.match("^(pydantic>.*<)", pkg) + if match: + extras[i] = f"{match.groups()[0]}2" version = re.search( @@ -216,10 +211,8 @@ def build_extension(self, ext: CMakeExtension) -> None: "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", ], - install_requires=requirements + extra_requirements, - # For now, the extra requirements are hard requirements. - # See issue #222: https://github.com/SINTEF/dlite/issues/222 - # extras_require={"all": extra_requirements}, + install_requires=requirements, + extras_require=extra_requirements, packages=["dlite"], scripts=[ str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"), diff --git a/requirements_full.txt b/requirements_full.txt index dea85c3e6..e518959c5 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -1,3 +1,4 @@ +# Optional requirements - used by various plugins, features or tests fortran-language-server>=1.12.0,<1.13 PyYAML>=5.4.1,<7 # psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 @@ -9,7 +10,8 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 -pydantic>=1.10.0,<2 -#pydantic>=1.10.0,<3 -#typing_extensions>=4.1,<5 +jinja2>=3.0,<4 +#pydantic>=1.10.0,<2 +pydantic>=1.10.0,<3 +typing_extensions>=4.1,<5 #jsonschema>=4.0,<4.18 From 4c015ed6317640abda58968a0b2e6ccd66d23827 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Fri, 18 Aug 2023 06:53:28 +0200 Subject: [PATCH 41/49] Always require pydantic<2 when building manylinux and musllinux wheels Cleaned up requirements Improved documentation of requirements --- .github/docker/Dockerfile-manylinux.template | 5 +++ .github/docker/Dockerfile-musllinux.template | 5 +++ .../build/runtime_dependencies.md | 33 +++++++++++++++---- doc/getting_started/installation.md | 19 +++++++++-- requirements_dev.txt | 2 +- requirements_full.txt | 1 - 6 files changed, 54 insertions(+), 11 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index bce0b8786..f2c25c283 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -28,6 +28,7 @@ ARG PY_MINORS="7 8 9 10 11" #COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt +COPY requirements_full.txt /tmp/requirements_dev.txt {{ EXTRA_PRE }} @@ -50,6 +51,9 @@ RUN yum update -y && \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ +# Change required version of pydantic to be <2 + sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \ +# Install required Python packages mkdir -p /ci/pip_cache && \ if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \ for minor in ${PY_MINORS}; do \ @@ -57,6 +61,7 @@ RUN yum update -y && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \ done {{ EXTRA_POST }} diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index fe392b84c..c74da0c78 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -25,6 +25,7 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest #COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt +COPY requirements_full.txt /tmp/requirements_dev.txt # Do not use distutils distributed with setuptools # This is due to base changes in the distutils API, removing msvccompiler, @@ -41,10 +42,14 @@ RUN apk add -u \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ +# Change required version of pydantic to be <2 + sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \ +# Install required Python packages mkdir -p /ci/pip_cache && \ for minor in 7 8 9 10; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \ done diff --git a/doc/getting_started/build/runtime_dependencies.md b/doc/getting_started/build/runtime_dependencies.md index a11464e97..c681e6ac4 100644 --- a/doc/getting_started/build/runtime_dependencies.md +++ b/doc/getting_started/build/runtime_dependencies.md @@ -8,24 +8,45 @@ On Ubuntu, they can be installed with sudo apt install python3 gfortran librdf libhdf5 When DLite is compiled with Python bindings, additional runtime features may be enabled by installing one of more of the following optional Python packages -- [tripper], optional, (used for property mappings) -- [PyYAML], optional (used for generic YAML storage plugin) -- [psycopg2], optional (used for generic PostgreSQL storage plugin) +- [tripper], used for property mappings +- [pint], used for units conversion in property mappings +- [pydantic], used for testing support for pydantic models +- [typing_extensions], needed by pydantic v2 +- [rdflib], used by `rdf` storage plugin +- [PyYAML], used by `yaml` storage plugin +- [psycopg2], used by `postgresql` storage plugin Note that in some cases a GSSAPI error is raised when using psycopg2 by pip installing psycopg2-binary. This is solved by installing from source as described in their documentation. -- [pandas], optional (used for csv storage plugin) -- [pymongo], optional, (used for mongodb storage plugin) -- [mongomock], optional, used for testing mongodb storage plugin. +- [pandas], used by the `csv` storage plugin +- [requests], used by `http` storage plugin +- [jinja2], used by `template` storage plugin +- [pymongo], used by the `mongodb` storage plugin These optional dependencies can be installed with + pip install -r requirements_full.txt + +Separate requirements can also be installed for development + pip install -r requirements_dev.txt +or for building the documentation + + pip install -r requirements_doc.txt + + [tripper]: https://pypi.org/project/tripper/ +[pint]: https://pint.readthedocs.io/en/stable/ +[pydantic]: https://docs.pydantic.dev/ +[typing_extensions]: https://github.com/python/typing_extensions +[rdflib]: https://rdflib.readthedocs.io/ [PyYAML]: https://pypi.org/project/PyYAML/ [psycopg2]: https://pypi.org/project/psycopg2/ [pandas]: https://pandas.pydata.org/ [pymongo]: https://github.com/mongodb/mongo-python-driver [mongomock]: https://github.com/mongomock/mongomock +[openpyxl]: https://openpyxl.readthedocs.io/ +[requests]: https://requests.readthedocs.io/ +[jinja2]: https://jinja.palletsprojects.com/ diff --git a/doc/getting_started/installation.md b/doc/getting_started/installation.md index a64c8ee62..9d098d506 100644 --- a/doc/getting_started/installation.md +++ b/doc/getting_started/installation.md @@ -10,21 +10,34 @@ Installing with pip If you are using Python, the easiest way to install DLite is with pip: ```shell -pip install DLite-Python +pip install DLite-Python[full] ``` -This will give you DLite together with all optional Python dependencies (see [runtime dependencies]). +The bracket `[full]` is optional, but ensures that you install DLite +together with all optional dependencies needed for additional features +and storage plugins. (see also [runtime dependencies]). Development installation ------------------------ If you to contribute or develop DLite, you should [build from source]. +Python dependencies for development can be installed with + +```shell +pip install DLite-Python[dev] +``` + +Install additional Python packages for building documentation with + +```shell +pip install DLite-Python[doc] +``` [Python]: https://www.python.org/ [Fortran]: https://en.wikipedia.org/wiki/Fortran [HDF5]: https://support.hdfgroup.org/HDF5/ [librdf]: https://librdf.org/ -[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime-dependencies.html +[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime_dependencies.html [build from source]: https://sintef.github.io/dlite/getting_started/build/build.html diff --git a/requirements_dev.txt b/requirements_dev.txt index b1560ce6b..772ed35a7 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,4 +3,4 @@ ipython>=7.34.0,<9 ipykernel>=6.0.1,<7 ipython_genutils~=0.2.0 mongomock>=4.1.2,<5 -pydantic>=1.9.0,<2 +openpyxl>=3.0.9,<3.2 diff --git a/requirements_full.txt b/requirements_full.txt index e518959c5..0370e214d 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -6,7 +6,6 @@ psycopg2-binary==2.9.5 pandas>=1.2,<2.1 rdflib>=4.2.1,<7 pint>=0.15,<1 -openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 From 1be867a4eb98547ab7fc517f3fc74ebdc5c14564 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Fri, 18 Aug 2023 07:28:45 +0200 Subject: [PATCH 42/49] Fixed typo --- .github/docker/Dockerfile-musllinux.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index c74da0c78..cb7e35419 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -36,7 +36,7 @@ RUN apk add -u \ redland \ rasqal \ hdf5 \ - swig && + swig && \ # Unpack static libraries # It's necessary to be in /opt/_internal because the internal libraries # exist here. From e52b1fdab870a1305fda44728ce99694a0e57635 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Fri, 18 Aug 2023 10:40:16 +0200 Subject: [PATCH 43/49] Fixed link in tips-and-tricks --- doc/contributors_guide/tips_and_tricks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/contributors_guide/tips_and_tricks.md b/doc/contributors_guide/tips_and_tricks.md index 8e1bcb1ff..ef4af3429 100644 --- a/doc/contributors_guide/tips_and_tricks.md +++ b/doc/contributors_guide/tips_and_tricks.md @@ -9,7 +9,7 @@ See the [Build and install to a virtual Python environment] in the installation Debugging tests failing inside docker on GitHub ----------------------------------------------- 1. Set up a local virtual environment with the same version of Python as in the failing test. - See [Build and install to a virtual Python environment]. + See [Build against Python environment]. 2. Enter your virtual environment with dlite and install cibuildwheel. workon dlite @@ -89,4 +89,4 @@ Debugging tests failing inside docker on GitHub [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/ -[Build and install to a virtual Python environment]: https://sintef.github.io/dlite/getting_started/installation.html#build-and-install-to-a-virtual-python-environment +[Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment From 0e011660cc874b21e58d76772c14df044a4e8b1c Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Fri, 18 Aug 2023 11:49:11 +0200 Subject: [PATCH 44/49] Skip yaml-dependent tests if yaml isn't installed --- bindings/python/tests/test_storage.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bindings/python/tests/test_storage.py b/bindings/python/tests/test_storage.py index 4b7c14212..43f972d4c 100755 --- a/bindings/python/tests/test_storage.py +++ b/bindings/python/tests/test_storage.py @@ -10,6 +10,12 @@ except ModuleNotFoundError: HAVE_PYTEST = False +try: + import yaml + HAVE_YAML = True +except ModuleNotFoundError: + HAVE_YAML = False + thisdir = os.path.abspath(os.path.dirname(__file__)) @@ -45,11 +51,7 @@ # Test yaml -try: - import yaml -except ImportError: - pass -else: +if HAVE_YAML: print('--- testing yaml') inst.save('yaml://inst.yaml?mode=w') del inst @@ -104,7 +106,8 @@ # Tests for issue #587 -bytearr = inst.to_bytes("yaml") +if HAVE_YAML: + bytearr = inst.to_bytes("yaml") #print(bytes(bytearr).decode()) if HAVE_PYTEST: with pytest.raises(dlite.DLiteError): From 6827e7f29c314dbff4fb76f8c3269b1580ae286f Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 21 Aug 2023 16:27:07 +0200 Subject: [PATCH 45/49] Added changes from code review --- .github/docker/Dockerfile-manylinux.template | 3 +-- .github/docker/Dockerfile-musllinux.template | 7 ++++--- README.md | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index f2c25c283..adab41cd1 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -60,8 +60,7 @@ RUN yum update -y && \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \ done {{ EXTRA_POST }} diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index cb7e35419..f6dc71d3a 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -23,6 +23,8 @@ # FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest +ARG PY_MINORS="7 8 9 10 11" + #COPY requirements.txt /tmp/requirements.txt COPY requirements_full.txt /tmp/requirements_full.txt COPY requirements_full.txt /tmp/requirements_dev.txt @@ -46,10 +48,9 @@ RUN apk add -u \ sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \ # Install required Python packages mkdir -p /ci/pip_cache && \ - for minor in 7 8 9 10; do \ + for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install -U pip && \ python3.${minor} -m pip install -U setuptools wheel && \ python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \ - python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \ + python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \ done diff --git a/README.md b/README.md index f4871cdac..e5287d066 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ pip install dlite-python[full] The bracket `[full]` is optional, but ensures that you install all optional dependencies together with DLite. +Without `[full]` you get a minimal DLite installation that only depends on +NumPy. For alternative installation methods, see the [installation instructions]. From a6c1b914ce9e3dfb9116a63f52a34b3cef87e610 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 22 Aug 2023 12:48:01 +0200 Subject: [PATCH 46/49] Updates from review --- README.md | 2 ++ requirements_full.txt | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 906a8c0fb..64693110e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ The bracket `[full]` is optional, but ensures that you install all optional dependencies together with DLite. Without `[full]` you get a minimal DLite installation that only depends on NumPy. +This would disable most storage plugins, except for the built-in +"json", "bson" and "rdf" (when compiled against Redland librdf). For alternative installation methods, see the [installation instructions]. diff --git a/requirements_full.txt b/requirements_full.txt index 0370e214d..6ddb93b7d 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -1,4 +1,4 @@ -# Optional requirements - used by various plugins, features or tests +# Optional requirements - used by various plugins or additional features like mappings fortran-language-server>=1.12.0,<1.13 PyYAML>=5.4.1,<7 # psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 @@ -10,7 +10,5 @@ pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 requests>=2.10,<3 jinja2>=3.0,<4 -#pydantic>=1.10.0,<2 pydantic>=1.10.0,<3 typing_extensions>=4.1,<5 -#jsonschema>=4.0,<4.18 From e119583ba23c807bf364d84cfce5db65e7bc95f6 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 22 Aug 2023 12:53:39 +0200 Subject: [PATCH 47/49] Combined commands in docker templates --- .github/docker/Dockerfile-manylinux.template | 3 +-- .github/docker/Dockerfile-musllinux.template | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index adab41cd1..a36856481 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -27,8 +27,7 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="7 8 9 10 11" #COPY requirements.txt /tmp/requirements.txt -COPY requirements_full.txt /tmp/requirements_full.txt -COPY requirements_full.txt /tmp/requirements_dev.txt +COPY requirements_full.txt requirements_dev.txt /tmp/ {{ EXTRA_PRE }} diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index f6dc71d3a..002919dcf 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -26,8 +26,7 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="7 8 9 10 11" #COPY requirements.txt /tmp/requirements.txt -COPY requirements_full.txt /tmp/requirements_full.txt -COPY requirements_full.txt /tmp/requirements_dev.txt +COPY requirements_full.txt requirements_dev.txt /tmp/ # Do not use distutils distributed with setuptools # This is due to base changes in the distutils API, removing msvccompiler, From 620808f97bd7dc6443f79be9eecb7d8689c1d69b Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 22 Aug 2023 17:22:35 +0200 Subject: [PATCH 48/49] Added note to requirements_full.txt --- requirements_full.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements_full.txt b/requirements_full.txt index 6ddb93b7d..7eec99048 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -1,4 +1,7 @@ # Optional requirements - used by various plugins or additional features like mappings +# +# Note: Some important indirect dependencies are not listed here. +# fortran-language-server>=1.12.0,<1.13 PyYAML>=5.4.1,<7 # psycopg2-binary can lead to segfault - so far all seems good with v2.9.5 From 9be8642c1847b19c0be5f0e024ba6a54516cd812 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Thu, 24 Aug 2023 08:25:09 +0200 Subject: [PATCH 49/49] Removed note from requirements_full.txt --- requirements_full.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements_full.txt b/requirements_full.txt index 7eec99048..6ddb93b7d 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -1,7 +1,4 @@ # Optional requirements - used by various plugins or additional features like mappings -# -# Note: Some important indirect dependencies are not listed here. -# fortran-language-server>=1.12.0,<1.13 PyYAML>=5.4.1,<7 # psycopg2-binary can lead to segfault - so far all seems good with v2.9.5