Merge pull request #85 from ukaea/nathan/deps-in-pyproject.toml

Nathan/deps in pyproject.toml

Author: NathanCummings

.github/workflows/ci.yml

@@ -38,15 +38,13 @@ jobs:
       run: |
         git lfs fetch
         git lfs pull
+        pip install --upgrade pip
         pip install uv
-        uv venv venv
-        source venv/bin/activate
-        uv pip install -r docs/requirements.txt
+        uv sync 
   
     - name: Run tests
       run: |
-        source venv/bin/activate
-        python -m pytest -rsx tests/ --data-path=/home/runner/work/fair-mast/fair-mast/tests/mock_data/index
+        uv run pytest -rs
 
   ruff-code-check:
     runs-on: ubuntu-latest

README.md

@@ -1,53 +1,29 @@
 # FAIR MAST Data Management System
 
-## Overview
-
-
 ## Development Setup
 
-### Mac Users:
+### Mac Users
 
-If you are using Mac for development, use [podman](https://podman.io/docs/installation) instead of docker. Follow the installation guide to set it up, then follow the below set up. 
+If you are using Mac for development, use [podman](https://podman.io/docs/installation) instead of docker. Follow the installation guide to set it up, then follow the below set up.
 
-### Linux/Windows Users:
+### Linux/Windows Users
 
 If using Linux or Windows, you need to make sure you have [docker](https://www.docker.com/get-started/) and `docker-compose` installed on your system.
 
 ### Setup
 
-We will be using the Python package manager [uv](https://astral.sh/blog/uv) to install our dependencies. As a first step, make sure this is installed with:
-```bash
-pip install uv
-```
 Secondly, clone the repository:
+
 ```bash
 git clone [email protected]:ukaea/fair-mast.git
 cd fair-mast
 ```
 
-You can use either `conda` or `venv` to set up the environment. Follow the below instructions depending on your preference. 
-### Option 1: Using Conda
-Assuming you already have conda installed on your system:
-```bash
-conda create -n mast python=3.11
-conda activate mast
-uv pip install -r docs/requirements.txt
-```
-
-### Option 2: Using venv
-Ensure you are using Python version `3.11`:
-```bash
-uv venv venv
-source venv/bin/activate
-uv pip install -r docs/requirements.txt
-```
-
-Use `uv --help` for additional commands, or refer to the documentation if needed.
-
 ### Start the Data Management System
+
 Run the development container to start the postgres database, fastapi, and minio containers locally. The development environment will watch the source directory and automatically reload changes to the API as you work.
 
-### Mac Users:
+#### Mac Users
 
 ```bash
 podman compose \
@@ -64,7 +40,7 @@ podman compose -f dev/docker/docker-compose.yml down
 podman volume rm --all
 ```
 
-### Linux/Windows Users:
+#### Linux/Windows Users
 
 ```bash
 docker-compose \
@@ -76,15 +52,14 @@ up \
 
 The following services will be started:
 
- - FastAPI REST & GraphQL Server - will start running at `http://localhost:8081`. 
-    - The REST API documentation is at `http://localhost:8081/redoc`. 
-    - The GraphQL API documentation is at `http://localhost:8081/graphql`.
- - Postgres Database Server - will start running at `http://localhost:5432`
- - Postgres Admin Server - will start running at `http://localhost:8081/pgadmin`
- - Minio S3 Storage Server - will start running at `http://localhost:9000`.
-    - The admin web GUI will be running at `http://localhost:8081/minio/ui`. 
+- FastAPI REST & GraphQL Server - will start running at `http://localhost:8081`.
+  - The REST API documentation is at `http://localhost:8081/redoc`.
+  - The GraphQL API documentation is at `http://localhost:8081/graphql`.
+- Postgres Database Server - will start running at `http://localhost:5432`
+- Postgres Admin Server - will start running at `http://localhost:5050`
 
 ### Populate the Database
+
 To create the database and populate it with content we need to get the metadata files. These are stored in the repository using [Git LFS](https://git-lfs.com).
 
 To retrieve these data files, follow the below instructions in your terminal:
@@ -95,35 +70,32 @@ git lfs fetch
 git lfs pull
 ```
 
-Assuming the files have been pulled successfully, the data files should exist within `tests/mock_data/mini` in the local directory. We can 
+Assuming the files have been pulled successfully, the data files should exist within `tests/mock_data/mini` in the local directory. We can
 create the database and ingest data using the following command:
 
-### Mac Users:
+#### Mac Users
 
 ```bash
-podman exec -it mast-api python -m src.api.create /code/data/mini
+podman exec -it mast-api python -m src.api.create /code/data/index
 ```
 
-### Linux/Windows Users:
+#### Linux/Windows Users
 
 ```bash
-docker exec -it mast-api python -m src.api.create /code/data/mini
+docker exec -it mast-api python -m src.api.create /code/data/index
 ```
 
 ### Running Unit Tests
+
 Verify everything is setup correctly by running the unit tests.
 
-To run the unit tests, input the following command inside your environment:
+Follow the below instructions to set up the environment.
 
 ```bash
-python -m pytest -rsx tests/ --data-path="INSERT FULL PATH TO DATA HERE"
+uv run pytest
 ```
 
-The data path will be will be along the lines of `~/fair-mast/tests/mock_data/mini`.
-
-This will run some unit tests for the REST and GraphQL APIs against a testing database, created from the data in `--data-path`. 
-
-### Production Deployment
+## Production Deployment
 
 To run the production container to start the postgres database, fastapi, and minio containers. This will also start an nginx proxy and make sure https is all setup
 
@@ -138,11 +110,19 @@ docker compose --env-file dev/docker/.env.dev  -f dev/docker/docker-compose.yml
 ```
 
 To also destory the volumes (including the metadatabase) you may add the volumes parameter:
+
 ```bash
 docker compose --env-file dev/docker/.env.dev  -f dev/docker/docker-compose.yml -f dev/docker/docker-compose-prod.yml down --volumes
 ```
 
-**Note** that every time you destory volumes, the production server will mint a new certificate for HTTPS. Lets Encrypt currently limits this to [5 per week](https://letsencrypt.org/docs/duplicate-certificate-limit/)
+**Note:** Every time you destory volumes, the production server will mint a new certificate for HTTPS. Lets Encrypt currently limits this to [5 per week](https://letsencrypt.org/docs/duplicate-certificate-limit/).
+
+You'll need to download and ingest the production data like so:
+
+```bash
+mkdir -p data/mast/meta
+rsync -vaP <CSD3-USERNAME>@login.hpc.cam.ac.uk:/rds/project/rds-sPGbyCAPsJI/archive/metadata data/
+```
 
 ```bash
 docker exec -it mast-api python -m src.api.create /code/data/index

dev/docker/api.Dockerfile

@@ -3,9 +3,9 @@ WORKDIR /code
 
 RUN pip install uv
 
-COPY docs/requirements.txt /code/requirements.txt
+COPY pyproject.toml /code/pyproject.toml
 
-RUN uv pip install --system --no-cache -r /code/requirements.txt
+RUN uv pip install . --system
 
 COPY ./src /code/src
 

dev/docker/docker-compose.yml

@@ -1,6 +1,4 @@
-version: '3.8'
 services:
-
   # Service for hosting the API.
   # This container runs the FastAPI REST and GraphQL APIs
   api:
@@ -32,41 +30,6 @@ services:
     networks:
       - dbnetwork
 
-  # Service for the Minio instance
-  # This container runs the minio S3 buckets and console
-  # minio:
-  #   image: minio/minio
-  #   container_name: minio
-  #   ports:
-  #     - "9000:9000"
-  #     - "9001:9001"
-  #   volumes:
-  #     - ./data/mast/s3data:/mnt/data
-  #   environment:
-  #     MINIO_ROOT_USER: ${MINIO_USER}
-  #     MINIO_ROOT_PASSWORD: ${MINIO_PASSWORD}
-  #     MINIO_VOLUMES: "/mnt/data"
-  #   command: ['server', '/mnt/data', '--console-address', ":9001"]
-  #   networks:
-  #     - dbnetwork
-
-  # # This service a bucket for the mast data is none exists.
-  # createbuckets:
-  #   image: minio/mc
-  #   depends_on:
-  #     - minio
-  #   volumes:
-  #    - ./data/:/data
-  #   entrypoint: >
-  #     /bin/sh -c "
-  #     /usr/bin/mc alias set srv https://minio:9000 minio99 minio123;
-  #     /usr/bin/mc mb srv/mast;
-  #     /usr/bin/mc anonymous set public srv/mast;
-  #     exit 0;
-  #     "
-  #   networks:
-  #     - dbnetwork
-
   # Service for the PostgreSQL Database
   # This container runs the PostgreSQL database which contains the metadata for MAST.
   meta_database:
@@ -85,7 +48,6 @@ services:
     networks:
       - dbnetwork
 
-
   # Service for pgadmin
   # This container runs a pgadmin instance for administering the database
   pgadmin:

docs/requirements.txt

@@ -3,8 +3,7 @@ name = "fair-mast"
 version = "1.0"
 description = "A data management system for Tokamak data"
 readme = "README.md"
-requires-python = ">=3.8"
-license = { name = "MIT" }
+requires-python = ">=3.9, <3.13"
 authors = [
 ]
 maintainers = [
@@ -15,11 +14,43 @@ maintainers = [
 
 classifiers = [
     "Development Status :: 4 - Beta",
-    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "License :: OSI Approved :: MIT License",
+]
+dependencies = [
+    "dask>=2024.8.0",
+    "fastapi-pagination>=0.12.31",
+    "fastapi>=0.115.2",
+    "jinja2>=3.1.4",
+    "numpy<2",
+    "pandas==2.0.3",
+    "psycopg2-binary>=2.9.10",
+    "pyarrow>=17.0.0",
+    "sqlakeyset>=2.0.1726021475",
+    "sqlalchemy-utils>=0.41.2",
+    "sqlalchemy<2",
+    "sqlmodel>=0.0.11",
+    "strawberry-graphql>=0.246.2",
+    "tqdm>=4.66.5",
+    "ujson>=5.10.0",
+    "uvicorn>=0.32.0",
 ]
 
 [project.urls]
 repository = "https://github.com/ukaea/fair-mast"
 
 [tool.ruff]
 exclude = ["docs/*.ipynb", "notebooks/*.ipynb", "src/metadata/*.ipynb", "src/api/static"]
+
+[tool.uv]
+dev-dependencies = [
+    "httpx>=0.27.2",
+    "pint>=0.24.3",
+    "pytest-benchmark>=4.0.0",
+    "pytest>=8.3.3",
+    "xarray>=2024.7.0",
+    "zarr>=2.18.2",
+]