diff --git a/CHANGELOG.md b/CHANGELOG.md index 837a3e0..db9f93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Keep it human-readable, your future self will thank you! - added downstream-ci pipeline and cd-pypi reusable workflow - Changelog release updater +- Create package documentation. + ### Changed - fix: added support for Python3.9. - fix: bug in graph cleaning method diff --git a/docs/_static/cutoff.jpg b/docs/_static/cutoff.jpg new file mode 100644 index 0000000..90f20dd Binary files /dev/null and b/docs/_static/cutoff.jpg differ diff --git a/docs/_static/hetero_data_graph.txt b/docs/_static/hetero_data_graph.txt new file mode 100644 index 0000000..b5b02f1 --- /dev/null +++ b/docs/_static/hetero_data_graph.txt @@ -0,0 +1,30 @@ +HeteroData( + data={ + x=[40320, 2], # coordinates in radians (lat in [-pi/2, pi/2], lon in [0, 2pi]) + node_type='ZarrDatasetNodes', + area_weight=[40320, 1], + }, + hidden={ + x=[10242, 2], # coordinates in radians (lat in [-pi/2, pi/2], lon in [0, 2pi]) + node_type='TriNodes', + area_weight=[10242, 1], + }, + (data, to, hidden)={ + edge_index=[2, 62980], + edge_type='CutOffEdges', + edge_length=[62980, 1], + edge_dirs=[62980, 2], + }, + (hidden, to, hidden)={ + edge_index=[2, 81900], + edge_type='MultiScaleEdges', + edge_length=[81900, 1], + edge_dirs=[81900, 2], + }, + (hidden, to, data)={ + edge_index=[2, 120960], + edge_type='KNNEdges', + edge_length=[120960, 1], + edge_dirs=[120960, 2], + } +) diff --git a/docs/cli/create.rst b/docs/cli/create.rst new file mode 100644 index 0000000..4f02101 --- /dev/null +++ b/docs/cli/create.rst @@ -0,0 +1,15 @@ +.. _cli-create: + +====== +create +====== + +Use this command to create a graph from a recipe file. + +The syntax of the recipe file is described in :doc:`building graphs <../graphs/introduction>`. + +.. argparse:: + :module: anemoi.graphs.__main__ + :func: create_parser + :prog: anemoi-graphs + :path: create diff --git a/docs/cli/describe.rst b/docs/cli/describe.rst new file mode 100644 index 0000000..18bc73b --- /dev/null +++ b/docs/cli/describe.rst @@ -0,0 +1,15 @@ +.. _cli-describe: + +======== +describe +======== + +Use this command to describe a graph stored in your filesystem. It will print graph information to the console. + +The syntax of the recipe file is described in :doc:`building graphs <../graphs/introduction>`. + +.. argparse:: + :module: anemoi.graphs.__main__ + :func: create_parser + :prog: anemoi-graphs + :path: describe diff --git a/docs/cli/inspect.rst b/docs/cli/inspect.rst new file mode 100644 index 0000000..394d529 --- /dev/null +++ b/docs/cli/inspect.rst @@ -0,0 +1,15 @@ +.. _cli-inspect: + +======== +inspect +======== + +Use this command to inspect a graph stored in your filesystem. + +The syntax of the recipe file is described in :doc:`building graphs <../graphs/introduction>`. + +.. argparse:: + :module: anemoi.graphs.__main__ + :func: create_parser + :prog: anemoi-graphs + :path: inspect diff --git a/docs/cli/introduction.rst b/docs/cli/introduction.rst new file mode 100644 index 0000000..4fa01db --- /dev/null +++ b/docs/cli/introduction.rst @@ -0,0 +1,29 @@ +.. _cli-introduction: + +============= +Introduction +============= + +When you install the `anemoi-graphs` package, this will also install command line tool +called ``anemoi-graphs`` which can be used to design and inspect weather graphs. + +The tool can provide help with the ``--help`` options: + +.. code-block:: bash + + % anemoi-graphs --help + +The commands are: + +.. toctree:: + :maxdepth: 1 + + create + describe + inspect + +.. argparse:: + :module: anemoi.graphs.__main__ + :func: create_parser + :prog: anemoi-graphs + :nosubcommands: diff --git a/docs/dev/code_structure.rst b/docs/dev/code_structure.rst new file mode 100644 index 0000000..3eed014 --- /dev/null +++ b/docs/dev/code_structure.rst @@ -0,0 +1,81 @@ +.. _dev-code_structure: + +################ + Code Structure +################ + +Understanding and maintaining the code structure is crucial for +sustainable development of Anemoi Graphs. This guide outlines best +practices for contributing to the codebase. + +****************************** + Subclassing for New Features +****************************** + +When creating a new feature, the recommended practice is to subclass +existing base classes rather than modifying them directly. This approach +preserves functionality for other users while allowing for +customization. + +Example: +======== + +In `anemoi/graphs/nodes/builder.py`, the `BaseNodeBuilder` class serves +as a foundation to define new sets of nodes. New node builders should +subclass this base class. + +******************* + File Organization +******************* + +When developing multiple new functions for a feature: + +#. Create a new file in the folder (e.g., + `edges/builder/.py`) to avoid confusion with base + functions. + +#. Group related functionality together for better organization and + maintainability. + +******************************** + Version Control Best Practices +******************************** + +#. Always use pre-commit hooks to ensure code quality and consistency. +#. Never commit directly to the `develop` branch. +#. Create a new branch for your feature or bug fix, e.g., + `feature/` or `bugfix/`. +#. Submit a Pull Request from your branch to `develop` for peer review + and testing. + +****************************** + Code Style and Documentation +****************************** + +#. Follow PEP 8 guidelines for Python code style, the pre-commit hooks + will help enforce this. +#. Write clear, concise docstrings for all classes and functions using + the Numpy style. +#. Use type hints to improve code readability and catch potential + errors. +#. Add inline comments for complex logic or algorithms. + +********* + Testing +********* + +#. Write unit tests for new features using pytest. +#. Ensure all existing tests pass before submitting a Pull Request. +#. Aim for high test coverage, especially for critical functionality. + +**************************** + Performance Considerations +**************************** + +#. Profile your code to identify performance bottlenecks. +#. Optimize critical paths and frequently called functions. +#. Consider using vectorized operations when working with large + datasets. + +By following these guidelines, you'll contribute to a maintainable and +robust codebase for Anemoi Graphs. diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst new file mode 100644 index 0000000..d04b5fd --- /dev/null +++ b/docs/dev/contributing.rst @@ -0,0 +1,151 @@ +.. _dev-contributing: + +############## + Contributing +############## + +Thank you for your interest in contributing to Anemoi Graphs! This guide +will help you get started with the development process. + +**************************************** + Setting Up the Development Environment +**************************************** + +#. Clone the repository: + + .. code:: bash + + git clone https://github.com/ecmwf/anemoi-graphs/ + cd anemoi-graphs + +#. Install dependencies: + + .. code:: bash + + # For all dependencies + pip install -e . + + # For development dependencies + pip install -e '.[dev]' + +#. (macOS only) Install pandoc for documentation building: + + .. code:: bash + + brew install pandoc + +****************** + Pre-Commit Hooks +****************** + +We use pre-commit hooks to ensure code quality and consistency. To set +them up: + +#. Install pre-commit hooks: + + .. code:: bash + + pre-commit install + +#. Run hooks on all files to verify installation: + + .. code:: bash + + pre-commit run --all-files + +******************* + Commit Guidelines +******************* + +Ideally, open an issue for the feature or bug fix you're working on +before starting development, to discuss the approach with maintainers. + +When committing code changes: + +#. Make small, focused commits with clear and concise messages. + +#. Follow the `Conventional Commits guidelines + `_, e.g., "feat:", "fix:", + "docs:", etc. + +#. Use present tense and imperative mood in commit messages (e.g., "Add + feature" not "Added feature"). + +#. Reference relevant issue numbers in commit messages when applicable. + +#. Update the ``CHANGELOG.md`` file with a human-friendly summary of + your changes. + +********************** + Pull Request Process +********************** + +#. Create a new branch for your feature or bug fix. +#. Make your changes and commit them using the guidelines above. +#. Push your branch to your fork on GitHub. +#. Open a Pull Request against the `develop` branch of the main + repository. +#. Ensure all tests pass and the code adheres to the project's style + guidelines. +#. Request a review from maintainers or other contributors. + +*************** + Running Tests +*************** + +We use pytest for our test suite. To run tests: + +.. code:: bash + + # Run all tests + pytest + + # Run tests in a specific file + pytest tests/test_.py + +Note: Some tests, like `test_gnn.py`, may run slower on CPU and are +better suited for GPU execution. + +************************ + Building Documentation +************************ + +You can build the documentation locally to preview changes before +submitting a Pull Request. We use Sphinx for documentation. + +You can install the dependencies for building the documentation with: + +.. code:: bash + + pip install '.[docs]' + +To build the documentation locally: + +.. code:: bash + + cd docs + make html + +The generated documentation will be in `docs/_build/html/index.html`. + +********************* + Code Review Process +********************* + +#. All code changes must be reviewed before merging. +#. Address any feedback or comments from reviewers promptly. +#. Once approved, a maintainer will merge your Pull Request. + +****************** + Reporting Issues +****************** + +If you encounter a bug or have a feature request: + +#. Check the existing issues to avoid duplicates. +#. If it's a new issue, create a detailed bug report or feature request. +#. Use clear, descriptive titles and provide as much relevant + information as possible. + +Thank you for contributing to Anemoi Graphs! Your efforts help improve +the project for everyone. diff --git a/docs/dev/testing.rst b/docs/dev/testing.rst new file mode 100644 index 0000000..68129b1 --- /dev/null +++ b/docs/dev/testing.rst @@ -0,0 +1,192 @@ +.. _dev-testing: + +######### + Testing +######### + +Comprehensive testing is crucial for maintaining the reliability and +stability of Anemoi Graphs. This guide outlines our testing strategy and +best practices for contributing tests. + +******************* + Testing Framework +******************* + +We use pytest as our primary testing framework. Pytest offers a simple +and powerful way to write and run tests. + +*************** + Writing Tests +*************** + +General Guidelines +================== + +#. Write tests for all new features and bug fixes. +#. Aim for high test coverage, especially for critical components. +#. Keep tests simple, focused, and independent of each other. +#. Use descriptive names for test functions, following the pattern + `test__`. + +Example Test Structure +====================== + +.. code:: python + + import pytest + from anemoi.graphs import SomeFeature + + + def test_some_feature_normal_input(): + feature = SomeFeature() + result = feature.process(normal_input) + assert result == expected_output + + + def test_some_feature_edge_case(): + feature = SomeFeature() + with pytest.raises(ValueError): + feature.process(invalid_input) + +**************** + Types of Tests +**************** + +1. Unit Tests +============= + +Test individual components in isolation. These should be the majority of +your tests. + +2. Integration Tests +==================== + +Test how different components work together. These are particularly +important for graph creation workflows. + +3. Functional Tests +=================== + +Test entire features or workflows from start to finish. These ensure +that the system works as expected from a user's perspective. + +4. Parametrized Tests +===================== + +Use pytest's parametrize decorator to run the same test with different +inputs: + +.. code:: python + + @pytest.mark.parametrize( + "input,expected", + [ + (2, 4), + (3, 9), + (4, 16), + ], + ) + def test_square(input, expected): + assert square(input) == expected + +You can also consider ``hypothesis`` for property-based testing. + +5. Fixtures +=========== + +Use fixtures to set up common test data or objects: + +.. code:: python + + @pytest.fixture + def sample_dataset(): + # Create and return a sample dataset + pass + + + def test_data_loading(sample_dataset): + # Use the sample_dataset fixture in your test + pass + +*************** + Running Tests +*************** + +To run all tests: + +.. code:: bash + + pytest + +To run tests in a specific file: + +.. code:: bash + + pytest tests/test_specific_feature.py + +To run tests with a specific mark: + +.. code:: bash + + pytest -m slow + +*************** + Test Coverage +*************** + +We use pytest-cov to measure test coverage. To run tests with coverage: + +.. code:: bash + + pytest --cov=anemoi_graphs + +Aim for at least 80% coverage for new features, and strive to maintain +or improve overall project coverage. + +************************ + Continuous Integration +************************ + +All tests are run automatically on our CI/CD pipeline for every pull +request. Ensure all tests pass before submitting your PR. + +********************* + Performance Testing +********************* + +For performance-critical components: + +#. Write benchmarks. +#. Compare performance before and after changes. +#. Set up performance regression tests in CI. + +********************** + Mocking and Patching +********************** + +Use unittest.mock or pytest-mock for mocking external dependencies or +complex objects: + +.. code:: python + + def test_api_call(mocker): + mock_response = mocker.Mock() + mock_response.json.return_value = {"data": "mocked"} + mocker.patch("requests.get", return_value=mock_response) + + result = my_api_function() + assert result == "mocked" + +**************** + Best Practices +**************** + +#. Keep tests fast: Optimize slow tests or mark them for separate + execution. +#. Use appropriate assertions: pytest provides a rich set of assertions. +#. Test edge cases and error conditions, not just the happy path. +#. Regularly review and update tests as the codebase evolves. +#. Document complex test setups or scenarios. + +By following these guidelines and continuously improving our test suite, +we can ensure the reliability and maintainability of Anemoi Graphs. diff --git a/docs/graphs/edge_attributes.rst b/docs/graphs/edge_attributes.rst new file mode 100644 index 0000000..c7d713b --- /dev/null +++ b/docs/graphs/edge_attributes.rst @@ -0,0 +1,44 @@ +.. _edge-attributes: + +#################### + Edges - Attributes +#################### + +There are 2 main edge attributes implemented in the `anemoi-graphs` +package: + +************* + Edge length +************* + +The `edge length` is a scalar value representing the distance between +the source and target nodes. This attribute is calculated using the +Haversine formula, which is a method of calculating the distance between +two points on the Earth's surface given their latitude and longitude +coordinates. + +.. code:: yaml + + edges: + - ... + edge_builder: ... + attributes: + edge_length: + _target_: anemoi.graphs.edges.attributes.EdgeLength + +**************** + Edge direction +**************** + +The `edge direction` is a 2D vector representing the direction of the +edge. This attribute is calculated from the difference between the +latitude and longitude coordinates of the source and target nodes. + +.. code:: yaml + + edges: + - ... + edge_builder: ... + attributes: + edge_length: + _target_: anemoi.graphs.edges.attributes.EdgeDirection diff --git a/docs/graphs/edges.rst b/docs/graphs/edges.rst new file mode 100644 index 0000000..57aebd2 --- /dev/null +++ b/docs/graphs/edges.rst @@ -0,0 +1,28 @@ +.. _graphs-edges: + +##################### + Edges - Connections +##################### + +Once the `nodes`, :math:`V`, are defined, you can create the `edges`, +:math:`E`, that will connect them. These connections are listed in the +``edges`` section of the recipe file, and they are created independently +for each (`source name`, `target name`) pair specified. + +.. code:: yaml + + edges: + - source_name: data + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.CutOff + cutoff_factor: 0.7 + +Below are the available methods for defining the edges: + +.. toctree:: + :maxdepth: 1 + + edges/cutoff + edges/knn + edges/multi_scale diff --git a/docs/graphs/edges/cutoff.rst b/docs/graphs/edges/cutoff.rst new file mode 100644 index 0000000..ba999e6 --- /dev/null +++ b/docs/graphs/edges/cutoff.rst @@ -0,0 +1,52 @@ +################ + Cut-off radius +################ + +The cut-off method is a method for establishing connections between two +sets of nodes. Given two sets of nodes, (`source`, `target`), the +cut-off method connects all source nodes, :math:`V_{source}`, in a +neighbourhood of the target nodes, :math:`V_{target}`. + +.. image:: ../../_static/cutoff.jpg + :alt: Cut-off radius image + :align: center + +The neighbourhood is defined by a `cut-off radius`, which is computed +as, + +.. math:: + + cutoff\_radius = cuttoff\_factor \times nodes\_reference\_dist + +where :math:`nodes\_reference\_dist` is the maximum distance between a +target node and its nearest source node. + +.. math:: + + nodes\_reference\_dist = \max_{x \in V_{target}} \left\{ \min_{y \in V_{source}, y \neq x} \left\{ d(x, y) \right\} \right\} + +where :math:`d(x, y)` is the `Haversine distance +`_ between nodes +:math:`x` and :math:`y`. The ``cutoff_factor`` is a parameter that can +be adjusted to increase or decrease the size of the neighbourhood, and +consequently the number of connections in the graph. + +To use this method to create your connections, you can use the following +YAML configuration: + +.. code:: yaml + + edges: + - source_name: source + target_name: destination + edge_builder: + _target_: anemoi.graphs.edges.CutOffEdges + cutoff_factor: 0.6 + +.. note:: + + The cut-off method is recommended for the encoder edges, to connect + all data nodes to hidden nodes. The optimal ``cutoff_factor`` value + will be the lowest value without orphan nodes. This optimal value + depends on the node distribution, so it is recommended to tune it for + each case. diff --git a/docs/graphs/edges/knn.rst b/docs/graphs/edges/knn.rst new file mode 100644 index 0000000..b90c048 --- /dev/null +++ b/docs/graphs/edges/knn.rst @@ -0,0 +1,25 @@ +##################### + K-Nearest Neighbors +##################### + +The knn method is a method for establishing connections between two sets +of nodes. Given two sets of nodes, (`source`, `target`), the knn method +connects all destination nodes, to their ``num_nearest_neighbours`` +nearest source nodes. + +To use this method to build your connections, you can use the following +YAML configuration: + +.. code:: yaml + + edges: + - source_name: source + target_name: destination + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + num_nearest_neighbours: 3 + +.. note:: + + The knn method is recommended for the decoder edges, to connect all + data nodes with the surrounding hidden nodes. diff --git a/docs/graphs/edges/multi_scale.rst b/docs/graphs/edges/multi_scale.rst new file mode 100644 index 0000000..7f5878a --- /dev/null +++ b/docs/graphs/edges/multi_scale.rst @@ -0,0 +1,35 @@ +################################################ + Multi-scale connections at refined icosahedron +################################################ + +The multi-scale connections can only be defined with the same source and +target nodes. Edges of different scales are defined based on the +refinement level of an icosahedron. The higher the refinement level, the +shorter the length of the edges. By default, all possible refinements +levels are considered. + +To use this method to build your connections, you can use the following +YAML configuration: + +.. code:: yaml + + edges: + - source_name: source + target_name: source + edge_builder: + _target_: anemoi.graphs.edges.MultiScaleEdges + x_hops: 1 + +where `x_hops` is the number of hops between two nodes of the same +refinement level to be considered neighbours, and then connected. + +.. note:: + + This method is used by data-driven weather models like GraphCast to + process the latent/hidden state. + +.. warning:: + + This connection method is only supported for building the connections + within a set of nodes defined with the ``TriNodes`` or ``HexNodes`` + classes. diff --git a/docs/graphs/introduction.rst b/docs/graphs/introduction.rst new file mode 100644 index 0000000..b0d6383 --- /dev/null +++ b/docs/graphs/introduction.rst @@ -0,0 +1,78 @@ +.. _graphs-introduction: + +############## + Introduction +############## + +The `anemoi-graphs` package allows you to design custom graphs for +training data-driven weather models. The graphs are built using a +`recipe`, which is a YAML file that specifies the nodes and edges of the +graph. + +********** + Concepts +********** + +nodes + A `node` represents a location (2D) on the earth's surface which may + contain additional `attributes`. + +data nodes + A set of nodes representing one or multiple datasets. The `data + nodes` may correspond to the input/output of our data-driven model. + They can be defined from Zarr datasets and this method supports all + :ref:`anemoi-datasets ` operations such + as `cutout` or `thinning`. + +hidden nodes + The `hidden nodes` capture intermediate representations of the model, + which are used to learn the dynamics of the system considered + (atmosphere, ocean, etc, ...). These nodes can be generated from + existing locations (Zarr datasets or NPZ files) or algorithmically + from iterative refinements of polygons over the globe. + +isolated nodes + A set of nodes that are not connected to any other nodes in the + graph. These nodes can be used to store additional information that + is not directly used in the training process. + +edges + An `edge` represents a connection between two nodes. The `edges` can + be used to define the flow of information between the nodes. Edges + may also contain `attributes` related to their length, direction or + other properties. + +***************** + Data structures +***************** + +The nodes :math:`V` correspond to locations on the earth's surface, and +they can be classified into 2 categories: + +- **Data nodes**: The `data nodes` represent the input/output of the + data-driven model, i.e. they are linked to existing datasets. +- **Hidden nodes**: These `hidden nodes` represent the latent space, + where the internal dynamics are learned. + +Several methods are currently supported to create your nodes. You can +use indistinctly any of these to create your `data` or `hidden` nodes. + +The `nodes` are defined in the ``nodes`` section of the recipe file. The +keys are the names of the sets of `nodes` that will later be used to +build the connections. Each `nodes` configuration must include a +``node_builder`` section describing how to define the `nodes`. The +following classes define different behaviour: + +- :doc:`node_coordinates/zarr_dataset` +- :doc:`node_coordinates/npz_file` +- :doc:`node_coordinates/tri_refined_icosahedron` +- :doc:`node_coordinates/hex_refined_icosahedron` +- :doc:`node_coordinates/healpix` + +In addition to the ``node_builder`` section, the `nodes` configuration +can contain an optional ``attributes`` section to define additional node +attributes (weights, mask, ...). For example, the weights can be used to +define the importance of each node in the loss function, or the masks +can be used to build connections only between subsets of nodes. + +- :doc:`node_attributes/weights` diff --git a/docs/graphs/node_attributes.rst b/docs/graphs/node_attributes.rst new file mode 100644 index 0000000..9856625 --- /dev/null +++ b/docs/graphs/node_attributes.rst @@ -0,0 +1,23 @@ +.. _graphs-node_attributes: + +#################### + Nodes - Attributes +#################### + +.. warning:: + + This is still a work in progress. More classes will be added in the + future. + +The nodes :math:`V` correspond to locations on the earth's surface. As +well as defining their locations, the `nodes` can contain additional +attributes, which should be defined in the ``attributes`` section of the +`nodes` configuration. For example, a `weights` attribute can be used to +define the importance of each node in the loss function, or a `masks` +attribute can be used to build connections only between subsets of +nodes. + +.. toctree:: + :maxdepth: 1 + + node_attributes/weights diff --git a/docs/graphs/node_attributes/weights.rst b/docs/graphs/node_attributes/weights.rst new file mode 100644 index 0000000..b3cfccd --- /dev/null +++ b/docs/graphs/node_attributes/weights.rst @@ -0,0 +1,10 @@ +######### + Weights +######### + +The `weights` are a node attribute useful for defining the importance of +a node in the loss function. You can set the weights to follow an +uniform distribution or to match the area associated with that node. + +.. literalinclude:: ../yaml/attributes_weights.yaml + :language: yaml diff --git a/docs/graphs/node_coordinates.rst b/docs/graphs/node_coordinates.rst new file mode 100644 index 0000000..aff5b70 --- /dev/null +++ b/docs/graphs/node_coordinates.rst @@ -0,0 +1,37 @@ +.. _graphs-node_coordinates: + +##################### + Nodes - Coordinates +##################### + +.. warning:: + + This is still a work in progress. More classes will be added in the + future. + +The `nodes` :math:`V` correspond to locations on the earth's surface. + +The `nodes` are defined in the ``nodes`` section of the recipe file. The +keys are the names of the sets of `nodes` that will later be used to +build the connections. Each `nodes` configuration must include a +``node_builder`` section describing how to define the `nodes`. + +The `nodes` can be defined based on the coordinates already available in +a file: + +.. toctree:: + :maxdepth: 1 + + node_coordinates/zarr_dataset + node_coordinates/npz_file + +or based on other algorithms. A commonn approach is to use an +icosahedron to project the earth's surface, and refine it iteratively to +reach the desired resolution. + +.. toctree:: + :maxdepth: 1 + + node_coordinates/tri_refined_icosahedron + node_coordinates/hex_refined_icosahedron + node_coordinates/healpix diff --git a/docs/graphs/node_coordinates/healpix.csv b/docs/graphs/node_coordinates/healpix.csv new file mode 100644 index 0000000..1783471 --- /dev/null +++ b/docs/graphs/node_coordinates/healpix.csv @@ -0,0 +1,11 @@ +Refinement level,Number of nodes,Resolution (km),Resolution (degrees) +0,12,6371,57.296 +1,48,3185.5,28.648 +2,192,1592.75,14.324 +3,768,796.375,7.162 +4,3072,398.187,3.581 +5,12288,199.094,1.790 +6,49152,99.547,0.895 +7,196608,49.773,0.448 +8,786432,24.887,0.224 +9,3145728,12.443,0.112 diff --git a/docs/graphs/node_coordinates/healpix.rst b/docs/graphs/node_coordinates/healpix.rst new file mode 100644 index 0000000..2f5fc17 --- /dev/null +++ b/docs/graphs/node_coordinates/healpix.rst @@ -0,0 +1,30 @@ +############### + HEALPix Nodes +############### + +This method allows us to define nodes based on the Hierarchical Equal +Area isoLatitude Pixelation of a sphere (HEALPix). The resolution of the +HEALPix grid is defined by the `resolution` parameter, which corresponds +to the number of refinements of the sphere. + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.HEALPixNodes + resolution: 3 + attributes: ... + +For reference, the following table shows the number of nodes and +resolution for each resolution: + +.. csv-table:: HEALPix refinements specifications + :file: ./healpix.csv + :header-rows: 1 + +.. warning:: + + This class will require the `healpy + `_ package to be installed. You can + install it with `pip install healpy`. diff --git a/docs/graphs/node_coordinates/hex_refined.csv b/docs/graphs/node_coordinates/hex_refined.csv new file mode 100644 index 0000000..dc78f58 --- /dev/null +++ b/docs/graphs/node_coordinates/hex_refined.csv @@ -0,0 +1,10 @@ +Refinement Level,Number of nodes,Avg. Hexagon Area (sq km) +0,122,4.250.546 +1,842,607.220 +2,5.882,86.745 +3,41.162,12.392 +4,288.122,1.770 +5,2.016.842,252 +6,14.117.882,36 +7,98.825.162,5.1 +8,691.776.122,0.7 diff --git a/docs/graphs/node_coordinates/hex_refined_icosahedron.rst b/docs/graphs/node_coordinates/hex_refined_icosahedron.rst new file mode 100644 index 0000000..7f665cd --- /dev/null +++ b/docs/graphs/node_coordinates/hex_refined_icosahedron.rst @@ -0,0 +1,36 @@ +############################### + Hexagonal refined Icosahedron +############################### + +This method allows us to define the nodes based on the Hexagonal +Hierarchical Geospatial Indexing System, which uses hexagons to divide +the sphere. Each refinement level divides each hexagon into seven +smaller hexagons. + +To define the `node coordinates` based on the hexagonal refinements of +an icosahedron, you can use the following YAML configuration: + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.HexNodes + resolution: 4 + attributes: ... + +where resolution is the number of refinements to be applied. + +.. csv-table:: Hexagonal Hierarchical refinements specifications + :file: ./hex_refined.csv + :header-rows: 1 + +Note that the refinement level is the parameter used to control the +resolution of the nodes, but the resolution also depends on the +refinement method. Then, for the same refinement level, ``HexNodes`` +will have a higher resolution than ``TriNodes``. + +.. warning:: + + This class will require the `h3 `_ package to be + installed. You can install it with `pip install h3`. diff --git a/docs/graphs/node_coordinates/npz_file.rst b/docs/graphs/node_coordinates/npz_file.rst new file mode 100644 index 0000000..266687d --- /dev/null +++ b/docs/graphs/node_coordinates/npz_file.rst @@ -0,0 +1,30 @@ +############### + From NPZ file +############### + +To define the `node coordinates` based on a NPZ file, you can use the +following YAML configuration: + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.NPZFileNodes + grids_definition_path: /path/to/folder/with/grids/ + resolution: o48 + +where `grids_definition_path` is the path to the folder containing the +grid definition files and `resolution` is the resolution of the grid to +be used. + +By default, the grid files are supposed to be in the `grids` folder in +the same directory as the recipe file. The grid definition files are +expected to be name `"grid_{resolution}.npz"`. + +.. note:: + + The NPZ file should contain the following keys: + + - `longitudes`: The longitudes of the grid. + - `latitudes`: The latitudes of the grid. diff --git a/docs/graphs/node_coordinates/tri_refined_icosahedron.rst b/docs/graphs/node_coordinates/tri_refined_icosahedron.rst new file mode 100644 index 0000000..44b3e44 --- /dev/null +++ b/docs/graphs/node_coordinates/tri_refined_icosahedron.rst @@ -0,0 +1,31 @@ +################################ + Triangular refined Icosahedron +################################ + +This class allows us to define nodes based on iterative refinements of +an icoshaedron with triangles. + +To define the `node coordinates` based on icosahedral refinements of an +icosahedron, you can use the following YAML configuration: + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.TriNodes + resolution: 4 + attributes: ... + +where resolution is the number of refinements to be applied to the +icosahedron. + +Note that the refinement level is the parameter used to control the +resolution of the nodes, but the resolution also depends on the +refinement method. Then, for the same refinement level, ``HexNodes`` +will have a higher resolution than ``TriNodes``. + +.. warning:: + + This class will require the `trimesh `_ package + to be installed. You can install it with `pip install trimesh`. diff --git a/docs/graphs/node_coordinates/zarr_dataset.rst b/docs/graphs/node_coordinates/zarr_dataset.rst new file mode 100644 index 0000000..3723c0e --- /dev/null +++ b/docs/graphs/node_coordinates/zarr_dataset.rst @@ -0,0 +1,39 @@ +################### + From Zarr dataset +################### + +This class builds a set of nodes from a Zarr dataset. The nodes are +defined by the coordinates of the dataset. The ZarrDataset class +supports operations compatible with :ref:`anemoi-datasets +`. + +To define the `node coordinates` based on a Zarr dataset, you can use +the following YAML configuration: + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.ZarrDatasetNodes + dataset: /path/to/dataset.zarr + attributes: ... + +where `dataset` is the path to the Zarr dataset. The +``ZarrDatasetNodes`` class supports operations compatible with +:ref:`anemoi-datasets `, such as "cutout". +Below, an example of how to use the "cutout" operation directly within +:ref:`anemoi-graphs `. + +.. code:: yaml + + nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.ZarrDatasetNodes + dataset: + cutout: + dataset: /path/to/lam_dataset.zarr + dataset: /path/to/boundary_forcing.zarr + adjust: "all" + attributes: ... diff --git a/docs/graphs/yaml/attributes_weights.yaml b/docs/graphs/yaml/attributes_weights.yaml new file mode 100644 index 0000000..f889ef7 --- /dev/null +++ b/docs/graphs/yaml/attributes_weights.yaml @@ -0,0 +1,10 @@ +nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.nodes.ZarrDatasetNodeBuilder + dataset: /path/to/dataset.zarr + attributes: + weights: + _target_: anemoi.graphs.nodes.weights.Area + norm: unit-max + hidden: ... diff --git a/docs/index.rst b/docs/index.rst index 46aaf9a..d5312bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,23 +22,106 @@ models from existing recipes but with their own data. This package provides a series of utility functions for used by the rest of the *Anemoi* packages. -- :doc:`installing` +- :doc:`overview` .. toctree:: :maxdepth: 1 :hidden: - installing + overview + +***************** + Building graphs +***************** + +- :doc:`graphs/introduction` +- :doc:`graphs/node_coordinates` +- :doc:`graphs/node_attributes` +- :doc:`graphs/edges` +- :doc:`graphs/edge_attributes` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Building graphs + + graphs/introduction + graphs/node_coordinates + graphs/node_attributes + graphs/edges + graphs/edge_attributes ********* Modules ********* +- :doc:`modules/node_builder` +- :doc:`modules/edge_builder` +- :doc:`modules/node_attributes` +- :doc:`modules/edge_attributes` +- :doc:`modules/graph_creator` +- :doc:`modules/graph_inspector` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Modules + + modules/node_builder + modules/edge_builder + modules/node_attributes + modules/edge_attributes + modules/graph_creator + modules/graph_inspector + +******************* + Command line tool +******************* + +- :doc:`cli/introduction` +- :doc:`cli/create` +- :doc:`cli/describe` +- :doc:`cli/inspect` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Command line tool + + cli/introduction + cli/create + cli/describe + cli/inspect + +************************** + Developing Anemoi Graphs +************************** + +- :doc:`dev/contributing` +- :doc:`dev/code_structure` +- :doc:`dev/testing` + .. toctree:: :maxdepth: 1 - :glob: + :hidden: + :caption: Developing Anemoi Graphs + + dev/contributing + dev/code_structure + dev/testing + +*********** + Tutorials +*********** + +- :doc:`usage/getting_started` + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Usage - modules/* + usage/getting_started ***************** Anemoi packages diff --git a/docs/installing.rst b/docs/installing.rst deleted file mode 100644 index e452a31..0000000 --- a/docs/installing.rst +++ /dev/null @@ -1,31 +0,0 @@ -############ - Installing -############ - -To install the package, you can use the following command: - -.. code:: bash - - pip install anemoi-graphs[...options...] - -The options are: - -- ``dev``: install the development dependencies -- ``all``: install all the dependencies - -************** - Contributing -************** - -.. code:: bash - - git clone ... - cd anemoi-graphs - pip install .[dev] - pip install -r docs/requirements.txt - -You may also have to install pandoc on MacOS: - -.. code:: bash - - brew install pandoc diff --git a/docs/modules/dates.rst b/docs/modules/dates.rst deleted file mode 100644 index 94713af..0000000 --- a/docs/modules/dates.rst +++ /dev/null @@ -1,8 +0,0 @@ -####### - dates -####### - -.. automodule:: anemoi.graphs.dates - :members: - :no-undoc-members: - :show-inheritance: diff --git a/docs/modules/edge_attributes.rst b/docs/modules/edge_attributes.rst new file mode 100644 index 0000000..9abb5e7 --- /dev/null +++ b/docs/modules/edge_attributes.rst @@ -0,0 +1,11 @@ +.. _modules-edge_attributes: + +################# + Edge attributes +################# + +.. automodule:: anemoi.graphs.edges.attributes + :members: + :exclude-members: BaseEdgeAttribute + :no-undoc-members: + :show-inheritance: diff --git a/docs/modules/edge_builder.rst b/docs/modules/edge_builder.rst new file mode 100644 index 0000000..1fa555b --- /dev/null +++ b/docs/modules/edge_builder.rst @@ -0,0 +1,11 @@ +.. _modules-edge_builder: + +############## + Edge builder +############## + +.. automodule:: anemoi.graphs.edges.builder + :members: + :exclude-members: BaseEdgeBuilder + :no-undoc-members: + :show-inheritance: diff --git a/docs/modules/graph_creator.rst b/docs/modules/graph_creator.rst new file mode 100644 index 0000000..7221c05 --- /dev/null +++ b/docs/modules/graph_creator.rst @@ -0,0 +1,14 @@ +.. _modules-graph_creator: + +############### + Graph Creator +############### + +This module is used to create custom graphs for data-driven weather +models. The graphs are built using a `recipe` that defines the structure +of the graph. + +.. automodule:: anemoi.graphs.create + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/modules/graph_inspector.rst b/docs/modules/graph_inspector.rst new file mode 100644 index 0000000..e7f1ede --- /dev/null +++ b/docs/modules/graph_inspector.rst @@ -0,0 +1,17 @@ +.. _modules-graph_inspector: + +################# + Graph Inspector +################# + +This module is used to inspect graphs. This inspection includes: + +- Distribution plots of node & edge attributes. +- Interactive plot of each subgraph. +- Interactive plot of isolated nodes. +- Description of the graph in the console. + +.. automodule:: anemoi.graphs.inspector + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/modules/node_attributes.rst b/docs/modules/node_attributes.rst new file mode 100644 index 0000000..3193409 --- /dev/null +++ b/docs/modules/node_attributes.rst @@ -0,0 +1,11 @@ +.. _modules-node_attributes: + +################# + Node attributes +################# + +.. automodule:: anemoi.graphs.nodes.attributes + :members: + :exclude-members: BaseWeights + :no-undoc-members: + :show-inheritance: diff --git a/docs/modules/node_builder.rst b/docs/modules/node_builder.rst new file mode 100644 index 0000000..9a83de5 --- /dev/null +++ b/docs/modules/node_builder.rst @@ -0,0 +1,11 @@ +.. _modules-node_builder: + +############## + Node builder +############## + +.. automodule:: anemoi.graphs.nodes.builder + :members: + :exclude-members: BaseNodeBuilder,IcosahedralNodes + :no-undoc-members: + :show-inheritance: diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..603709b --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,127 @@ +.. _overview: + +########## + Overview +########## + +A graph :math:`G = (V, E)` is a collection of nodes/vertices :math:`V` +and edges :math:`E` that connect the nodes. The nodes can represent +locations in the globe. + +In weather models, the nodes :math:`V` can generally be classified into +2 categories: + +- **Data nodes**: The `data nodes` represent the input/output of the + data-driven model, so they are linked to existing datasets. +- **Hidden nodes**: These `hidden nodes` represent the latent space, + where the internal dynamics are learned. + +Similarly, the edges :math:`V` can be classified into 3 categories: + +- **Encoder edges**: These `encoder edges` connect the `data` nodes + with the `hidden` nodes to encode the input data into the latent + space. + +- **Processor edges**: These `processor edges` connect the `hidden` + nodes with the `hidden` nodes to process the latent space. + +- **Decoder edges**: These `decoder edges` connect the `hidden` nodes + with the `data` nodes to decode the latent space into the output + data. + +When building the graph with `anemoi-graphs`, there is no difference +between these categories. However, it is important to keep this +distinction in mind when designing a weather graph to be used in a +data-driven model with :ref:`anemoi-training +`. + +******************* + Design principles +******************* + +In particular, when designing a graph for a weather model, the following +guidelines should be followed: + +- Use a coarser resolution for the `hidden nodes`. This will reduce the + computational cost of training and inference. +- All input nodes should be connected to the `hidden nodes`. This will + ensure that all available information can be used. +- In the encoder edges, minimise the number of connections to the + `hidden nodes`. This will reduce the computational cost. +- All output nodes should have incoming connections from a few + surrounding `hidden nodes`. +- The number of incoming connections in each set of nodes should be be + similar to make the training more stable. +- Think whether or not your use case requires long-range connections + between the `hidden nodes` or not. + +**************** + Data structure +**************** + +The graphs generated by :ref:`anemoi-utils ` +are represented as a `pytorch_geometric.data.HeteroData +`_ +object. They include all the attributes specified in the recipe file and +the node/edge type. The node/edge type represents the node/edge builder +used to create the set of nodes/edges. + +.. literalinclude:: _static/hetero_data_graph.txt + :language: console + +The `HeteroData` object contains some useful attributes such as +`node_types` and `edge_types` which output the nodes and edges defined +in the respective graph. + +.. code:: console + + >>> graph.node_types + ['data', 'hidden'] + + >>> graph.edge_types + [("data", "to", "hidden"), ("hidden", "to", "hidden"), ("hidden", "to", "data")] + +In addition, you can inspect the attributes of the nodes and edges using +the `node_attrs` and `edge_attrs` methods. + +.. code:: console + + >>> graph["data"].node_attrs() + ["x", "area_weight"] + + >>> graph[("data", "to", "hidden")].edge_attrs() + ['edge_index', 'edge_length', 'edge_dirs'] + +************ + Installing +************ + +To install the package, you can use the following command: + +.. code:: bash + + pip install anemoi-graphs[...options...] + +The options are: + +- ``dev``: install the development dependencies +- ``docs``: install the dependencies for the documentation +- ``test``: install the dependencies for testing +- ``all``: install all the dependencies + +************** + Contributing +************** + +.. code:: bash + + git clone ... + cd anemoi-graphs + pip install .[dev] + pip install -r docs/requirements.txt + +You may also have to install pandoc on MacOS: + +.. code:: bash + + brew install pandoc diff --git a/docs/usage/getting_started.rst b/docs/usage/getting_started.rst new file mode 100644 index 0000000..b4998e1 --- /dev/null +++ b/docs/usage/getting_started.rst @@ -0,0 +1,117 @@ +.. _usage-getting-started: + +################# + Getting started +################# + +************** + First recipe +************** + +The simplest use case is to build an encoder-processor-decoder graph for +a global weather model. In this case, the recipe must contain a +``nodes`` section where the keys will be the names of the sets of +`nodes`, that will later be used to build the connections. Each `nodes` +configuration must include a ``node_builder`` section describing how to +generate the `nodes`, and it may include an optional ``attributes`` +section to define additional attributes (weights, mask, ...). + +.. literalinclude:: yaml/nodes.yaml + :language: yaml + +Once the `nodes` have been defined, you need to create the edges between +them through which information will flow. To this aim, the recipe file +must contain a ``edges`` section. These connections are defined between +pairs of `nodes` (source and target, specified by `source_name` and +`target_name`). + +There are several methods to build these edges such as cutoff +(`CutOffEdges`) or nearest neighbours (`KNNEdges`). For an +encoder-processor-decoder graph you will need to build two sets of +`edges`. The first set of edges will connect the `data` nodes with the +`hidden` nodes to encode the input data into the latent space, normally +referred to as the `encoder edges` and represented here by the first +element of the ``edges`` section. The second set of `edges` will connect +the `hidden` nodes with the `data` nodes to decode the latent space into +the output data, normally referred to as `decoder edges` and represented +here by the second element of the ``edges`` section. + +.. literalinclude:: yaml/global_wo-proc.yaml + :language: yaml + +.. figure:: schemas/global_wo-proc.png + :alt: Schema of global graph (without processor connections) + :align: center + +To create the graph, run the following command: + +.. code:: console + + $ anemoi-graphs create recipe.yaml graph.pt + +Once the build is complete, you can inspect the dataset using the +following command: + +.. code:: console + + $ anemoi-graphs inspect graph.pt + +This will generate the following graph: + +.. literalinclude:: yaml/global_wo-proc.txt + :language: console + +.. note:: + + Note that that the resulting graph will only work with a Transformer + processor because there are no connections between the `hidden + nodes`. + +****************************** + Adding processor connections +****************************** + +To add connections within the ``hidden`` nodes, to be used in the +processor, you need to add a new set of `edges` to the recipe file. +These connections are normally referred to as `processor edges` and are +represented here by the third element of the ``edges`` section. + +.. literalinclude:: yaml/global.yaml + :language: yaml + +.. figure:: schemas/global.png + :alt: Schema of global graph + :align: center + +This will generate the following graph: + +.. literalinclude:: yaml/global.txt + :language: console + +******************* + Adding attributes +******************* + +When training a data-driven weather model, it is common to add +attributes to the nodes or edges. For example, you may want to add node +attributes to weight the loss function, or add edge attributes to +represent the direction of the edges. + +To add attributes to the `nodes`, you must include the `attributes` +section in the `nodes` configuration. The attributes can be defined as a +list of dictionaries, where each dictionary contains the name of the +attribute and the type of the attribute. + +.. literalinclude:: yaml/nodes_with-attrs.yaml + :language: yaml + +To add the extra features to the edges of the graph, you need to set +them in the ``attributes`` section. + +.. literalinclude:: yaml/global_with-attrs.yaml + :language: yaml + +This will generate the following graph: + +.. literalinclude:: yaml/global_with-attrs.txt + :language: console diff --git a/docs/usage/schemas/global.excalidraw b/docs/usage/schemas/global.excalidraw new file mode 100644 index 0000000..eca62b2 --- /dev/null +++ b/docs/usage/schemas/global.excalidraw @@ -0,0 +1,479 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "diamond", + "version": 1574, + "versionNonce": 1952312215, + "index": "akG", + "isDeleted": false, + "id": "Znt9M8pxS0HpC9GiKp7f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -686.275390625, + "y": 8.697265625, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 189.40234375, + "height": 168.08203124999994, + "seed": 216165446, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "id": "9zgm6lmwdP5Lhkwjvggxr", + "type": "text" + }, + { + "id": "X87zkD0RgTBG4qEdx6-6a", + "type": "arrow" + } + ], + "updated": 1718866666194, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1514, + "versionNonce": 994799639, + "index": "akV", + "isDeleted": false, + "id": "9zgm6lmwdP5Lhkwjvggxr", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -620.644775390625, + "y": 67.7177734375, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 58.43994140625, + "height": 50, + "seed": 395899782, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718864334184, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Hidden\nnodes", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Znt9M8pxS0HpC9GiKp7f2", + "originalText": "Hidden\nnodes", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "diamond", + "version": 1473, + "versionNonce": 1197881228, + "index": "al", + "isDeleted": false, + "id": "Qo-9em1mLQnX3Epd04wAU", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -680.701171875, + "y": 298.513671875, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 189.40234375, + "height": 168.08203124999994, + "seed": 500381062, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "bjYlghPRGa9M149mWesMW" + }, + { + "id": "X87zkD0RgTBG4qEdx6-6a", + "type": "arrow" + } + ], + "updated": 1718794669385, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1430, + "versionNonce": 91521241, + "index": "am", + "isDeleted": false, + "id": "bjYlghPRGa9M149mWesMW", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -617.6405715942383, + "y": 357.5341796875, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 63.57997131347656, + "height": 50, + "seed": 2048824518, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718864334184, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Data \nnodes", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Qo-9em1mLQnX3Epd04wAU", + "originalText": "Data nodes", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "arrow", + "version": 265, + "versionNonce": 1943685428, + "index": "b0M", + "isDeleted": false, + "id": "X87zkD0RgTBG4qEdx6-6a", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0.014815383044963326, + "x": -634.25390625, + "y": 337.84375, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 64.01953125, + "height": 191.93359375, + "seed": 980493452, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1718794678935, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Qo-9em1mLQnX3Epd04wAU", + "focus": -0.22730420345522181, + "gap": 3.2174693078751844 + }, + "endBinding": { + "elementId": "Znt9M8pxS0HpC9GiKp7f2", + "focus": 0.06334851014220298, + "gap": 5.576512041887561 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -64.01953125, + -97.4453125 + ], + [ + -1.421875, + -191.93359375 + ] + ] + }, + { + "type": "arrow", + "version": 547, + "versionNonce": 1330253748, + "index": "b0N", + "isDeleted": false, + "id": "o1-27eSnLA7FWSCeI8_hf", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 3.1473660224451887, + "x": -480.7864185018202, + "y": 335.0726012960115, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 64.01953125, + "height": 191.93359375, + "seed": 69264820, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1718794699487, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -64.01953125, + -97.4453125 + ], + [ + -1.421875, + -191.93359375 + ] + ] + }, + { + "type": "text", + "version": 104, + "versionNonce": 447960375, + "index": "b0O", + "isDeleted": false, + "id": "w4_mrYxHlHgpbd01zu-1J", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -747.6875, + "y": 229.17578125, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 36.3828125, + "height": 25, + "seed": 778583092, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718864334184, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A)", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "A)", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "type": "text", + "version": 34, + "versionNonce": 482409913, + "index": "b0P", + "isDeleted": false, + "id": "8jI1ivvTgDU6LdGAtBLst", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -463.83203125, + "y": 219.03125, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 21.29998779296875, + "height": 25, + "seed": 335072396, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718864334184, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B)", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "B)", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "EGBSbXlhXBORmnsVFvEts", + "type": "arrow", + "x": -678.5792230220354, + "y": 13.811080021389312, + "width": 87.03339052302621, + "height": 48.768129202436036, + "angle": 5.148276390964529, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0Q", + "roundness": { + "type": 2 + }, + "seed": 1974783065, + "version": 1072, + "versionNonce": 1886357911, + "isDeleted": false, + "boundElements": null, + "updated": 1718866748897, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 39.15328310546727, + -48.768129202436036 + ], + [ + 87.03339052302621, + -1.8211131319121323 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null + }, + { + "type": "arrow", + "version": 1384, + "versionNonce": 918275127, + "index": "b0R", + "isDeleted": false, + "id": "uEVsPEcX9f8SeEA9Q79-X", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 1.194283386269774, + "x": -597.5325644796139, + "y": 13.995674352769438, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 87.03339052302621, + "height": 48.768129202436036, + "seed": 1101268537, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1718866755747, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 39.15328310546727, + -48.768129202436036 + ], + [ + 87.03339052302621, + -1.8211131319121323 + ] + ] + }, + { + "type": "text", + "version": 113, + "versionNonce": 1667036473, + "index": "b0S", + "isDeleted": false, + "id": "VQ-NFIzfci1S3-cTTm0_G", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -606.0249938964844, + "y": -82.9609375, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 19.639984130859375, + "height": 25, + "seed": 1604276025, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718866732266, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "C)", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "C)", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/docs/usage/schemas/global.png b/docs/usage/schemas/global.png new file mode 100644 index 0000000..2c2a429 Binary files /dev/null and b/docs/usage/schemas/global.png differ diff --git a/docs/usage/schemas/global_wo-proc.excalidraw b/docs/usage/schemas/global_wo-proc.excalidraw new file mode 100644 index 0000000..7b28c79 --- /dev/null +++ b/docs/usage/schemas/global_wo-proc.excalidraw @@ -0,0 +1,344 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "diamond", + "version": 1572, + "versionNonce": 1863238796, + "index": "akG", + "isDeleted": false, + "id": "Znt9M8pxS0HpC9GiKp7f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -686.275390625, + "y": 8.697265625, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 189.40234375, + "height": 168.08203124999994, + "seed": 216165446, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "id": "9zgm6lmwdP5Lhkwjvggxr", + "type": "text" + }, + { + "id": "X87zkD0RgTBG4qEdx6-6a", + "type": "arrow" + } + ], + "updated": 1718794669385, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1508, + "versionNonce": 358396172, + "index": "akV", + "isDeleted": false, + "id": "9zgm6lmwdP5Lhkwjvggxr", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -620.644775390625, + "y": 67.7177734375, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 58.43994140625, + "height": 50, + "seed": 395899782, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718794797984, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Hidden\nnodes", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Znt9M8pxS0HpC9GiKp7f2", + "originalText": "Hidden\nnodes", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "diamond", + "version": 1473, + "versionNonce": 1197881228, + "index": "al", + "isDeleted": false, + "id": "Qo-9em1mLQnX3Epd04wAU", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -680.701171875, + "y": 298.513671875, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 189.40234375, + "height": 168.08203124999994, + "seed": 500381062, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "bjYlghPRGa9M149mWesMW" + }, + { + "id": "X87zkD0RgTBG4qEdx6-6a", + "type": "arrow" + } + ], + "updated": 1718794669385, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1424, + "versionNonce": 13350324, + "index": "am", + "isDeleted": false, + "id": "bjYlghPRGa9M149mWesMW", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -617.6405715942383, + "y": 357.5341796875, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 63.57997131347656, + "height": 50, + "seed": 2048824518, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1718794797984, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Data \nnodes", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Qo-9em1mLQnX3Epd04wAU", + "originalText": "Data nodes", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "X87zkD0RgTBG4qEdx6-6a", + "type": "arrow", + "x": -634.25390625, + "y": 337.84375, + "width": 64.01953125, + "height": 191.93359375, + "angle": 0.014815383044963326, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0M", + "roundness": { + "type": 2 + }, + "seed": 980493452, + "version": 265, + "versionNonce": 1943685428, + "isDeleted": false, + "boundElements": null, + "updated": 1718794678935, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -64.01953125, + -97.4453125 + ], + [ + -1.421875, + -191.93359375 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "Qo-9em1mLQnX3Epd04wAU", + "focus": -0.22730420345522181, + "gap": 3.2174693078751844 + }, + "endBinding": { + "elementId": "Znt9M8pxS0HpC9GiKp7f2", + "focus": 0.06334851014220298, + "gap": 5.576512041887561 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "arrow", + "version": 547, + "versionNonce": 1330253748, + "index": "b0N", + "isDeleted": false, + "id": "o1-27eSnLA7FWSCeI8_hf", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 3.1473660224451887, + "x": -480.7864185018202, + "y": 335.0726012960115, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 64.01953125, + "height": 191.93359375, + "seed": 69264820, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1718794699487, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -64.01953125, + -97.4453125 + ], + [ + -1.421875, + -191.93359375 + ] + ] + }, + { + "id": "w4_mrYxHlHgpbd01zu-1J", + "type": "text", + "x": -747.6875, + "y": 229.17578125, + "width": 36.3828125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0O", + "roundness": null, + "seed": 778583092, + "version": 98, + "versionNonce": 841301132, + "isDeleted": false, + "boundElements": null, + "updated": 1718794797984, + "link": null, + "locked": false, + "text": "A)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "A)", + "autoResize": false, + "lineHeight": 1.25 + }, + { + "id": "8jI1ivvTgDU6LdGAtBLst", + "type": "text", + "x": -463.83203125, + "y": 219.03125, + "width": 21.29998779296875, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0P", + "roundness": null, + "seed": 335072396, + "version": 28, + "versionNonce": 1531159092, + "isDeleted": false, + "boundElements": null, + "updated": 1718794797984, + "link": null, + "locked": false, + "text": "B)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "B)", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/docs/usage/schemas/global_wo-proc.png b/docs/usage/schemas/global_wo-proc.png new file mode 100644 index 0000000..71ffdde Binary files /dev/null and b/docs/usage/schemas/global_wo-proc.png differ diff --git a/docs/usage/yaml/global.txt b/docs/usage/yaml/global.txt new file mode 100644 index 0000000..96da1e1 --- /dev/null +++ b/docs/usage/yaml/global.txt @@ -0,0 +1,21 @@ +┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈ +📦 Path : graph.pt +🔢 Format version: 0.0.1 + +💽 Size : 3.1 MiB (3,283,650) + + Nodes name │ Num. nodes | Attribute dim | Min. latitude | Max. latitude | Min. longitude | Max. longitude + ─────────────┼────────────┼───────────────┼───────────────┼───────────────┼────────────────┼──────────────── + data | 10,840 | 0 | -3.135 | 3.140 | 0.02 | 6.13 + hidden | 6,200 | 0 | -3.141 | 3.137 | 0.01 | 6.14 + ─────────────┴────────────┴───────────────┴───────────────┴───────────────┴────────────────┴──────────────── + + + Source │ Destination │ Num. edges │ Attribute dim | Min. length │ Max. length │ Mean length │ Std dev + ─────────────┼──────────────┼─────────────┼───────────────┼─────────────┼─────────────┼─────────────┼───────── + data │ hidden │ 13508 │ 1 | 0.3116 │ 25.79 │ 11.059531 │ 5.5856 + hidden │ data │ 40910 │ 1 | 0.2397 │ 21.851 │ 12.270924 │ 4.2347 + hidden │ hidden │ 32010 │ 1 | 0.2397 │ 21.851 │ 10.270924 │ 4.2347 + ─────────────┴──────────────┴─────────────┴───────────────|─────────────┴─────────────┴─────────────┴───────── +🔋 Graph ready, last update 7 seconds ago. +📊 Statistics ready. diff --git a/docs/usage/yaml/global.yaml b/docs/usage/yaml/global.yaml new file mode 100644 index 0000000..a0bc3bb --- /dev/null +++ b/docs/usage/yaml/global.yaml @@ -0,0 +1,23 @@ +nodes: + data: ... + hidden: ... + +edges: + # A) Encoder connections + - source_name: data + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.CutOffEdges + cutoff_factor: 0.7 + # B) Decoder connections + - source_name: hidden + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + nearest_neighbours: 3 + # C) Processor connections + - source_name: hidden + target_name: data + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + nearest_neighbours: 3 diff --git a/docs/usage/yaml/global_with-attrs.txt b/docs/usage/yaml/global_with-attrs.txt new file mode 100644 index 0000000..57d15a7 --- /dev/null +++ b/docs/usage/yaml/global_with-attrs.txt @@ -0,0 +1,21 @@ +┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈ +📦 Path : graph.pt +🔢 Format version: 0.0.1 + +💽 Size : 3.1 MiB (3,283,650) + + Nodes name │ Num. nodes | Attribute dim | Min. latitude | Max. latitude | Min. longitude | Max. longitude + ─────────────┼────────────┼───────────────┼───────────────┼───────────────┼────────────────┼──────────────── + data | 10,840 | 1 | -3.135 | 3.140 | 0.02 | 6.13 + hidden | 6,200 | 1 | -3.141 | 3.137 | 0.01 | 6.14 + ─────────────┴────────────┴───────────────┴───────────────┴───────────────┴────────────────┴──────────────── + + + Source │ Destination │ Num. edges │ Attribute dim | Min. length │ Max. length │ Mean length │ Std dev + ─────────────┼──────────────┼─────────────┼───────────────┼─────────────┼─────────────┼─────────────┼───────── + data │ hidden │ 13508 │ 3 | 0.3116 │ 25.79 │ 11.059531 │ 5.5856 + hidden │ data │ 40910 │ 3 | 0.2397 │ 21.851 │ 12.270924 │ 4.2347 + hidden │ hidden │ 32010 │ 3 | 0.2397 │ 21.851 │ 10.270924 │ 4.2347 + ─────────────┴──────────────┴─────────────┴───────────────|─────────────┴─────────────┴─────────────┴───────── +🔋 Graph ready, last update 7 seconds ago. +📊 Statistics ready. diff --git a/docs/usage/yaml/global_with-attrs.yaml b/docs/usage/yaml/global_with-attrs.yaml new file mode 100644 index 0000000..fc292d0 --- /dev/null +++ b/docs/usage/yaml/global_with-attrs.yaml @@ -0,0 +1,32 @@ +nodes: + data: ... + hidden: ... + +edges: + # A) Encoder connections + - source_name: data + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.CutOffEdges + cutoff_factor: 0.7 + attributes: + edge_length: + _target_: anemoi.graphs.edges.attributes.EdgeLength + # B) Decoder connections + - source_name: hidden + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + nearest_neighbours: 3 + attributes: + edge_length: + _target_: anemoi.graphs.edges.attributes.EdgeLength + # C) Processor connections + - source_name: hidden + target_name: data + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + nearest_neighbours: 3 + attributes: + edge_length: + _target_: anemoi.graphs.edges.attributes.EdgeLength diff --git a/docs/usage/yaml/global_wo-proc.png b/docs/usage/yaml/global_wo-proc.png new file mode 100644 index 0000000..71ffdde Binary files /dev/null and b/docs/usage/yaml/global_wo-proc.png differ diff --git a/docs/usage/yaml/global_wo-proc.txt b/docs/usage/yaml/global_wo-proc.txt new file mode 100644 index 0000000..c27dafc --- /dev/null +++ b/docs/usage/yaml/global_wo-proc.txt @@ -0,0 +1,20 @@ +┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈ +📦 Path : graph.pt +🔢 Format version: 0.0.1 + +💽 Size : 3.1 MiB (3,283,650) + + Nodes name │ Num. nodes | Attribute dim | Min. latitude | Max. latitude | Min. longitude | Max. longitude + ─────────────┼────────────┼───────────────┼───────────────┼───────────────┼────────────────┼──────────────── + data | 10,840 | 4 | -3.135 | 3.140 | 0.02 | 6.13 + hidden | 6,200 | 4 | -3.141 | 3.137 | 0.01 | 6.14 + ─────────────┴────────────┴───────────────┴───────────────┴───────────────┴────────────────┴──────────────── + + + Source │ Destination │ Num. edges │ Attribute dim | Min. length │ Max. length │ Mean length │ Std dev + ─────────────┼──────────────┼─────────────┼───────────────┼─────────────┼─────────────┼─────────────┼───────── + data │ hidden │ 13508 │ 1 | 0.3116 │ 25.79 │ 11.059531 │ 5.5856 + hidden │ data │ 40910 │ 1 | 0.2397 │ 21.851 │ 12.270924 │ 4.2347 + ─────────────┴──────────────┴─────────────┴───────────────|─────────────┴─────────────┴─────────────┴───────── +🔋 Graph ready, last update 17 seconds ago. +📊 Statistics ready. diff --git a/docs/usage/yaml/global_wo-proc.yaml b/docs/usage/yaml/global_wo-proc.yaml new file mode 100644 index 0000000..c1e3ad3 --- /dev/null +++ b/docs/usage/yaml/global_wo-proc.yaml @@ -0,0 +1,17 @@ +nodes: + data: ... + hidden: ... + +edges: + # A) Encoder connections + - source_name: data + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.CutOffEdges + cutoff_factor: 0.7 + # B) Decoder connections + - source_name: hidden + target_name: hidden + edge_builder: + _target_: anemoi.graphs.edges.KNNEdges + nearest_neighbours: 3 diff --git a/docs/usage/yaml/nodes.yaml b/docs/usage/yaml/nodes.yaml new file mode 100644 index 0000000..7c97851 --- /dev/null +++ b/docs/usage/yaml/nodes.yaml @@ -0,0 +1,10 @@ +nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.ZarrDatasetNodes + dataset: /path/to/dataset.zarr + hidden: + node_builder: + _target_: anemoi.graphs.nodes.NPZFileNodes + grid_definition_path: /path/to/grids/ + resolution: o48 diff --git a/docs/usage/yaml/nodes_with-attrs.yaml b/docs/usage/yaml/nodes_with-attrs.yaml new file mode 100644 index 0000000..44ded27 --- /dev/null +++ b/docs/usage/yaml/nodes_with-attrs.yaml @@ -0,0 +1,10 @@ +nodes: + data: + node_builder: + _target_: anemoi.graphs.nodes.ZarrDatasetNodes + dataset: /path/to/dataset.zarr + attributes: + weights: + _target_: anemoi.graphs.nodes.attributes.AreaWeights + norm: unit-max + hidden: ... diff --git a/src/anemoi/graphs/edges/attributes.py b/src/anemoi/graphs/edges/attributes.py index 3aff47a..ccae3fd 100644 --- a/src/anemoi/graphs/edges/attributes.py +++ b/src/anemoi/graphs/edges/attributes.py @@ -63,10 +63,8 @@ class EdgeDirection(BaseEdgeAttribute): Methods ------- - get_raw_values(graph, source_name, target_name) - Compute directions between nodes connected by edges. compute(graph, source_name, target_name) - Compute directional attributes. + Compute direction of all edges. """ def __init__(self, norm: str | None = None, luse_rotated_features: bool = True) -> None: @@ -109,8 +107,6 @@ class EdgeLength(BaseEdgeAttribute): Methods ------- - get_raw_values(graph, source_name, target_name) - Compute haversine distance between nodes connected by edges. compute(graph, source_name, target_name) Compute edge lengths attributes. """ diff --git a/src/anemoi/graphs/edges/builder.py b/src/anemoi/graphs/edges/builder.py index 5fbdb13..389faec 100644 --- a/src/anemoi/graphs/edges/builder.py +++ b/src/anemoi/graphs/edges/builder.py @@ -139,8 +139,6 @@ class KNNEdges(BaseEdgeBuilder): Methods ------- - get_adjacency_matrix(source_nodes, target_nodes) - Compute the adjacency matrix for the KNN method. register_edges(graph) Register the edges in the graph. register_attributes(graph, config) @@ -197,10 +195,6 @@ class CutOffEdges(BaseEdgeBuilder): Methods ------- - get_cutoff_radius(graph, mask_attr) - Compute the cut-off radius. - get_adjacency_matrix(source_nodes, target_nodes) - Get the adjacency matrix for the cut-off method. register_edges(graph) Register the edges in the graph. register_attributes(graph, config) @@ -218,7 +212,8 @@ def __init__(self, source_name: str, target_name: str, cutoff_factor: float): def get_cutoff_radius(self, graph: HeteroData, mask_attr: torch.Tensor | None = None): """Compute the cut-off radius. - The cut-off radius is computed as the product of the target nodes reference distance and the cut-off factor. + The cut-off radius is computed as the product of the target nodes + reference distance and the cut-off factor. Parameters ---------- @@ -267,7 +262,27 @@ def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStor class MultiScaleEdges(BaseEdgeBuilder): - """Base class for multi-scale edges in the nodes of a graph.""" + """Base class for multi-scale edges in the nodes of a graph. + + Attributes + ---------- + source_name : str + The name of the source nodes. + target_name : str + The name of the target nodes. + x_hops : int + Number of hops (in the refined icosahedron) between two nodes to connect + them with an edge. + + Methods + ------- + register_edges(graph) + Register the edges in the graph. + register_attributes(graph, config) + Register attributes in the edges of the graph. + update_graph(graph, attrs_config) + Update the graph with the edges. + """ def __init__(self, source_name: str, target_name: str, x_hops: int): super().__init__(source_name, target_name) diff --git a/src/anemoi/graphs/nodes/attributes.py b/src/anemoi/graphs/nodes/attributes.py index e0ed1d8..11009d5 100644 --- a/src/anemoi/graphs/nodes/attributes.py +++ b/src/anemoi/graphs/nodes/attributes.py @@ -55,7 +55,13 @@ def compute(self, graph: HeteroData, nodes_name: str, *args, **kwargs) -> torch. class UniformWeights(BaseWeights): - """Implements a uniform weight for the nodes.""" + """Implements a uniform weight for the nodes. + + Methods + ------- + compute(self, graph, nodes_name) + Compute the area attributes for each node. + """ def get_raw_values(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray: """Compute the weights. @@ -87,9 +93,7 @@ class AreaWeights(BaseWeights): Methods ------- - get_raw_values(nodes, *args, **kwargs) - Compute the area associated to each node. - compute(nodes, *args, **kwargs) + compute(self, graph, nodes_name) Compute the area attributes for each node. """ diff --git a/src/anemoi/graphs/nodes/builder.py b/src/anemoi/graphs/nodes/builder.py index 2afdfc8..54753c4 100644 --- a/src/anemoi/graphs/nodes/builder.py +++ b/src/anemoi/graphs/nodes/builder.py @@ -120,13 +120,11 @@ class ZarrDatasetNodes(BaseNodeBuilder): Methods ------- - get_coordinates() - Get the lat-lon coordinates of the nodes. - register_nodes(graph, name) + register_nodes(graph) Register the nodes in the graph. - register_attributes(graph, name, config) + register_attributes(graph, config) Register the attributes in the nodes of the graph specified. - update_graph(graph, name, attr_config) + update_graph(graph, attr_config) Update the graph with new nodes and attributes. """ @@ -160,13 +158,11 @@ class NPZFileNodes(BaseNodeBuilder): Methods ------- - get_coordinates() - Get the lat-lon coordinates of the nodes. - register_nodes(graph, name) + register_nodes(graph) Register the nodes in the graph. - register_attributes(graph, name, config) + register_attributes(graph, config) Register the attributes in the nodes of the graph specified. - update_graph(graph, name, attr_config) + update_graph(graph, attr_config) Update the graph with new nodes and attributes. """ @@ -233,14 +229,52 @@ def register_attributes(self, graph: HeteroData, config: DotDict) -> HeteroData: class TriNodes(IcosahedralNodes): - """It depends on the trimesh Python library.""" + """Nodes based on iterative refinements of an icosahedron. + + It depends on the trimesh Python library. + + Attributes + ---------- + resolutions : list[int] + Refinement level of the mesh. + name : str + The name of the nodes. + + Methods + ------- + register_nodes(graph) + Register the nodes in the graph. + register_attributes(graph, config) + Register the attributes in the nodes of the graph specified. + update_graph(graph, attr_config) + Update the graph with new nodes and attributes. + """ def create_nodes(self) -> np.ndarray: return create_icosahedral_nodes(resolutions=self.resolutions) class HexNodes(IcosahedralNodes): - """It depends on the h3 Python library.""" + """Nodes based on iterative refinements of an icosahedron. + + It depends on the h3 Python library. + + Attributes + ---------- + resolutions : list[int] + Refinement level of the mesh. + name : str + The name of the nodes. + + Methods + ------- + register_nodes(graph) + Register the nodes in the graph. + register_attributes(graph, config) + Register the attributes in the nodes of the graph specified. + update_graph(graph, attr_config) + Update the graph with new nodes and attributes. + """ def create_nodes(self) -> np.ndarray: return create_hexagonal_nodes(self.resolutions) @@ -260,8 +294,6 @@ class HEALPixNodes(BaseNodeBuilder): Methods ------- - get_coordinates() - Get the lat-lon coordinates of the nodes. register_nodes(graph, name) Register the nodes in the graph. register_attributes(graph, name, config) diff --git a/src/anemoi/graphs/nodes/weights.py b/src/anemoi/graphs/nodes/weights.py new file mode 100644 index 0000000..25419cc --- /dev/null +++ b/src/anemoi/graphs/nodes/weights.py @@ -0,0 +1,61 @@ +import logging +from abc import ABC +from abc import abstractmethod +from typing import Optional + +import numpy as np +import torch +from scipy.spatial import SphericalVoronoi +from torch_geometric.data.storage import NodeStorage + +from anemoi.graphs.generate.transforms import to_sphere_xyz +from anemoi.graphs.normalizer import NormalizerMixin + +logger = logging.getLogger(__name__) + + +class BaseWeights(ABC, NormalizerMixin): + """Base class for the weights of the nodes.""" + + def __init__(self, norm: Optional[str] = None): + self.norm = norm + + @abstractmethod + def compute(self, nodes: NodeStorage, *args, **kwargs): ... + + def get_weights(self, *args, **kwargs) -> torch.Tensor: + weights = self.compute(*args, **kwargs) + if weights.ndim == 1: + weights = weights[:, np.newaxis] + norm_weights = self.normalize(weights) + return torch.tensor(norm_weights, dtype=torch.float32) + + +class UniformWeights(BaseWeights): + """Implements a uniform weight for the nodes.""" + + def compute(self, nodes: NodeStorage) -> np.ndarray: + return np.ones(nodes.num_nodes) + + +class AreaWeights(BaseWeights): + """Implements the area of the nodes as the weights.""" + + def __init__(self, norm: str = "unit-max", radius: float = 1.0, centre: np.ndarray = np.array([0, 0, 0])): + super().__init__(norm=norm) + + # Weighting of the nodes + self.radius = radius + self.centre = centre + + def compute(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray: + latitudes, longitudes = nodes.x[:, 0], nodes.x[:, 1] + points = to_sphere_xyz((latitudes, longitudes)) + sv = SphericalVoronoi(points, self.radius, self.centre) + area_weights = sv.calculate_areas() + logger.debug( + "There are %d of weights, which (unscaled) add up a total weight of %.2f.", + len(area_weights), + np.array(area_weights).sum(), + ) + return area_weights