Merge pull request #1 from aynumosir/python-binding

Add Python binding

.editorconfig

@@ -0,0 +1,25 @@
+# EditorConfig helps developers define and maintain consistent coding styles between different editors and IDEs
+
+# Top-most EditorConfig file
+root = true
+
+# Python files
+[*.py]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.rs, *.js]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{yaml,json,toml}]
+indent_style = space
+indent_size = 2

.envrc

@@ -1 +1,3 @@
 use flake
+
+source .venv/bin/activate

.github/workflows/python.yaml

@@ -0,0 +1,126 @@
+# This file is autogenerated by maturin v1.5.0
+# To update, run
+#
+#    maturin generate-ci github -m ./ainu-utils-python/Cargo.toml -o ./.github/workflows/python.yaml --platform linux macos --pytest
+#
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+    tags:
+      - '*'
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [x86_64, x86, aarch64, armv7, s390x, ppc64le]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter --manifest-path ./ainu-utils-python/Cargo.toml
+          sccache: 'true'
+          manylinux: auto
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.target }}
+          path: dist
+      - name: pytest
+        if: ${{ startsWith(matrix.target, 'x86_64') }}
+        shell: bash
+        run: |
+          set -e
+          pip install ainu-utils --find-links dist --force-reinstall
+          pip install pytest
+          cd ./ainu-utils-python && pytest
+      - name: pytest
+        if: ${{ !startsWith(matrix.target, 'x86') && matrix.target != 'ppc64' }}
+        uses: uraimo/[email protected]
+        with:
+          arch: ${{ matrix.target }}
+          distro: ubuntu22.04
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update
+            apt-get install -y --no-install-recommends python3 python3-pip
+            pip3 install -U pip pytest
+          run: |
+            set -e
+            pip3 install ainu-utils --find-links dist --force-reinstall
+            cd ./ainu-utils-python && pytest
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter --manifest-path ./ainu-utils-python/Cargo.toml
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.target }}
+          path: dist
+      - name: pytest
+        if: ${{ !startsWith(matrix.target, 'aarch64') }}
+        shell: bash
+        run: |
+          set -e
+          pip install ainu-utils --find-links dist --force-reinstall
+          pip install pytest
+          cd ./ainu-utils-python && pytest
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist --manifest-path ./ainu-utils-python/Cargo.toml
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: dist
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [linux, macos, sdist]
+    steps:
+      - uses: actions/download-artifact@v4
+      - name: Publish to PyPI
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+        with:
+          command: upload
+          args: --non-interactive --skip-existing wheels-*/*

.github/workflows/rust.yaml

@@ -8,16 +8,18 @@ on:
   pull_request:
     branches:
       - "*"
+    paths: 
+      - "ainu-utils/**"
+
+defaults:
+  run:
+    working-directory: ainu-utils
 
 # https://github.com/taiki-e/cargo-llvm-cov/tree/main?tab=readme-ov-file#continuous-integration
 jobs:
   test:
     runs-on: ubuntu-latest
 
-    defaults:
-      run:
-        working-directory: ainu-utils
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/rust_release.yaml

@@ -8,16 +8,15 @@ on:
   push:
     tags:
       - v*
-  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: ainu-utils
 
 jobs:
   publish:
     runs-on: ubuntu-latest
 
-    defaults:
-      run:
-        working-directory: ainu-utils
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.gitignore

@@ -12,3 +12,5 @@ Cargo.lock
 
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
+
+.venv

.vscode/settings.json

@@ -0,0 +1,23 @@
+{
+  "cSpell.enabled": true,
+
+  "files.exclude": {
+    "**/.venv": true,
+    "**/.mypy_cache": true,
+    "**/.ruff_cache": true,
+    "**/.pytest_cache": true,
+    "**/*.egg-info": true,
+    "**/__pycache__": true
+  },
+
+  // https://github.com/astral-sh/ruff-vscode?tab=readme-ov-file#configuring-vs-code
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.codeActionsOnSave": {
+      "source.fixAll": "always"
+    }
+  },
+
+  // https://github.com/microsoft/vscode-mypy/issues/157
+  "mypy-type-checker.reportingScope": "workspace"
+}

ainu-utils-python/Cargo.toml

@@ -4,9 +4,10 @@ version = "0.1.0"
 edition = "2021"
 
 [lib]
-name = "ainu_utils_python"
+name = "ainu_utils"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = "0.19.0"
+pyo3 = "0.20.0"
 ainu-utils = { path = "../ainu-utils" }
+

ainu-utils-python/ainu_utils.pyi

@@ -0,0 +1,4 @@
+# https://www.maturin.rs/project_layout#adding-python-type-information
+def segment(text: str) -> list[str]: ...
+
+test_number: int

ainu-utils-python/pyproject.toml

@@ -1,11 +1,22 @@
-[project]
-name = "ainu-utils-python"
-requires-python = ">=3.8"
-dynamic = ["version"]
-
 [build-system]
 requires = ["maturin>=1.4,<2.0"]
 build-backend = "maturin"
 
 [tool.maturin]
 features = ["pyo3/extension-module"]
+
+[project]
+name = "ainu-utils"
+description = "A collection of utility for with the Ainu language"
+requires-python = ">=3.8"
+version = "0.1.0"
+license = "MIT"
+
+[project.optional-dependencies]
+test = [
+	"pytest==8.1.1",
+]
+dev = [
+	"ruff==0.3.3",
+	"mypy==1.9.0",
+]

ainu-utils-python/src/lib.rs

@@ -1,14 +1,15 @@
-extern crate ainu_utils as ainu;
+extern crate ainu_utils as ainu_utils_rust;
 
 use pyo3::prelude::*;
 
 #[pyfunction]
-fn tokenize(text: &str) -> Vec<String> {
-    ainu::tokenizer::tokenize(text)
+fn segment(text: &str) -> Vec<String> {
+    ainu_utils_rust::segmenter::segment(text)
 }
 
 #[pymodule]
 fn ainu_utils(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(tokenize, m)?)?;
+    m.add_function(wrap_pyfunction!(segment, m)?)?;
+    m.add("test_number", 123)?;
     Ok(())
 }

ainu-utils-python/tests/test_tokenize.py

@@ -0,0 +1,6 @@
+import ainu_utils
+
+
+def test_tokenize():
+    result = ainu_utils.segment("irankarapte. e=iwanke ya?")
+    assert result == ["irankarapte", ".", "e=", "iwanke", "ya", "?"]

ainu-utils/examples/tokenize.rs

@@ -1,11 +1,11 @@
-use ainu_utils::tokenizer::tokenize;
+use ainu_utils::segmenter::segment;
 use std::env;
 
 fn main() {
     let args: Vec<String> = env::args().collect();
     let text = &args[1];
 
-    let tokens = tokenize(text);
+    let tokens = segment(text);
 
     println!("{:?}", tokens);
 }

ainu-utils/src/lib.rs

@@ -1,2 +1,2 @@
 pub mod number;
-pub mod tokenizer;
+pub mod segmenter;