Skip to content

Commit b7c4e2f

Browse files
neetneet
authored
Merge pull request #1 from aynumosir/python-binding
Add Python binding
2 parents 1d4794e + aadde38 commit b7c4e2f

File tree

16 files changed

+250
-47
lines changed

16 files changed

+250
-47
lines changed

‎.editorconfig

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# EditorConfig helps developers define and maintain consistent coding styles between different editors and IDEs
2+
3+
# Top-most EditorConfig file
4+
root = true
5+
6+
# Python files
7+
[*.py]
8+
indent_style = space
9+
indent_size = 4
10+
end_of_line = lf
11+
charset = utf-8
12+
trim_trailing_whitespace = true
13+
insert_final_newline = true
14+
15+
[*.rs, *.js]
16+
indent_style = space
17+
indent_size = 2
18+
end_of_line = lf
19+
charset = utf-8
20+
trim_trailing_whitespace = true
21+
insert_final_newline = true
22+
23+
[*.{yaml,json,toml}]
24+
indent_style = space
25+
indent_size = 2

‎.envrc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
use flake
2+
3+
source .venv/bin/activate

‎.github/workflows/python.yaml

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# This file is autogenerated by maturin v1.5.0
2+
# To update, run
3+
#
4+
# maturin generate-ci github -m ./ainu-utils-python/Cargo.toml -o ./.github/workflows/python.yaml --platform linux macos --pytest
5+
#
6+
name: CI
7+
8+
on:
9+
push:
10+
branches:
11+
- main
12+
- master
13+
tags:
14+
- '*'
15+
pull_request:
16+
workflow_dispatch:
17+
18+
permissions:
19+
contents: read
20+
21+
jobs:
22+
linux:
23+
runs-on: ubuntu-latest
24+
strategy:
25+
matrix:
26+
target: [x86_64, x86, aarch64, armv7, s390x, ppc64le]
27+
steps:
28+
- uses: actions/checkout@v4
29+
- uses: actions/setup-python@v5
30+
with:
31+
python-version: '3.10'
32+
- name: Build wheels
33+
uses: PyO3/maturin-action@v1
34+
with:
35+
target: ${{ matrix.target }}
36+
args: --release --out dist --find-interpreter --manifest-path ./ainu-utils-python/Cargo.toml
37+
sccache: 'true'
38+
manylinux: auto
39+
- name: Upload wheels
40+
uses: actions/upload-artifact@v4
41+
with:
42+
name: wheels-linux-${{ matrix.target }}
43+
path: dist
44+
- name: pytest
45+
if: ${{ startsWith(matrix.target, 'x86_64') }}
46+
shell: bash
47+
run: |
48+
set -e
49+
pip install ainu-utils --find-links dist --force-reinstall
50+
pip install pytest
51+
cd ./ainu-utils-python && pytest
52+
- name: pytest
53+
if: ${{ !startsWith(matrix.target, 'x86') && matrix.target != 'ppc64' }}
54+
uses: uraimo/[email protected]
55+
with:
56+
arch: ${{ matrix.target }}
57+
distro: ubuntu22.04
58+
githubToken: ${{ github.token }}
59+
install: |
60+
apt-get update
61+
apt-get install -y --no-install-recommends python3 python3-pip
62+
pip3 install -U pip pytest
63+
run: |
64+
set -e
65+
pip3 install ainu-utils --find-links dist --force-reinstall
66+
cd ./ainu-utils-python && pytest
67+
68+
macos:
69+
runs-on: macos-latest
70+
strategy:
71+
matrix:
72+
target: [x86_64, aarch64]
73+
steps:
74+
- uses: actions/checkout@v4
75+
- uses: actions/setup-python@v5
76+
with:
77+
python-version: '3.10'
78+
- name: Build wheels
79+
uses: PyO3/maturin-action@v1
80+
with:
81+
target: ${{ matrix.target }}
82+
args: --release --out dist --find-interpreter --manifest-path ./ainu-utils-python/Cargo.toml
83+
sccache: 'true'
84+
- name: Upload wheels
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: wheels-macos-${{ matrix.target }}
88+
path: dist
89+
- name: pytest
90+
if: ${{ !startsWith(matrix.target, 'aarch64') }}
91+
shell: bash
92+
run: |
93+
set -e
94+
pip install ainu-utils --find-links dist --force-reinstall
95+
pip install pytest
96+
cd ./ainu-utils-python && pytest
97+
98+
sdist:
99+
runs-on: ubuntu-latest
100+
steps:
101+
- uses: actions/checkout@v4
102+
- name: Build sdist
103+
uses: PyO3/maturin-action@v1
104+
with:
105+
command: sdist
106+
args: --out dist --manifest-path ./ainu-utils-python/Cargo.toml
107+
- name: Upload sdist
108+
uses: actions/upload-artifact@v4
109+
with:
110+
name: wheels-sdist
111+
path: dist
112+
113+
release:
114+
name: Release
115+
runs-on: ubuntu-latest
116+
if: "startsWith(github.ref, 'refs/tags/')"
117+
needs: [linux, macos, sdist]
118+
steps:
119+
- uses: actions/download-artifact@v4
120+
- name: Publish to PyPI
121+
uses: PyO3/maturin-action@v1
122+
env:
123+
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
124+
with:
125+
command: upload
126+
args: --non-interactive --skip-existing wheels-*/*

‎.github/workflows/rust.yaml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,18 @@ on:
88
pull_request:
99
branches:
1010
- "*"
11+
paths:
12+
- "ainu-utils/**"
13+
14+
defaults:
15+
run:
16+
working-directory: ainu-utils
1117

1218
# https://github.com/taiki-e/cargo-llvm-cov/tree/main?tab=readme-ov-file#continuous-integration
1319
jobs:
1420
test:
1521
runs-on: ubuntu-latest
1622

17-
defaults:
18-
run:
19-
working-directory: ainu-utils
20-
2123
steps:
2224
- name: Checkout
2325
uses: actions/checkout@v4

‎.github/workflows/rust_release.yaml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,15 @@ on:
88
push:
99
tags:
1010
- v*
11-
workflow_dispatch:
11+
12+
defaults:
13+
run:
14+
working-directory: ainu-utils
1215

1316
jobs:
1417
publish:
1518
runs-on: ubuntu-latest
1619

17-
defaults:
18-
run:
19-
working-directory: ainu-utils
20-
2120
steps:
2221
- name: Checkout repository
2322
uses: actions/checkout@v4

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ Cargo.lock
1212

1313
# MSVC Windows builds of rustc generate these, which store debugging information
1414
*.pdb
15+
16+
.venv

‎.vscode/settings.json

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"cSpell.enabled": true,
3+
4+
"files.exclude": {
5+
"**/.venv": true,
6+
"**/.mypy_cache": true,
7+
"**/.ruff_cache": true,
8+
"**/.pytest_cache": true,
9+
"**/*.egg-info": true,
10+
"**/__pycache__": true
11+
},
12+
13+
// https://github.com/astral-sh/ruff-vscode?tab=readme-ov-file#configuring-vs-code
14+
"[python]": {
15+
"editor.defaultFormatter": "charliermarsh.ruff",
16+
"editor.codeActionsOnSave": {
17+
"source.fixAll": "always"
18+
}
19+
},
20+
21+
// https://github.com/microsoft/vscode-mypy/issues/157
22+
"mypy-type-checker.reportingScope": "workspace"
23+
}

‎ainu-utils-python/Cargo.toml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ version = "0.1.0"
44
edition = "2021"
55

66
[lib]
7-
name = "ainu_utils_python"
7+
name = "ainu_utils"
88
crate-type = ["cdylib"]
99

1010
[dependencies]
11-
pyo3 = "0.19.0"
11+
pyo3 = "0.20.0"
1212
ainu-utils = { path = "../ainu-utils" }
13+

‎ainu-utils-python/ainu_utils.pyi

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# https://www.maturin.rs/project_layout#adding-python-type-information
2+
def segment(text: str) -> list[str]: ...
3+
4+
test_number: int

‎ainu-utils-python/pyproject.toml

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,22 @@
1-
[project]
2-
name = "ainu-utils-python"
3-
requires-python = ">=3.8"
4-
dynamic = ["version"]
5-
61
[build-system]
72
requires = ["maturin>=1.4,<2.0"]
83
build-backend = "maturin"
94

105
[tool.maturin]
116
features = ["pyo3/extension-module"]
7+
8+
[project]
9+
name = "ainu-utils"
10+
description = "A collection of utility for with the Ainu language"
11+
requires-python = ">=3.8"
12+
version = "0.1.0"
13+
license = "MIT"
14+
15+
[project.optional-dependencies]
16+
test = [
17+
"pytest==8.1.1",
18+
]
19+
dev = [
20+
"ruff==0.3.3",
21+
"mypy==1.9.0",
22+
]

‎ainu-utils-python/src/lib.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1-
extern crate ainu_utils as ainu;
1+
extern crate ainu_utils as ainu_utils_rust;
22

33
use pyo3::prelude::*;
44

55
#[pyfunction]
6-
fn tokenize(text: &str) -> Vec<String> {
7-
ainu::tokenizer::tokenize(text)
6+
fn segment(text: &str) -> Vec<String> {
7+
ainu_utils_rust::segmenter::segment(text)
88
}
99

1010
#[pymodule]
1111
fn ainu_utils(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
12-
m.add_function(wrap_pyfunction!(tokenize, m)?)?;
12+
m.add_function(wrap_pyfunction!(segment, m)?)?;
13+
m.add("test_number", 123)?;
1314
Ok(())
1415
}

‎ainu-utils-python/tests/test_tokenize.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import ainu_utils
2+
3+
4+
def test_tokenize():
5+
result = ainu_utils.segment("irankarapte. e=iwanke ya?")
6+
assert result == ["irankarapte", ".", "e=", "iwanke", "ya", "?"]

‎ainu-utils/examples/tokenize.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
use ainu_utils::tokenizer::tokenize;
1+
use ainu_utils::segmenter::segment;
22
use std::env;
33

44
fn main() {
55
let args: Vec<String> = env::args().collect();
66
let text = &args[1];
77

8-
let tokens = tokenize(text);
8+
let tokens = segment(text);
99

1010
println!("{:?}", tokens);
1111
}

‎ainu-utils/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
pub mod number;
2-
pub mod tokenizer;
2+
pub mod segmenter;

0 commit comments

Comments
 (0)