Skip to content

Commit 8735ff5

Browse files
baek2smjijoongmoon
baek2sm
authored and
jijoongmoon
committed
onnx interpreter for llama
The execution result of the example app is as follows (llama 1b): ``` ================================================================================ Layer name Layer type Output dimension Input layer ================================================================================ onnx__add_6 input 1:1:1:1 -------------------------------------------------------------------------------- onnx__add_6/generat multiout 1:1:1:1 onnx__add_6 -------------------------------------------------------------------------------- sin input 1:1:1:64 -------------------------------------------------------------------------------- sin/generated_out_0 multiout 1:1:1:64 sin -------------------------------------------------------------------------------- ... -------------------------------------------------------------------------------- model_norm_cast_1 cast 1:1:1:2048 model_norm_mul -------------------------------------------------------------------------------- model_norm_mul_1 multiply 1:1:1:2048 model_norm_weight model_norm_cast_1 -------------------------------------------------------------------------------- lm_head_matmul matmul 1:1:1:50304 model_norm_mul_1 onnx__matmul_3531 ================================================================================ ``` **Self evaluation:** 1. Build test: [*]Passed [ ]Failed [ ]Skipped 2. Run test: [*]Passed [ ]Failed [ ]Skipped Signed-off-by: Seungbaek Hong <[email protected]>
1 parent 7a28b42 commit 8735ff5

21 files changed

+661
-142
lines changed

‎Applications/ONNX/jni/add_example.onnx‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎Applications/ONNX/jni/attention_example.onnx‎

-2.93 KB
Binary file not shown.

‎Applications/ONNX/jni/llama_example.onnx‎

322 KB
Binary file not shown.

‎Applications/ONNX/jni/main.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,7 @@ int main() {
2121
auto model = ml::train::createModel();
2222

2323
try {
24-
std::string path =
25-
"../../../../Applications/ONNX/jni/attention_example.onnx";
24+
std::string path = "../../../../Applications/ONNX/jni/llama_example.onnx";
2625
model->load(path, ml::train::ModelFormat::MODEL_FORMAT_ONNX);
2726
} catch (const std::exception &e) {
2827
std::cerr << "Error during load: " << e.what() << "\n";

‎api/ccapi/include/layer.h‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,10 @@ enum LayerType {
4949
LAYER_TANGENT = ML_TRAIN_LAYER_TYPE_TANGENT, /**< Tangent Layer type */
5050
LAYER_MATMUL =
5151
ML_TRAIN_LAYER_TYPE_MATMUL, /**< Matrix multiplication Layer type */
52-
LAYER_CAST = ML_TRAIN_LAYER_TYPE_CAST, /**< Cast Layer type */
53-
LAYER_GATHER = ML_TRAIN_LAYER_TYPE_GATHER, /**< Gather Layer type */
54-
LAYER_SLICE = ML_TRAIN_LAYER_TYPE_SLICE, /**< Slice Layer type */
55-
LAYER_NEGATIVE = ML_TRAIN_LAYER_TYPE_NEGATIVE, /**< Negative Layer type */
52+
LAYER_CAST = ML_TRAIN_LAYER_TYPE_CAST, /**< Cast Layer type */
53+
LAYER_GATHER = ML_TRAIN_LAYER_TYPE_GATHER, /**< Gather Layer type */
54+
LAYER_SLICE = ML_TRAIN_LAYER_TYPE_SLICE, /**< Slice Layer type */
55+
LAYER_NEG = ML_TRAIN_LAYER_TYPE_NEG, /**< Negative Layer type */
5656
LAYER_FC = ML_TRAIN_LAYER_TYPE_FC, /**< Fully Connected Layer type */
5757
LAYER_SWIGLU = ML_TRAIN_LAYER_TYPE_SWIGLU, /**< Swiglu Layer type */
5858
LAYER_BN = ML_TRAIN_LAYER_TYPE_BN, /**< Batch Normalization Layer type */
@@ -418,14 +418,6 @@ MatMulLayer(const std::vector<std::string> &properties = {}) {
418418
return createLayer(LayerType::LAYER_MATMUL, properties);
419419
}
420420

421-
/**
422-
* @brief Helper function to create neg layer
423-
*/
424-
inline std::unique_ptr<Layer>
425-
NegLayer(const std::vector<std::string> &properties = {}) {
426-
return createLayer(LayerType::LAYER_NEG, properties);
427-
}
428-
429421
/**
430422
* @brief Helper function to create cast layer
431423
*/
@@ -455,7 +447,7 @@ SliceLayer(const std::vector<std::string> &properties = {}) {
455447
*/
456448
inline std::unique_ptr<Layer>
457449
NegativeLayer(const std::vector<std::string> &properties = {}) {
458-
return createLayer(LayerType::LAYER_NEGATIVE, properties);
450+
return createLayer(LayerType::LAYER_NEG, properties);
459451
}
460452

461453
/**

‎api/nntrainer-api-common.h‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -74,20 +74,20 @@ typedef enum {
7474
ML_TRAIN_LAYER_TYPE_TRANSPOSE = 36, /**< Transpose Layer type */
7575
ML_TRAIN_LAYER_TYPE_CONV2D_TRANSPOSE =
7676
37, /**< Convolution 2D Transpose Layer (Since 9.0) */
77-
ML_TRAIN_LAYER_TYPE_POW = 38, /**< Pow Layer type (Since 9.0)*/
78-
ML_TRAIN_LAYER_TYPE_TENSOR = 39, /**< Tensor Layer type (Since 9.0)*/
79-
ML_TRAIN_LAYER_TYPE_SQRT = 40, /**< SQRT Layer type (Since 9.0)*/
80-
ML_TRAIN_LAYER_TYPE_SINE = 41, /**< Sine Layer type (Since 9.0)*/
81-
ML_TRAIN_LAYER_TYPE_COSINE = 42, /**< Cosine Layer type (Since 9.0)*/
82-
ML_TRAIN_LAYER_TYPE_TANGENT = 43, /**< Tangent Layer type (Since 9.0)*/
83-
ML_TRAIN_LAYER_TYPE_MATMUL = 44, /**< MatMul Layer type (Since 9.0)*/
84-
ML_TRAIN_LAYER_TYPE_CAST = 45, /**< Cast Layer type (Since 9.0)*/
85-
ML_TRAIN_LAYER_TYPE_GATHER = 46, /**< Gather Layer type (Since 9.0)*/
86-
ML_TRAIN_LAYER_TYPE_SLICE = 47, /**< SQRT Layer type (Since 9.0)*/
87-
ML_TRAIN_LAYER_TYPE_NEGATIVE = 48, /**< Negative Layer type (Since 9.0)*/
88-
ML_TRAIN_LAYER_TYPE_REDUCE_SUM = 49, /**< ReduceSum Layer type (Since 9.0) */
77+
ML_TRAIN_LAYER_TYPE_POW = 38, /**< Pow Layer type (Since 9.0)*/
78+
ML_TRAIN_LAYER_TYPE_TENSOR = 39, /**< Tensor Layer type (Since 9.0)*/
79+
ML_TRAIN_LAYER_TYPE_SQRT = 40, /**< SQRT Layer type (Since 9.0)*/
80+
ML_TRAIN_LAYER_TYPE_SINE = 41, /**< Sine Layer type (Since 9.0)*/
81+
ML_TRAIN_LAYER_TYPE_COSINE = 42, /**< Cosine Layer type (Since 9.0)*/
82+
ML_TRAIN_LAYER_TYPE_TANGENT = 43, /**< Tangent Layer type (Since 9.0)*/
83+
ML_TRAIN_LAYER_TYPE_MATMUL = 44, /**< MatMul Layer type (Since 9.0)*/
8984
ML_TRAIN_LAYER_TYPE_CHANNEL_SHUFFLE =
90-
50, /**< Channel Shuffle Layer type (Since 9.0)*/
85+
45, /**< Channel Shuffle Layer type (Since 9.0)*/
86+
ML_TRAIN_LAYER_TYPE_NEG = 46, /**< Negative Layer type (Since 9.0)*/
87+
ML_TRAIN_LAYER_TYPE_GATHER = 47, /**< Gather Layer type (Since 9.0)*/
88+
ML_TRAIN_LAYER_TYPE_REDUCE_SUM = 48, /**< ReduceSum Layer type (Since 9.0) */
89+
ML_TRAIN_LAYER_TYPE_SLICE = 49, /**< SQRT Layer type (Since 9.0)*/
90+
ML_TRAIN_LAYER_TYPE_CAST = 50, /**< Cast Layer type (Since 9.0)*/
9191
ML_TRAIN_LAYER_TYPE_REDUCE_MEAN =
9292
51, /**< ReduceMean Layer type (Since 10.0) */
9393
ML_TRAIN_LAYER_TYPE_RESHAPE = 52, /**< Reshape Layer type (Since 10.0) */

‎docs/how-to-run-onnx-model.md‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ int main() {
7777
auto model = ml::train::createModel();
7878

7979
try {
80-
std::string path = "../../../../Applications/ONNX/jni/add_example.onnx";
80+
std::string path = "../../../../Applications/ONNX/jni/llama_example.onnx";
8181
model->load(path, ml::train::ModelFormat::MODEL_FORMAT_ONNX);
8282
} catch (const std::exception &e) {
8383
std::cerr << "Error during load: " << e.what() << "\n";
@@ -113,7 +113,7 @@ cd build/Applications/ONNX/jni
113113

114114
It will:
115115

116-
- Load the provided ONNX model (`add_example.onnx`)
116+
- Load the provided ONNX model (`llama_example.onnx`)
117117
- Compile and initialize the model
118118
- Print a summary of the model structure
119119

@@ -123,12 +123,25 @@ Then the output should look like this:
123123
================================================================================
124124
Layer name Layer type Output dimension Input layer
125125
================================================================================
126-
input input 1:1:1:2
126+
onnx__add_6 input 1:1:1:1
127127
--------------------------------------------------------------------------------
128-
bias weight 1:1:1:2
128+
onnx__add_6/generat multiout 1:1:1:1 onnx__add_6
129129
--------------------------------------------------------------------------------
130-
add add 1:1:1:2 input
131-
bias
130+
sin input 1:1:1:64
131+
--------------------------------------------------------------------------------
132+
sin/generated_out_0 multiout 1:1:1:64 sin
133+
--------------------------------------------------------------------------------
134+
135+
...
136+
137+
--------------------------------------------------------------------------------
138+
model_norm_cast_1 cast 1:1:1:2048 model_norm_mul
139+
--------------------------------------------------------------------------------
140+
model_norm_mul_1 multiply 1:1:1:2048 model_norm_weight
141+
model_norm_cast_1
142+
--------------------------------------------------------------------------------
143+
lm_head_matmul matmul 1:1:1:50304 model_norm_mul_1
144+
onnx__matmul_3531
132145
================================================================================
133146
```
134147

‎nntrainer/app_context.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
* manages the global configuration of the current environment
99
* @see https://github.com/nnstreamer/nntrainer
1010
* @author Jihoon Lee <[email protected]>
11-
* @bug No known bugs except for NYI items
11+
* @bug No known bugs except for NYI items
1212
*
1313
*/
1414

@@ -304,7 +304,13 @@ void AppContext::add_default_object() {
304304
LayerType::LAYER_TANGENT);
305305
registerFactory(nntrainer::createLayer<MatMulLayer>, MatMulLayer::type,
306306
LayerType::LAYER_MATMUL);
307-
registerFactory(nntrainer::createLayer<NegLayer>, NegLayer::type,
307+
registerFactory(nntrainer::createLayer<CastLayer>, CastLayer::type,
308+
LayerType::LAYER_CAST);
309+
registerFactory(nntrainer::createLayer<GatherLayer>, GatherLayer::type,
310+
LayerType::LAYER_GATHER);
311+
registerFactory(nntrainer::createLayer<SliceLayer>, SliceLayer::type,
312+
LayerType::LAYER_SLICE);
313+
registerFactory(nntrainer::createLayer<NegativeLayer>, NegativeLayer::type,
308314
LayerType::LAYER_NEG);
309315
registerFactory(nntrainer::createLayer<FullyConnectedLayer>,
310316
FullyConnectedLayer::type, LayerType::LAYER_FC);

0 commit comments

Comments
 (0)