A simple example of using ggml-backend and ggml-alloc

[FSSRepo]

This only multiplies 2 2x2 matrices. Remember that ggml_mul_mat internally transposes B matrix.

$$ \begin{bmatrix} 2 & 8 \\ 5 & 1 \\ \end{bmatrix} \times \begin{bmatrix} 10 & 9 \\ 5 & 9 \\ \end{bmatrix} = \begin{bmatrix} 92 & 59 \\ 82 & 34 \\ \end{bmatrix} $$

This serves only the purpose of teaching others how to use the backends and ggml-alloc in a simplified manner, rather than a complex and confusing example.

#include "ggml.h"
#include "ggml/ggml-alloc.h"
#include "ggml/ggml-backend.h"

// #define GGML_USE_CUBLAS uncomment this to use cuda backend, make sure build ggml lib with GGML_CUBLAS=ON

#ifdef GGML_USE_CUBLAS
#include "ggml-cuda.h"
#endif

#ifdef GGML_USE_METAL
#include "ggml-metal.h"
#endif

#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>

struct test_model {
    struct ggml_tensor * a;
    struct ggml_tensor * b;
    ggml_backend_t backend = NULL;
    ggml_backend_buffer_t buffer;
    struct ggml_context * ctx;
};

void load_model(test_model & model, float* a, float* b, bool use_gpu = false) {
    size_t buffer_size = 0;
    {
        buffer_size += 2 * 2 * ggml_type_sizef(GGML_TYPE_F32); // tensor a
        buffer_size += 2 * 2 * ggml_type_sizef(GGML_TYPE_F32); // tensor b
        buffer_size += 512; // overhead
    }

    printf("%s: ggml tensor size    = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
    printf("%s: backend buffer size = %d bytes\n", __func__, (int) buffer_size);

    int num_tensors = 2;
    struct ggml_init_params params {
            /*.mem_size   =*/ ggml_tensor_overhead() * num_tensors,
            /*.mem_buffer =*/ NULL,
            /*.no_alloc   =*/ true,
    };

    // initialize the backend
#ifdef GGML_USE_CUBLAS
    if (use_gpu) {
        fprintf(stderr, "%s: using CUDA backend\n", __func__);
        model.backend = ggml_backend_cuda_init();
        if (!model.backend) {
            fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
        }
    }
#endif

#ifdef GGML_USE_METAL
    if (use_gpu) {
        fprintf(stderr, "%s: using Metal backend\n", __func__);
        ggml_metal_log_set_callback(ggml_log_callback_default, nullptr);
        model.backend = ggml_backend_metal_init();
        if (!model.backend) {
            fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
        }
    }
#endif

    if(!model.backend) {
        // fallback to CPU backend
        model.backend = ggml_backend_cpu_init();
    }

    model.buffer = ggml_backend_alloc_buffer(model.backend, buffer_size);

    // create context
    model.ctx = ggml_init(params);

    // create tensors
    model.a = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, 2, 2);
    model.b = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, 2, 2);

    // create a allocator
    ggml_allocr * alloc = ggml_allocr_new_from_buffer(model.buffer);

    // alloc memory
    ggml_allocr_alloc(alloc, model.a);

    // load data to buffer
    if(ggml_backend_is_cpu(model.backend)
#ifdef GGML_USE_METAL
                || ggml_backend_is_metal(model.backend)
#endif
    ) {
        memcpy(model.a->data, a, ggml_nbytes(model.a));
    } else {
        ggml_backend_tensor_set(model.a, a, 0, ggml_nbytes(model.a)); // cuda requires copy the data directly to device
    }

    // alloc memory
    ggml_allocr_alloc(alloc, model.b);

    if(ggml_backend_is_cpu(model.backend)
#ifdef GGML_USE_METAL
                || ggml_backend_is_metal(model.backend)
#endif
    ) {
        memcpy(model.b->data, b, ggml_nbytes(model.b));
    } else {
        ggml_backend_tensor_set(model.b, b, 0, ggml_nbytes(model.b));  // cuda requires copy the data directly to device
    }

    ggml_allocr_free(alloc);
}

struct ggml_cgraph * build_graph(const test_model& model, struct ggml_allocr * allocr) {
    static size_t buf_size = ggml_tensor_overhead()*GGML_MAX_NODES + ggml_graph_overhead();
    static std::vector<uint8_t> buf(buf_size);

    struct ggml_init_params params0 = {
        /*.mem_size   =*/ buf_size,
        /*.mem_buffer =*/ buf.data(),
        /*.no_alloc   =*/ true, // the tensors will be allocated later by ggml_allocr_alloc_graph()
    };

    // create a temporally context to build the graph
    struct ggml_context * ctx0 = ggml_init(params0);

    struct ggml_cgraph  * gf = ggml_new_graph(ctx0);

    struct ggml_tensor* result = ggml_mul_mat(ctx0, model.a, model.b);

    ggml_build_forward_expand(gf, result);

    // delete the temporally context used to build the graph
    ggml_free(ctx0);
    return gf;
}

struct ggml_tensor* compute(const test_model & model, struct ggml_allocr * allocr) {
    // reset the allocator to free all the memory allocated during the previous inference
    ggml_allocr_reset(allocr);

    struct ggml_cgraph * gf = build_graph(model, allocr);
    
    // allocate tensors
    ggml_allocr_alloc_graph(allocr, gf);
    int n_threads = 1;

    if (ggml_backend_is_cpu(model.backend)) {
        ggml_backend_cpu_set_n_threads(model.backend, n_threads);
    }

#ifdef GGML_USE_METAL
    if (ggml_backend_is_metal(model.backend)) {
        ggml_backend_metal_set_n_cb(model.backend, n_threads);
    }
#endif

    ggml_backend_graph_compute(model.backend, gf);

    //ggml_graph_print(gf);

    // in this case, the output tensor is the last one in the graph
    return gf->nodes[gf->n_nodes - 1];
}

int main(void)
{
    ggml_time_init();
    // Initialize a data
    float matrixA[4] = {
        2, 8,
        5, 1
    };

    // transpose of [10, 5, 9, 9]
    float matrixB[4] = {
        10, 9,
        5, 9
    };

    test_model model;
    load_model(model, matrixA, matrixB, true);
    
    ggml_backend_buffer_t buf_compute; // for compute
    struct ggml_allocr * allocr = NULL;

    {
        size_t align = ggml_backend_get_alignment(model.backend);
        allocr = ggml_allocr_new_measure(align);

        //create the worst case graph for memory usage estimation
        struct ggml_cgraph * gf = build_graph(model, allocr);
        size_t mem_size = ggml_allocr_alloc_graph(allocr, gf);
        ggml_allocr_free(allocr);

        // compute the required memory
        buf_compute = ggml_backend_alloc_buffer(model.backend, mem_size);
        allocr = ggml_allocr_new_from_buffer(buf_compute);
        fprintf(stderr, "%s: compute buffer size: %.4f KB\n", __func__, mem_size/1024.0);
    }

    struct ggml_tensor * result = compute(model, allocr);

    float* out_data = new float[2 * 2];

    ggml_backend_tensor_get(result, out_data, 0, ggml_nbytes(result));

    // expected result: 92.00 59.00 82.00 34.00
    printf("mult mat:\n%.2f %.2f %.2f %.2f\n",
                out_data[0], out_data[1], out_data[2], out_data[3]);

   // free memory
    ggml_free(model.ctx);

    ggml_backend_buffer_free(model.buffer);
    ggml_backend_buffer_free(buf_compute);
    ggml_backend_free(model.backend);
    return 0;
}

Output with cuda backend:

load_model: ggml tensor size    = 320 bytes
load_model: backend buffer size = 544 bytes
load_model: using CUDA backend
ggml_init_cublas: found 1 CUDA devices:
  Device 0: NVIDIA GeForce RTX 3050 Laptop GPU, compute capability 8.6
main: compute buffer size: 0.1250 KB
mult mat:
92.00 59.00 82.00 34.00

Output with cpu backend (default):

load_model: ggml tensor size    = 320 bytes
load_model: backend buffer size = 544 bytes
main: compute buffer size: 0.0625 KB
mult mat:
92.00 59.00 82.00 34.00

[ggerganov]

You should open a PR and add this as an example / test

[bobqianic]

Can you help me understand where the following functions are implemented: ggml_tensor_overhead(), ggml_tensor_overhead(), and ggml_nbytes()? They appear to have some practical functionality, but after searching through the codebase, I only found their declarations in ggml.h. I couldn't locate any actual implementation. Am I missing something?

[bobqianic]

Okay, I found it, it was my problem. Actually, I searched directly through the search box on GitHub, but couldn't find it. I opened ggml.c and used the browser's built-in search tool and then I found it. GitHub obviously has a bug.

[slaren]

ggml_tensor_overhead() (and similarly ggml_graph_overhead()) return the number of bytes required to allocate one tensor or one graph. Essentially it's sizeof(struct ggml_tensor) padded to the alignment size, excluding the actual tensor data. ggml_nbytes() returns the size of the data of the tensor. The _overhead() functions are usually used to calculate the size of the buffer of a ggml_context.

[saraalrawi]

[FSSRepo]

I don't know much about how to do training in ggml, I think there's a member who already tried to train a VAE with ggml. See this discussion

[ggerganov]

@saraalrawi Have you looked into test3.c - could be a good starting point for optimization experiments