Skip to content

🚀 [Feature]: ObjectBox storage support #1531

@karnadii

Description

@karnadii

Feature Description

implement driver for ObjectBox from github.com/objectbox/objectbox-go

Additional Context (optional)

No response

Code Snippet (optional)

package main

import "github.com/gofiber/storage/%package%"

func main() {
  // Steps to reproduce
}

Checklist:

  • I agree to follow Fiber's Code of Conduct.
    I have checked for existing issues that describe my suggestion prior to opening this one.
    I understand that improperly formatted feature requests may be closed without explanation.

Activity

karnadii

karnadii commented on Nov 19, 2024

@karnadii
Author

I have make some small implementation
but I just learned go two days ago so I don't know what the best practice is

package middleware

import (
	"math/rand/v2"
	"time"

	"github.com/objectbox/objectbox-go/objectbox"
)

//go:generate go run github.com/objectbox/objectbox-go/cmd/objectbox-gogen

type CacheEntry struct {
	Id        uint64 `objectbox:"id"`
	Key       string `objectbox:"index"`
	Value     []byte
	ExpiresAt int64
}

type ObjectBoxStorage struct {
	ob  *objectbox.ObjectBox
	box *CacheEntryBox
}

func NewObjectBoxStorage() (*ObjectBoxStorage, error) {
	ob, err := objectbox.NewBuilder().Model(ObjectBoxModel()).Build()
	if err != nil {
		return nil, err
	}

	storage := &ObjectBoxStorage{
		ob:  ob,
		box: BoxForCacheEntry(ob),
	}

	// Run cleanup every hour
	go func() {
		ticker := time.NewTicker(1 * time.Hour)
		for range ticker.C {
			storage.cleanupExpired()
		}
	}()

	return storage, nil
}

func (s *ObjectBoxStorage) Get(key string) ([]byte, error) {
	if rand.Float32() < 0.1 {
		s.cleanupExpired()
	}

	query := s.box.Query(CacheEntry_.Key.Equals(key, true), CacheEntry_.ExpiresAt.GreaterThan(time.Now().Unix()))
	entries, err := query.Find()
	if err != nil {
		return nil, err
	}

	if len(entries) == 0 {
		return nil, nil
	}
	return entries[0].Value, nil

}

func (s *ObjectBoxStorage) Set(key string, val []byte, exp time.Duration) error {
	entry := &CacheEntry{
		Key:       key,
		Value:     val,
		ExpiresAt: time.Now().Add(exp).Unix(),
	}
	_, err := s.box.Put(entry)
	return err
}

func (s *ObjectBoxStorage) Delete(key string) error {
	query := s.box.Query(CacheEntry_.Key.Equals(key, true))
	entries, err := query.Find()
	if err != nil {
		return err
	}

	for _, entry := range entries {
		if err := s.box.Remove(entry); err != nil {
			return err
		}
	}

	return nil
}

func (s *ObjectBoxStorage) Reset() error {
	return s.box.RemoveAll()
}

func (s *ObjectBoxStorage) Close() error {
	s.ob.Close()
	return nil
}

func (s *ObjectBoxStorage) cleanupExpired() {
	query := s.box.Query(CacheEntry_.ExpiresAt.LessThan(time.Now().Unix()))
	entries, err := query.Find()
	if err != nil {
		return
	}
	s.box.ObjectBox.RunInWriteTx(func() error {
		for _, entry := range entries {
			s.box.Remove(entry)
		}
		return nil
	})

}
gaby

gaby commented on Nov 19, 2024

@gaby
Member

I will take a look later to see how much effort is this.

self-assigned this
on Nov 19, 2024
karnadii

karnadii commented on Nov 28, 2024

@karnadii
Author

I couldn't wait so after looking for other storage implementation, I write this.

package objectbox

import "time"

// Config defines the configuration options for ObjectBox storage.
type Config struct {
	// Directory is the path where the database is stored.
	// Optional, defaults to "objectbox"
	Directory string

	// MaxSizeInKb sets the maximum size of the database in kilobytes.
	// Optional, defaults to 1GB (1024 * 1024 * 1024)
	MaxSizeInKb uint64

	// MaxReaders defines the maximum number of concurrent readers.
	// Optional, defaults to 126
	MaxReaders uint

	// Reset determines if existing keys should be cleared on startup.
	// Optional, defaults to false
	Reset bool

	// CleanerInterval sets the frequency for deleting expired keys.
	// Optional, defaults to 60 seconds
	CleanerInterval time.Duration
}

var DefaultConfig = Config{
	Directory:       "objectbox_db",
	MaxSizeInKb:     1024 * 1024, // 1GByte
	MaxReaders:      126,
	Reset:           false,
	CleanerInterval: 60 * time.Second,
}

func getConfig(config ...Config) Config {
	if len(config) < 1 {
		return DefaultConfig
	}

	cfg := config[0]

	// Set default values

	if cfg.Directory == "" {
		cfg.Directory = DefaultConfig.Directory
	}

	if cfg.MaxSizeInKb == 0 {
		cfg.MaxSizeInKb = DefaultConfig.MaxSizeInKb
	}

	if cfg.MaxReaders == 0 {
		cfg.MaxReaders = DefaultConfig.MaxReaders
	}

	if int(cfg.CleanerInterval.Seconds()) == 0 {
		cfg.CleanerInterval = DefaultConfig.CleanerInterval
	}

	return cfg

}
package objectbox

import (
	"time"

	"github.com/objectbox/objectbox-go/objectbox"
)

//go:generate go run github.com/objectbox/objectbox-go/cmd/objectbox-gogen

// Cache represents a single cache entry in the storage.
type Cache struct {
	Id        uint64 `objectbox:"id"`
	Key       string `objectbox:"index,unique"`
	Value     []byte
	ExpiresAt int64 `objectbox:"index"`
}

// Storage handles the ObjectBox database operations and cleanup routines.
type Storage struct {
	ob   *objectbox.ObjectBox
	box  *CacheBox
	done chan struct{}
}

// New creates a new Storage instance with the provided configuration.
// It initializes the ObjectBox database and starts the cleanup routine.
func New(config ...Config) *Storage {
	cfg := getConfig(config...)

	ob, err := objectbox.NewBuilder().Model(ObjectBoxModel()).MaxSizeInKb(cfg.MaxSizeInKb).MaxReaders(cfg.MaxReaders).Directory(cfg.Directory).Build()
	if err != nil {
		return nil
	}

	if cfg.Reset {
		box := BoxForCache(ob)
		box.RemoveAll()
	}

	storage := &Storage{
		ob:   ob,
		box:  BoxForCache(ob),
		done: make(chan struct{}),
	}

	go storage.cleanerTicker(cfg.CleanerInterval)

	return storage
}

// Get retrieves a value from cache by its key.
// Returns nil if key doesn't exist or has expired.
func (s *Storage) Get(key string) ([]byte, error) {
	if len(key) < 1 {
		return nil, nil
	}

	query := s.box.Query(Cache_.Key.Equals(key, true),
		objectbox.Any(
			Cache_.ExpiresAt.Equals(0),
			Cache_.ExpiresAt.GreaterThan(time.Now().Unix()),
		))
	caches, err := query.Find()

	if err != nil {
		return nil, err
	}

	if len(caches) < 1 {
		return nil, nil
	}

	return caches[0].Value, nil

}

// Set stores a value in cache with the specified key and expiration.
// If expiration is 0, the entry won't expire.
func (s *Storage) Set(key string, value []byte, exp time.Duration) error {
	if len(key) <= 0 || len(value) <= 0 {
		return nil
	}

	// Since objectbox go doen't support conflict strategy,
	// we need to check if the key already exists
	// and update the value if it does. Thus we need to
	// get the id of the cache first and then update the cache
	// with the new value with the same id.
	query := s.box.Query(Cache_.Key.Equals(key, true))
	cachesIds, err := query.FindIds()
	if err != nil {
		return err
	}

	// if the id is 0 it will create new cache
	// otherwise it will update the existing entry
	var id uint64 = 0
	if len(cachesIds) > 0 {
		id = cachesIds[0]
	}

	var expAt int64

	if exp > 0 { // Changed from exp != 0 to exp > 0
		expAt = time.Now().Add(exp).Unix()
	}

	cache := &Cache{
		Id:        id,
		Key:       key,
		Value:     value,
		ExpiresAt: expAt,
	}

	_, err = s.box.Put(cache)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an entry from cache by its key.
func (s *Storage) Delete(key string) error {
	if len(key) <= 0 {
		return nil
	}

	query := s.box.Query(Cache_.Key.Equals(key, true))
	cachesIds, err := query.FindIds()
	if err != nil {
		return err
	}

	if len(cachesIds) < 1 {
		return nil
	}

	if err := s.box.RemoveId(cachesIds[0]); err != nil {
		return err
	}

	return nil

}

// Reset removes all entries from the cache.
func (s *Storage) Reset() error {
	return s.box.RemoveAll()
}

// Close shuts down the storage, stopping the cleanup routine
// and closing the database connection.
func (s *Storage) Close() error {
	close(s.done)
	s.ob.Close()
	return nil
}

// cleaneStorage removes all expired cache entries.
func (s *Storage) cleaneStorage() {
	s.box.Query(Cache_.ExpiresAt.LessThan(time.Now().Unix())).Remove()

}

// cleanerTicker runs periodic cleanup of expired entries.
func (s *Storage) cleanerTicker(interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			s.cleaneStorage()
		case <-s.done:
			return
		}
	}
}
gaby

gaby commented on Dec 1, 2024

@gaby
Member

@karnadii Awesome, do you allow us to use this implementation to make an official driver for it on this repo?

karnadii

karnadii commented on Dec 1, 2024

@karnadii
Author

@gaby yes please, the updated code is here https://github.com/karnadii/storage/tree/objectbox/objectbox
I just don't know how to setup the test action. please use the code and made appropriate changes as you please.

gaby

gaby commented on Dec 2, 2024

@gaby
Member

Will do this week, thanks! 💪

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

    Participants

    @gaby@karnadii

    Issue actions

      🚀 [Feature]: ObjectBox storage support · Issue #1531 · gofiber/storage