Merge pull request #8 from mjschwenne/grackle

Grackle adoptation for gokv tutorial

Author: upamanyus

.envrc

@@ -0,0 +1 @@
+use flake

.github/commit-emails.toml

@@ -0,0 +1,2 @@
+# commit-emails.xyz config
+to = "[email protected],[email protected]"

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI 
+
+on:
+  push:
+    branches:
+      - main
+      - grackle
+  pull_request:
+
+jobs:
+  update-grackle:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v4
+        with:
+          submodules: true 
+      - name: Install protoc
+        run: sudo apt-get install protobuf-compiler -y
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with: 
+          go-version: "1.22"
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Check Grackle 
+        run: |
+          ./update-grackle.py
+          git diff --exit-code

.gitignore

@@ -1,3 +1,4 @@
 kvdur
 kvdur_log
 *.prof
+.direnv

Makefile

@@ -7,4 +7,5 @@ check:
 	go test ./...
 
 fix:
+	./update-grackle.py
 	gofmt -w -s .

cachekv/cachekv.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message cacheValue {
+  string v = 1;
+  uint64 l = 2;
+}

fencing/ctr/ctr.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+enum Error {
+  ENone = 0;
+  Estale = 1;
+}
+
+message putArgs {
+  uint64 epoch = 1;
+  uint64 v = 2;
+}
+
+message getArgs {
+  uint64 epoch = 1;
+}
+
+message getReply {
+  Error err = 1;
+  uint64 val = 2;
+}

flake.lock

@@ -0,0 +1,62 @@
+{
+  description = "A Flake for Applying Grackle to gokv";
+
+  inputs = {
+    nixpkgs.url = "nixpkgs";
+  };
+
+  outputs = {nixpkgs, ...}: let
+    system = "x86_64-linux";
+  in {
+    devShells."${system}".default = let
+      pkgs = import nixpkgs {
+        inherit system;
+      };
+      goose = pkgs.buildGoModule {
+        name = "goose";
+        src = pkgs.fetchFromGitHub {
+          owner = "goose-lang";
+          repo = "goose";
+          rev = "a4f2f84193d34f56dd84fc623adc43a6441da1eb";
+          sha256 = "1b1dfa1qsv2h7hy5x20zhic2npr5gz1zp76m1lab4v490adxj2rx";
+        };
+        vendorHash = "sha256-HCJ8v3TSv4UrkOsRuENWVz5Z7zQ1UsOygx0Mo7MELzY=";
+      };
+      grackle = pkgs.buildGoModule {
+        name = "grackle";
+        src = pkgs.fetchFromGitHub {
+          owner = "mjschwenne";
+          repo = "grackle";
+          rev = "101412356cdfbcad78f8aaa724101312928c4978";
+          sha256 = "06zf2bvrbbjhgrd6994h3wcaml7m83m6f9r61pj7y09xq9nw10br";
+        };
+        vendorHash = "sha256-Wk2v0HSAkrzxHJvCfbw6xOn0OQ1xukvYjDxk3c2LmH8=";
+        checkPhase = false;
+      };
+    in
+      pkgs.mkShell {
+        # create an environment with the required coq libraries
+        packages = with pkgs; [
+          # Go deps
+          go
+          gopls
+          goose
+          grackle
+
+          # Protobuf deps
+          protobuf
+          protoc-gen-go
+          proto-contrib
+          protoscope
+
+          # nix tools
+          nix-prefetch-git
+          nix-prefetch
+          update-nix-fetchgit
+        ];
+
+        shellHook = ''
+        '';
+      };
+  };
+}

flake.nix

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+message u64ToU64 {
+  uint64 k = 1;
+  uint64 v = 2;
+}
+
+message mapU64ToU64 {
+  repeated u64ToU64 entries = 1;
+}
+
+message u64ToBytes {
+  uint64 k = 1;
+  bytes v = 2;
+}
+
+message mapU64ToBytes {
+  repeated u64ToBytes entries = 1;
+}

map_marshal/map.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+message u64ToString {
+  uint64 k = 1;
+  string v = 2;
+}
+
+message mapU64ToString {
+  repeated u64ToString entries = 1;
+}

map_string_marshal/map_string.proto

@@ -0,0 +1,62 @@
+syntax = "proto3";
+
+import "../map_marshal/map.proto";
+
+enum Error {
+  ENone = 0;
+  EDontHaveShard = 1;
+}
+
+enum KvOp {
+  KV_FreshCID = 0;
+  KV_Put = 1;
+  KV_Get = 2;
+  KV_Conditional_Put = 3;
+  KV_Ins_Shard = 4;
+  KV_Mov_Shard = 5;
+}
+
+message putRequest {
+  uint64 key = 1;
+  bytes value = 2;
+}
+
+message putReply {
+  Error err = 1;
+}
+
+message getRequest {
+  uint64 key = 1;
+}
+
+message getReply {
+  Error err = 1;
+  bytes value = 2;
+}
+
+message conditionalPutRequest {
+  uint64 key = 1;
+  bytes expectedValue = 2;
+  bytes newValue = 3;
+}
+
+message conditionalPutReply {
+  Error err = 1;
+  bool success = 2;
+}
+
+message installShardRequest {
+  uint64 sid = 1;
+  // This might be tricky since now the proto files aren't standalone
+  mapU64ToBytes kvs = 2;
+}
+
+message moveShardRequest {
+  uint64 sid = 1;
+  // De-alias HostName to uint64
+  uint64 hostName = 2;
+}
+
+message shardMap {
+  repeated uint64 shards = 1;
+}

memkv/memkv.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+message prepareReply {
+  bool success = 1;
+  // De-alias Entry to uint64
+  repeated uint64 log = 2;
+  uint64 pn = 3;
+}
+
+message proposeArgs {
+  uint64 pn = 1;
+  uint64 commiteIndex = 2;
+  // De-alias Entry to uint64
+  repeated uint64 log = 3;
+}

paxi/comulti/comulti.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+message config {
+  // De-alias grove_ffi.Address to uint64
+  repeated uint64 memebers = 1;
+  // De-alias grove_ffi.Address to uint64
+  repeated uint64 nextMembers = 2;
+}
+
+message monotonicValue {
+  uint64 version = 1;
+  bytes val = 2;
+  config conf = 3;
+}
+
+message prepareReply {
+  // Maybe should be enum?
+  uint64 err = 1;
+  uint64 term = 2;
+  monotonicValue val = 3;
+}
+
+ message proposeArgs {
+   uint64 term = 1;
+   monotonicValue val = 2;
+ }
+
+message tryCommitReply {
+  uint64 err = 1;
+  uint64 version = 2;
+}

paxi/reconf/reconf.proto

@@ -0,0 +1,6 @@
+[project]
+name = "gokv"
+version = "2024.0.0"
+dependencies = [
+  "ruff==0.8.1",
+]

pyproject.toml

@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+message appendArgs {
+  uint64 epoch = 1;
+  // De-aliasing LogEntry to []byte
+  bytes entry = 2;
+  uint64 index = 3;
+}
+
+message configuration {
+  // De-aliasing grove_ffi.Address to uint64
+  repeated uint64 replicas = 1;
+}
+
+message becomeReplicaArgs {
+  uint64 epoch = 1;
+  uint64 startIndex = 2;
+  // De-aliasing LogEntry to []byte
+  repeated bytes log = 3;
+}
+
+message becomePrimaryArgs {
+  uint64 epoch = 1;
+  configuration conf = 2;
+}
+
+message getLogReply {
+  // De-aliasing Error to uint64. Should this be an enum?
+  uint64 err = 1;
+  // De-aliasing LogEntry to []byte
+  repeated bytes log = 2;
+  uint64 startIndex = 3;
+}

reconfig/replica/replica.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message configuration {
+  // De-alias grove_ffi.Address to uint64
+  repeated uint64 addrs = 1;
+}