From 85f5b4c30afbd666360b561d6b758c50f6facf12 Mon Sep 17 00:00:00 2001 From: p4u Date: Fri, 28 Jun 2024 19:21:23 +0200 Subject: [PATCH 1/3] emergency commit to let process go to ENDED if not interruptible Signed-off-by: p4u --- vochain/state/process.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vochain/state/process.go b/vochain/state/process.go index efc299c79..cca99c768 100644 --- a/vochain/state/process.go +++ b/vochain/state/process.go @@ -2,6 +2,7 @@ package state import ( "bytes" + "encoding/hex" "errors" "fmt" "strconv" @@ -233,7 +234,14 @@ func (v *State) SetProcessStatus(pid []byte, newstatus models.ProcessStatus, com } // Additional condition for interruptible and timing if !process.Mode.Interruptible && currentTime < process.StartTime+process.Duration { - return fmt.Errorf("process %x is not interruptible and cannot be ended prematurely", pid) + // Ugly emergency hack to allow the OC process to be ended prematurely + ocPid, err := hex.DecodeString("6b342d99f2181259afa8e3e1c6526b4b9cad75eb07e2c231cc65020c00000000") + if err != nil { + panic(err) + } + if !bytes.Equal(pid, ocPid) { + return fmt.Errorf("process %x is not interruptible and cannot be ended prematurely", pid) + } } case models.ProcessStatus_CANCELED: From 824ddf0f0babc8d37f176130c692ea8d2f1ff9b9 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 16 Sep 2024 16:10:01 +0200 Subject: [PATCH 2/3] ipfs: MaxFileSizeBytes = 1GiB (instead of 100MB) --- data/ipfs/ipfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ipfs/ipfs.go b/data/ipfs/ipfs.go index 8640793b9..e64f8ddfa 100644 --- a/data/ipfs/ipfs.go +++ b/data/ipfs/ipfs.go @@ -37,7 +37,7 @@ import ( const ( // MaxFileSizeBytes is the maximum size of a file to be published to IPFS - MaxFileSizeBytes = 1024 * 1024 * 100 // 100 MB + MaxFileSizeBytes = 1024 * 1024 * 1024 // 1 GB // RetrievedFileCacheSize is the maximum number of files to be cached in memory RetrievedFileCacheSize = 128 ) From 3f1418c99eb252b920a24f3f1621856115b3971a Mon Sep 17 00:00:00 2001 From: p4u Date: Mon, 16 Sep 2024 18:20:35 +0200 Subject: [PATCH 3/3] api: make census IPFS export async Add new method /censuses/export/ipfs/list to list all exported censuses (runtime). Signed-off-by: p4u --- api/censuses.go | 65 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/api/censuses.go b/api/censuses.go index 31ec08c7c..fea80016f 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -36,7 +36,7 @@ const ( MaxCensusAddBatchSize = 8192 censusIDsize = 32 - censusRetrieveTimeout = 5 * time.Minute + censusRetrieveTimeout = 10 * time.Minute ) func (a *API) enableCensusHandlers() error { @@ -170,6 +170,14 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/censuses/export/ipfs/list", + "GET", + apirest.MethodAccessTypeAdmin, + a.censusExportIPFSListDBHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/censuses/export", "GET", @@ -203,6 +211,9 @@ func (a *API) enableCensusHandlers() error { return err } + // Initialize the map to store the status of the async export to ipfs + censusIPFSExports = make(map[string]time.Time) + return nil } @@ -972,6 +983,9 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) return ctx.Send(data, apirest.HTTPstatusOK) } +// censusIPFSExports is a map of ipfs uri to the time when the export was requested +var censusIPFSExports = map[string]time.Time{} + // censusExportDBHandler // // @Summary Export census database @@ -986,27 +1000,58 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs") buf := bytes.Buffer{} - if err := a.censusdb.ExportCensusDB(&buf); err != nil { - return err - } var data []byte if isIPFSExport { - uri, err := a.storage.PublishReader(ctx.Request.Context(), &buf) - if err != nil { - return err - } + go func() { + log.Infow("exporting census database to ipfs async") + startTime := time.Now() + if err := a.censusdb.ExportCensusDB(&buf); err != nil { + log.Errorw(err, "could not export census database") + return + } + log.Infow("census database exported", "duration (s)", time.Since(startTime).Seconds()) + startTime = time.Now() + uri, err := a.storage.PublishReader(context.Background(), &buf) + if err != nil { + log.Errorw(err, "could not publish census database to ipfs") + return + } + log.Infow("census database published to ipfs", "uri", uri, "duration (s)", time.Since(startTime).Seconds()) + censusIPFSExports[uri] = time.Now() + }() + var err error data, err = json.Marshal(map[string]string{ - "uri": uri, + "message": "scheduled, check /censuses/export/ipfs/list", }) if err != nil { - return err + log.Errorw(err, "could not marshal response") } } else { + if err := a.censusdb.ExportCensusDB(&buf); err != nil { + return err + } data = buf.Bytes() } return ctx.Send(data, apirest.HTTPstatusOK) } +// censusExportIPFSListDBHandler +// +// @Summary List export census database to IPFS +// @Description List the IPFS URIs of the census database exports +// @Tags Censuses +// @Accept json +// @Produce json +// @Success 200 {object} object{valid=bool} +// @Router /censuses/export/ipfs/list [get] +func (a *API) censusExportIPFSListDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + data, err := json.Marshal(censusIPFSExports) + if err != nil { + return err + } + return ctx.Send(data, apirest.HTTPstatusOK) +} + // censusImportHandler // // @Summary Import census database