F/ipfs export async

api/censuses.go

@@ -36,7 +36,7 @@ const (
 	MaxCensusAddBatchSize = 8192
 
 	censusIDsize          = 32
-	censusRetrieveTimeout = 5 * time.Minute
+	censusRetrieveTimeout = 10 * time.Minute
 )
 
 func (a *API) enableCensusHandlers() error {
@@ -170,6 +170,14 @@ func (a *API) enableCensusHandlers() error {
 	); err != nil {
 		return err
 	}
+	if err := a.Endpoint.RegisterMethod(
+		"/censuses/export/ipfs/list",
+		"GET",
+		apirest.MethodAccessTypeAdmin,
+		a.censusExportIPFSListDBHandler,
+	); err != nil {
+		return err
+	}
 	if err := a.Endpoint.RegisterMethod(
 		"/censuses/export",
 		"GET",
@@ -203,6 +211,9 @@ func (a *API) enableCensusHandlers() error {
 		return err
 	}
 
+	// Initialize the map to store the status of the async export to ipfs
+	censusIPFSExports = make(map[string]time.Time)
+
 	return nil
 }
 
@@ -972,6 +983,9 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 	return ctx.Send(data, apirest.HTTPstatusOK)
 }
 
+// censusIPFSExports is a map of ipfs uri to the time when the export was requested
+var censusIPFSExports = map[string]time.Time{}
+
 // censusExportDBHandler
 //
 //	@Summary		Export census database
@@ -986,27 +1000,58 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
 	isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs")
 	buf := bytes.Buffer{}
-	if err := a.censusdb.ExportCensusDB(&buf); err != nil {
-		return err
-	}
 	var data []byte
 	if isIPFSExport {
-		uri, err := a.storage.PublishReader(ctx.Request.Context(), &buf)
-		if err != nil {
-			return err
-		}
+		go func() {
+			log.Infow("exporting census database to ipfs async")
+			startTime := time.Now()
+			if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+				log.Errorw(err, "could not export census database")
+				return
+			}
+			log.Infow("census database exported", "duration (s)", time.Since(startTime).Seconds())
+			startTime = time.Now()
+			uri, err := a.storage.PublishReader(context.Background(), &buf)
+			if err != nil {
+				log.Errorw(err, "could not publish census database to ipfs")
+				return
+			}
+			log.Infow("census database published to ipfs", "uri", uri, "duration (s)", time.Since(startTime).Seconds())
+			censusIPFSExports[uri] = time.Now()
+		}()
+		var err error
 		data, err = json.Marshal(map[string]string{
-			"uri": uri,
+			"message": "scheduled, check /censuses/export/ipfs/list",
 		})
 		if err != nil {
-			return err
+			log.Errorw(err, "could not marshal response")
 		}
 	} else {
+		if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+			return err
+		}
 		data = buf.Bytes()
 	}
 	return ctx.Send(data, apirest.HTTPstatusOK)
 }
 
+// censusExportIPFSListDBHandler
+//
+//	@Summary		List export census database to IPFS
+//	@Description	List the IPFS URIs of the census database exports
+//	@Tags			Censuses
+//	@Accept			json
+//	@Produce		json
+//	@Success		200		{object}	object{valid=bool}
+//	@Router			/censuses/export/ipfs/list [get]
+func (a *API) censusExportIPFSListDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
+	data, err := json.Marshal(censusIPFSExports)
+	if err != nil {
+		return err
+	}
+	return ctx.Send(data, apirest.HTTPstatusOK)
+}
+
 // censusImportHandler
 //
 //	@Summary		Import census database

data/ipfs/ipfs.go

@@ -37,7 +37,7 @@ import (
 
 const (
 	// MaxFileSizeBytes is the maximum size of a file to be published to IPFS
-	MaxFileSizeBytes = 1024 * 1024 * 100 // 100 MB
+	MaxFileSizeBytes = 1024 * 1024 * 1024 // 1 GB
 	// RetrievedFileCacheSize is the maximum number of files to be cached in memory
 	RetrievedFileCacheSize = 128
 )

vochain/state/process.go

@@ -2,6 +2,7 @@ package state
 
 import (
 	"bytes"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"strconv"
@@ -234,7 +235,14 @@ func (v *State) SetProcessStatus(pid []byte, newstatus models.ProcessStatus, com
 		}
 		// Additional condition for interruptible and timing
 		if !process.Mode.Interruptible && currentTime < process.StartTime+process.Duration {
-			return fmt.Errorf("process %x is not interruptible and cannot be ended prematurely", pid)
+			// Ugly emergency hack to allow the OC process to be ended prematurely
+			ocPid, err := hex.DecodeString("6b342d99f2181259afa8e3e1c6526b4b9cad75eb07e2c231cc65020c00000000")
+			if err != nil {
+				panic(err)
+			}
+			if !bytes.Equal(pid, ocPid) {
+				return fmt.Errorf("process %x is not interruptible and cannot be ended prematurely", pid)
+			}
 		}
 
 	case models.ProcessStatus_CANCELED: