Put route, bug fixes

Author: MasloMaslane

proxy/handlers/files.go

@@ -1,14 +1,23 @@
 package handlers
 
 import (
-	"github.com/sio2project/ft-to-s3/v1/utils"
-
 	"net/http"
+	"strings"
+
+	"github.com/sio2project/ft-to-s3/v1/utils"
 )
 
 func Files(w http.ResponseWriter, r *http.Request, logger *utils.LoggerObject, bucketName string) {
 	if r.Method == http.MethodPut {
 		Put(w, r, logger, bucketName)
+	} else if r.Method == http.MethodGet {
+		if strings.HasPrefix(r.URL.Path, "/files/") {
+			GetFiles(w, r, logger, bucketName)
+		} else if strings.HasPrefix(r.URL.Path, "/list/") {
+			GetList(w, r, logger, bucketName)
+		} else {
+			w.WriteHeader(http.StatusBadRequest)
+		}
 	} else {
 		w.WriteHeader(http.StatusMethodNotAllowed)
 	}

proxy/handlers/get_files.go

@@ -0,0 +1,33 @@
+package handlers
+
+import (
+	"github.com/sio2project/ft-to-s3/v1/storage"
+	"github.com/sio2project/ft-to-s3/v1/utils"
+	"io"
+	"net/http"
+	"strconv"
+)
+
+func GetFiles(w http.ResponseWriter, r *http.Request, logger *utils.LoggerObject, bucketName string) {
+	path := r.URL.Path[len("/files/"):]
+	result := storage.Get(bucketName, logger, path)
+	if result.Err != nil {
+		logger.Error("Error", result.Err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if !result.Found {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	if result.Gziped {
+		w.Header().Set("Content-Encoding", "gzip")
+	}
+	w.Header().Set("Logical-Size", strconv.FormatInt(result.LogicalSize, 10))
+	w.Header().Set("Last-Modified", toRFC2822(result.LastModified))
+	w.WriteHeader(http.StatusOK)
+	_, err := io.Copy(w, result.File)
+	if err != nil {
+		logger.Error("Error", err)
+	}
+}

proxy/handlers/get_list.go

@@ -0,0 +1,10 @@
+package handlers
+
+import (
+	"github.com/sio2project/ft-to-s3/v1/utils"
+	"net/http"
+)
+
+func GetList(w http.ResponseWriter, r *http.Request, logger *utils.LoggerObject, bucketName string) {
+	//path := r.URL.Path[len("/list/"):]
+}

storage/main.go

@@ -3,15 +3,16 @@ package storage
 import (
 	"bytes"
 	"context"
-	"io"
-
 	"github.com/minio/minio-go/v7"
 	"github.com/sio2project/ft-to-s3/v1/db"
 	"github.com/sio2project/ft-to-s3/v1/utils"
+	"io"
 )
 
 func Store(bucketName string, logger *utils.LoggerObject, path string, reader io.Reader, version int64, size int64,
 	compressed bool, sha256Digest string, logicalSize int64) (int64, error) {
+	logger.Debug("storage.Store called on", bucketName+":"+path)
+
 	session := db.GetSession()
 	defer session.Close()
 	fileMutex := db.GetMutex(session, bucketName+":"+path)
@@ -25,33 +26,37 @@ func Store(bucketName string, logger *utils.LoggerObject, path string, reader io
 	if version <= dbModified {
 		return dbModified, nil
 	}
+	logger.Debug("Version is greater than dbModified")
 
 	oldFile, err := db.GetHashForPath(bucketName, path)
 	if err != nil {
 		return 0, err
 	}
 
 	options := minio.PutObjectOptions{}
+	var data bytes.Buffer
 	if sha256Digest == "" || logicalSize == -1 {
-		var data []byte
+		var tempData []byte
+		teeReader := io.TeeReader(reader, &data)
 		if compressed {
-			data, err = utils.ReadGzip(reader)
+			tempData, err = utils.ReadGzip(teeReader)
 			if err != nil {
 				return 0, err
 			}
 		} else {
-			data, err = io.ReadAll(reader)
+			tempData, err = io.ReadAll(teeReader)
 			if err != nil {
 				return 0, err
 			}
 		}
 
-		sha256Digest = utils.Sha256Checksum(data)
-		logicalSize = int64(len(data))
-		reader = bytes.NewReader(data)
+		sha256Digest = utils.Sha256Checksum(tempData)
+		logicalSize = int64(len(tempData))
 	}
 	if compressed {
+		logger.Debug("Setting ContentEncoding to gzip")
 		options.ContentEncoding = "gzip"
+		options.ContentType = "application/gzip"
 	}
 
 	refCount, err := db.GetRefCount(bucketName, sha256Digest)
@@ -60,10 +65,10 @@ func Store(bucketName string, logger *utils.LoggerObject, path string, reader io
 	}
 
 	if refCount == 0 {
-		logger.Debug("Storing with options ", options)
+		logger.Debug("Storing with options", options)
 		minioClient := GetClient()
 
-		_, err = minioClient.PutObject(context.Background(), bucketName, sha256Digest, reader, size, options)
+		_, err = minioClient.PutObject(context.Background(), bucketName, sha256Digest, &data, size, options)
 		if err != nil {
 			return 0, err
 		}
@@ -98,3 +103,48 @@ func deleteByHash(bucketName string, logger *utils.LoggerObject, path string, lo
 	logger.Debug("DeleteByHash called on ", path)
 	return nil
 }
+
+func Get(bucketName string, logger *utils.LoggerObject, path string) *GetResult {
+	logger.Debug("storage.Get called on", bucketName+":"+path)
+
+	fileHash, err := db.GetHashForPath(bucketName, path)
+	if err != nil {
+		return &GetResult{Err: err}
+	}
+	if fileHash == "" {
+		return &GetResult{Found: false}
+	}
+
+	lastModified, err := db.GetModified(bucketName, path)
+	if err != nil {
+		return &GetResult{Err: err}
+	}
+
+	minioClient := GetClient()
+	info, err := minioClient.StatObject(context.Background(), bucketName, fileHash, minio.StatObjectOptions{})
+	if err != nil {
+		minioErr := minio.ToErrorResponse(err)
+		if minioErr.Code == "NoSuchKey" {
+			return &GetResult{Found: false}
+		}
+		return &GetResult{Err: err}
+	}
+	gziped := info.ContentType == "application/gzip"
+
+	reader, err := minioClient.GetObject(context.Background(), bucketName, fileHash, minio.GetObjectOptions{})
+	if err != nil {
+		minioErr := minio.ToErrorResponse(err)
+		if minioErr.Code == "NoSuchKey" {
+			return &GetResult{Found: false}
+		}
+		return &GetResult{Err: err}
+	}
+
+	return &GetResult{
+		Found:        true,
+		File:         reader,
+		Gziped:       gziped,
+		LastModified: lastModified,
+		LogicalSize:  info.Size,
+	}
+}

storage/structs.go

@@ -0,0 +1,12 @@
+package storage
+
+import "io"
+
+type GetResult struct {
+	Found        bool
+	File         io.Reader
+	Gziped       bool
+	LastModified int64
+	LogicalSize  int64
+	Err          error
+}