Merge pull request #34 from aspacca/v2_package_registry

download zip package and load field yml from it

Author: Andrea Spacca

pkg/genlib/fields/cache.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"golang.org/x/sync/semaphore"
 	"sync"
-
-	"github.com/elastic/go-ucfg/yaml"
 )
 
 const (
@@ -98,48 +96,3 @@ func (f *Cache) LoadFields(ctx context.Context, integration, stream, version str
 
 	return flds, nil
 }
-
-func (f *Cache) LoadManifest(ctx context.Context, integration, stream, version string) (*Manifest, error) {
-	var err error
-
-	t := tuple{
-		integration: integration,
-		stream:      stream,
-		version:     version,
-	}
-
-	f.mut.RLock()
-	manifest, ok := f.manifest[t]
-	f.mut.RUnlock()
-
-	if ok {
-		return &manifest, nil
-	}
-
-	// Pull the manifest file
-	url, err := makeManifestURL(f.baseUrl, integration, stream, version)
-	if err != nil {
-		return nil, err
-	}
-
-	data, err := getFromURL(ctx, url.String())
-	if err != nil {
-		return nil, err
-	}
-
-	// deserialize
-	cfg, err := yaml.NewConfig(data)
-	if err != nil {
-		return nil, err
-	}
-	err = cfg.Unpack(&manifest)
-	if err != nil {
-		return nil, err
-	}
-
-	f.mut.Lock()
-	f.manifest[t] = manifest
-	f.mut.Unlock()
-
-	return &manifest, nil
-}

pkg/genlib/fields/load.go

@@ -1,10 +1,13 @@
 package fields
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -26,12 +29,7 @@ const (
 
 func LoadFields(ctx context.Context, baseURL, integration, dataStream, version string) (Fields, error) {
 
-	packageURL, err := makePackageURL(baseURL, integration, version)
-	if err != nil {
-		return nil, err
-	}
-
-	fieldsContent, err := getFieldsFiles(ctx, packageURL, dataStream)
+	fieldsContent, err := getFieldsFiles(ctx, baseURL, integration, dataStream, version)
 	if err != nil {
 		return nil, err
 	}
@@ -61,42 +59,92 @@ func makePackageURL(baseURL, integration, version string) (*url.URL, error) {
 	return u, nil
 }
 
-func getFieldsFiles(ctx context.Context, packageURL *url.URL, dataStream string) ([]byte, error) {
-	body, err := getFromURL(ctx, packageURL.String())
+func makeDownloadURL(baseURL, donwloadPath string) (*url.URL, error) {
+
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	u.Path = path.Join(u.Path, donwloadPath)
+	return u, nil
+}
+
+func getFieldsFiles(ctx context.Context, baseURL, integration, dataStream, version string) ([]byte, error) {
+	packageURL, err := makePackageURL(baseURL, integration, version)
 	if err != nil {
 		return nil, err
 	}
 
-	var assetsPayload struct {
-		Assets []string `json:"assets"`
+	r, err := getFromURL(ctx, packageURL.String())
+	if err != nil {
+		return nil, err
 	}
 
-	if err = json.Unmarshal(body, &assetsPayload); err != nil {
+	var downloadPayload struct {
+		Download string `json:"download"`
+	}
+
+	body, err := ioutil.ReadAll(r)
+	if err = json.Unmarshal(body, &downloadPayload); err != nil {
 		return nil, err
 	}
 
-	fieldsFilesURL := make([]string, 0)
-	prefixFieldsPath := path.Join(packageURL.Path, dataStreamSlug, dataStream, fieldsSlug)
-	for _, assetPath := range assetsPayload.Assets {
-		if !strings.HasPrefix(assetPath, prefixFieldsPath) {
-			continue
+	downloadURL, err := makeDownloadURL(baseURL, downloadPayload.Download)
+	r, err = getFromURL(ctx, downloadURL.String())
+	defer func(r io.ReadCloser) {
+		if r != nil {
+			_ = r.Close()
 		}
+	}(r)
+
+	if err != nil {
+		return nil, err
+	}
 
-		fieldsFilesURL = append(fieldsFilesURL, assetPath)
+	zipContent, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
 	}
 
+	archive, err := zip.NewReader(bytes.NewReader(zipContent), int64(len(zipContent)))
+	if err != nil {
+		return nil, err
+	}
+
+	prefixFieldsPath := path.Join(fmt.Sprintf("%s-%s", integration, version), dataStreamSlug, dataStream, fieldsSlug)
+
 	var fieldsContent string
-	for _, fieldsFileURL := range fieldsFilesURL {
-		packageURL.Path = fieldsFileURL
+	for _, z := range archive.File {
+		if z.FileInfo().IsDir() {
+			continue
+		}
 
-		body, err := getFromURL(ctx, packageURL.String())
+		if !strings.HasPrefix(z.Name, prefixFieldsPath) {
+			continue
+		}
+
+		fieldsFileName := z.Name
+		zr, err := z.Open()
 		if err != nil {
+			if zr != nil {
+				_ = zr.Close()
+			}
 			return nil, err
 		}
 
-		key := strings.TrimSuffix(filepath.Base(fieldsFileURL), filepath.Ext(fieldsFileURL))
+		fieldsFileContent, err := ioutil.ReadAll(zr)
+		if err != nil {
+			if zr != nil {
+				_ = zr.Close()
+			}
+			return nil, err
+		}
+
+		_ = zr.Close()
+		key := strings.TrimSuffix(filepath.Base(fieldsFileName), filepath.Ext(fieldsFileName))
 		keyEntry := fmt.Sprintf("- key: %s\n  fields:\n", key)
-		for _, line := range strings.Split(string(body), "\n") {
+		for _, line := range strings.Split(string(fieldsFileContent), "\n") {
 			keyEntry += `    ` + line + "\n"
 		}
 
@@ -106,7 +154,7 @@ func getFieldsFiles(ctx context.Context, packageURL *url.URL, dataStream string)
 	return []byte(fieldsContent), nil
 }
 
-func getFromURL(ctx context.Context, srcURL string) ([]byte, error) {
+func getFromURL(ctx context.Context, srcURL string) (io.ReadCloser, error) {
 
 	req, err := http.NewRequestWithContext(ctx, "GET", srcURL, nil)
 
@@ -116,18 +164,21 @@ func getFromURL(ctx context.Context, srcURL string) ([]byte, error) {
 
 	client := &http.Client{}
 	resp, err := client.Do(req)
-
 	if err != nil {
+		if resp.Body != nil {
+			_ = resp.Body.Close()
+		}
 		return nil, err
 	}
 
-	defer resp.Body.Close()
-
 	if resp.StatusCode != http.StatusOK {
+		if resp.Body != nil {
+			_ = resp.Body.Close()
+		}
 		return nil, ErrNotFound
 	}
 
-	return ioutil.ReadAll(resp.Body)
+	return resp.Body, nil
 }
 
 func makeManifestURL(baseURL, integration, stream, version string) (*url.URL, error) {

pkg/genlib/fields/version.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"io/ioutil"
 	"net/url"
 	"path"
 	"strings"
@@ -17,7 +18,7 @@ func MapVersion(ctx context.Context, baseUrl, integration, kibanaVersion string)
 		return "", err
 	}
 
-	body, err := getFromURL(ctx, searchUrl.String())
+	r, err := getFromURL(ctx, searchUrl.String())
 	if err != nil {
 		return "", err
 	}
@@ -26,6 +27,12 @@ func MapVersion(ctx context.Context, baseUrl, integration, kibanaVersion string)
 		Version string `json:"version"`
 	}
 
+	body, err := ioutil.ReadAll(r)
+	if err != nil {
+		_ = r.Close()
+		return "", err
+	}
+
 	if err = json.Unmarshal(body, &payload); err != nil {
 		return "", err
 	}