feature: "tagless import" feature mode (#4)

In order to build from lookahead sources, srpmproc must support the new gitlab address for centos 9 stream as well as manipulating the sources to expected format.

Resolves #2.


- Options for tagless import and new stream lookaside format added
- Git Mode now supports scanning for branch head in addition to the "normal" pulling of specific version tags
- Alternate ProcessRPM added for Tagless mode in process.go (still a WIP)
- Tagless mode converts a repo to the "traditional" format (SPECS/ + SOURCES/ + <pkg>.metadata )
- Tagless mode will build a dummy srpm to determine NVR information (rpmbuild + rpm shell commands) (will use this to tag imports in the target git)
- Limitation:  Tagless imports only pull the latest head from a branch
- CentOS-Stream import branches are converted from stream-<MODULE_VERSION>-<RHEL_VERSION> to the more familiar r9s-stream-<VERSION>
- stream-style YAML is detected and converted for modules, similar to the older modulemd.src.txt files
- This new pattern is for "tagless mode" only, previous tagged imports (from git.centos.org) should not be affected

Author: skip77

cmd/srpmproc/main.go

@@ -56,6 +56,8 @@ var (
 	basicPassword        string
 	packageVersion       string
 	packageRelease       string
+	taglessMode          bool
+	altLookAside         bool
 )
 
 var root = &cobra.Command{
@@ -92,7 +94,10 @@ func mn(_ *cobra.Command, _ []string) {
 		HttpPassword:         basicPassword,
 		PackageVersion:       packageVersion,
 		PackageRelease:       packageRelease,
+		TaglessMode:          taglessMode,
+		AltLookAside:         altLookAside,
 	})
+
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -142,6 +147,8 @@ func main() {
 	root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
 	root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
 	root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
+	root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode:  If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
+	root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

pkg/data/process.go

@@ -57,4 +57,6 @@ type ProcessData struct {
 	Log                  *log.Logger
 	PackageVersion       string
 	PackageRelease       string
+	TaglessMode          bool
+	AltLookAside         bool
 }

pkg/misc/regex.go

@@ -5,6 +5,7 @@ import (
 	"github.com/rocky-linux/srpmproc/pkg/data"
 	"path/filepath"
 	"regexp"
+	"strings"
 )
 
 func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
@@ -31,3 +32,22 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
 
 	return regexp.MustCompile(regex)
 }
+
+// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine
+// if we are ok with importing that reference.  We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
+// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
+func TaglessRefOk(tag string, pd *data.ProcessData) bool {
+
+	// First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
+	if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
+		return true
+	}
+
+	// Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is
+	// (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0")
+	if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) {
+		return true
+	}
+
+	return false
+}

pkg/modes/git.go

@@ -116,7 +116,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
 				if exists != nil && exists.when.After(tag.Tagger.When) {
 					return nil
 				}
-
 				latestTags[match[2]] = &remoteTarget{
 					remote: refSpec,
 					when:   tag.Tagger.When,
@@ -126,11 +125,39 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
 		return nil
 	}
 
+	// In case of "tagless mode", we need to get the head ref of the branch instead
+	// This is a kind of alternative implementation of the above tagAdd assignment
+	refAdd := func(tag *object.Tag) error {
+		if misc.TaglessRefOk(tag.Name, pd) {
+			pd.Log.Printf("Tagless mode:  Identified tagless commit for import: %s\n", tag.Name)
+			refSpec := fmt.Sprintf(tag.Name)
+
+			// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
+			// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
+			tmpRef := strings.Split(refSpec, "/")
+			tmpBranchName := tmpRef[(len(tmpRef) - 1)]
+
+			latestTags[tmpBranchName] = &remoteTarget{
+				remote: refSpec,
+				when:   tag.Tagger.When,
+			}
+		}
+		return nil
+	}
+
 	tagIter, err := repo.TagObjects()
+
 	if err != nil {
 		return nil, fmt.Errorf("could not get tag objects: %v", err)
 	}
-	_ = tagIter.ForEach(tagAdd)
+
+	// tagless mode means we use "refAdd" (add commit by reference)
+	// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
+	if pd.TaglessMode {
+		_ = tagIter.ForEach(refAdd)
+	} else {
+		_ = tagIter.ForEach(tagAdd)
+	}
 
 	listOpts := &git.ListOptions{
 		Auth: pd.Authenticator,
@@ -157,17 +184,26 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
 		if err != nil {
 			continue
 		}
-		_ = tagAdd(&object.Tag{
-			Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
-			Tagger: commit.Committer,
-		})
+
+		// Call refAdd instead of tagAdd in the case of TaglessMode enabled
+		if pd.TaglessMode {
+			_ = refAdd(&object.Tag{
+				Name:   string(ref.Name()),
+				Tagger: commit.Committer,
+			})
+		} else {
+			_ = tagAdd(&object.Tag{
+				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
+				Tagger: commit.Committer,
+			})
+		}
+
 	}
 
 	for _, branch := range latestTags {
 		pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
 		branches = append(branches, *branch)
 	}
-
 	sort.Sort(branches)
 
 	var sortedBranches []string
@@ -185,54 +221,66 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
 }
 
 func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
+
 	remote, err := md.Repo.Remote("upstream")
-	if err != nil {
+
+	if err != nil && !pd.TaglessMode {
 		return fmt.Errorf("could not get upstream remote: %v", err)
 	}
 
 	var refspec config.RefSpec
 	var branchName string
 
-	if strings.HasPrefix(md.TagBranch, "refs/heads") {
-		refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
-		branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
-	} else {
-		match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
-		branchName = match[2]
-		refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
-	}
-	pd.Log.Printf("checking out upstream refspec %s", refspec)
-	fetchOpts := &git.FetchOptions{
-		Auth:       pd.Authenticator,
-		RemoteName: "upstream",
-		RefSpecs:   []config.RefSpec{refspec},
-		Tags:       git.AllTags,
-		Force:      true,
-	}
-	err = remote.Fetch(fetchOpts)
-	if err != nil && err != git.NoErrAlreadyUpToDate {
-		if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
-			fetchOpts.Auth = nil
-			err = remote.Fetch(fetchOpts)
-			if err != nil && err != git.NoErrAlreadyUpToDate {
+	// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
+	// and don't need to perform any checkout or fetch operations
+	if !pd.TaglessMode {
+		if strings.HasPrefix(md.TagBranch, "refs/heads") {
+			refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
+			branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
+		} else {
+			match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
+			branchName = match[2]
+			refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
+			fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName)
+		}
+		pd.Log.Printf("checking out upstream refspec %s", refspec)
+
+		fetchOpts := &git.FetchOptions{
+			Auth:       pd.Authenticator,
+			RemoteName: "upstream",
+			RefSpecs:   []config.RefSpec{refspec},
+			Tags:       git.AllTags,
+			Force:      true,
+		}
+		err = remote.Fetch(fetchOpts)
+		if err != nil && err != git.NoErrAlreadyUpToDate {
+			if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
+				fetchOpts.Auth = nil
+				err = remote.Fetch(fetchOpts)
+				if err != nil && err != git.NoErrAlreadyUpToDate {
+					return fmt.Errorf("could not fetch upstream: %v", err)
+				}
+			} else {
 				return fmt.Errorf("could not fetch upstream: %v", err)
 			}
-		} else {
-			return fmt.Errorf("could not fetch upstream: %v", err)
 		}
-	}
 
-	err = md.Worktree.Checkout(&git.CheckoutOptions{
-		Branch: plumbing.ReferenceName(md.TagBranch),
-		Force:  true,
-	})
-	if err != nil {
-		return fmt.Errorf("could not checkout source from git: %v", err)
+		err = md.Worktree.Checkout(&git.CheckoutOptions{
+			Branch: plumbing.ReferenceName(md.TagBranch),
+			Force:  true,
+		})
+		if err != nil {
+			return fmt.Errorf("could not checkout source from git: %v", err)
+		}
+
+		_, err = md.Worktree.Add(".")
+		if err != nil {
+			return fmt.Errorf("could not add Worktree: %v", err)
+		}
 	}
 
-	_, err = md.Worktree.Add(".")
-	if err != nil {
-		return fmt.Errorf("could not add Worktree: %v", err)
+	if pd.TaglessMode {
+		branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
 	}
 
 	metadataPath := ""
@@ -292,7 +340,32 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
 				body = fromBlobStorage
 				pd.Log.Printf("downloading %s from blob storage", hash)
 			} else {
-				url := fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
+
+				url := ""
+				// Alternate lookaside logic:  if enabled, we pull from a new URL pattern
+				if !pd.AltLookAside {
+					url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
+				} else {
+					// We first need the hash algorithm based on length of hash:
+					hashType := "sha512"
+					switch len(hash) {
+					case 128:
+						hashType = "sha512"
+					case 64:
+						hashType = "sha256"
+					case 40:
+						hashType = "sha1"
+					case 32:
+						hashType = "md5"
+					}
+
+					// need the name of the file without "SOURCES/":
+					fileName := strings.Split(path, "/")[1]
+
+					// Alt. lookaside url is of the form: <cdn> / <name> / <filename> / <hashtype> / <hash> / <filename>
+					url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName)
+				}
+
 				pd.Log.Printf("downloading %s", url)
 
 				req, err := http.NewRequest("GET", url, nil)

pkg/srpmproc/patch.go

@@ -32,6 +32,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
@@ -286,13 +287,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
 		return nil
 	}
 
-	mdTxtPath := "SOURCES/modulemd.src.txt"
-	f, err := md.Worktree.Filesystem.Open(mdTxtPath)
-	if err != nil {
-		mdTxtPath = "SOURCES/modulemd.txt"
+	mdTxtPath := ""
+	var f billy.File
+
+	// tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import)
+	// if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt
+	if pd.TaglessMode {
+		mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name)
+		f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+		if err != nil {
+			mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name)
+			f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+			if err != nil {
+				return fmt.Errorf("could not open modulemd file: %v", err)
+			}
+		}
+	} else {
+		mdTxtPath = "SOURCES/modulemd.src.txt"
 		f, err = md.Worktree.Filesystem.Open(mdTxtPath)
 		if err != nil {
-			return fmt.Errorf("could not open modulemd file: %v", err)
+			mdTxtPath = "SOURCES/modulemd.txt"
+			f, err = md.Worktree.Filesystem.Open(mdTxtPath)
+			if err != nil {
+				return fmt.Errorf("could not open modulemd file: %v", err)
+			}
 		}
 	}
 
@@ -307,11 +325,13 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
 	}
 
 	// Get stream branch from tag
-	match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
-	streamBranch := strings.Split(match[2], "-")
-	// Force stream to be the same as stream name in branch
-	module.Data.Stream = streamBranch[len(streamBranch)-1]
-
+	// (in tagless mode we are trusting the "Stream: <VERSION>" text in the source YAML to be accurate)
+	if !pd.TaglessMode {
+		match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
+		streamBranch := strings.Split(match[2], "-")
+		// Force stream to be the same as stream name in branch
+		module.Data.Stream = streamBranch[len(streamBranch)-1]
+	}
 	log.Println("This module contains the following rpms:")
 	for name := range module.Data.Components.Rpms {
 		pd.Log.Printf("\t- %s", name)