Merge pull request #50 from NamelessOne91/legacy_pkgs_handling

Sync all repo packages by default, use a flag to skip legacy ones

Author: NamelessOne91

cmd/sync.go

@@ -77,7 +77,7 @@ var (
 	}
 	thisRepo           string
 	archs              string
-	syncLegacyPackages bool
+	skipLegacyPackages bool
 )
 
 // Config maps the configuration in minima.yaml
@@ -93,8 +93,8 @@ func syncersFromConfig(configString string) ([]*get.Syncer, error) {
 	if err != nil {
 		return nil, err
 	}
-	//---passing the flag value to a global variable in get package, to trigger syncing of i586 rpms inside x86_64
-	get.Legacy = syncLegacyPackages
+	//---passing the flag value to a global variable in get package, to disables syncing of i586 and i686 rpms (usually inside x86_64)
+	get.SkipLegacy = skipLegacyPackages
 
 	if config.SCC.Username != "" {
 		if thisRepo != "" {
@@ -123,11 +123,6 @@ func syncersFromConfig(configString string) ([]*get.Syncer, error) {
 			return nil, err
 		}
 
-		archs := map[string]bool{}
-		for _, archString := range httpRepo.Archs {
-			archs[archString] = true
-		}
-
 		var storage get.Storage
 		switch config.Storage.Type {
 		case "file":
@@ -138,7 +133,7 @@ func syncersFromConfig(configString string) ([]*get.Syncer, error) {
 				return nil, err
 			}
 		}
-		syncers = append(syncers, get.NewSyncer(*repoURL, archs, storage))
+		syncers = append(syncers, get.NewSyncer(*repoURL, storage))
 	}
 
 	return syncers, nil
@@ -162,5 +157,5 @@ func init() {
 	// local flags
 	syncCmd.Flags().StringVarP(&thisRepo, "repository", "r", "", "flag that can specifies a single repo (example: SLES11-SP4-Updates)")
 	syncCmd.Flags().StringVarP(&archs, "arch", "a", "", "flag that specifies covered archs in the given repo")
-	syncCmd.Flags().BoolVarP(&syncLegacyPackages, "legacypackages", "l", false, "flag that triggers mirroring of i586 pkgs in x86_64 repos")
+	syncCmd.Flags().BoolVarP(&skipLegacyPackages, "nolegacy", "l", false, "flag that disables mirroring of i586 and i686 pkgs")
 }

get/syncer.go

@@ -101,20 +101,31 @@ var (
 			Noarch:               "all",
 		},
 	}
-	Legacy bool
+	SkipLegacy bool
 )
 
 // Syncer syncs repos from an HTTP source to a Storage
 type Syncer struct {
 	// URL of the repo this syncer syncs
 	URL     url.URL
-	archs   map[string]bool
 	storage Storage
 }
 
+// Decision encodes what to do with a file
+type Decision int
+
+const (
+	// Download means the Syncer will download a file
+	Download Decision = iota
+	// Recycle means the Syncer will copy an existing file without downloading
+	Recycle
+	// Skip means the Syncer detected an already-existing file and has nothing to do
+	Skip
+)
+
 // NewSyncer creates a new Syncer
-func NewSyncer(url url.URL, archs map[string]bool, storage Storage) *Syncer {
-	return &Syncer{url, archs, storage}
+func NewSyncer(url url.URL, storage Storage) *Syncer {
+	return &Syncer{url, storage}
 }
 
 // StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
@@ -190,11 +201,6 @@ func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
 	return
 }
 
-// downloadStore downloads a repo-relative path into a file
-func (r *Syncer) downloadStore(path string, description string) error {
-	return r.downloadStoreApply(path, "", description, 0, util.Nop)
-}
-
 // downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
 func (r *Syncer) downloadStoreApply(relativePath string, checksum string, description string, hash crypto.Hash, f util.ReaderConsumer) error {
 	log.Printf("Downloading %v...", description)
@@ -232,6 +238,7 @@ func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesTo
 			log.Printf(data[i].Location.Href)
 			metadataLocation := data[i].Location.Href
 			metadataChecksum := data[i].Checksum
+
 			decision := r.decide(metadataLocation, metadataChecksum, checksumMap)
 			switch decision {
 			case Download:
@@ -404,39 +411,30 @@ func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum,
 	if err != nil {
 		return
 	}
+
 	compType := strings.Trim(filepath.Ext(path), ".")
 	primary, err := repoType.DecodePackages(reader, compType)
 	if err != nil {
 		return
 	}
 
-	allArchs := len(r.archs) == 0
 	for _, pack := range primary.Packages {
-		if allArchs || pack.Arch == repoType.Noarch || r.archs[pack.Arch] || (r.archs["i586"] && pack.Arch == "i686") || (Legacy && (r.archs["x86_64"] && (pack.Arch == "i586" || pack.Arch == "i686"))) {
-			decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
-			switch decision {
-			case Download:
-				packagesToDownload = append(packagesToDownload, pack)
-			case Recycle:
-				packagesToRecycle = append(packagesToRecycle, pack)
-			}
+		if SkipLegacy && (pack.Arch == "i586" || pack.Arch == "i686") {
+			fmt.Println("Skipping legacy package:", pack.Location.Href)
+			continue
+		}
+
+		decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
+		switch decision {
+		case Download:
+			packagesToDownload = append(packagesToDownload, pack)
+		case Recycle:
+			packagesToRecycle = append(packagesToRecycle, pack)
 		}
 	}
 	return
 }
 
-// Decision encodes what to do with a file
-type Decision int
-
-const (
-	// Download means the Syncer will download a file
-	Download Decision = iota
-	// Recycle means the Syncer will copy an existing file without downloading
-	Recycle
-	// Skip means the Syncer detected an already-existing file and has nothing to do
-	Skip
-)
-
 func (r *Syncer) decide(location string, checksum XMLChecksum, checksumMap map[string]XMLChecksum) Decision {
 	previousChecksum, foundInChecksumMap := checksumMap[location]
 

get/syncer_test.go

@@ -18,15 +18,12 @@ func TestStoreRepo(t *testing.T) {
 		t.Error(err)
 	}
 
-	archs := map[string]bool{
-		"x86_64": true,
-	}
 	storage := NewFileStorage(directory)
 	url, err := url.Parse("http://localhost:8080/repo")
 	if err != nil {
 		t.Error(err)
 	}
-	syncer := NewSyncer(*url, archs, storage)
+	syncer := NewSyncer(*url, storage)
 
 	// first sync
 	err = syncer.StoreRepo()
@@ -75,15 +72,12 @@ func TestStoreRepoZstd(t *testing.T) {
 		t.Error(err)
 	}
 
-	archs := map[string]bool{
-		"x86_64": true,
-	}
 	storage := NewFileStorage(directory)
 	url, err := url.Parse("http://localhost:8080/zstrepo")
 	if err != nil {
 		t.Error(err)
 	}
-	syncer := NewSyncer(*url, archs, storage)
+	syncer := NewSyncer(*url, storage)
 
 	// first sync
 	err = syncer.StoreRepo()
@@ -131,15 +125,12 @@ func TestStoreDebRepo(t *testing.T) {
 		t.Error(err)
 	}
 
-	archs := map[string]bool{
-		"amd64": true,
-	}
 	storage := NewFileStorage(directory)
 	url, err := url.Parse("http://localhost:8080/deb_repo")
 	if err != nil {
 		t.Error(err)
 	}
-	syncer := NewSyncer(*url, archs, storage)
+	syncer := NewSyncer(*url, storage)
 
 	// first sync
 	err = syncer.StoreRepo()