Skip to content

map: make special extensions configurable, remove legacy csv #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 3, 2025
Merged

map: make special extensions configurable, remove legacy csv #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@
// ServerList is a list of ServerConfig structs, which gets parsed into servers.
ServerList []ServerConfig `mapstructure:"servers"`

// Special extensions for the download map
SpecialExtensions map[string]string `mapstructure:"specialExtensions"`

// ReloadFunc is called when a reload is done via http api.
ReloadFunc func()

Expand Down Expand Up @@ -82,7 +85,7 @@
}
}

func Remove[V comparable](collection []V, value V) []V {

Check failure on line 88 in config.go

View workflow job for this annotation

GitHub Actions / build 

exported function Remove should have comment or be unexported
return lo.Filter(collection, func(item V, _ int) bool {
return item != value
})
Expand Down Expand Up @@ -163,7 +166,7 @@
r.config.TopChoices = len(r.servers)
}

// Check if on the config is declared or use default logic
// Check if on the config is declared or use default logic
if r.config.SameCityThreshold == 0 {
r.config.SameCityThreshold = 200000.0
}
Expand Down Expand Up @@ -230,7 +233,7 @@
"path": u.Path,
"latitude": s.Latitude,
"longitude": s.Longitude,
"country": s.Country,
"country": s.Country,
}).Info("Added server")
}
}(i, server, u)
Expand Down Expand Up @@ -314,7 +317,7 @@
return nil
}
log.WithField("file", mapFile).Info("Loading download map")
newMap, err := loadMapFile(mapFile)
newMap, err := loadMapFile(mapFile, r.config.SpecialExtensions)
if err != nil {
return err
}
Expand Down
11 changes: 10 additions & 1 deletion dlrouter.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,13 @@ servers:
- server: stpete-mirror.armbian.com/apt/
- server: xogium.performanceservers.nl/apt/
- server: github.com/armbian/mirror/releases/download/
continent: GITHUB
continent: GITHUB

specialExtensions:
boot-sms.img.xz: -boot-sms
boot-boe.img.xz: -boot-boe
boot-csot.img.xz: -boot-csot
rootfs.img.xz: -rootfs
img.qcow2: -qcow2
img.qcow2.xz: -qcow2
boot.bin.xz: -uboot-bin
62 changes: 16 additions & 46 deletions map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package redirector

import (
"encoding/csv"
"encoding/json"
"errors"
"io"
Expand All @@ -12,6 +11,7 @@ import (

log "github.com/sirupsen/logrus"

"golang.org/x/exp/maps"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
Expand All @@ -20,7 +20,7 @@ import (
var ErrUnsupportedFormat = errors.New("unsupported map format")

// loadMapFile loads a file as a map
func loadMapFile(file string) (map[string]string, error) {
func loadMapFile(file string, specialExtensions map[string]string) (map[string]string, error) {
f, err := os.Open(file)

if err != nil {
Expand All @@ -32,40 +32,13 @@ func loadMapFile(file string) (map[string]string, error) {
ext := path.Ext(file)

switch ext {
case ".csv":
return loadMapCSV(f)
case ".json":
return loadMapJSON(f)
return loadMapJSON(f, specialExtensions)
}

return nil, ErrUnsupportedFormat
}

// loadMapCSV loads a pipe separated file of mappings
func loadMapCSV(f io.Reader) (map[string]string, error) {
m := make(map[string]string)

r := csv.NewReader(f)

r.Comma = '|'

for {
row, err := r.Read()

if err != nil {
if err == io.EOF {
break
}

return nil, err
}

m[strings.TrimLeft(row[0], "/")] = strings.TrimLeft(row[1], "/")
}

return m, nil
}

// Map represents a JSON format of an asset list
type Map struct {
Assets []ReleaseFile `json:"assets"`
Expand All @@ -88,11 +61,14 @@ type ReleaseFile struct {

var distroCaser = cases.Title(language.Und)

var imageExtensions = []string{"img.xz", "img.qcow2", "boot.bin.xz"}

// loadMapJSON loads a map file from JSON, based on the format specified in the github issue.
// See: https://github.com/armbian/os/pull/129
func loadMapJSON(f io.Reader) (map[string]string, error) {
func loadMapJSON(f io.Reader, specialExtensions map[string]string) (map[string]string, error) {
// Avoid panics
if specialExtensions == nil {
specialExtensions = make(map[string]string)
}

m := make(map[string]string)

var data Map
Expand Down Expand Up @@ -135,21 +111,15 @@ func loadMapJSON(f io.Reader) (map[string]string, error) {
}

// Check special case for some extensions
switch {
case strings.Contains(file.Extension, "boot-sms.img.xz"):
sb.WriteString("-boot-sms")
case strings.Contains(file.Extension, "boot-boe.img.xz"):
sb.WriteString("-boot-boe")
case strings.Contains(file.Extension, "boot-csot.img.xz"):
sb.WriteString("-boot-csot")
case strings.Contains(file.Extension, "rootfs.img.xz"):
sb.WriteString("-rootfs")
case strings.Contains(file.Extension, "img.qcow2"):
sb.WriteString("-qcow2")
case strings.Contains(file.Extension, "boot.bin.xz"):
sb.WriteString("-uboot-bin")
for k, v := range specialExtensions {
if strings.Contains(file.Extension, k) {
sb.WriteString(v)
}
}

imageExtensions := maps.Keys(specialExtensions)
imageExtensions = append(imageExtensions, "img.xz") // extra allocation, but it's fine

// Add board into the map without an extension
for _, ext := range imageExtensions {
if strings.HasSuffix(file.Extension, ext) {
Expand Down
22 changes: 13 additions & 9 deletions map_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,17 @@ import (
. "github.com/onsi/gomega"
)

var _ = Describe("Map", func() {
It("Should successfully load the map from a CSV/Pipe separated file", func() {
m, err := loadMapCSV(strings.NewReader(`bananapi/Bullseye_current|bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz|Aug 26 2021|332M`))
var testExtensions = map[string]string{
"boot-sms.img.xz": "-boot-sms",
"boot-boe.img.xz": "-boot-boe",
"boot-csot.img.xz": "-boot-csot",
"rootfs.img.xz": "-rootfs",
"img.qcow2": "-qcow2",
"img.qcow2.xz": "-qcow2",
"boot.bin.xz": "-uboot-bin",
}

Expect(err).To(BeNil())
Expect(m["bananapi/Bullseye_current"]).To(Equal("bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz"))
})
var _ = Describe("Map", func() {
It("Should successfully load the map from a JSON file", func() {
data := `{
"assets": [
Expand All @@ -34,7 +38,7 @@ var _ = Describe("Map", func() {
]
}`

m, err := loadMapJSON(strings.NewReader(data))
m, err := loadMapJSON(strings.NewReader(data), testExtensions)

Expect(err).To(BeNil())
Expect(m["aml-s9xx-box/Bookworm_current_server"]).To(Equal("/aml-s9xx-box/archive/Armbian_23.11.1_Aml-s9xx-box_bookworm_current_6.1.63.img.xz"))
Expand Down Expand Up @@ -85,7 +89,7 @@ var _ = Describe("Map", func() {
]
}`

m, err := loadMapJSON(strings.NewReader(data))
m, err := loadMapJSON(strings.NewReader(data), testExtensions)

Expect(err).To(BeNil())
Expect(m["khadas-vim1/Bookworm_current_xfce"]).To(Equal("/khadas-vim1/archive/Armbian_23.11.1_Khadas-vim1_bookworm_current_6.1.63_xfce_desktop.img.xz"))
Expand Down Expand Up @@ -200,7 +204,7 @@ var _ = Describe("Map", func() {
]
}`

m, err := loadMapJSON(strings.NewReader(data))
m, err := loadMapJSON(strings.NewReader(data), testExtensions)

Expect(err).To(BeNil())
Expect(m["khadas-vim4/Bookworm_legacy_server"]).To(Equal("/khadas-vim4/archive/Armbian_23.11.1_Khadas-vim4_bookworm_legacy_5.4.180.oowow.img.xz"))
Expand Down
Loading