Merge pull request #97 from TheThingsNetwork/feature/43-firefly

Support migration from the Firefly LNS

Author: KrishnaIyer

CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Added
 
+- Firefly source.
+
 ### Changed
 
 ### Deprecated

README.md

@@ -13,7 +13,7 @@ Binaries are available on [GitHub](https://github.com/TheThingsNetwork/lorawan-s
 - [x] The Things Network Stack V2
 - [x] [ChirpStack Network Server](https://www.chirpstack.io/)
 - [x] [The Things Stack](https://www.github.com/TheThingsNetwork/lorawan-stack/)
-- [ ] [Firefly](https://fireflyiot.com/)
+- [x] [Firefly](https://fireflyiot.com/)
 - [ ] [LORIOT Network Server](https://www.loriot.io/)
 
 Support for different sources is done by creating Source plugins. List available sources with:
@@ -238,6 +238,75 @@ $ ttn-lw-migrate tts application 'my-app-id' --dry-run --verbose > devices.json
 $ ttn-lw-migrate tts application 'my-app-id' > devices.json
 ```
 
+## Firefly
+
+### Configuration
+
+Configure with environment variables, or command-line arguments.
+
+See `ttn-lw-migrate firefly {device|application} --help` for more details.
+
+The following example shows how to set options via environment variables.
+
+```bash
+$ export FIREFLY_HOST=example.com       # Host of the Firefly API
+$ export FIREFLY_API_KEY=abcdefgh       # Firefly API Key
+$ export APP_ID=my-test-app             # Application ID for the exported devices
+$ export JOIN_EUI=1111111111111111      # JoinEUI for the exported devices
+$ export FREQUENCY_PLAN_ID=EU_863_870   # Frequency Plan ID for the exported devices
+$ export MAC_VERSION=1.0.2b             # LoRaWAN MAC version for the exported devices
+```
+
+### Notes
+
+- The export process will halt if any error occurs.
+- Use the `--invalidate-keys` option to invalidate the root and/or session keys of the devices on the Firefly server. This is necessary to prevent both networks from communicating with the same device. The last byte of the keys will be incremented by 0x01. This enables an easy rollback if necessary. Setting this flag to false (default) would result in a "dry run", where the devices are exported but they will still be able to communicate with the Firefly server.
+
+### Export Devices
+
+To export a single device using its Device EUI (e.g. `1111111111111112`):
+
+```bash
+# dry run first, verify that no errors occur
+$ ttn-lw-migrate firefly device 1111111111111112 --verbose > devices.json
+# export device
+$ ttn-lw-migrate firefly device 1111111111111112 --invalidate-keys > devices.json
+```
+
+In order to export a large number of devices, create a file named `device_euis.txt` with one device EUI per line:
+
+```txt
+1111111111111112
+FF11111111111134
+ABCD111111111100
+```
+
+And then export with:
+
+```bash
+# dry run first, verify that no errors occur
+$ ttn-lw-migrate firefly device --verbose < device_ids.txt > devices.json
+# export devices
+$ ttn-lw-migrate firefly device --invalidate-keys < device_ids.txt > devices.json
+```
+
+### Export All Devices
+
+The Firefly LNS does not strictly enforce device to application relationships.
+
+Setting the `--all` flag will export **all devices that are accessible by the API key**. The `application` command without the `--all` flag does nothing.
+
+> Note: Please be cautious while using this command as this might invalidate all the keys of all the devices.
+
+To export all devices accessible by the API Key,
+
+```bash
+# dry run first, verify that no errors occur
+$ ttn-lw-migrate firefly application --all --verbose > devices.json
+# export all devices
+$ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json
+```
+
 ## Development Environment
 
 Requires Go version 1.16 or higher. [Download Go](https://golang.org/dl/).

cmd/firefly/firefly.go

@@ -0,0 +1,27 @@
+// Copyright © 2023 The Things Network Foundation, The Things Industries B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package firefly
+
+import (
+	"go.thethings.network/lorawan-stack-migrate/pkg/commands"
+	_ "go.thethings.network/lorawan-stack-migrate/pkg/source/firefly"
+)
+
+const sourceName = "firefly"
+
+// FireflyCmd represents the firefly source.
+var FireflyCmd = commands.Source(sourceName,
+	"Export devices from Digimondo's Firefly",
+)

cmd/root.go

@@ -20,16 +20,14 @@ import (
 
 	"github.com/spf13/cobra"
 	"go.thethings.network/lorawan-stack-migrate/cmd/chirpstack"
+	"go.thethings.network/lorawan-stack-migrate/cmd/firefly"
 	"go.thethings.network/lorawan-stack-migrate/cmd/ttnv2"
 	"go.thethings.network/lorawan-stack-migrate/cmd/tts"
 	"go.thethings.network/lorawan-stack-migrate/pkg/export"
 	"go.thethings.network/lorawan-stack-migrate/pkg/source"
-	"go.thethings.network/lorawan-stack/v3/pkg/log"
-	"go.thethings.network/lorawan-stack/v3/pkg/rpcmiddleware/rpclog"
 )
 
 var (
-	logger    *log.Logger
 	ctx       = context.Background()
 	exportCfg = export.Config{}
 	rootCfg   = &source.RootConfig
@@ -39,26 +37,8 @@ var (
 
 		SilenceUsage: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			logLevel := log.InfoLevel
-			if verbose, _ := cmd.Flags().GetBool("verbose"); verbose {
-				logLevel = log.DebugLevel
-			}
-			logHandler, err := log.NewZap("console")
-			if err != nil {
-				return err
-			}
-			logger = log.NewLogger(
-				logHandler,
-				log.WithLevel(logLevel),
-			)
-			rpclog.ReplaceGrpcLogger(logger)
-			ctx = log.NewContext(ctx, logger)
-
 			exportCfg.DevIDPrefix, _ = cmd.Flags().GetString("dev-id-prefix")
-			exportCfg.EUIForID, _ = cmd.Flags().GetBool("set-eui-as-id")
-			ctx = export.NewContext(ctx, exportCfg)
-
-			cmd.SetContext(ctx)
+			cmd.SetContext(export.NewContext(ctx, exportCfg))
 			return nil
 		},
 	}
@@ -86,6 +66,11 @@ func init() {
 		"frequency-plans-url",
 		"https://raw.githubusercontent.com/TheThingsNetwork/lorawan-frequency-plans/master",
 		"URL for fetching frequency plans")
+	rootCmd.PersistentFlags().String(
+		"dev-id-prefix",
+		"",
+		"(optional) value to be prefixed to the resulting device IDs",
+	)
 
 	rootCmd.AddGroup(&cobra.Group{
 		ID:    "sources",
@@ -95,4 +80,5 @@ func init() {
 	rootCmd.AddCommand(ttnv2.TTNv2Cmd)
 	rootCmd.AddCommand(tts.TTSCmd)
 	rootCmd.AddCommand(chirpstack.ChirpStackCmd)
+	rootCmd.AddCommand(firefly.FireflyCmd)
 }

pkg/commands/export.go

@@ -16,23 +16,15 @@ package commands
 
 import (
 	"io"
-	"os"
 
 	"github.com/spf13/cobra"
 	"go.thethings.network/lorawan-stack-migrate/pkg/export"
+	"go.thethings.network/lorawan-stack-migrate/pkg/iterator"
 	"go.thethings.network/lorawan-stack-migrate/pkg/source"
 	"go.thethings.network/lorawan-stack/v3/pkg/log"
 )
 
 func Export(cmd *cobra.Command, args []string, f func(s source.Source, item string) error) error {
-	var iter Iterator
-	switch len(args) {
-	case 0:
-		iter = NewReaderIterator(os.Stdin, '\n')
-	default:
-		iter = NewListIterator(args)
-	}
-
 	s, err := source.NewSource(cmd.Context())
 	if err != nil {
 		return err
@@ -43,6 +35,14 @@ func Export(cmd *cobra.Command, args []string, f func(s source.Source, item stri
 		}
 	}()
 
+	var iter iterator.Iterator
+	switch len(args) {
+	case 0:
+		iter = s.Iterator(cmd.Name() == "application")
+	default:
+		iter = iterator.NewListIterator(args)
+	}
+
 	for {
 		item, err := iter.Next()
 		switch err {

pkg/commands/source.go

@@ -127,14 +127,14 @@ func ExecuteParentPersistentPreRun(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 	p := cmd.Parent()
-
 	if f := p.PersistentPreRunE; f != nil {
 		if err := f(p, args); err != nil {
 			return err
 		}
 	} else if f := p.PersistentPreRun; f != nil {
 		f(p, args)
 	}
+	cmd.SetContext(p.Context())
 	return nil
 }
 

pkg/export/errors.go

@@ -22,4 +22,5 @@ var (
 	errInvalidFields         = errors.DefineInvalidArgument("invalid_fields", "invalid fields for device `{device_id}`")
 	errDevIDExceedsMaxLength = errors.Define("dev_id_exceeds_max_length", "device ID `{id}` exceeds max length")
 	errAppIDExceedsMaxLength = errors.Define("app_id_exceeds_max_length", "application ID `{id}` exceeds max length")
+	errNoExportedIDorEUI     = errors.Define("no_exported_id_or_eui", "device `{device_id}` has no exported ID or EUI")
 )

pkg/export/export.go

@@ -15,14 +15,14 @@
 package export
 
 import (
+	"encoding/hex"
 	"fmt"
 	"os"
 	"strings"
 
+	"go.thethings.network/lorawan-stack-migrate/pkg/source"
 	"go.thethings.network/lorawan-stack/v3/pkg/jsonpb"
 	"go.thethings.network/lorawan-stack/v3/pkg/ttnpb"
-
-	"go.thethings.network/lorawan-stack-migrate/pkg/source"
 )
 
 const (
@@ -36,7 +36,6 @@ func toJSON(dev *ttnpb.EndDevice) ([]byte, error) {
 }
 
 type Config struct {
-	EUIForID    bool
 	DevIDPrefix string
 }
 
@@ -46,9 +45,13 @@ func (cfg Config) ExportDev(s source.Source, devID string) error {
 		return errExport.WithAttributes("device_id", devID).WithCause(err)
 	}
 	oldID := dev.Ids.DeviceId
+	eui := dev.Ids.DevEui
 
-	if eui := dev.Ids.DevEui; cfg.EUIForID && eui != nil {
-		dev.Ids.DeviceId = strings.ToLower(string(eui))
+	if oldID == "" {
+		if eui == nil {
+			return errNoExportedIDorEUI.WithAttributes("device_id", devID)
+		}
+		dev.Ids.DeviceId = strings.ToLower(hex.EncodeToString(eui))
 	}
 	if cfg.DevIDPrefix != "" {
 		dev.Ids.DeviceId = fmt.Sprintf("%s-%s", cfg.DevIDPrefix, dev.Ids.DeviceId)
@@ -59,7 +62,7 @@ func (cfg Config) ExportDev(s source.Source, devID string) error {
 		return errDevIDExceedsMaxLength.WithAttributes("id", id)
 	}
 
-	if dev.Ids.DeviceId != oldID {
+	if dev.Ids.DeviceId != oldID && oldID != "" {
 		if dev.Attributes == nil {
 			dev.Attributes = make(map[string]string)
 		}

pkg/commands/iterator.go pkg/iterator/iterator.gopkg/commands/iterator.go renamed to pkg/iterator/iterator.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package commands
+package iterator
 
 import (
 	"bufio"
@@ -61,3 +61,17 @@ func (r *readerIterator) Next() (string, error) {
 	}
 	return strings.TrimSpace(s), err
 }
+
+// noopIterator is a no-op iterator.
+type noopIterator struct {
+}
+
+// NewNoopIterator returns a new no-op iterator.
+func NewNoopIterator() Iterator {
+	return &noopIterator{}
+}
+
+// Next implements Iterator
+func (n *noopIterator) Next() (string, error) {
+	return "", io.EOF
+}

pkg/commands/iterator_test.go pkg/iterator/iterator_test.gopkg/commands/iterator_test.go renamed to pkg/iterator/iterator_test.go

@@ -1,4 +1,18 @@
-package commands_test
+// Copyright © 2023 The Things Network Foundation, The Things Industries B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iterator_test
 
 import (
 	"bytes"
@@ -8,11 +22,11 @@ import (
 
 	"github.com/smartystreets/assertions"
 	"github.com/smartystreets/assertions/should"
-	"go.thethings.network/lorawan-stack-migrate/pkg/commands"
+	"go.thethings.network/lorawan-stack-migrate/pkg/iterator"
 )
 
 func TestListIterator(t *testing.T) {
-	it := commands.NewListIterator([]string{"one", "two", "three"})
+	it := iterator.NewListIterator([]string{"one", "two", "three"})
 	a := assertions.New(t)
 
 	s, err := it.Next()
@@ -36,7 +50,7 @@ func TestListIterator(t *testing.T) {
 func TestReaderIterator(t *testing.T) {
 	for _, sep := range []string{"\n", "\r\n"} {
 		buf := []byte(strings.Join([]string{"one", "two", "three"}, sep))
-		it := commands.NewReaderIterator(bytes.NewBuffer(buf), '\n')
+		it := iterator.NewReaderIterator(bytes.NewBuffer(buf), '\n')
 		a := assertions.New(t)
 
 		s, err := it.Next()