Merge branch 'develop'

.gitlab-ci.yml

@@ -16,7 +16,7 @@ cache:
 
 tests:
   stage: test
-  image: golang:1.12
+  image: golang:1.13
   services:
     - thethingsnetwork/rabbitmq
     - redis
@@ -30,7 +30,7 @@ tests:
 
 binaries:
   stage: build
-  image: golang:1.12
+  image: golang:1.13
   script:
     - mkdir release
     - export CI_BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
@@ -48,7 +48,7 @@ sign:
     - master@thethingsnetwork/ttn
     - develop@thethingsnetwork/ttn
   stage: sign
-  image: golang:1.12
+  image: golang:1.13
   script:
     - pushd release
     - shasum -a 256 $(ls) > checksums

.travis.yml

@@ -8,7 +8,7 @@ services:
     - docker
 
 go:
-    - "1.12.x"
+    - "1.13.x"
 
 env:
   global:

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM alpine:3.10
 RUN apk --update --no-cache add ca-certificates
 ADD ./release/ttn-linux-amd64 /usr/local/bin/ttn
 RUN chmod 755 /usr/local/bin/ttn

README.md

@@ -23,7 +23,7 @@ Although we're all about building an open, public network, we understand that so
 
 First, you'll have to prepare your development environment. Follow the steps below to set up your development machine.
 
-1. Make sure you have [Go](https://golang.org) installed (recommended version 1.11, version 1.8 or later is known to work).
+1. Make sure you have [Go](https://golang.org) installed (version 1.11 or later).
 2. Set up your [Go environment](https://golang.org/doc/code.html#GOPATH)
 3. Install the [protobuf compiler (`protoc`)](https://github.com/google/protobuf/releases)
 4. Install `make`. On Linux install `build-essential`. On macOS, `make` comes with XCode or the developer tools. On Windows you can get `make` from [https://gnuarmeclipse.github.io/windows-build-tools/](https://gnuarmeclipse.github.io/windows-build-tools/)

api/go.mod

@@ -2,6 +2,8 @@ module github.com/TheThingsNetwork/ttn/api
 
 go 1.11
 
+replace github.com/TheThingsNetwork/ttn/utils/errors => ../utils/errors
+
 require (
 	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
 	github.com/TheThingsNetwork/api v0.0.0-20190516085542-c732802571cf

api/go.sum

@@ -4,12 +4,11 @@ github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6
 github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/TheThingsNetwork/api v0.0.0-20190516085542-c732802571cf h1:1LvY2VUOy+/Sb5O8eE2HIecvUZupe7kLoquUpeajuPs=
 github.com/TheThingsNetwork/api v0.0.0-20190516085542-c732802571cf/go.mod h1:XO2/81Njy3JefUFXtTg63YGJJ2wUyMRyYm6LaYfu8jE=
+github.com/TheThingsNetwork/go-utils v0.0.0-20190331064810-aa2a11bd5910/go.mod h1:ux1QZ2JrltnJxuZ0VoNbJP1k8Eb5jiMkR3RGDXyBS+o=
 github.com/TheThingsNetwork/go-utils v0.0.0-20190516083235-bdd4967fab4e h1:JEt3G2ONKGfW1YDp2A8Q/+kZIuIB117nkit0+GDJN04=
 github.com/TheThingsNetwork/go-utils v0.0.0-20190516083235-bdd4967fab4e/go.mod h1:9uzg7Jk8ywYqL+xUEhTNrJcs68Nafj4qTaz/zB+STwg=
 github.com/TheThingsNetwork/ttn/api v0.0.0-20190516081709-034d40b328bd/go.mod h1:UCRXmEaShvS/wHOf2RcoY2vKUGJnrYrotBA6LZzYdFM=
 github.com/TheThingsNetwork/ttn/core/types v0.0.0-20190516081709-034d40b328bd/go.mod h1:VVWTaeAJHezuE+c0Vk0AJ4R6KSLg50H1y3RB7vGhGOA=
-github.com/TheThingsNetwork/ttn/utils/errors v0.0.0-20190516081709-034d40b328bd h1:ITXOJpmUR4Jhp3Xb/xNUIJH4WR0h2/NsxZkSDzFIFiU=
-github.com/TheThingsNetwork/ttn/utils/errors v0.0.0-20190516081709-034d40b328bd/go.mod h1:e8FjzgvhAVx9+iqPloB4v7QM0rmv+r5ysRn9kWFamG4=
 github.com/TheThingsNetwork/ttn/utils/random v0.0.0-20190516081709-034d40b328bd/go.mod h1:ktVq1/rYkTlgilBixqCtltTh29rOUnZET4g50xoKlpE=
 github.com/TheThingsNetwork/ttn/utils/security v0.0.0-20190516081709-034d40b328bd/go.mod h1:aaYF12LufW5Xs4Z2C6UOrCzpkyoBjw+rmHCcNYgb1JU=
 github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI=

api/ratelimit/ratelimit.go

@@ -54,3 +54,8 @@ func (r *Registry) Limit(id string) bool {
 func (r *Registry) Wait(id string) time.Duration {
 	return r.getOrCreate(id, r.newFunc).Take(1)
 }
+
+// WaitMaxDuration returns the time to wait until available, but with a max
+func (r *Registry) WaitMaxDuration(id string, max time.Duration) (time.Duration, bool) {
+	return r.getOrCreate(id, r.newFunc).TakeMaxDuration(1, max)
+}

cmd/discovery.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	goruntime "runtime"
 	"syscall"
 
 	pb "github.com/TheThingsNetwork/api/discovery"
@@ -47,6 +48,7 @@ var discoveryCmd = &cobra.Command{
 			Addr:     viper.GetString("discovery.redis-address"),
 			Password: viper.GetString("discovery.redis-password"),
 			DB:       viper.GetInt("discovery.redis-db"),
+			PoolSize: 10 * goruntime.NumCPU(),
 		})
 
 		if err := connectRedis(client); err != nil {

cmd/docs/README.md

@@ -116,6 +116,7 @@ ttn gen-keypair generates a public/private keypair
       --http-port int                     The port where the gRPC proxy should listen (default 8084)
       --mqtt-address string               MQTT host and port. Leave empty to disable MQTT
       --mqtt-address-announce string      MQTT address to announce (takes value of server-address-announce if empty while enabled)
+      --mqtt-fields                       Enable MQTT Fields
       --mqtt-password string              MQTT password
       --mqtt-username string              MQTT username
       --redis-address string              Redis host and port (default "localhost:6379")

cmd/handler.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	goruntime "runtime"
 	"syscall"
 
 	pb "github.com/TheThingsNetwork/api/handler"
@@ -51,6 +52,7 @@ var handlerCmd = &cobra.Command{
 			Addr:     viper.GetString("handler.redis-address"),
 			Password: viper.GetString("handler.redis-password"),
 			DB:       viper.GetInt("handler.redis-db"),
+			PoolSize: 10 * goruntime.NumCPU(),
 		})
 
 		if err := connectRedis(client); err != nil {
@@ -78,7 +80,7 @@ var handlerCmd = &cobra.Command{
 				viper.GetString("handler.mqtt-username"),
 				viper.GetString("handler.mqtt-password"),
 				viper.GetString("handler.mqtt-address"),
-			)
+			).WithMQTTFields(viper.GetBool("handler.mqtt-fields"))
 
 			mqttPort, err := parse.Port(viper.GetString("handler.mqtt-address"))
 			if err != nil {
@@ -191,10 +193,12 @@ func init() {
 	handlerCmd.Flags().String("mqtt-address-announce", "", "MQTT address to announce (takes value of server-address-announce if empty while enabled)")
 	handlerCmd.Flags().String("mqtt-username", "", "MQTT username")
 	handlerCmd.Flags().String("mqtt-password", "", "MQTT password")
+	handlerCmd.Flags().Bool("mqtt-fields", true, "Enable MQTT Fields")
 	viper.BindPFlag("handler.mqtt-address", handlerCmd.Flags().Lookup("mqtt-address"))
 	viper.BindPFlag("handler.mqtt-address-announce", handlerCmd.Flags().Lookup("mqtt-address-announce"))
 	viper.BindPFlag("handler.mqtt-username", handlerCmd.Flags().Lookup("mqtt-username"))
 	viper.BindPFlag("handler.mqtt-password", handlerCmd.Flags().Lookup("mqtt-password"))
+	viper.BindPFlag("handler.mqtt-fields", handlerCmd.Flags().Lookup("mqtt-fields"))
 
 	handlerCmd.Flags().String("amqp-address", "", "AMQP host and port. Leave empty to disable AMQP")
 	handlerCmd.Flags().String("amqp-address-announce", "", "AMQP address to announce (takes value of server-address-announce if empty while enabled)")

cmd/networkserver.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"runtime"
 	"strings"
 	"syscall"
 
@@ -41,6 +42,7 @@ var networkserverCmd = &cobra.Command{
 			Addr:     viper.GetString("networkserver.redis-address"),
 			Password: viper.GetString("networkserver.redis-password"),
 			DB:       viper.GetInt("networkserver.redis-db"),
+			PoolSize: 10 * runtime.NumCPU(),
 		})
 
 		if err := connectRedis(client); err != nil {

core/broker/manager_server.go

@@ -34,8 +34,10 @@ func (b *brokerManager) validateClient(ctx context.Context) (*claims.Claims, err
 	if err != nil {
 		return nil, err
 	}
-	if b.clientRate.Limit(claims.Subject) {
-		return claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client reached")
+	if wait, ok := b.clientRate.WaitMaxDuration(claims.Subject, 500*time.Millisecond); ok {
+		time.Sleep(wait)
+	} else {
+		return claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client %q reached", claims.Subject)
 	}
 	return claims, nil
 }
@@ -138,7 +140,7 @@ func (b *broker) RegisterManager(s *grpc.Server) {
 		devAddrManager: pb_lorawan.NewDevAddrManagerClient(b.nsConn),
 	}
 
-	server.clientRate = ratelimit.NewRegistry(5000, time.Hour)
+	server.clientRate = ratelimit.NewRegistry(5, time.Second)
 
 	pb.RegisterBrokerManagerServer(s, server)
 	lorawan.RegisterDeviceManagerServer(s, server)

core/discovery/connectivity.go

@@ -108,20 +108,17 @@ func (d *discovery) updateStatus() error {
 						for {
 							if status.ClientConn.WaitForStateChange(status.ctx, state) { // blocking
 								status.mu.Lock()
+								previousState := state
 								state = status.ClientConn.GetState()
 								status.lastStateChange = time.Now()
-								switch state {
-								case connectivity.Idle:
-									d.Ctx.Infof("%s %s connection idle", id.serviceName, id.id)
-								case connectivity.Connecting:
-									d.Ctx.Infof("%s %s connecting", id.serviceName, id.id)
-								case connectivity.Ready:
-									d.Ctx.Infof("%s %s connection ready", id.serviceName, id.id)
+								if state == connectivity.Ready || previousState == connectivity.Ready {
 									status.lastAvailable = status.lastStateChange
+								}
+								switch state {
+								case connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Shutdown:
+									d.Ctx.Infof("Connection to %s %s went from %s to %s", id.serviceName, id.id, previousState, state)
 								case connectivity.TransientFailure:
-									d.Ctx.Warnf("%s %s connection failure", id.serviceName, id.id)
-								case connectivity.Shutdown:
-									d.Ctx.Infof("%s %s connection shutdown", id.serviceName, id.id)
+									d.Ctx.Warnf("Connection to %s %s went from %s to %s", id.serviceName, id.id, previousState, state)
 								}
 								status.mu.Unlock()
 							} else { // context canceled

core/handler/handler.go

@@ -26,6 +26,7 @@ type Handler interface {
 	component.ManagementInterface
 
 	WithMQTT(username, password string, brokers ...string) Handler
+	WithMQTTFields(enabled bool) Handler
 	WithAMQP(username, password, host, exchange string) Handler
 	WithDeviceAttributes(attribute ...string) Handler
 
@@ -60,13 +61,14 @@ type handler struct {
 
 	downlink chan *pb_broker.DownlinkMessage
 
-	mqttClient   mqtt.Client
-	mqttUsername string
-	mqttPassword string
-	mqttBrokers  []string
-	mqttEnabled  bool
-	mqttUp       chan *types.UplinkMessage
-	mqttEvent    chan *types.DeviceEvent
+	mqttClient        mqtt.Client
+	mqttUsername      string
+	mqttPassword      string
+	mqttBrokers       []string
+	mqttEnabled       bool
+	mqttFieldsEnabled bool
+	mqttUp            chan *types.UplinkMessage
+	mqttEvent         chan *types.DeviceEvent
 
 	amqpClient   amqp.Client
 	amqpUsername string
@@ -97,6 +99,11 @@ func (h *handler) WithMQTT(username, password string, brokers ...string) Handler
 	return h
 }
 
+func (h *handler) WithMQTTFields(enabled bool) Handler {
+	h.mqttFieldsEnabled = enabled
+	return h
+}
+
 func (h *handler) WithAMQP(username, password, host, exchange string) Handler {
 	h.amqpUsername = username
 	h.amqpPassword = password

core/handler/manager_server.go

@@ -64,11 +64,15 @@ func (h *handlerManager) validateTTNAuthAppContext(ctx context.Context, appID st
 	if err != nil {
 		return ctx, nil, err
 	}
-	if h.clientRate.Limit(claims.Subject) {
-		return ctx, claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client reached")
+	if wait, ok := h.clientRate.WaitMaxDuration(claims.Subject, 500*time.Millisecond); ok {
+		time.Sleep(wait)
+	} else {
+		return ctx, claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client %q reached", claims.Subject)
 	}
-	if h.applicationRate.Limit(appID) {
-		return ctx, claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for application reached")
+	if wait, ok := h.applicationRate.WaitMaxDuration(appID, 500*time.Millisecond); ok {
+		time.Sleep(wait)
+	} else {
+		return ctx, claims, grpc.Errorf(codes.ResourceExhausted, "Rate limit for application %q reached", appID)
 	}
 	return ctx, claims, nil
 }
@@ -557,8 +561,8 @@ func (h *handler) RegisterManager(s *grpc.Server) {
 		devAddrManager: pb_lorawan.NewDevAddrManagerClient(h.ttnBrokerConn),
 	}
 
-	server.applicationRate = ratelimit.NewRegistry(5000, time.Hour)
-	server.clientRate = ratelimit.NewRegistry(5000, time.Hour)
+	server.applicationRate = ratelimit.NewRegistry(5, time.Second)
+	server.clientRate = ratelimit.NewRegistry(5, time.Second)
 
 	pb_handler.RegisterHandlerManagerServer(s, server)
 	pb_handler.RegisterApplicationManagerServer(s, server)

core/handler/mqtt.go

@@ -58,7 +58,7 @@ func (h *handler) HandleMQTT(username, password string, mqttBrokers ...string) e
 					ctx.Warn("Uplink publish timeout")
 				}
 			}(ctx)
-			if len(up.PayloadFields) > 0 {
+			if h.mqttFieldsEnabled && len(up.PayloadFields) > 0 {
 				fieldsToken := h.mqttClient.PublishUplinkFields(up.AppID, up.DevID, up.PayloadFields)
 				go func(ctx ttnlog.Interface) {
 					if fieldsToken.WaitTimeout(MQTTTimeout) {

core/networkserver/adr.go

@@ -271,7 +271,7 @@ func getAdrReqPayloads(dev *device.Device, frequencyPlan *band.FrequencyPlan, dr
 		} else {
 			for i, ch := range frequencyPlan.UplinkChannels {
 				for _, dr := range ch.DataRates {
-					if dr == drIdx {
+					if dr == drIdx && i < 8 { // We can enable up to 8 channels.
 						payloads[0].ChMask[i] = true
 					}
 				}
@@ -300,7 +300,7 @@ func getAdrReqPayloads(dev *device.Device, frequencyPlan *band.FrequencyPlan, dr
 		} else {
 			for i, ch := range frequencyPlan.UplinkChannels {
 				for _, dr := range ch.DataRates {
-					if dr == drIdx {
+					if dr == drIdx && i < 7 { // We can enable up to 7 channels.
 						payloads[0].ChMask[i] = true
 					}
 				}

core/networkserver/manager_server.go

@@ -41,8 +41,10 @@ func (n *networkServerManager) getDevice(ctx context.Context, in *pb_lorawan.Dev
 	if err != nil {
 		return nil, err
 	}
-	if n.clientRate.Limit(claims.Subject) {
-		return nil, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client reached")
+	if wait, ok := n.clientRate.WaitMaxDuration(claims.Subject, 500*time.Millisecond); ok {
+		time.Sleep(wait)
+	} else {
+		return nil, grpc.Errorf(codes.ResourceExhausted, "Rate limit for client %q reached", claims.Subject)
 	}
 	dev, err := n.networkServer.devices.Get(in.AppEUI, in.DevEUI)
 	if err != nil {
@@ -195,7 +197,7 @@ func (n *networkServerManager) GetStatus(ctx context.Context, in *pb.StatusReque
 func (n *networkServer) RegisterManager(s *grpc.Server) {
 	server := &networkServerManager{networkServer: n}
 
-	server.clientRate = ratelimit.NewRegistry(5000, time.Hour)
+	server.clientRate = ratelimit.NewRegistry(5, time.Second)
 
 	pb.RegisterNetworkServerManagerServer(s, server)
 	pb_lorawan.RegisterDeviceManagerServer(s, server)

core/proxy/go.mod

@@ -2,8 +2,12 @@ module github.com/TheThingsNetwork/ttn/core/proxy
 
 go 1.11
 
+replace github.com/TheThingsNetwork/ttn/utils/testing => ../../utils/testing
+
 require (
 	github.com/TheThingsNetwork/go-utils v0.0.0-20190516083235-bdd4967fab4e
 	github.com/TheThingsNetwork/ttn/utils/testing v0.0.0-20190520084050-7adf4a69a7c3
+	github.com/gogo/protobuf v1.2.1
+	github.com/grpc-ecosystem/grpc-gateway v1.9.0
 	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3
 )