Merge pull request #1353 from iotaledger/develop

Merge v0.6.3 changes to master

.github/workflows/docker-testing.yml

@@ -0,0 +1,21 @@
+name: Manually Triggered Docker Image
+
+on: workflow_dispatch
+
+jobs:
+
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+    steps:
+      -   name: Checkout repository
+          uses: actions/checkout@v2
+
+      -   name: Publish to Docker Hub
+          uses: docker/build-push-action@92e71463491f2d026a477188b8ad3a0fdd9d672c
+          with:
+            repository: iotaledger/goshimmer
+            username: '${{ secrets.IOTALEDGER_HUB_DOCKER_LOGIN }}'
+            password: '${{ secrets.IOTALEDGER_HUB_DOCKER_PASSWORD }}'
+            tags: testing

CHANGELOG.md

@@ -1,3 +1,14 @@
+# v0.6.3 - 2021-05-25
+* Improve congestion control
+* Fix builds
+* Fix dRNG shutdown deadlock
+* Fix panic on booking
+* Fix locking management on message factory
+* Fix several visualizer bugs
+* Update Grafana pie charts
+* Update snapshot file with Pollen UTXO at 2021-05-25 16:54 UTC
+* **Breaking**: bumps network and database versions
+
 # v0.6.2 - 2021-05-24
 * Add Congestion Control
 * Add global snapshot

docker-compose.yml

@@ -1,51 +1,79 @@
-version: "3"
+version: '3.3'
 
-services:
+networks:
+  outside:
+    external:
+      name: shimmer
 
+services:
   goshimmer:
-    network_mode: host
-    image: iotaledger/goshimmer
+    image: iotaledger/goshimmer:latest
+    container_name: goshimmer
+    hostname: goshimmer
     stop_grace_period: 2m
-    build:
-      context: ./
-      dockerfile: Dockerfile
-    container_name: iota_goshimmer
-    command: >
-      --node.enablePlugins=prometheus
     # Mount volumes:
     # make sure to give read/write access to the folder ./mainnetdb (e.g., chmod -R 777 ./mainnetdb)
     # optionally, you can mount a config.json into the container
     volumes:
-      - ./mainnetdb/:/tmp/mainnetdb/:rw
-      - ./config.json:/config.json:ro
-    # Expose ports:
-    # gossip:       - "14666:14666/tcp"
-    # autopeering:  - "14626:14626/udp"
-    # webAPI:       - "8080:8080/tcp"
-    # dashboard:    - "8081:8081/tcp"
+      - "./db:/tmp/mainnetdb:rw"
+      - "/etc/localtime:/etc/localtime:ro"
     ports:
-      - "14666:14666/tcp"
-      - "14626:14626/udp"
-      - "9311:9311/tcp" # prometheus exporter
-      - "8080:8080/tcp" # webApi
-      - "8081:8081/tcp" # dashboard
+      # Autopeering
+      - "0.0.0.0:14626:14626/udp"
+      # Gossip
+      - "0.0.0.0:14666:14666/tcp"
+      # FPC
+      - "0.0.0.0:10895:10895/tcp"
+      # HTTP API
+      - "0.0.0.0:8080:8080/tcp"
+      # Dashboard
+      - "0.0.0.0:8081:8081/tcp"
+      # pprof profiling
+      - "0.0.0.0:6061:6061/tcp"
+    environment:
+      - ANALYSIS_CLIENT_SERVERADDRESS=ressims.iota.cafe:21888
+      - AUTOPEERING_PORT=14626
+      - DASHBOARD_BINDADDRESS=0.0.0.0:8081
+      - GOSSIP_PORT=14666
+      - WEBAPI_BINDADDRESS=0.0.0.0:8080
+      - PROFILING_BINDADDRESS=0.0.0.0:6061
+      - NETWORKDELAY_ORIGINPUBLICKEY=9DB3j9cWYSuEEtkvanrzqkzCQMdH1FGv3TawJdVbDxkd
+      - FPC_BINDADDRESS=0.0.0.0:10895
+      - PROMETHEUS_BINDADDRESS=0.0.0.0:9311
+    command: >
+      --skip-config=true
+      --autopeering.entryNodes=2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entryshimmer.tanglebay.com:14646
+      --node.disablePlugins=
+      --node.enablePlugins=remotelog,networkdelay,spammer,prometheus
+      --logger.level=info
+      --logger.disableEvents=false
+      --logger.remotelog.serverAddress=ressims.iota.cafe:5213
+      --drng.pollen.instanceId=1
+      --drng.pollen.threshold=3
+      --drng.pollen.committeeMembers=AheLpbhRs1XZsRF8t8VBwuyQh9mqPHXQvthV5rsHytDG,FZ28bSTidszUBn8TTCAT9X1nVMwFNnoYBmZ1xfafez2z,GT3UxryW4rA9RN9ojnMGmZgE2wP7psagQxgVdA4B9L1P,4pB5boPvvk2o5MbMySDhqsmC2CtUdXyotPPEpb7YQPD7,64wCsTZpmKjRVHtBKXiFojw7uw3GszumfvC4kHdWsHga
+      --drng.xteam.instanceId=1339
+      --drng.xteam.threshold=4
+      --drng.xteam.committeeMembers=GUdTwLDb6t6vZ7X5XzEnjFNDEVPteU7tVQ9nzKLfPjdo,68vNzBFE9HpmWLb2x4599AUUQNuimuhwn3XahTZZYUHt,Dc9n3JxYecaX3gpxVnWb4jS3KVz1K1SgSK1KpV1dzqT1,75g6r4tqGZhrgpDYZyZxVje1Qo54ezFYkCw94ELTLhPs,CN1XLXLHT9hv7fy3qNhpgNMD6uoHFkHtaNNKyNVCKybf,7SmttyqrKMkLo5NPYaiFoHs8LE6s7oCoWCQaZhui8m16,CypSmrHpTe3WQmCw54KP91F5gTmrQEL7EmTX38YStFXx
+    networks:
+      - outside
 
   prometheus:
-    network_mode: host
     image: prom/prometheus:latest
     container_name: prometheus
     restart: unless-stopped
     ports:
-      - 9090:9090
+      - "9090:9090/tcp"
     command:
       - --config.file=/etc/prometheus/prometheus.yml
     volumes:
       - ./tools/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/data:/prometheus:rw
     depends_on:
       - goshimmer
+    networks:
+      - outside
 
   grafana:
-    network_mode: host
     image: grafana/grafana:latest
     container_name: grafana
     restart: unless-stopped
@@ -54,7 +82,9 @@ services:
       # environment variables for grafana within docker
       - GF_PATHS_PROVISIONING=/var/lib/grafana/provisioning
     ports:
-      - 3000:3000
+      - "3000:3000/tcp"
     user: "472"
     volumes:
-      - ./tools/monitoring/grafana:/var/lib/grafana:rw
+      - ./tools/monitoring/grafana:/var/lib/grafana:rw
+    networks:
+      - outside

packages/drng/ticker.go

@@ -41,7 +41,10 @@ func (t *Ticker) UpdateRandomness(r Randomness) {
 	for len(t.fromRandomnessEvent) > 0 {
 		<-t.fromRandomnessEvent
 	}
-	t.fromRandomnessEvent <- r
+	select {
+	case t.fromRandomnessEvent <- r:
+	default:
+	}
 }
 
 // Start starts the Ticker.

packages/tangle/approvalweightmanager.go

@@ -43,6 +43,7 @@ func NewApprovalWeightManager(tangle *Tangle) (approvalWeightManager *ApprovalWe
 	approvalWeightManager = &ApprovalWeightManager{
 		Events: &ApprovalWeightManagerEvents{
 			MessageProcessed: events.NewEvent(MessageIDCaller),
+			MessageFinalized: events.NewEvent(MessageIDCaller),
 		},
 		tangle:               tangle,
 		lastConfirmedMarkers: make(map[markers.SequenceID]markers.Index),
@@ -576,6 +577,7 @@ var (
 // ApprovalWeightManagerEvents represents events happening in the ApprovalWeightManager.
 type ApprovalWeightManagerEvents struct {
 	MessageProcessed   *events.Event
+	MessageFinalized   *events.Event
 	BranchConfirmation *events.ThresholdEvent
 	MarkerConfirmation *events.ThresholdEvent
 }

packages/tangle/approvalweightmanager_test.go

@@ -942,6 +942,7 @@ func newEventMock(t *testing.T, approvalWeightManager *ApprovalWeightManager) *e
 
 	// attach all events
 	e.attach(approvalWeightManager.Events.MessageProcessed, e.MessageProcessed)
+	e.attach(approvalWeightManager.Events.MessageFinalized, func(MessageID) {})
 
 	// assure that all available events are mocked
 	numEvents := reflect.ValueOf(approvalWeightManager.Events).Elem().NumField()

packages/tangle/booker.go

@@ -69,6 +69,12 @@ func (b *Booker) Setup() {
 func (b *Booker) BookMessage(messageID MessageID) (err error) {
 	b.tangle.Storage.Message(messageID).Consume(func(message *Message) {
 		b.tangle.Storage.MessageMetadata(messageID).Consume(func(messageMetadata *MessageMetadata) {
+			// don't book the same message more than once!
+			if messageMetadata.IsBooked() {
+				err = errors.Errorf("message already booked %s", messageID)
+				return
+			}
+
 			branchIDOfPayload, bookingErr := b.bookPayload(message)
 			if bookingErr != nil {
 				err = errors.Errorf("failed to book payload of %s: %w", messageID, bookingErr)

packages/tangle/messagefactory.go

@@ -12,6 +12,7 @@ import (
 	"github.com/iotaledger/hive.go/kvstore"
 
 	"github.com/iotaledger/goshimmer/packages/clock"
+	"github.com/iotaledger/goshimmer/packages/ledgerstate"
 	"github.com/iotaledger/goshimmer/packages/tangle/payload"
 )
 
@@ -85,21 +86,21 @@ func (f *MessageFactory) IssuePayload(p payload.Payload, parentsCount ...int) (*
 	if err != nil {
 		err = errors.Errorf("could not create sequence number: %w", err)
 		f.Events.Error.Trigger(err)
+		f.issuanceMutex.Unlock()
 		return nil, err
 	}
 
 	countStrongParents := 2
 	if len(parentsCount) > 0 {
 		countStrongParents = parentsCount[0]
 	}
-
 	strongParents, weakParents, err := f.selector.Tips(p, countStrongParents, 2)
 	if err != nil {
 		err = errors.Errorf("tips could not be selected: %w", err)
 		f.Events.Error.Trigger(err)
+		f.issuanceMutex.Unlock()
 		return nil, err
 	}
-
 	issuingTime := f.getIssuingTime(strongParents, weakParents)
 
 	issuerPublicKey := f.localIdentity.PublicKey()
@@ -109,11 +110,14 @@ func (f *MessageFactory) IssuePayload(p payload.Payload, parentsCount ...int) (*
 
 	nonce, err := f.doPOW(strongParents, weakParents, issuingTime, issuerPublicKey, sequenceNumber, p)
 	for err != nil && time.Since(startTime) < f.powTimeout {
-		strongParents, weakParents, err = f.selector.Tips(p, countStrongParents, 2)
-		if err != nil {
-			err = errors.Errorf("tips could not be selected: %w", err)
-			f.Events.Error.Trigger(err)
-			return nil, err
+		if p.Type() != ledgerstate.TransactionType {
+			strongParents, weakParents, err = f.selector.Tips(p, countStrongParents, 2)
+			if err != nil {
+				err = errors.Errorf("tips could not be selected: %w", err)
+				f.Events.Error.Trigger(err)
+				f.issuanceMutex.Unlock()
+				return nil, err
+			}
 		}
 
 		issuingTime = f.getIssuingTime(strongParents, weakParents)
@@ -123,6 +127,7 @@ func (f *MessageFactory) IssuePayload(p payload.Payload, parentsCount ...int) (*
 	if err != nil {
 		err = errors.Errorf("pow failed: %w", err)
 		f.Events.Error.Trigger(err)
+		f.issuanceMutex.Unlock()
 		return nil, err
 	}
 	f.issuanceMutex.Unlock()

packages/tangle/tangle.go

@@ -12,6 +12,7 @@ import (
 	"github.com/iotaledger/hive.go/identity"
 	"github.com/iotaledger/hive.go/kvstore"
 	"github.com/iotaledger/hive.go/kvstore/mapdb"
+	"github.com/iotaledger/hive.go/timeutil"
 	"github.com/iotaledger/hive.go/typeutils"
 	"github.com/mr-tron/base58"
 
@@ -51,6 +52,7 @@ type Tangle struct {
 
 	setupParserOnce sync.Once
 	fifoScheduling  typeutils.AtomicBool
+	shutdownSignal  chan struct{}
 }
 
 // New is the constructor for the Tangle.
@@ -61,6 +63,7 @@ func New(options ...Option) (tangle *Tangle) {
 			MessageInvalid:  events.NewEvent(MessageIDCaller),
 			Error:           events.NewEvent(events.ErrorCaller),
 		},
+		shutdownSignal: make(chan struct{}),
 	}
 
 	tangle.Configure(options...)
@@ -142,19 +145,25 @@ func (t *Tangle) Setup() {
 	// pass solid messages to the scheduler
 	t.Solidifier.Events.MessageSolid.Attach(events.NewClosure(t.schedule))
 
+	var switchSchedulerOnce sync.Once
 	// switch the scheduler, the first time we get synced
 	t.TimeManager.Events.SyncChanged.Attach(events.NewClosure(func(e *SyncChangedEvent) {
-		if !e.Synced {
+		// only switch, if we became synced and the FIFO scheduler is currently running
+		if !e.Synced || !t.fifoScheduling.IsSet() {
 			return
 		}
-		// switch the scheduler exactly once
-		if t.fifoScheduling.SetToIf(true, false) {
+		switchSchedulerOnce.Do(func() {
 			// switching the scheduler takes some time, so we must not do it directly in the event func
 			go func() {
-				t.FIFOScheduler.Shutdown() // schedule remaining messages
-				t.Scheduler.Start()        // start the actual scheduler
+				// delay the actual switch by the sync time windows, the SyncChange event will be triggered
+				// at least this duration before the most recent messages have been solidified
+				if timeutil.Sleep(t.Options.SyncTimeWindow, t.shutdownSignal) {
+					t.fifoScheduling.SetTo(false) // stop adding new messages to the FIFO scheduler
+					t.FIFOScheduler.Shutdown()    // schedule remaining messages
+					t.Scheduler.Start()           // start the actual scheduler
+				}
 			}()
-		}
+		})
 	}))
 }
 
@@ -206,6 +215,8 @@ func (t *Tangle) Prune() (err error) {
 
 // Shutdown marks the tangle as stopped, so it will not accept any new messages (waits for all backgroundTasks to finish).
 func (t *Tangle) Shutdown() {
+	close(t.shutdownSignal)
+
 	t.MessageFactory.Shutdown()
 	t.RateSetter.Shutdown()
 	t.FIFOScheduler.Shutdown()

plugins/autopeering/discovery/parameters.go

@@ -5,7 +5,7 @@ import "github.com/iotaledger/hive.go/configuration"
 // Parameters contains the configuration parameters used by the message layer.
 var Parameters = struct {
 	// NetworkVersion defines the config flag of the network version.
-	NetworkVersion int `default:"30" usage:"autopeering network version"`
+	NetworkVersion int `default:"31" usage:"autopeering network version"`
 
 	// EntryNodes defines the config flag of the entry nodes.
 	EntryNodes []string `default:"[email protected]:15626,5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entryshimmer.tanglebay.com:14646" usage:"list of trusted entry nodes for auto peering"`

plugins/banner/plugin.go

@@ -17,7 +17,7 @@ var (
 	once   sync.Once
 
 	// AppVersion version number
-	AppVersion = "v0.6.2"
+	AppVersion = "v0.6.3"
 	// SimplifiedAppVersion is the version number without commit hash
 	SimplifiedAppVersion = simplifiedVersion(AppVersion)
 )

plugins/dashboard/frontend/src/app/components/Visualizer.tsx

@@ -60,7 +60,7 @@ export class Visualizer extends React.Component<Props, any> {
 
     render() {
         let {
-            vertices, confirmed_count, selected,
+            vertices, finalized_count, selected,
             selected_approvers_count, selected_approvees_count,
             verticesLimit, tips_count, paused, search
         } = this.props.visualizerStore;
@@ -129,11 +129,15 @@ export class Visualizer extends React.Component<Props, any> {
                     <Col xs={{span: 5, offset: 2}}>
                         <p>                            
                             <Badge pill style={{background: "#b9b7bd", color: "white"}}>
-                                Pending
+                                MSG Pending
                             </Badge>
                             {' '}
                             <Badge pill style={{background: "#6c71c4", color: "white"}}>
-                                Non-TX Confirmed
+                                MSG Confirmed
+                            </Badge>
+                            {' '}
+                            <Badge pill style={{background: "#393e46", color: "white"}}>
+                                TX Pending
                             </Badge>
                             {' '}
                             <Badge pill style={{background: "#fad02c", color: "white"}}>
@@ -149,7 +153,7 @@ export class Visualizer extends React.Component<Props, any> {
                             </Badge>
                             <br/>
                             Vertices: {vertices.size}, Tips: {tips_count},
-                            Confirmed/UnConfirmed: {confirmed_count}/{vertices.size - confirmed_count},{' '}
+                            Confirmed/UnConfirmed: {finalized_count}/{vertices.size - finalized_count},{' '}
                             MPS: {last_mps_metric.mps}
                             <br/>
                             Selected: {selected ?