Merge branch 'matrix-org:main' into main

.github/workflows/dendrite.yml

@@ -67,6 +67,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Install libolm
+        run: sudo apt-get install libolm-dev libolm3
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -101,6 +103,8 @@ jobs:
           --health-retries 5
     steps:
       - uses: actions/checkout@v3
+      - name: Install libolm
+        run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
@@ -232,6 +236,8 @@ jobs:
           --health-retries 5
     steps:
       - uses: actions/checkout@v3
+      - name: Install libolm
+        run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
         uses: actions/setup-go@v3
         with:

.gitignore

@@ -74,4 +74,7 @@ complement/
 docs/_site
 
 media_store/
-build
+build
+
+# golang workspaces
+go.work*

appservice/appservice.go

@@ -20,10 +20,9 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/gomatrixserverlib"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/query"
@@ -86,7 +85,7 @@ func NewInternalAPI(
 func generateAppServiceAccount(
 	userAPI userapi.AppserviceUserAPI,
 	as config.ApplicationService,
-	serverName gomatrixserverlib.ServerName,
+	serverName spec.ServerName,
 ) error {
 	var accRes userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{

appservice/appservice_test.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/dendrite/test/testrig"
 )
@@ -326,7 +326,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 	room := test.NewRoom(t, alice)
 
 	// Invite Bob
-	room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
 		"membership": "invite",
 	}, test.WithStateKey(bob.ID))
 
@@ -345,7 +345,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 				t.Fatal(err)
 			}
 			for _, ev := range txn.Events {
-				if ev.Type != gomatrixserverlib.MRoomMember {
+				if ev.Type != spec.MRoomMember {
 					continue
 				}
 				// Usually we would check the event content for the membership, but since

appservice/consumers/roomserver.go

@@ -26,9 +26,11 @@ import (
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -103,7 +105,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 	ctx context.Context, state *appserviceState, msgs []*nats.Msg,
 ) bool {
 	log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
-	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
+	events := make([]*types.HeaderedEvent, 0, len(msgs))
 	for _, msg := range msgs {
 		// Only handle events we care about
 		receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
@@ -173,13 +175,13 @@ func (s *OutputRoomEventConsumer) onMessage(
 // endpoint. It will block for the backoff period if necessary.
 func (s *OutputRoomEventConsumer) sendEvents(
 	ctx context.Context, state *appserviceState,
-	events []*gomatrixserverlib.HeaderedEvent,
+	events []*types.HeaderedEvent,
 	txnID string,
 ) error {
 	// Create the transaction body.
 	transaction, err := json.Marshal(
 		ApplicationServiceTransaction{
-			Events: synctypes.HeaderedToClientEvents(events, synctypes.FormatAll),
+			Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll),
 		},
 	)
 	if err != nil {
@@ -188,7 +190,7 @@ func (s *OutputRoomEventConsumer) sendEvents(
 
 	// If txnID is not defined, generate one from the events.
 	if txnID == "" {
-		txnID = fmt.Sprintf("%d_%d", events[0].Event.OriginServerTS(), len(transaction))
+		txnID = fmt.Sprintf("%d_%d", events[0].PDU.OriginServerTS(), len(transaction))
 	}
 
 	// Send the transaction to the appservice.
@@ -230,7 +232,7 @@ func (s *appserviceState) backoffAndPause(err error) error {
 // event falls within one of a given application service's namespaces.
 //
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
-func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
+func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
 	switch {
 	case appservice.URL == "":
 		return false
@@ -240,7 +242,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 		return true
 	}
 
-	if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
+	if event.Type() == spec.MRoomMember && event.StateKey() != nil {
 		if appservice.IsInterestedInUserID(*event.StateKey()) {
 			return true
 		}
@@ -268,7 +270,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 
 // appserviceJoinedAtEvent returns a boolean depending on whether a given
 // appservice has membership at the time a given event was created.
-func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
+func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
 	// TODO: This is only checking the current room state, not the state at
 	// the event in question. Pretty sure this is what Synapse does too, but
 	// until we have a lighter way of checking the state before the event that
@@ -286,15 +288,15 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			switch {
 			case ev.StateKey == nil:
 				continue
-			case ev.Type != gomatrixserverlib.MRoomMember:
+			case ev.Type != spec.MRoomMember:
 				continue
 			}
 			var membership gomatrixserverlib.MemberContent
 			err = json.Unmarshal(ev.Content, &membership)
 			switch {
 			case err != nil:
 				continue
-			case membership.Membership == gomatrixserverlib.Join:
+			case membership.Membership == spec.Join:
 				if appservice.IsInterestedInUserID(*ev.StateKey) {
 					return true
 				}

build/dendritejs-pinecone/main.go

@@ -171,7 +171,7 @@ func startup() {
 	cfg.Global.TrustedIDServers = []string{}
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.PrivateKey = sk
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 

build/gobind-pinecone/monolith.go

@@ -35,6 +35,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/pinecone/types"
 	"github.com/sirupsen/logrus"
 
@@ -140,9 +141,9 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) {
 	}
 }
 
-func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error) {
-	var nodeKey gomatrixserverlib.ServerName
-	if userID, err := gomatrixserverlib.NewUserID(nodeID, false); err == nil {
+func getServerKeyFromString(nodeID string) (spec.ServerName, error) {
+	var nodeKey spec.ServerName
+	if userID, err := spec.NewUserID(nodeID, false); err == nil {
 		hexKey, decodeErr := hex.DecodeString(string(userID.Domain()))
 		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
 			return "", fmt.Errorf("UserID domain is not a valid ed25519 public key: %v", userID.Domain())
@@ -154,15 +155,15 @@ func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error)
 		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
 			return "", fmt.Errorf("Relay server uri is not a valid ed25519 public key: %v", nodeID)
 		} else {
-			nodeKey = gomatrixserverlib.ServerName(nodeID)
+			nodeKey = spec.ServerName(nodeID)
 		}
 	}
 
 	return nodeKey, nil
 }
 
 func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
-	relays := []gomatrixserverlib.ServerName{}
+	relays := []spec.ServerName{}
 	for _, uri := range strings.Split(uris, ",") {
 		uri = strings.TrimSpace(uri)
 		if len(uri) == 0 {
@@ -188,7 +189,7 @@ func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
 		m.p2pMonolith.RelayRetriever.SetRelayServers(relays)
 	} else {
 		relay.UpdateNodeRelayServers(
-			gomatrixserverlib.ServerName(nodeKey),
+			spec.ServerName(nodeKey),
 			relays,
 			m.p2pMonolith.ProcessCtx.Context(),
 			m.p2pMonolith.GetFederationAPI(),
@@ -215,7 +216,7 @@ func (m *DendriteMonolith) GetRelayServers(nodeID string) string {
 			relaysString += string(relay)
 		}
 	} else {
-		request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(nodeKey)}
+		request := api.P2PQueryRelayServersRequest{Server: spec.ServerName(nodeKey)}
 		response := api.P2PQueryRelayServersResponse{}
 		err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.ProcessCtx.Context(), &request, &response)
 		if err != nil {
@@ -291,7 +292,7 @@ func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, err
 	pubkey := m.p2pMonolith.Router.PublicKey()
 	userID := userutil.MakeUserID(
 		localpart,
-		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
+		spec.ServerName(hex.EncodeToString(pubkey[:])),
 	)
 	userReq := &userapiAPI.PerformAccountCreationRequest{
 		AccountType: userapiAPI.AccountTypeUser,
@@ -342,7 +343,7 @@ func (m *DendriteMonolith) Start() {
 
 	prefix := hex.EncodeToString(pk)
 	cfg := monolith.GenerateDefaultConfig(sk, m.StorageDirectory, m.CacheDirectory, prefix)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.InMemory = false
 	// NOTE : disabled for now since there is a 64 bit alignment panic on 32 bit systems

build/gobind-pinecone/monolith_test.go

@@ -18,11 +18,14 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
 func TestMonolithStarts(t *testing.T) {
-	monolith := DendriteMonolith{}
+	monolith := DendriteMonolith{
+		StorageDirectory: t.TempDir(),
+		CacheDirectory:   t.TempDir(),
+	}
 	monolith.Start()
 	monolith.PublicKey()
 	monolith.Stop()
@@ -60,7 +63,10 @@ func TestMonolithSetRelayServers(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		monolith := DendriteMonolith{}
+		monolith := DendriteMonolith{
+			StorageDirectory: t.TempDir(),
+			CacheDirectory:   t.TempDir(),
+		}
 		monolith.Start()
 
 		inputRelays := tc.relays
@@ -110,7 +116,7 @@ func TestParseServerKey(t *testing.T) {
 		name        string
 		serverKey   string
 		expectedErr bool
-		expectedKey gomatrixserverlib.ServerName
+		expectedKey spec.ServerName
 	}{
 		{
 			name:        "valid userid as key",

build/gobind-yggdrasil/monolith.go

@@ -33,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	_ "golang.org/x/mobile/bind"
@@ -134,7 +135,7 @@ func (m *DendriteMonolith) Start() {
 		Generate:       true,
 		SingleDatabase: true,
 	})
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.ServerName = spec.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))

clientapi/admin_test.go

@@ -18,8 +18,8 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
 
@@ -123,7 +123,7 @@ func TestPurgeRoom(t *testing.T) {
 	room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
 
 	// Invite Bob
-	room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room.CreateAndInsert(t, aliceAdmin, spec.MRoomMember, map[string]interface{}{
 		"membership": "invite",
 	}, test.WithStateKey(bob.ID))
 
@@ -142,8 +142,8 @@ func TestPurgeRoom(t *testing.T) {
 
 		// this starts the JetStream consumers
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
-		federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
-		rsAPI.SetFederationAPI(nil, nil)
+		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
+		rsAPI.SetFederationAPI(fsAPI, nil)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
@@ -194,7 +194,7 @@ func TestAdminEvacuateRoom(t *testing.T) {
 	room := test.NewRoom(t, aliceAdmin)
 
 	// Join Bob
-	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
 		"membership": "join",
 	}, test.WithStateKey(bob.ID))
 
@@ -291,10 +291,10 @@ func TestAdminEvacuateUser(t *testing.T) {
 	room2 := test.NewRoom(t, aliceAdmin)
 
 	// Join Bob
-	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
 		"membership": "join",
 	}, test.WithStateKey(bob.ID))
-	room2.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+	room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
 		"membership": "join",
 	}, test.WithStateKey(bob.ID))