diff --git a/.dockerignore b/.dockerignore index 76547e9ee3..80ccf937c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ bin -*.wasm -.git \ No newline at end of file +*.wasm \ No newline at end of file diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 85fd355f2d..772b45cb24 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -123,7 +123,7 @@ jobs: with: # Optional: pass GITHUB_TOKEN to avoid rate limiting. token: ${{ secrets.GITHUB_TOKEN }} - - run: go test -json -v ./... 2>&1 | gotestfmt + - run: go test -json -v ./... 2>&1 | gotestfmt -hide all env: POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -255,7 +255,7 @@ jobs: key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-stable-test-race- - - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt + - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all env: POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -280,6 +280,8 @@ jobs: with: go-version: "stable" cache: true + - name: Docker version + run: docker version - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - name: Test upgrade (PostgreSQL) @@ -300,6 +302,8 @@ jobs: with: go-version: "stable" cache: true + - name: Docker version + run: docker version - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - name: Test upgrade (PostgreSQL) @@ -436,7 +440,7 @@ jobs: # Run Complement - run: | set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt + go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all shell: bash name: Run Complement Tests env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0c3053a563..8448d8e23c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,8 +57,8 @@ jobs: id: docker_build_monolith uses: docker/build-push-action@v3 with: - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache + cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max context: . build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index a9c1718a04..9a5eb2b62f 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -32,9 +32,10 @@ jobs: version: v3.10.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.4.1 + uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: config: helm/cr.yaml charts_dir: helm/ + mark_as_latest: false diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index e76cc82f37..509861860e 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -128,7 +128,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest + go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v3 for dendrite uses: actions/checkout@v3 with: diff --git a/CHANGES.md b/CHANGES.md index 8052efd8a1..bdb6a796eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,105 @@ # Changelog +## Dendrite 0.13.3 (2023-09-28) + +### Fixes: + +- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir)) +- Invitations are now correctly pushed to devices +- A bug which could result in the corruption of `m.direct` account data has been fixed + +### Features + +- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response +- Room version 11 is now supported +- Clients can request the `federation` `event_format` when creating filters +- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + +### Other + +- Dendrite now requires Go 1.20 if building from source + +## Dendrite 0.13.2 (2023-08-23) + +### Fixes: + +- Migrations in SQLite are now prepared on the correct context (transaction or database) +- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander)) +- Event size checks are more in line with Synapse +- Requests to `/messages` have been optimized, possibly reducing database round trips +- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed) +- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage +- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist +- Background federated joins should now be fixed and not timeout after a short time +- Database connections are now correctly re-used +- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging) +- Don't expose information about the system when trying to download files that don't exist + +### Features + +- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + - Lookup correct prev events in the sync API + - Populate `prev_sender` correctly in the sync API + - Event federation should work better +- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite +- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported +- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger)) +- The default room version to use when creating rooms can now be configured using `room_server.default_room_version` + +## Dendrite 0.13.1 (2023-07-06) + +This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended. + +When deduplicating state events, we were checking if the event in question was already in a state snapshot. If it was in a previous state snapshot, we would +then remove it from the list of events to store. If this happened, we were, unfortunately, skipping the next event to check. This resulted in +events getting stored in state snapshots where they may not be needed. When we now compared two of those state snapshots, one of them +contained the skipped event, while the other didn't. This difference possibly shouldn't exist, resulting in unexpected state resets and explains +reports of missing state events as well. + +Rooms where a state reset occurred earlier should, hopefully, reconcile over time. + +### Fixes: + +- A long-standing "off-by-one" error has been fixed, which could result in state resets +- Roomserver Prometheus Metrics are available again + +### Features + +- Updated dependencies + - Internal NATS Server has been updated from v2.9.15 to v2.9.19 + +## Dendrite 0.13.0 (2023-06-30) + +### Features + +- Results in responses to `/search` now highlight words more accurately and not only the search terms as before +- Support for connecting to appservices listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb)) +- Admin APIs for token authenticated registration have been added (contributed by [santhoshivan23](https://github.com/santhoshivan23)) +- Initial support for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + - This is **highly experimental**, things like changing usernames/avatars, inviting users, upgrading rooms isn't working + +### Fixes + +- `m.upload.size` is now optional, finally allowing uploads with unlimited file size +- A bug while resolving server names has been fixed (contributed by [anton-molyboha](https://github.com/anton-molyboha)) +- Application services should only receive one invitation instead of 2 (or worse), which could result in state resets previously +- Several admin endpoints are now using `POST` instead of `GET` +- `/delete_devices` now uses user-interactive authentication +- Several "membership" (e.g `/kick`, `/ban`) endpoints are using less heavy database queries to check if the user is allowed to perform this action +- `/3pid` endpoints are now available on `/v3` instead of the `/unstable` prefix +- Upgrading rooms ignores state events of other users, which could result in failed upgrades before +- Uploading key backups with a wrong version now returns `M_WRONG_ROOM_KEYS_VERSION` +- A potential state reset when joining the same room multiple times in short sequence has been fixed +- A bug where we returned the full event as `redacted_because` in redaction events has been fixed +- The `displayname` and `avatar_url` can now be set to empty strings +- Unsafe hotserving of files has been fixed (contributed by [joshqou](https://github.com/joshqou)) +- Joining new rooms would potentially return "redacted" events, due to history visibility not being set correctly, this could result in events being rejected +- Backfilling resulting in `unsuported room version ''` should now be solved + +### Other + +- Huge refactoring of Dendrite and gomatrixserverlib + ## Dendrite 0.12.0 (2023-03-13) ### Features diff --git a/Dockerfile b/Dockerfile index 2487ea5bca..4ee20933a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # base installs required dependencies and runs go mod download to cache dependencies # FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base -RUN apk --update --no-cache add bash build-base curl +RUN apk --update --no-cache add bash build-base curl git # # build creates all needed binaries diff --git a/README.md b/README.md index 295203eb48..bde19b07ee 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ Dendrite is **beta** software, which means: -- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. +- Dendrite is ready for early adopters. We recommend running Dendrite with a PostgreSQL database. - Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features. - Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite. @@ -21,7 +21,7 @@ This does not mean: - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. - Dendrite is feature-complete. There may be client or federation APIs that are not implemented. -- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support. +- Dendrite is ready for massive homeserver deployments. There is no high-availability/clustering support. Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. @@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for more information on requirements. -To build Dendrite, you will need Go 1.18 or later. +To build Dendrite, you will need Go 1.20 or later. For a usable federating Dendrite deployment, you will also need: @@ -47,7 +47,7 @@ For a usable federating Dendrite deployment, you will also need: Also recommended are: - A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms -- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf) +- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/dendrite-sample.conf) The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment. @@ -60,7 +60,7 @@ The following instructions are enough to get Dendrite started as a non-federatin ```bash $ git clone https://github.com/matrix-org/dendrite $ cd dendrite -$ ./build.sh +$ go build -o bin/ ./cmd/... # Generate a Matrix signing key for federation (required) $ ./bin/generate-keys --private-key matrix_key.pem @@ -85,7 +85,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l ## Progress -We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver +We use a script called "Are We Synapse Yet" which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 878ca5666e..ddc24477bf 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -134,7 +134,6 @@ func TestAppserviceInternalAPI(t *testing.T) { } as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} - t.Cleanup(func() { ctx.ShutdownDendrite() ctx.WaitForShutdown() @@ -144,6 +143,7 @@ func TestAppserviceInternalAPI(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) @@ -238,6 +238,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index c02d904041..e8b9211c4e 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -128,7 +128,7 @@ func (s *OutputRoomEventConsumer) onMessage( if len(output.NewRoomEvent.AddsStateEventIDs) > 0 { newEventID := output.NewRoomEvent.Event.EventID() eventsReq := &api.QueryEventsByIDRequest{ - RoomID: output.NewRoomEvent.Event.RoomID(), + RoomID: output.NewRoomEvent.Event.RoomID().String(), EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)), } eventsRes := &api.QueryEventsByIDResponse{} @@ -181,7 +181,9 @@ func (s *OutputRoomEventConsumer) sendEvents( // Create the transaction body. transaction, err := json.Marshal( ApplicationServiceTransaction{ - Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll), + Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }), }, ) if err != nil { @@ -233,12 +235,18 @@ func (s *appserviceState) backoffAndPause(err error) error { // // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool { + user := "" + userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) + if err == nil { + user = userID.String() + } + switch { case appservice.URL == "": return false - case appservice.IsInterestedInUserID(event.Sender()): + case appservice.IsInterestedInUserID(user): return true - case appservice.IsInterestedInRoomID(event.RoomID()): + case appservice.IsInterestedInRoomID(event.RoomID().String()): return true } @@ -249,7 +257,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont } // Check all known room aliases of the room the event came from - queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} + queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()} var queryRes api.GetAliasesForRoomIDResponse if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { @@ -260,7 +268,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont } else { log.WithFields(log.Fields{ "appservice": appservice.ID, - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), }).WithError(err).Errorf("Unable to get aliases for room") } @@ -276,7 +284,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e // until we have a lighter way of checking the state before the event that // doesn't involve state res, then this is probably OK. membershipReq := &api.QueryMembershipsForRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), JoinedOnly: true, } membershipRes := &api.QueryMembershipsForRoomResponse{} @@ -305,7 +313,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e } else { log.WithFields(log.Fields{ "appservice": appservice.ID, - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), }).WithError(err).Errorf("Unable to get membership for room") } return false diff --git a/appservice/query/query.go b/appservice/query/query.go index ca8d7b3a32..5c736f379e 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -217,7 +217,7 @@ func (a *AppServiceQueryAPI) Locations( } if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { - log.WithError(err).Error("unable to get 'locations' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service") continue } @@ -252,7 +252,7 @@ func (a *AppServiceQueryAPI) User( } if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { - log.WithError(err).Error("unable to get 'user' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service") continue } @@ -290,7 +290,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, as := range a.Cfg.Derived.ApplicationServices { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } @@ -320,7 +320,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, p := range as.Protocols { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } existing, ok := response[p] diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 585374738d..80c0dbff6d 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -944,4 +944,12 @@ rmv remote user can join room with version 10 rmv User can invite remote user to room with version 10 rmv Remote user can backfill in a room with version 10 rmv Can reject invites over federation for rooms with version 10 -rmv Can receive redactions from regular users over federation in room version 10 \ No newline at end of file +rmv Can receive redactions from regular users over federation in room version 10 +rmv User can create and send/receive messages in a room with version 11 +rmv local user can join room with version 11 +rmv User can invite local user to room with version 11 +rmv remote user can join room with version 11 +rmv User can invite remote user to room with version 11 +rmv Remote user can backfill in a room with version 11 +rmv Can reject invites over federation for rooms with version 11 +rmv Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/build.cmd b/build.cmd deleted file mode 100644 index 9e90622c8b..0000000000 --- a/build.cmd +++ /dev/null @@ -1,51 +0,0 @@ -@echo off - -:ENTRY_POINT - setlocal EnableDelayedExpansion - - REM script base dir - set SCRIPTDIR=%~dp0 - set PROJDIR=%SCRIPTDIR:~0,-1% - - REM Put installed packages into ./bin - set GOBIN=%PROJDIR%\bin - - set FLAGS= - - REM Check if sources are under Git control - if not exist ".git" goto :CHECK_BIN - - REM set BUILD=`git rev-parse --short HEAD \\ ""` - FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO ( - set BUILD=%%X - ) - - REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""` - FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO ( - set BRANCHRAW=%%X - set BRANCH=!BRANCHRAW:/=! - ) - if "%BRANCH%" == "main" set BRANCH= - - set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD% - -:CHECK_BIN - if exist "bin" goto :ALL_SET - mkdir "bin" - -:ALL_SET - set CGO_ENABLED=1 - for /D %%P in (cmd\*) do ( - go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P" - ) - - set CGO_ENABLED=0 - set GOOS=js - set GOARCH=wasm - go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone - - goto :DONE - -:DONE - echo Done - endlocal \ No newline at end of file diff --git a/build.sh b/build.sh deleted file mode 100755 index f8b5001bff..0000000000 --- a/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -eu - -# Put installed packages into ./bin -export GOBIN=$PWD/`dirname $0`/bin - -if [ -d ".git" ] -then - export BUILD=`git rev-parse --short HEAD || ""` - export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` - if [ "$BRANCH" = main ] - then - export BRANCH="" - fi - - export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" -else - export FLAGS="" -fi - -mkdir -p bin - -CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... - -# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index 90f5151670..ab50cf3184 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/docker/README.md b/build/docker/README.md index b66cb864b1..8d69b9af16 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -6,23 +6,20 @@ They can be found on Docker Hub: - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments -## Dockerfiles +## Dockerfile -The `Dockerfile` is a multistage file which can build all four Dendrite -images depending on the supplied `--target`. From the root of the Dendrite +The `Dockerfile` is a multistage file which can build Dendrite. From the root of the Dendrite repository, run: ``` -docker build . --target monolith -t matrixdotorg/dendrite-monolith -docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone -docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil +docker build . -t matrixdotorg/dendrite-monolith ``` -## Compose files +## Compose file -There are two sample `docker-compose` files: +There is one sample `docker-compose` files: -- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment +- `docker-compose.yml` which runs a Dendrite deployment with Postgres ## Configuration @@ -55,7 +52,7 @@ Create your config based on the [`dendrite-sample.yaml`](https://github.com/matr Then start the deployment: ``` -docker-compose -f docker-compose.monolith.yml up +docker-compose -f docker-compose.yml up ``` ## Building the images diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml deleted file mode 100644 index 1a8fe4eee4..0000000000 --- a/build/docker/docker-compose.monolith.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: "3.4" -services: - postgres: - hostname: postgres - image: postgres:14 - restart: always - volumes: - - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh - # To persist your PostgreSQL databases outside of the Docker image, - # to prevent data loss, modify the following ./path_to path: - - ./path_to/postgresql:/var/lib/postgresql/data - environment: - POSTGRES_PASSWORD: itsasecret - POSTGRES_USER: dendrite - healthcheck: - test: ["CMD-SHELL", "pg_isready -U dendrite"] - interval: 5s - timeout: 5s - retries: 5 - networks: - - internal - - monolith: - hostname: monolith - image: matrixdotorg/dendrite-monolith:latest - command: [ - "--tls-cert=server.crt", - "--tls-key=server.key" - ] - ports: - - 8008:8008 - - 8448:8448 - volumes: - - ./config:/etc/dendrite - - ./media:/var/dendrite/media - depends_on: - - postgres - networks: - - internal - restart: unless-stopped - -networks: - internal: - attachable: true diff --git a/build/docker/docker-compose.yml b/build/docker/docker-compose.yml new file mode 100644 index 0000000000..9397673f85 --- /dev/null +++ b/build/docker/docker-compose.yml @@ -0,0 +1,52 @@ +version: "3.4" + +services: + postgres: + hostname: postgres + image: postgres:15-alpine + restart: always + volumes: + # This will create a docker volume to persist the database files in. + # If you prefer those files to be outside of docker, you'll need to change this. + - dendrite_postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: itsasecret + POSTGRES_USER: dendrite + POSTGRES_DATABASE: dendrite + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dendrite"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - internal + + monolith: + hostname: monolith + image: matrixdotorg/dendrite-monolith:latest + ports: + - 8008:8008 + - 8448:8448 + volumes: + - ./config:/etc/dendrite + # The following volumes use docker volumes, change this + # if you prefer to have those files outside of docker. + - dendrite_media:/var/dendrite/media + - dendrite_jetstream:/var/dendrite/jetstream + - dendrite_search_index:/var/dendrite/searchindex + depends_on: + postgres: + condition: service_healthy + networks: + - internal + restart: unless-stopped + +networks: + internal: + attachable: true + +volumes: + dendrite_postgres_data: + dendrite_media: + dendrite_jetstream: + dendrite_search_index: \ No newline at end of file diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh deleted file mode 100755 index 27d2a4df43..0000000000 --- a/build/docker/postgres/create_db.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -for db in userapi_accounts mediaapi syncapi roomserver keyserver federationapi appservice mscs; do - createdb -U dendrite -O dendrite dendrite_$db -done diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 69a815321f..66667b03ca 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -2,6 +2,7 @@ package clientapi import ( "context" + "fmt" "net/http" "net/http/httptest" "reflect" @@ -23,12 +24,654 @@ import ( "github.com/matrix-org/util" "github.com/tidwall/gjson" + capi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" uapi "github.com/matrix-org/dendrite/userapi/api" ) +func TestAdminCreateToken(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RegistrationRequiresToken = true + defer close() + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + testCases := []struct { + name string + requestingUser *test.User + requestOpt test.HTTPRequestOpt + wantOK bool + withHeader bool + }{ + { + name: "Missing auth", + requestingUser: bob, + wantOK: false, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token1", + }, + ), + }, + { + name: "Bob is denied access", + requestingUser: bob, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token2", + }, + ), + }, + { + name: "Alice can create a token without specifyiing any information", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{}), + }, + { + name: "Alice can to create a token specifying a name", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token3", + }, + ), + }, + { + name: "Alice cannot to create a token that already exists", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token3", + }, + ), + }, + { + name: "Alice can create a token specifying valid params", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token4", + "uses_allowed": 5, + "expiry_time": time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond), + }, + ), + }, + { + name: "Alice cannot create a token specifying invalid name", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token@", + }, + ), + }, + { + name: "Alice cannot create a token specifying invalid uses_allowed", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token5", + "uses_allowed": -1, + }, + ), + }, + { + name: "Alice cannot create a token specifying invalid expiry_time", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "token": "token6", + "expiry_time": time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond), + }, + ), + }, + { + name: "Alice cannot to create a token specifying invalid length", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "length": 80, + }, + ), + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/registrationTokens/new") + if tc.requestOpt != nil { + req = test.NewRequest(t, http.MethodPost, "/_dendrite/admin/registrationTokens/new", tc.requestOpt) + } + if tc.withHeader { + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) + } + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func TestAdminListRegistrationTokens(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RegistrationRequiresToken = true + defer close() + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + } + tokens := []capi.RegistrationToken{ + { + Token: getPointer("valid"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + { + Token: getPointer("invalid"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + } + for _, tkn := range tokens { + tkn := tkn + userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn) + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + testCases := []struct { + name string + requestingUser *test.User + valid string + isValidSpecified bool + wantOK bool + withHeader bool + }{ + { + name: "Missing auth", + requestingUser: bob, + wantOK: false, + isValidSpecified: false, + }, + { + name: "Bob is denied access", + requestingUser: bob, + wantOK: false, + withHeader: true, + isValidSpecified: false, + }, + { + name: "Alice can list all tokens", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + }, + { + name: "Alice can list all valid tokens", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + valid: "true", + isValidSpecified: true, + }, + { + name: "Alice can list all invalid tokens", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + valid: "false", + isValidSpecified: true, + }, + { + name: "No response when valid has a bad value", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + valid: "trueee", + isValidSpecified: true, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + var path string + if tc.isValidSpecified { + path = fmt.Sprintf("/_dendrite/admin/registrationTokens?valid=%v", tc.valid) + } else { + path = "/_dendrite/admin/registrationTokens" + } + req := test.NewRequest(t, http.MethodGet, path) + if tc.withHeader { + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) + } + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func TestAdminGetRegistrationToken(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RegistrationRequiresToken = true + defer close() + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + } + tokens := []capi.RegistrationToken{ + { + Token: getPointer("alice_token1"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + { + Token: getPointer("alice_token2"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + } + for _, tkn := range tokens { + tkn := tkn + userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn) + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + testCases := []struct { + name string + requestingUser *test.User + token string + wantOK bool + withHeader bool + }{ + { + name: "Missing auth", + requestingUser: bob, + wantOK: false, + }, + { + name: "Bob is denied access", + requestingUser: bob, + wantOK: false, + withHeader: true, + }, + { + name: "Alice can GET alice_token1", + token: "alice_token1", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + }, + { + name: "Alice can GET alice_token2", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + token: "alice_token2", + }, + { + name: "Alice cannot GET a token that does not exists", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token3", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token) + req := test.NewRequest(t, http.MethodGet, path) + if tc.withHeader { + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) + } + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func TestAdminDeleteRegistrationToken(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RegistrationRequiresToken = true + defer close() + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + } + tokens := []capi.RegistrationToken{ + { + Token: getPointer("alice_token1"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + { + Token: getPointer("alice_token2"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + } + for _, tkn := range tokens { + tkn := tkn + userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn) + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + testCases := []struct { + name string + requestingUser *test.User + token string + wantOK bool + withHeader bool + }{ + { + name: "Missing auth", + requestingUser: bob, + wantOK: false, + }, + { + name: "Bob is denied access", + requestingUser: bob, + wantOK: false, + withHeader: true, + }, + { + name: "Alice can DELETE alice_token1", + token: "alice_token1", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + }, + { + name: "Alice can DELETE alice_token2", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + token: "alice_token2", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token) + req := test.NewRequest(t, http.MethodDelete, path) + if tc.withHeader { + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) + } + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func TestAdminUpdateRegistrationToken(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RegistrationRequiresToken = true + defer close() + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + tokens := []capi.RegistrationToken{ + { + Token: getPointer("alice_token1"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + { + Token: getPointer("alice_token2"), + UsesAllowed: getPointer(int32(10)), + ExpiryTime: getPointer(time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond)), + Pending: getPointer(int32(0)), + Completed: getPointer(int32(0)), + }, + } + for _, tkn := range tokens { + tkn := tkn + userAPI.PerformAdminCreateRegistrationToken(ctx, &tkn) + } + testCases := []struct { + name string + requestingUser *test.User + method string + token string + requestOpt test.HTTPRequestOpt + wantOK bool + withHeader bool + }{ + { + name: "Missing auth", + requestingUser: bob, + wantOK: false, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": 10, + }, + ), + }, + { + name: "Bob is denied access", + requestingUser: bob, + wantOK: false, + withHeader: true, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": 10, + }, + ), + }, + { + name: "Alice can UPDATE a token's uses_allowed property", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": 10, + }), + }, + { + name: "Alice can UPDATE a token's expiry_time property", + requestingUser: aliceAdmin, + wantOK: true, + withHeader: true, + token: "alice_token2", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "expiry_time": time.Now().Add(5*24*time.Hour).UnixNano() / int64(time.Millisecond), + }, + ), + }, + { + name: "Alice can UPDATE a token's uses_allowed and expiry_time property", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": 20, + "expiry_time": time.Now().Add(10*24*time.Hour).UnixNano() / int64(time.Millisecond), + }, + ), + }, + { + name: "Alice CANNOT update a token with invalid properties", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token2", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": -5, + "expiry_time": time.Now().Add(-1*5*24*time.Hour).UnixNano() / int64(time.Millisecond), + }, + ), + }, + { + name: "Alice CANNOT UPDATE a token that does not exist", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token9", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": 100, + }, + ), + }, + { + name: "Alice can UPDATE token specifying uses_allowed as null - Valid for infinite uses", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "uses_allowed": nil, + }, + ), + }, + { + name: "Alice can UPDATE token specifying expiry_time AS null - Valid for infinite time", + requestingUser: aliceAdmin, + wantOK: false, + withHeader: true, + token: "alice_token1", + requestOpt: test.WithJSONBody(t, map[string]interface{}{ + "expiry_time": nil, + }, + ), + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + path := fmt.Sprintf("/_dendrite/admin/registrationTokens/%s", tc.token) + req := test.NewRequest(t, http.MethodPut, path) + if tc.requestOpt != nil { + req = test.NewRequest(t, http.MethodPut, path, tc.requestOpt) + } + if tc.withHeader { + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) + } + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func getPointer[T any](s T) *T { + return &s +} + func TestAdminResetPassword(t *testing.T) { aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser)) @@ -48,6 +691,7 @@ func TestAdminResetPassword(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. @@ -133,18 +777,23 @@ func TestPurgeRoom(t *testing.T) { cfg, processCtx, close := testrig.CreateConfig(t, dbType) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - defer close() + defer func() { + // give components the time to process purge requests + time.Sleep(time.Millisecond * 50) + close() + }() routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -209,12 +858,13 @@ func TestAdminEvacuateRoom(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -309,12 +959,13 @@ func TestAdminEvacuateUser(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -403,6 +1054,7 @@ func TestAdminMarkAsStale(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/api/api.go b/clientapi/api/api.go index 23974c8658..28ff593fcc 100644 --- a/clientapi/api/api.go +++ b/clientapi/api/api.go @@ -21,3 +21,11 @@ type ExtraPublicRoomsProvider interface { // Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately. Rooms() []fclient.PublicRoom } + +type RegistrationToken struct { + Token *string `json:"token"` + UsesAllowed *int32 `json:"uses_allowed"` + Pending *int32 `json:"pending"` + Completed *int32 `json:"completed"` + ExpiryTime *int64 `json:"expiry_time"` +} diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index b339818a4c..82ec9fea2b 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -120,6 +120,7 @@ func TestGetPutDevices(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -168,6 +169,7 @@ func TestDeleteDevice(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -272,6 +274,7 @@ func TestDeleteDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -920,13 +923,17 @@ func TestCapabilities(t *testing.T) { } } + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + expectedMap := map[string]interface{}{ "capabilities": map[string]interface{}{ "m.change_password": map[string]bool{ "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": defaultRoomVersion, "available": versionsMap, }, }, @@ -947,6 +954,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -993,6 +1001,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. @@ -1090,6 +1099,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1265,6 +1275,7 @@ func TestPushRules(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -1651,6 +1662,7 @@ func TestKeys(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -2112,6 +2124,7 @@ func TestKeyBackup(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 7eacf9cc9c..81afc3b13f 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -145,8 +145,16 @@ func SaveReadMarker( userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, ) util.JSONResponse { + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("userID for this device is invalid"), + } + } + // Verify that the user is a member of this room - resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if resErr != nil { return *resErr } diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 8dd662a1b4..519666076e 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "net/http" + "regexp" + "strconv" "time" "github.com/gorilla/mux" @@ -16,14 +18,254 @@ import ( "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" + "golang.org/x/exp/constraints" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/internal/httputil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/userapi/api" + userapi "github.com/matrix-org/dendrite/userapi/api" ) +var validRegistrationTokenRegex = regexp.MustCompile("^[[:ascii:][:digit:]_]*$") + +func AdminCreateNewRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse { + if !cfg.RegistrationRequiresToken { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("Registration via tokens is not enabled on this homeserver"), + } + } + request := struct { + Token string `json:"token"` + UsesAllowed *int32 `json:"uses_allowed,omitempty"` + ExpiryTime *int64 `json:"expiry_time,omitempty"` + Length int32 `json:"length"` + }{} + + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)), + } + } + + token := request.Token + usesAllowed := request.UsesAllowed + expiryTime := request.ExpiryTime + length := request.Length + + if len(token) == 0 { + if length == 0 { + // length not provided in request. Assign default value of 16. + length = 16 + } + // token not present in request body. Hence, generate a random token. + if length <= 0 || length > 64 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("length must be greater than zero and not greater than 64"), + } + } + token = util.RandomString(int(length)) + } + + if len(token) > 64 { + //Token present in request body, but is too long. + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("token must not be longer than 64"), + } + } + + isTokenValid := validRegistrationTokenRegex.Match([]byte(token)) + if !isTokenValid { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("token must consist only of characters matched by the regex [A-Za-z0-9-_]"), + } + } + // At this point, we have a valid token, either through request body or through random generation. + if usesAllowed != nil && *usesAllowed < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"), + } + } + if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("expiry_time must not be in the past"), + } + } + pending := int32(0) + completed := int32(0) + // If usesAllowed or expiryTime is 0, it means they are not present in the request. NULL (indicating unlimited uses / no expiration will be persisted in DB) + registrationToken := &clientapi.RegistrationToken{ + Token: &token, + UsesAllowed: usesAllowed, + Pending: &pending, + Completed: &completed, + ExpiryTime: expiryTime, + } + created, err := userAPI.PerformAdminCreateRegistrationToken(req.Context(), registrationToken) + if !created { + return util.JSONResponse{ + Code: http.StatusConflict, + JSON: map[string]string{ + "error": fmt.Sprintf("token: %s already exists", token), + }, + } + } + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: err, + } + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{ + "token": token, + "uses_allowed": getReturnValue(usesAllowed), + "pending": pending, + "completed": completed, + "expiry_time": getReturnValue(expiryTime), + }, + } +} + +func getReturnValue[t constraints.Integer](in *t) any { + if in == nil { + return nil + } + return *in +} + +func AdminListRegistrationTokens(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse { + queryParams := req.URL.Query() + returnAll := true + valid := true + validQuery, ok := queryParams["valid"] + if ok { + returnAll = false + validValue, err := strconv.ParseBool(validQuery[0]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("invalid 'valid' query parameter"), + } + } + valid = validValue + } + tokens, err := userAPI.PerformAdminListRegistrationTokens(req.Context(), returnAll, valid) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.ErrorUnknown, + } + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{ + "registration_tokens": tokens, + }, + } +} + +func AdminGetRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + tokenText := vars["token"] + token, err := userAPI.PerformAdminGetRegistrationToken(req.Context(), tokenText) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)), + } + } + return util.JSONResponse{ + Code: 200, + JSON: token, + } +} + +func AdminDeleteRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + tokenText := vars["token"] + err = userAPI.PerformAdminDeleteRegistrationToken(req.Context(), tokenText) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: err, + } + } + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } +} + +func AdminUpdateRegistrationToken(req *http.Request, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + tokenText := vars["token"] + request := make(map[string]*int64) + if err = json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON(fmt.Sprintf("Failed to decode request body: %s", err)), + } + } + newAttributes := make(map[string]interface{}) + usesAllowed, ok := request["uses_allowed"] + if ok { + // Only add usesAllowed to newAtrributes if it is present and valid + if usesAllowed != nil && *usesAllowed < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("uses_allowed must be a non-negative integer or null"), + } + } + newAttributes["usesAllowed"] = usesAllowed + } + expiryTime, ok := request["expiry_time"] + if ok { + // Only add expiryTime to newAtrributes if it is present and valid + if expiryTime != nil && spec.Timestamp(*expiryTime).Time().Before(time.Now()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("expiry_time must not be in the past"), + } + } + newAttributes["expiryTime"] = expiryTime + } + if len(newAttributes) == 0 { + // No attributes to update. Return existing token + return AdminGetRegistrationToken(req, cfg, userAPI) + } + updatedToken, err := userAPI.PerformAdminUpdateRegistrationToken(req.Context(), tokenText, newAttributes) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound(fmt.Sprintf("token: %s not found", tokenText)), + } + } + return util.JSONResponse{ + Code: 200, + JSON: *updatedToken, + } +} + func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { @@ -123,7 +365,8 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De } } request := struct { - Password string `json:"password"` + Password string `json:"password"` + LogoutDevices bool `json:"logout_devices"` }{} if err = json.NewDecoder(req.Body).Decode(&request); err != nil { return util.JSONResponse{ @@ -146,7 +389,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De Localpart: localpart, ServerName: serverName, Password: request.Password, - LogoutDevices: true, + LogoutDevices: request.LogoutDevices, } updateRes := &api.PerformPasswordUpdateResponse{} if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil { diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go index f6603be8b6..2d6b72d3ea 100644 --- a/clientapi/routing/aliases.go +++ b/clientapi/routing/aliases.go @@ -55,9 +55,16 @@ func GetAliases( visibility = content.HistoryVisibility } if visibility != spec.WorldReadable { + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("userID doesn't have power level to change visibility"), + } + } queryReq := api.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: device.UserID, + UserID: *deviceUserID, } var queryRes api.QueryMembershipForUserResponse if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil { diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index fa50fa1aa5..38e5bd4672 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -17,6 +17,7 @@ package routing import ( "net/http" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -24,7 +25,7 @@ import ( // GetCapabilities returns information about the server's supported feature set // and other relevant capabilities to an authenticated user. -func GetCapabilities() util.JSONResponse { +func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { versionsMap := map[gomatrixserverlib.RoomVersion]string{} for v, desc := range version.SupportedRoomVersions() { if desc.Stable() { @@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse { "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": rsAPI.DefaultRoomVersion(), "available": versionsMap, }, }, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 7a7a85e85f..47e3ba1c30 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -22,17 +22,13 @@ import ( "strings" "time" - "github.com/getsentry/sentry-go" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/types" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -41,32 +37,19 @@ import ( // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom type createRoomRequest struct { - Invite []string `json:"invite"` - Name string `json:"name"` - Visibility string `json:"visibility"` - Topic string `json:"topic"` - Preset string `json:"preset"` - CreationContent json.RawMessage `json:"creation_content"` - InitialState []fledglingEvent `json:"initial_state"` - RoomAliasName string `json:"room_alias_name"` - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - PowerLevelContentOverride json.RawMessage `json:"power_level_content_override"` - IsDirect bool `json:"is_direct"` + Invite []string `json:"invite"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Topic string `json:"topic"` + Preset string `json:"preset"` + CreationContent json.RawMessage `json:"creation_content"` + InitialState []gomatrixserverlib.FledglingEvent `json:"initial_state"` + RoomAliasName string `json:"room_alias_name"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + PowerLevelContentOverride json.RawMessage `json:"power_level_content_override"` + IsDirect bool `json:"is_direct"` } -const ( - presetPrivateChat = "private_chat" - presetTrustedPrivateChat = "trusted_private_chat" - presetPublicChat = "public_chat" -) - -const ( - historyVisibilityShared = "shared" - // TODO: These should be implemented once history visibility is implemented - // historyVisibilityWorldReadable = "world_readable" - // historyVisibilityInvited = "invited" -) - func (r createRoomRequest) Validate() *util.JSONResponse { whitespace := "\t\n\x0b\x0c\r " // https://docs.python.org/2/library/string.html#string.whitespace // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L81 @@ -78,12 +61,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } for _, userID := range r.Invite { - // TODO: We should put user ID parsing code into gomatrixserverlib and use that instead - // (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 ) - // It should be a struct (with pointers into a single string to avoid copying) and - // we should update all refs to use UserID types rather than strings. - // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92 - if _, _, err := gomatrixserverlib.SplitID('@', userID); err != nil { + if _, err := spec.NewUserID(userID, true); err != nil { return &util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("user id must be in the form @localpart:domain"), @@ -91,7 +69,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } switch r.Preset { - case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "": + case spec.PresetPrivateChat, spec.PresetTrustedPrivateChat, spec.PresetPublicChat, "": default: return &util.JSONResponse{ Code: http.StatusBadRequest, @@ -129,13 +107,6 @@ type createRoomResponse struct { RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec } -// fledglingEvent is a helper representation of an event used when creating many events in succession. -type fledglingEvent struct { - Type string `json:"type"` - StateKey string `json:"state_key"` - Content interface{} `json:"content"` -} - // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *api.Device, @@ -143,12 +114,12 @@ func CreateRoom( profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - var r createRoomRequest - resErr := httputil.UnmarshalJSONRequest(req, &r) + var createRequest createRoomRequest + resErr := httputil.UnmarshalJSONRequest(req, &createRequest) if resErr != nil { return *resErr } - if resErr = r.Validate(); resErr != nil { + if resErr = createRequest.Validate(); resErr != nil { return *resErr } evTime, err := httputil.ParseTSParam(req) @@ -158,46 +129,51 @@ func CreateRoom( JSON: spec.InvalidParam(err.Error()), } } - return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime) + return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime) } // createRoom implements /createRoom -// nolint: gocyclo func createRoom( ctx context.Context, - r createRoomRequest, device *api.Device, + createRequest createRoomRequest, device *api.Device, cfg *config.ClientAPI, profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, ) util.JSONResponse { - _, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID) + userID, err := spec.NewUserID(device.UserID, true) if err != nil { - util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") + util.GetLogger(ctx).WithError(err).Error("invalid userID") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, } } - if !cfg.Matrix.IsLocalServerName(userDomain) { + if !cfg.Matrix.IsLocalServerName(userID.Domain()) { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)), + JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userID.Domain())), } } - // TODO (#267): Check room ID doesn't clash with an existing one, and we - // probably shouldn't be using pseudo-random strings, maybe GUIDs? - roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain) - logger := util.GetLogger(ctx) - userID := device.UserID + + // TODO: Check room ID doesn't clash with an existing one, and we + // probably shouldn't be using pseudo-random strings, maybe GUIDs? + roomID, err := spec.NewRoomID(fmt.Sprintf("!%s:%s", util.RandomString(16), userID.Domain())) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("invalid roomID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } // Clobber keys: creator, room_version - roomVersion := roomserverVersion.DefaultRoomVersion() - if r.RoomVersion != "" { - candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) + roomVersion := rsAPI.DefaultRoomVersion() + if createRequest.RoomVersion != "" { + candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion) _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) if roomVersionError != nil { return util.JSONResponse{ @@ -208,17 +184,13 @@ func createRoom( roomVersion = candidateVersion } - // TODO: visibility/presets/raw initial state - // TODO: Create room alias association - // Make sure this doesn't fall into an application service's namespace though! - logger.WithFields(log.Fields{ - "userID": userID, - "roomID": roomID, + "userID": userID.String(), + "roomID": roomID.String(), "roomVersion": roomVersion, }).Info("Creating new room") - profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID.String(), asAPI, profileAPI) if err != nil { util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") return util.JSONResponse{ @@ -227,427 +199,39 @@ func createRoom( } } - createContent := map[string]interface{}{} - if len(r.CreationContent) > 0 { - if err = json.Unmarshal(r.CreationContent, &createContent); err != nil { - util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed") - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("invalid create content"), - } - } - } - createContent["creator"] = userID - createContent["room_version"] = roomVersion - powerLevelContent := eventutil.InitialPowerLevelsContent(userID) - joinRuleContent := gomatrixserverlib.JoinRuleContent{ - JoinRule: spec.Invite, - } - historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{ - HistoryVisibility: historyVisibilityShared, - } + userDisplayName := profile.DisplayName + userAvatarURL := profile.AvatarURL - if r.PowerLevelContentOverride != nil { - // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults - err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed") - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("malformed power_level_content_override"), - } - } - } + keyID := cfg.Matrix.KeyID + privateKey := cfg.Matrix.PrivateKey - var guestsCanJoin bool - switch r.Preset { - case presetPrivateChat: - joinRuleContent.JoinRule = spec.Invite - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - guestsCanJoin = true - case presetTrustedPrivateChat: - joinRuleContent.JoinRule = spec.Invite - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - for _, invitee := range r.Invite { - powerLevelContent.Users[invitee] = 100 - } - guestsCanJoin = true - case presetPublicChat: - joinRuleContent.JoinRule = spec.Public - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - } + req := roomserverAPI.PerformCreateRoomRequest{ + InvitedUsers: createRequest.Invite, + RoomName: createRequest.Name, + Visibility: createRequest.Visibility, + Topic: createRequest.Topic, + StatePreset: createRequest.Preset, + CreationContent: createRequest.CreationContent, + InitialState: createRequest.InitialState, + RoomAliasName: createRequest.RoomAliasName, + RoomVersion: roomVersion, + PowerLevelContentOverride: createRequest.PowerLevelContentOverride, + IsDirect: createRequest.IsDirect, - createEvent := fledglingEvent{ - Type: spec.MRoomCreate, - Content: createContent, - } - powerLevelEvent := fledglingEvent{ - Type: spec.MRoomPowerLevels, - Content: powerLevelContent, - } - joinRuleEvent := fledglingEvent{ - Type: spec.MRoomJoinRules, - Content: joinRuleContent, - } - historyVisibilityEvent := fledglingEvent{ - Type: spec.MRoomHistoryVisibility, - Content: historyVisibilityContent, - } - membershipEvent := fledglingEvent{ - Type: spec.MRoomMember, - StateKey: userID, - Content: gomatrixserverlib.MemberContent{ - Membership: spec.Join, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - }, + UserDisplayName: userDisplayName, + UserAvatarURL: userAvatarURL, + KeyID: keyID, + PrivateKey: privateKey, + EventTime: evTime, } - var nameEvent *fledglingEvent - var topicEvent *fledglingEvent - var guestAccessEvent *fledglingEvent - var aliasEvent *fledglingEvent - - if r.Name != "" { - nameEvent = &fledglingEvent{ - Type: spec.MRoomName, - Content: eventutil.NameContent{ - Name: r.Name, - }, - } - } - - if r.Topic != "" { - topicEvent = &fledglingEvent{ - Type: spec.MRoomTopic, - Content: eventutil.TopicContent{ - Topic: r.Topic, - }, - } - } - - if guestsCanJoin { - guestAccessEvent = &fledglingEvent{ - Type: spec.MRoomGuestAccess, - Content: eventutil.GuestAccessContent{ - GuestAccess: "can_join", - }, - } - } - - var roomAlias string - if r.RoomAliasName != "" { - roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain) - // check it's free TODO: This races but is better than nothing - hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{ - Alias: roomAlias, - IncludeAppservices: false, - } - - var aliasResp roomserverAPI.GetRoomIDForAliasResponse - err = rsAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if aliasResp.RoomID != "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.RoomInUse("Room ID already exists."), - } - } - - aliasEvent = &fledglingEvent{ - Type: spec.MRoomCanonicalAlias, - Content: eventutil.CanonicalAlias{ - Alias: roomAlias, - }, - } - } - - var initialStateEvents []fledglingEvent - for i := range r.InitialState { - if r.InitialState[i].StateKey != "" { - initialStateEvents = append(initialStateEvents, r.InitialState[i]) - continue - } - - switch r.InitialState[i].Type { - case spec.MRoomCreate: - continue - - case spec.MRoomPowerLevels: - powerLevelEvent = r.InitialState[i] - - case spec.MRoomJoinRules: - joinRuleEvent = r.InitialState[i] - - case spec.MRoomHistoryVisibility: - historyVisibilityEvent = r.InitialState[i] - - case spec.MRoomGuestAccess: - guestAccessEvent = &r.InitialState[i] - - case spec.MRoomName: - nameEvent = &r.InitialState[i] - - case spec.MRoomTopic: - topicEvent = &r.InitialState[i] - - default: - initialStateEvents = append(initialStateEvents, r.InitialState[i]) - } - } - - // send events into the room in order of: - // 1- m.room.create - // 2- room creator join member - // 3- m.room.power_levels - // 4- m.room.join_rules - // 5- m.room.history_visibility - // 6- m.room.canonical_alias (opt) - // 7- m.room.guest_access (opt) - // 8- other initial state items - // 9- m.room.name (opt) - // 10- m.room.topic (opt) - // 11- invite events (opt) - with is_direct flag if applicable TODO - // 12- 3pid invite events (opt) TODO - // This differs from Synapse slightly. Synapse would vary the ordering of 3-7 - // depending on if those events were in "initial_state" or not. This made it - // harder to reason about, hence sticking to a strict static ordering. - // TODO: Synapse has txn/token ID on each event. Do we need to do this here? - eventsToMake := []fledglingEvent{ - createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent, - } - if guestAccessEvent != nil { - eventsToMake = append(eventsToMake, *guestAccessEvent) - } - eventsToMake = append(eventsToMake, initialStateEvents...) - if nameEvent != nil { - eventsToMake = append(eventsToMake, *nameEvent) - } - if topicEvent != nil { - eventsToMake = append(eventsToMake, *topicEvent) - } - if aliasEvent != nil { - // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. - // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. - eventsToMake = append(eventsToMake, *aliasEvent) - } - - // TODO: invite events - // TODO: 3pid invite events - - verImpl, err := gomatrixserverlib.GetRoomVersion(roomVersion) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("unknown room version"), - } - } - - var builtEvents []*types.HeaderedEvent - authEvents := gomatrixserverlib.NewAuthEvents(nil) - for i, e := range eventsToMake { - depth := i + 1 // depth starts at 1 - - builder := verImpl.NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ - Sender: userID, - RoomID: roomID, - Type: e.Type, - StateKey: &e.StateKey, - Depth: int64(depth), - }) - err = builder.SetContent(e.Content) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if i > 0 { - builder.PrevEvents = []string{builtEvents[i-1].EventID()} - } - var ev gomatrixserverlib.PDU - if err = builder.AddAuthEvents(&authEvents); err != nil { - util.GetLogger(ctx).WithError(err).Error("AddAuthEvents failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - ev, err = builder.Build(evTime, userDomain, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("buildEvent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil { - util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - // Add the event to the list of auth events - builtEvents = append(builtEvents, &types.HeaderedEvent{PDU: ev}) - err = authEvents.AddEvent(ev) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - } - - inputs := make([]roomserverAPI.InputRoomEvent, 0, len(builtEvents)) - for _, event := range builtEvents { - inputs = append(inputs, roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, - Event: event, - Origin: userDomain, - SendAsServer: roomserverAPI.DoNotSendToOtherServers, - }) - } - if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil { - util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - // TODO(#269): Reserve room alias while we create the room. This stops us - // from creating the room but still failing due to the alias having already - // been taken. - if roomAlias != "" { - aliasReq := roomserverAPI.SetRoomAliasRequest{ - Alias: roomAlias, - RoomID: roomID, - UserID: userID, - } - - var aliasResp roomserverAPI.SetRoomAliasResponse - err = rsAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - if aliasResp.AliasExists { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.RoomInUse("Room alias already exists."), - } - } - } - - // If this is a direct message then we should invite the participants. - if len(r.Invite) > 0 { - // Build some stripped state for the invite. - var globalStrippedState []fclient.InviteV2StrippedState - for _, event := range builtEvents { - // Chosen events from the spec: - // https://spec.matrix.org/v1.3/client-server-api/#stripped-state - switch event.Type() { - case spec.MRoomCreate: - fallthrough - case spec.MRoomName: - fallthrough - case spec.MRoomAvatar: - fallthrough - case spec.MRoomTopic: - fallthrough - case spec.MRoomCanonicalAlias: - fallthrough - case spec.MRoomEncryption: - fallthrough - case spec.MRoomMember: - fallthrough - case spec.MRoomJoinRules: - ev := event.PDU - globalStrippedState = append( - globalStrippedState, - fclient.NewInviteV2StrippedState(ev), - ) - } - } - - // Process the invites. - var inviteEvent *types.HeaderedEvent - for _, invitee := range r.Invite { - // Build the invite event. - inviteEvent, err = buildMembershipEvent( - ctx, invitee, "", profileAPI, device, spec.Invite, - roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI, - ) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") - continue - } - inviteStrippedState := append( - globalStrippedState, - fclient.NewInviteV2StrippedState(inviteEvent.PDU), - ) - // Send the invite event to the roomserver. - event := inviteEvent - err = rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{ - Event: event, - InviteRoomState: inviteStrippedState, - RoomVersion: event.Version(), - SendAsServer: string(userDomain), - }) - switch e := err.(type) { - case roomserverAPI.ErrInvalidID: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.Unknown(e.Error()), - } - case roomserverAPI.ErrNotAllowed: - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden(e.Error()), - } - case nil: - default: - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - sentry.CaptureException(err) - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - } - } - - if r.Visibility == spec.Public { - // expose this room in the published room list - if err = rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{ - RoomID: roomID, - Visibility: spec.Public, - }); err != nil { - util.GetLogger(ctx).WithError(err).Error("failed to publish room") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } + roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req) + if createRes != nil { + return *createRes } response := createRoomResponse{ - RoomID: roomID, + RoomID: roomID.String(), RoomAlias: roomAlias, } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index c786f8cc4e..9077276620 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -181,13 +181,39 @@ func SetLocalAlias( return *resErr } - queryReq := roomserverAPI.SetRoomAliasRequest{ - UserID: device.UserID, - RoomID: r.RoomID, - Alias: alias, + roomID, err := spec.NewRoomID(r.RoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } } - var queryRes roomserverAPI.SetRoomAliasResponse - if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { + + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } else if senderID == nil { + util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias) + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -195,7 +221,7 @@ func SetLocalAlias( } } - if queryRes.AliasExists { + if aliasAlreadyExists { return util.JSONResponse{ Code: http.StatusConflict, JSON: spec.Unknown("The alias " + alias + " already exists."), @@ -215,27 +241,88 @@ func RemoveLocalAlias( alias string, rsAPI roomserverAPI.ClientRoomserverAPI, ) util.JSONResponse { - queryReq := roomserverAPI.RemoveRoomAliasRequest{ - Alias: alias, - UserID: device.UserID, + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{Err: "UserID for device is invalid"}, + } + } + + roomIDReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: alias} + roomIDRes := roomserverAPI.GetRoomIDForAliasResponse{} + err = rsAPI.GetRoomIDForAlias(req.Context(), &roomIDReq, &roomIDRes) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The alias does not exist."), + } + } + + validRoomID, err := spec.NewRoomID(roomIDRes.RoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The alias does not exist."), + } } - var queryRes roomserverAPI.RemoveRoomAliasResponse - if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { + + // This seems like the kind of auth check that should be done in the roomserver, but + // if this check fails (user is not in the room), then there will be no SenderID for the user + // for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID + // as meaning they are not in the room, since lacking a sender ID could be caused by other bugs. + // TODO: maybe have QuerySenderIDForUser return richer errors? + var queryResp roomserverAPI.QueryMembershipForUserResponse + err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + RoomID: validRoomID.String(), + UserID: *userID, + }, &queryResp) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + if !queryResp.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You do not have permission to remove this alias."), + } + } + + deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The alias does not exist."), + } + } + // TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors + if deviceSenderID == nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias) + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed") return util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, + JSON: spec.Unknown("internal server error"), } } - if !queryRes.Found { + if !aliasFound { return util.JSONResponse{ Code: http.StatusNotFound, JSON: spec.NotFound("The alias does not exist."), } } - if !queryRes.Removed { + if !aliasRemoved { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("You do not have permission to remove this alias."), @@ -288,7 +375,30 @@ func SetVisibility( req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device, roomID string, ) util.JSONResponse { - resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID) + deviceUserID, err := spec.NewUserID(dev.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("userID for this device is invalid"), + } + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("RoomID is invalid"), + } + } + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) + if err != nil || senderID == nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("failed to find senderID for this user"), + } + } + + resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if resErr != nil { return *resErr } @@ -301,7 +411,7 @@ func SetVisibility( }}, } var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse - err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) if err != nil || len(queryEventsRes.StateEvents) == 0 { util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return util.JSONResponse{ @@ -312,7 +422,7 @@ func SetVisibility( // NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU) - if power.UserLevel(dev.UserID) < power.EventLevel(spec.MRoomCanonicalAlias, true) { + if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("userID doesn't have power level to change visibility"), diff --git a/clientapi/routing/joined_rooms.go b/clientapi/routing/joined_rooms.go index f664183f87..3fe0d8b4d0 100644 --- a/clientapi/routing/joined_rooms.go +++ b/clientapi/routing/joined_rooms.go @@ -33,23 +33,36 @@ func GetJoinedRooms( device *userapi.Device, rsAPI api.ClientRoomserverAPI, ) util.JSONResponse { - var res api.QueryRoomsForUserResponse - err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &res) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + rooms, err := rsAPI.QueryRoomsForUser(req.Context(), *deviceUserID, "join") if err != nil { util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, + JSON: spec.Unknown("internal server error"), } } - if res.RoomIDs == nil { - res.RoomIDs = []string{} + + var roomIDStrs []string + if rooms == nil { + roomIDStrs = []string{} + } else { + roomIDStrs = make([]string, len(rooms)) + for i, roomID := range rooms { + roomIDStrs[i] = roomID.String() + } } + return util.JSONResponse{ Code: http.StatusOK, - JSON: getJoinedRoomsResponse{res.RoomIDs}, + JSON: getJoinedRoomsResponse{roomIDStrs}, } } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 4b67b09f01..933ea8d3af 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/roomserver" @@ -34,9 +35,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc // Create the users in the userapi for _, u := range []*test.User{alice, bob, charlie} { @@ -63,7 +64,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { IsDirect: true, Topic: "testing", Visibility: "public", - Preset: presetPublicChat, + Preset: spec.PresetPublicChat, RoomAliasName: "alias", Invite: []string{bob.ID}, }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) @@ -78,7 +79,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { IsDirect: true, Topic: "testing", Visibility: "public", - Preset: presetPublicChat, + Preset: spec.PresetPublicChat, Invite: []string{charlie.ID}, }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse) diff --git a/clientapi/routing/leaveroom.go b/clientapi/routing/leaveroom.go index fbf148264b..7e8c066eb2 100644 --- a/clientapi/routing/leaveroom.go +++ b/clientapi/routing/leaveroom.go @@ -29,10 +29,18 @@ func LeaveRoomByID( rsAPI roomserverAPI.ClientRoomserverAPI, roomID string, ) util.JSONResponse { + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("device userID is invalid"), + } + } + // Prepare to ask the roomserver to perform the room join. leaveReq := roomserverAPI.PerformLeaveRequest{ RoomID: roomID, - UserID: device.UserID, + Leaver: *userID, } leaveRes := roomserverAPI.PerformLeaveResponse{} diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index bff676826d..252017db2e 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -47,6 +47,7 @@ func TestLogin(t *testing.T) { routers := httputil.NewRouters() caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for /login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 4f2a0e3943..8b8cc47bcc 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -16,14 +16,12 @@ package routing import ( "context" + "crypto/ed25519" "fmt" "net/http" "time" "github.com/getsentry/sentry-go" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/spec" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -34,6 +32,9 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -55,7 +56,29 @@ func SendBan( } } - errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to ban this user, bad userID"), + } + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("RoomID is invalid"), + } + } + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) + if err != nil || senderID == nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to ban this user, unknown senderID"), + } + } + + errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if errRes != nil { return *errRes } @@ -64,7 +87,7 @@ func SendBan( if errRes != nil { return *errRes } - allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban + allowedToBan := pl.UserLevel(*senderID) >= pl.Ban if !allowedToBan { return util.JSONResponse{ Code: http.StatusForbidden, @@ -131,7 +154,29 @@ func SendKick( } } - errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"), + } + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("RoomID is invalid"), + } + } + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) + if err != nil || senderID == nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"), + } + } + + errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if errRes != nil { return *errRes } @@ -140,7 +185,7 @@ func SendKick( if errRes != nil { return *errRes } - allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick + allowedToKick := pl.UserLevel(*senderID) >= pl.Kick if !allowedToKick { return util.JSONResponse{ Code: http.StatusForbidden, @@ -148,10 +193,17 @@ func SendKick( } } + bodyUserID, err := spec.NewUserID(body.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("body userID is invalid"), + } + } var queryRes roomserverAPI.QueryMembershipForUserResponse - err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: body.UserID, + UserID: *bodyUserID, }, &queryRes) if err != nil { return util.ErrorResponse(err) @@ -183,15 +235,30 @@ func SendUnban( } } - errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"), + } + } + + errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if errRes != nil { return *errRes } + bodyUserID, err := spec.NewUserID(body.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("body userID is invalid"), + } + } var queryRes roomserverAPI.QueryMembershipForUserResponse - err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: body.UserID, + UserID: *bodyUserID, }, &queryRes) if err != nil { return util.ErrorResponse(err) @@ -242,7 +309,15 @@ func SendInvite( } } - errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"), + } + } + + errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if errRes != nil { return *errRes } @@ -262,22 +337,55 @@ func sendInvite( rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, ) (util.JSONResponse, error) { - event, err := buildMembershipEvent( - ctx, userID, reason, profileAPI, device, spec.Invite, - roomID, false, cfg, evTime, rsAPI, asAPI, - ) + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("RoomID is invalid"), + }, err + } + inviter, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + }, err + } + invitee, err := spec.NewUserID(userID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("UserID is invalid"), + }, err + } + profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + }, err + } + identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) if err != nil { - util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, }, err } - err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{ - Event: event, + InviteInput: roomserverAPI.InviteInput{ + RoomID: *validRoomID, + Inviter: *inviter, + Invitee: *invitee, + DisplayName: profile.DisplayName, + AvatarURL: profile.AvatarURL, + Reason: reason, + IsDirect: false, + KeyID: identity.KeyID, + PrivateKey: identity.PrivateKey, + EventTime: evTime, + }, InviteRoomState: nil, // ask the roomserver to draw up invite room state for us - RoomVersion: event.Version(), SendAsServer: string(device.UserDomain()), }) @@ -308,6 +416,42 @@ func sendInvite( }, nil } +func buildMembershipEventDirect( + ctx context.Context, + targetSenderID spec.SenderID, reason string, userDisplayName, userAvatarURL string, + sender spec.SenderID, senderDomain spec.ServerName, + membership, roomID string, isDirect bool, + keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, evTime time.Time, + rsAPI roomserverAPI.ClientRoomserverAPI, +) (*types.HeaderedEvent, error) { + targetSenderString := string(targetSenderID) + proto := gomatrixserverlib.ProtoEvent{ + SenderID: string(sender), + RoomID: roomID, + Type: "m.room.member", + StateKey: &targetSenderString, + } + + content := gomatrixserverlib.MemberContent{ + Membership: membership, + DisplayName: userDisplayName, + AvatarURL: userAvatarURL, + Reason: reason, + IsDirect: isDirect, + } + + if err := proto.SetContent(content); err != nil { + return nil, err + } + + identity := &fclient.SigningIdentity{ + ServerName: senderDomain, + KeyID: keyID, + PrivateKey: privateKey, + } + return eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil) +} + func buildMembershipEvent( ctx context.Context, targetUserID, reason string, profileAPI userapi.ClientUserAPI, @@ -321,31 +465,39 @@ func buildMembershipEvent( return nil, err } - proto := gomatrixserverlib.ProtoEvent{ - Sender: device.UserID, - RoomID: roomID, - Type: "m.room.member", - StateKey: &targetUserID, + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return nil, err } - - content := gomatrixserverlib.MemberContent{ - Membership: membership, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - Reason: reason, - IsDirect: isDirect, + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, err + } + senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID) + if err != nil { + return nil, err + } else if senderID == nil { + return nil, fmt.Errorf("no sender ID for %s in %s", *userID, *validRoomID) } - if err = proto.SetContent(content); err != nil { + targetID, err := spec.NewUserID(targetUserID, true) + if err != nil { return nil, err } + targetSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *targetID) + if err != nil { + return nil, err + } else if targetSenderID == nil { + return nil, fmt.Errorf("no sender ID for %s in %s", *targetID, *validRoomID) + } - identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) + identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID) if err != nil { return nil, err } - return eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil) + return buildMembershipEventDirect(ctx, *targetSenderID, reason, profile.DisplayName, profile.AvatarURL, + *senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI) } // loadProfile lookups the profile of a given user from the database and returns @@ -443,7 +595,7 @@ func checkAndProcessThreepid( return } -func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse { +func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID spec.UserID, roomID string) *util.JSONResponse { var membershipRes roomserverAPI.QueryMembershipForUserResponse err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, @@ -471,12 +623,21 @@ func SendForget( ) util.JSONResponse { ctx := req.Context() logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID) + + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, bad userID"), + } + } + var membershipRes roomserverAPI.QueryMembershipForUserResponse membershipReq := roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: device.UserID, + UserID: *deviceUserID, } - err := rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes) + err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes) if err != nil { logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user") return util.JSONResponse{ diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 76129f0a89..564cd588aa 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -16,6 +16,7 @@ package routing import ( "context" + "fmt" "net/http" "time" @@ -104,12 +105,6 @@ func SetAvatarURL( if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } - if r.AvatarURL == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("'avatar_url' must be supplied."), - } - } localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { @@ -151,7 +146,7 @@ func SetAvatarURL( } } - response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime) + response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime) if err != nil { return response } @@ -199,12 +194,6 @@ func SetDisplayName( if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } - if r.DisplayName == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("'displayname' must be supplied."), - } - } localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { @@ -246,7 +235,7 @@ func SetDisplayName( } } - response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime) + response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime) if err != nil { return response } @@ -260,13 +249,17 @@ func SetDisplayName( func updateProfile( ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device, profile *authtypes.Profile, - userID string, cfg *config.ClientAPI, evTime time.Time, + userID string, evTime time.Time, ) (util.JSONResponse, error) { - var res api.QueryRoomsForUserResponse - err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &res) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + }, err + } + + rooms, err := rsAPI.QueryRoomsForUser(ctx, *deviceUserID, "join") if err != nil { util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed") return util.JSONResponse{ @@ -275,6 +268,11 @@ func updateProfile( }, err } + roomIDStrs := make([]string, len(rooms)) + for i, room := range rooms { + roomIDStrs[i] = room.String() + } + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") @@ -285,7 +283,7 @@ func updateProfile( } events, err := buildMembershipEvents( - ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI, + ctx, roomIDStrs, *profile, userID, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -356,19 +354,33 @@ func getProfile( func buildMembershipEvents( ctx context.Context, - device *userapi.Device, roomIDs []string, - newProfile authtypes.Profile, userID string, cfg *config.ClientAPI, + newProfile authtypes.Profile, userID string, evTime time.Time, rsAPI api.ClientRoomserverAPI, ) ([]*types.HeaderedEvent, error) { evs := []*types.HeaderedEvent{} + fullUserID, err := spec.NewUserID(userID, true) + if err != nil { + return nil, err + } for _, roomID := range roomIDs { + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, err + } + senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) + if err != nil { + return nil, err + } else if senderID == nil { + return nil, fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID) + } + senderIDString := string(*senderID) proto := gomatrixserverlib.ProtoEvent{ - Sender: userID, + SenderID: senderIDString, RoomID: roomID, Type: "m.room.member", - StateKey: &userID, + StateKey: &senderIDString, } content := gomatrixserverlib.MemberContent{ @@ -378,16 +390,21 @@ func buildMembershipEvents( content.DisplayName = newProfile.DisplayName content.AvatarURL = newProfile.AvatarURL - if err := proto.SetContent(content); err != nil { + if err = proto.SetContent(content); err != nil { + return nil, err + } + + user, err := spec.NewUserID(userID, true) + if err != nil { return nil, err } - identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) + identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *user) if err != nil { return nil, err } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil) + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index ed70e5c5cc..f331a73c72 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -34,7 +34,8 @@ import ( ) type redactionContent struct { - Reason string `json:"reason"` + Reason string `json:"reason"` + Redacts string `json:"redacts"` } type redactionResponse struct { @@ -47,11 +48,43 @@ func SendRedaction( txnID *string, txnCache *transactions.Cache, ) util.JSONResponse { - resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + deviceUserID, userIDErr := spec.NewUserID(device.UserID, true) + if userIDErr != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("userID doesn't have power level to redact"), + } + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("RoomID is invalid"), + } + } + senderID, queryErr := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) + if queryErr != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("userID doesn't have power level to redact"), + } + } + + resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if resErr != nil { return *resErr } + // if user is member of room, and sender ID is nil, then this user doesn't have a pseudo ID for some reason, + // which is unexpected. + if senderID == nil { + util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + if txnID != nil { // Try to fetch response from transactionsCache if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { @@ -66,7 +99,7 @@ func SendRedaction( JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence? } } - if ev.RoomID() != roomID { + if ev.RoomID().String() != roomID { return util.JSONResponse{ Code: 400, JSON: spec.NotFound("cannot redact event in another room"), @@ -76,7 +109,7 @@ func SendRedaction( // "Users may redact their own events, and any user with a power level greater than or equal // to the redact power level of the room may redact events there" // https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-redact-eventid-txnid - allowedToRedact := ev.Sender() == device.UserID + allowedToRedact := ev.SenderID() == *senderID if !allowedToRedact { plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomPowerLevels, @@ -88,8 +121,8 @@ func SendRedaction( JSON: spec.Forbidden("You don't have permission to redact this event, no power_levels event in this room."), } } - pl, err := plEvent.PowerLevels() - if err != nil { + pl, plErr := plEvent.PowerLevels() + if plErr != nil { return util.JSONResponse{ Code: 403, JSON: spec.Forbidden( @@ -97,7 +130,7 @@ func SendRedaction( ), } } - allowedToRedact = pl.UserLevel(device.UserID) >= pl.Redact + allowedToRedact = pl.UserLevel(*senderID) >= pl.Redact } if !allowedToRedact { return util.JSONResponse{ @@ -114,12 +147,17 @@ func SendRedaction( // create the new event and set all the fields we can proto := gomatrixserverlib.ProtoEvent{ - Sender: device.UserID, - RoomID: roomID, - Type: spec.MRoomRedaction, - Redacts: eventID, + SenderID: string(*senderID), + RoomID: roomID, + Type: spec.MRoomRedaction, + Redacts: eventID, } - err := proto.SetContent(r) + + // Room version 11 expects the "redacts" field on the + // content field, so add it here as well + r.Redacts = eventID + + err = proto.SetContent(r) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed") return util.JSONResponse{ @@ -128,7 +166,7 @@ func SendRedaction( } } - identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) + identity, err := rsAPI.SigningIdentityFor(req.Context(), *validRoomID, *deviceUserID) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -137,7 +175,7 @@ func SendRedaction( } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) + e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, &identity, time.Now(), rsAPI, &queryRes) if errors.Is(err, eventutil.ErrRoomNoExists{}) { return util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 2a88ec3806..0a1986cf71 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -415,6 +415,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) for _, tc := range testCases { @@ -594,6 +595,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" @@ -634,6 +636,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) expectedDisplayName := "rabbit" diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go new file mode 100644 index 0000000000..2884d2c32d --- /dev/null +++ b/clientapi/routing/room_hierarchy.go @@ -0,0 +1,180 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "strconv" + "sync" + + "github.com/google/uuid" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +// For storing pagination information for room hierarchies +type RoomHierarchyPaginationCache struct { + cache map[string]roomserverAPI.RoomHierarchyWalker + mu sync.Mutex +} + +// Create a new, empty, pagination cache. +func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { + return RoomHierarchyPaginationCache{ + cache: map[string]roomserverAPI.RoomHierarchyWalker{}, + } +} + +// Get a cached page, or nil if there is no associated page in the cache. +func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker { + c.mu.Lock() + defer c.mu.Unlock() + line, ok := c.cache[token] + if ok { + return &line + } else { + return nil + } +} + +// Add a cache line to the pagination cache. +func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string { + c.mu.Lock() + defer c.mu.Unlock() + token := uuid.NewString() + c.cache[token] = line + return token +} + +// Query the hierarchy of a room/space +// +// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy +func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch req.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + limit := 1000 // Default to 1000 + limitStr := req.URL.Query().Get("limit") + if limitStr != "" { + var maybeLimit int + maybeLimit, err = strconv.Atoi(limitStr) + if err != nil || maybeLimit < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"), + } + } + limit = maybeLimit + if limit > 1000 { + limit = 1000 // Maximum limit of 1000 + } + } + + maxDepth := -1 // '-1' representing no maximum depth + maxDepthStr := req.URL.Query().Get("max_depth") + if maxDepthStr != "" { + var maybeMaxDepth int + maybeMaxDepth, err = strconv.Atoi(maxDepthStr) + if err != nil || maybeMaxDepth < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"), + } + } + maxDepth = maybeMaxDepth + } + + from := req.URL.Query().Get("from") + + var walker roomserverAPI.RoomHierarchyWalker + if from == "" { // No pagination token provided, so start new hierarchy walker + walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth) + } else { // Attempt to resume cached walker + cachedWalker := paginationCache.Get(from) + + if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"), + } + } + + walker = *cachedWalker + } + + discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit) + + if err != nil { + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request") + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + } + + nextBatch := "" + // nextWalker will be nil if there's no more rooms left to walk + if nextWalker != nil { + nextBatch = paginationCache.AddLine(*nextWalker) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: RoomHierarchyClientResponse{ + Rooms: discoveredRooms, + NextBatch: nextBatch, + }, + } + +} + +// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy +type RoomHierarchyClientResponse struct { + Rooms []fclient.RoomHierarchyRoom `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 2a2fa66550..d4aa1d08d1 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -20,14 +20,16 @@ import ( "strings" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/setup/base" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" + + "github.com/matrix-org/dendrite/setup/base" + userapi "github.com/matrix-org/dendrite/userapi/api" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" @@ -42,6 +44,19 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" ) +type WellKnownClientHomeserver struct { + BaseUrl string `json:"base_url"` +} + +type WellKnownSlidingSyncProxy struct { + Url string `json:"url"` +} + +type WellKnownClientResponse struct { + Homeserver WellKnownClientHomeserver `json:"m.homeserver"` + SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"` +} + // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. // @@ -84,22 +99,32 @@ func Setup( unstableFeatures["org.matrix."+msc] = true } + // singleflight protects /join endpoints from being invoked + // multiple times from the same user and room, otherwise + // a state reset can occur. This also avoids unneeded + // state calculations. + // TODO: actually fix this in the roomserver, as there are + // possibly other ways that can result in a stat reset. + sf := singleflight.Group{} + if cfg.Matrix.WellKnownClientName != "" { logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName) + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy) + } wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse { + response := WellKnownClientResponse{ + Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName}, + } + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{ + Url: cfg.Matrix.WellKnownSlidingSyncProxy, + } + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct { - HomeserverName struct { - BaseUrl string `json:"base_url"` - } `json:"m.homeserver"` - }{ - HomeserverName: struct { - BaseUrl string `json:"base_url"` - }{ - BaseUrl: cfg.Matrix.WellKnownClientName, - }, - }, + JSON: response, } })).Methods(http.MethodGet, http.MethodOptions) } @@ -152,6 +177,36 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) } + dendriteAdminRouter.Handle("/admin/registrationTokens/new", + httputil.MakeAdminAPI("admin_registration_tokens_new", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return AdminCreateNewRegistrationToken(req, cfg, userAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + dendriteAdminRouter.Handle("/admin/registrationTokens", + httputil.MakeAdminAPI("admin_list_registration_tokens", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return AdminListRegistrationTokens(req, cfg, userAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + dendriteAdminRouter.Handle("/admin/registrationTokens/{token}", + httputil.MakeAdminAPI("admin_get_registration_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + switch req.Method { + case http.MethodGet: + return AdminGetRegistrationToken(req, cfg, userAPI) + case http.MethodPut: + return AdminUpdateRegistrationToken(req, cfg, userAPI) + case http.MethodDelete: + return AdminDeleteRegistrationToken(req, cfg, userAPI) + default: + return util.MatrixErrorResponse( + 404, + string(spec.ErrorNotFound), + "unknown method", + ) + } + }), + ).Methods(http.MethodGet, http.MethodPut, http.MethodDelete, http.MethodOptions) dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}", httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { @@ -248,6 +303,8 @@ func Setup( // Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing! v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() + v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter() + unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter() v3mux.Handle("/createRoom", @@ -264,9 +321,17 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return JoinRoomByIDOrAlias( - req, device, rsAPI, userAPI, vars["roomIDOrAlias"], - ) + // Only execute a join for roomIDOrAlias and UserID once. If there is a join in progress + // it waits for it to complete and returns that result for subsequent requests. + resp, _, _ := sf.Do(vars["roomIDOrAlias"]+device.UserID, func() (any, error) { + return JoinRoomByIDOrAlias( + req, device, rsAPI, userAPI, vars["roomIDOrAlias"], + ), nil + }) + // once all joins are processed, drop them from the cache. Further requests + // will be processed as usual. + sf.Forget(vars["roomIDOrAlias"] + device.UserID) + return resp.(util.JSONResponse) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) @@ -300,9 +365,17 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return JoinRoomByIDOrAlias( - req, device, rsAPI, userAPI, vars["roomID"], - ) + // Only execute a join for roomID and UserID once. If there is a join in progress + // it waits for it to complete and returns that result for subsequent requests. + resp, _, _ := sf.Do(vars["roomID"]+device.UserID, func() (any, error) { + return JoinRoomByIDOrAlias( + req, device, rsAPI, userAPI, vars["roomID"], + ), nil + }) + // once all joins are processed, drop them from the cache. Further requests + // will be processed as usual. + sf.Forget(vars["roomID"] + device.UserID) + return resp.(util.JSONResponse) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/leave", @@ -449,6 +522,19 @@ func Setup( }, httputil.WithAllowGuests()), ).Methods(http.MethodPut, http.MethodOptions) + // Defined outside of handler to persist between calls + // TODO: clear based on some criteria + roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache() + v1mux.Handle("/rooms/{roomID}/hierarchy", + httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache) + }, httputil.WithAllowGuests()), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { if r := rateLimits.Limit(req, nil); r != nil { return *r @@ -1185,7 +1271,7 @@ func Setup( if r := rateLimits.Limit(req, device); r != nil { return *r } - return GetCapabilities() + return GetCapabilities(rsAPI) }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index bc14642f8e..69131966bd 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -23,19 +23,19 @@ import ( "sync" "time" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" ) // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid @@ -68,6 +68,8 @@ var sendEventDuration = prometheus.NewHistogramVec( // /rooms/{roomID}/send/{eventType} // /rooms/{roomID}/send/{eventType}/{txnID} // /rooms/{roomID}/state/{eventType}/{stateKey} +// +// nolint: gocyclo func SendEvent( req *http.Request, device *userapi.Device, @@ -91,6 +93,30 @@ func SendEvent( } } + // Translate user ID state keys to room keys in pseudo ID rooms + if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil { + parsedRoomID, innerErr := spec.NewRoomID(roomID) + if innerErr != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + + newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID) + }) + if innerErr != nil { + // TODO: work out better logic for failure cases (e.g. sender ID not found) + util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + stateKey = newStateKey + } + // create a mutex for the specific user in the specific room // this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order userID := device.UserID @@ -121,6 +147,17 @@ func SendEvent( delete(r, "join_authorised_via_users_server") } + // for power level events we need to replace the userID with the pseudoID + if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && eventType == spec.MRoomPowerLevels { + err = updatePowerLevels(req, r, roomID, rsAPI) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{Err: err.Error()}, + } + } + } + evTime, err := httputil.ParseTSParam(req) if err != nil { return util.JSONResponse{ @@ -129,7 +166,7 @@ func SendEvent( } } - e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, cfg, rsAPI, evTime) + e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime) if resErr != nil { return *resErr } @@ -225,6 +262,35 @@ func SendEvent( return res } +func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error { + users, ok := r["users"] + if !ok { + return nil + } + userMap := users.(map[string]interface{}) + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return err + } + for user, level := range userMap { + uID, err := spec.NewUserID(user, true) + if err != nil { + continue // we're modifying the map in place, so we're going to have invalid userIDs after the first iteration + } + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID) + if err != nil { + return err + } else if senderID == nil { + util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID) + continue + } + userMap[string(*senderID)] = level + delete(userMap, user) + } + r["users"] = userMap + return nil +} + // stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse // with the existing event_id, making this an idempotent request. func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse { @@ -261,21 +327,47 @@ func generateSendEvent( r map[string]interface{}, device *userapi.Device, roomID, eventType string, stateKey *string, - cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI, evTime time.Time, ) (gomatrixserverlib.PDU, *util.JSONResponse) { // parse the incoming http request - userID := device.UserID + fullUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("Bad userID"), + } + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("RoomID is invalid"), + } + } + senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) + if err != nil { + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.NotFound("internal server error"), + } + } else if senderID == nil { + // TODO: is it always the case that lack of a sender ID means they're not joined? + // And should this logic be deferred to the roomserver somehow? + return nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("not joined to room"), + } + } // create the new event and set all the fields we can proto := gomatrixserverlib.ProtoEvent{ - Sender: userID, + SenderID: string(*senderID), RoomID: roomID, Type: eventType, StateKey: stateKey, } - err := proto.SetContent(r) + err = proto.SetContent(r) if err != nil { util.GetLogger(ctx).WithError(err).Error("proto.SetContent failed") return nil, &util.JSONResponse{ @@ -284,7 +376,7 @@ func generateSendEvent( } } - identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) + identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusInternalServerError, @@ -293,7 +385,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes) + e, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, &queryRes) switch specificErr := err.(type) { case nil: case eventutil.ErrRoomNoExists: @@ -331,7 +423,9 @@ func generateSendEvent( stateEvents[i] = queryRes.StateEvents[i].PDU } provider := gomatrixserverlib.NewAuthEvents(gomatrixserverlib.ToPDUs(stateEvents)) - if err = gomatrixserverlib.Allowed(e.PDU, &provider); err != nil { + if err = gomatrixserverlib.Allowed(e.PDU, &provider, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, *validRoomID, senderID) + }); err != nil { return nil, &util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client? @@ -348,7 +442,7 @@ func generateSendEvent( JSON: spec.BadJSON("Cannot unmarshal the event content."), } } - if content["replacement_room"] == e.RoomID() { + if content["replacement_room"] == e.RoomID().String() { return nil, &util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidParam("Cannot send tombstone event that points to the same room."), diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go new file mode 100644 index 0000000000..9cdd753587 --- /dev/null +++ b/clientapi/routing/sendevent_test.go @@ -0,0 +1,275 @@ +package routing + +import ( + "context" + "crypto/ed25519" + "fmt" + "io" + "net/http" + "strings" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "gotest.tools/v3/assert" +) + +// Mock roomserver API for testing +// +// Currently pretty specialised for the pseudo ID test, so will need +// editing if future (other) sendevent tests are using this. +type sendEventTestRoomserverAPI struct { + rsapi.ClientRoomserverAPI + t *testing.T + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + roomState []*types.HeaderedEvent + + // userID -> room key + senderMapping map[string]ed25519.PrivateKey + + savedInputRoomEvents []rsapi.InputRoomEvent +} + +func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error { + res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateTuples { + for _, stateEv := range s.roomState { + if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey { + res.StateEvents[stateKeyTuple] = stateEv + } + } + } + return nil +} + +func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { + if req.RoomID == s.roomIDStr { + res.RoomExists = true + res.RoomVersion = s.roomVersion + + res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState)) + copy(res.StateEvents, s.roomState) + + res.LatestEvents = []string{} + res.Depth = 1 + return nil + } else { + s.t.Logf("room event/state queried for %s", req.RoomID) + return fmt.Errorf("unknown room") + } + +} + +func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderIDFromPseudoIDKey(roomKey) + return &sender, nil + } else { + return nil, nil + } + } else { + senderID := spec.SenderIDFromUserID(userID) + return &senderID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + for uID, roomKey := range s.senderMapping { + if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + } else { + userID := senderID.ToUserID() + if userID == nil { + return nil, fmt.Errorf("bad sender ID") + } + return userID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) { + if s.roomIDStr == roomID.String() { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[sender.String()] + if !ok { + s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String()) + return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String()) + } + return fclient.SigningIdentity{PrivateKey: roomKey}, nil + } else { + return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil + } + } + + return fclient.SigningIdentity{}, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { + s.savedInputRoomEvents = req.InputRoomEvents +} + +// Test that user ID state keys are translated correctly +func Test_SendEvent_PseudoIDStateKeys(t *testing.T) { + nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + + senderKeySeed := make([]byte, 32) + senderUserID := "@testuser:domain" + senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed) + senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey)) + + eventType := "com.example.test" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: senderUserID, + } + + t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID), + } + + roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: nonpseudoIDRoomVersion, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID) + } + if *stateKey != senderUserID { + t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID) + } + }) + + t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID), + } + + roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: pseudoIDRoomVersion, + senderMapping: map[string]ed25519.PrivateKey{ + senderUserID: senderPrivKey, + }, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID) + } + if *stateKey != senderPseudoID { + t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID) + } + }) +} + +func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) { + events := make([]*types.HeaderedEvent, len(eventsJSON)) + + roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer) + if err != nil { + return nil, fmt.Errorf("no roomver impl: %s", err.Error()) + } + + for i, eventJSON := range eventsJSON { + pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false) + if evErr != nil { + return nil, fmt.Errorf("failed to make event: %s", err.Error()) + } + ev := types.HeaderedEvent{PDU: pdu} + events[i] = &ev + } + + return events, nil +} diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index c5b29297a6..979bced3b2 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -43,8 +43,16 @@ func SendTyping( } } + deviceUserID, err := spec.NewUserID(userID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("userID doesn't have power level to change visibility"), + } + } + // Verify that the user is a member of this room - resErr := checkMemberInRoom(req.Context(), rsAPI, userID, roomID) + resErr := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) if resErr != nil { return *resErr } diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index ad50cc80b5..5deb559df2 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -28,7 +28,6 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/roomserver/version" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -52,6 +51,7 @@ type sendServerNoticeRequest struct { StateKey string `json:"state_key,omitempty"` } +// nolint:gocyclo // SendServerNotice sends a message to a specific user. It can only be invoked by an admin. func SendServerNotice( req *http.Request, @@ -94,34 +94,42 @@ func SendServerNotice( } } + userID, err := spec.NewUserID(r.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid user ID"), + } + } + // get rooms for specified user - allUserRooms := []string{} - userRooms := api.QueryRoomsForUserResponse{} + allUserRooms := []spec.RoomID{} // Get rooms the user is either joined, invited or has left. for _, membership := range []string{"join", "invite", "leave"} { - if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: r.UserID, - WantMembership: membership, - }, &userRooms); err != nil { + userRooms, queryErr := rsAPI.QueryRoomsForUser(ctx, *userID, membership) + if queryErr != nil { return util.ErrorResponse(err) } - allUserRooms = append(allUserRooms, userRooms.RoomIDs...) + allUserRooms = append(allUserRooms, userRooms...) } // get rooms of the sender - senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName) - senderRooms := api.QueryRoomsForUserResponse{} - if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: senderUserID, - WantMembership: "join", - }, &senderRooms); err != nil { + senderUserID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName), true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + senderRooms, err := rsAPI.QueryRoomsForUser(ctx, *senderUserID, "join") + if err != nil { return util.ErrorResponse(err) } // check if we have rooms in common - commonRooms := []string{} + commonRooms := []spec.RoomID{} for _, userRoomID := range allUserRooms { - for _, senderRoomID := range senderRooms.RoomIDs { + for _, senderRoomID := range senderRooms { if userRoomID == senderRoomID { commonRooms = append(commonRooms, senderRoomID) } @@ -134,12 +142,12 @@ func SendServerNotice( var ( roomID string - roomVersion = version.DefaultRoomVersion() + roomVersion = rsAPI.DefaultRoomVersion() ) // create a new room for the user if len(commonRooms) == 0 { - powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID) + powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID.String()) powerLevelContent.Users[r.UserID] = -10 // taken from Synapse pl, err := json.Marshal(powerLevelContent) if err != nil { @@ -155,7 +163,7 @@ func SendServerNotice( Invite: []string{r.UserID}, Name: cfgNotices.RoomName, Visibility: "private", - Preset: presetPrivateChat, + Preset: spec.PresetPrivateChat, CreationContent: cc, RoomVersion: roomVersion, PowerLevelContentOverride: pl, @@ -187,9 +195,17 @@ func SendServerNotice( } } else { // we've found a room in common, check the membership - roomID = commonRooms[0] + deviceUserID, err := spec.NewUserID(r.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("userID doesn't have power level to change visibility"), + } + } + + roomID = commonRooms[0].String() membershipRes := api.QueryMembershipForUserResponse{} - err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: r.UserID, RoomID: roomID}, &membershipRes) + err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: *deviceUserID, RoomID: roomID}, &membershipRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("unable to query membership for user") return util.JSONResponse{ @@ -212,7 +228,7 @@ func SendServerNotice( "body": r.Content.Body, "msgtype": r.Content.MsgType, } - e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, cfgClient, rsAPI, time.Now()) + e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now()) if resErr != nil { logrus.Errorf("failed to send message: %+v", resErr) return *resErr @@ -234,7 +250,7 @@ func SendServerNotice( ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{ - &types.HeaderedEvent{PDU: e}, + {PDU: e}, }, device.UserDomain(), cfgClient.Matrix.ServerName, @@ -341,7 +357,7 @@ func getSenderDevice( if len(deviceRes.Devices) > 0 { // If there were changes to the profile, create a new membership event if displayNameChanged || avatarChanged { - _, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now()) + _, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, time.Now()) if err != nil { return nil, err } diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 319f4eba51..18f9a0e9cd 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -99,9 +99,17 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a if !worldReadable { // The room isn't world-readable so try to work out based on the // user's membership if we want the latest state or not. - err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{ + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("UserID is invalid") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("Device UserID is invalid"), + } + } + err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: device.UserID, + UserID: *userID, }, &membershipRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser") @@ -142,7 +150,9 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a for _, ev := range stateRes.StateEvents { stateEvents = append( stateEvents, - synctypes.ToClientEvent(ev, synctypes.FormatAll), + synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, ev), ) } } else { @@ -162,9 +172,16 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a } } for _, ev := range stateAfterRes.StateEvents { + clientEvent, err := synctypes.ToClientEvent(ev, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed converting to ClientEvent") + continue + } stateEvents = append( stateEvents, - synctypes.ToClientEvent(ev, synctypes.FormatAll), + *clientEvent, ) } } @@ -188,6 +205,37 @@ func OnIncomingStateTypeRequest( var worldReadable bool var wantLatestState bool + roomVer, err := rsAPI.QueryRoomVersionForRoom(ctx, roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)), + } + } + + // Translate user ID state keys to room keys in pseudo ID rooms + if roomVer == gomatrixserverlib.RoomVersionPseudoIDs { + parsedRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("invalid room ID"), + } + } + newStateKey, err := synctypes.FromClientStateKey(*parsedRoomID, stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + return rsAPI.QuerySenderIDForUser(ctx, roomID, userID) + }) + if err != nil { + // TODO: work out better logic for failure cases (e.g. sender ID not found) + util.GetLogger(ctx).WithError(err).Error("synctypes.FromClientStateKey failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + stateKey = *newStateKey + } + // Always fetch visibility so that we can work out whether to show // the latest events or the last event from when the user was joined. // Then include the requested event type and state key, assuming it @@ -249,11 +297,19 @@ func OnIncomingStateTypeRequest( // membershipRes will only be populated if the room is not world-readable. var membershipRes api.QueryMembershipForUserResponse if !worldReadable { + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("UserID is invalid") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("Device UserID is invalid"), + } + } // The room isn't world-readable so try to work out based on the // user's membership if we want the latest state or not. - err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{ + err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: device.UserID, + UserID: *userID, }, &membershipRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser") @@ -335,7 +391,9 @@ func OnIncomingStateTypeRequest( } stateEvent := stateEventInStateResp{ - ClientEvent: synctypes.ToClientEvent(event, synctypes.FormatAll), + ClientEvent: synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, event), } var res interface{} diff --git a/clientapi/routing/state_test.go b/clientapi/routing/state_test.go new file mode 100644 index 0000000000..93b0437231 --- /dev/null +++ b/clientapi/routing/state_test.go @@ -0,0 +1,253 @@ +package routing + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "gotest.tools/v3/assert" +) + +var () + +type stateTestRoomserverAPI struct { + rsapi.RoomserverInternalAPI + t *testing.T + roomState map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + userIDStr string + // userID -> senderID + senderMapping map[string]string +} + +func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s stateTestRoomserverAPI) QueryLatestEventsAndState( + ctx context.Context, + req *rsapi.QueryLatestEventsAndStateRequest, + res *rsapi.QueryLatestEventsAndStateResponse, +) error { + res.RoomExists = req.RoomID == s.roomIDStr + if !res.RoomExists { + return nil + } + + res.StateEvents = []*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateToFetch { + val, ok := s.roomState[stateKeyTuple] + if ok && val != nil { + res.StateEvents = append(res.StateEvents, val) + } + } + + return nil +} + +func (s stateTestRoomserverAPI) QueryMembershipForUser( + ctx context.Context, + req *rsapi.QueryMembershipForUserRequest, + res *rsapi.QueryMembershipForUserResponse, +) error { + if req.UserID.String() == s.userIDStr { + res.HasBeenInRoom = true + res.IsInRoom = true + res.RoomExists = true + res.Membership = spec.Join + } + + return nil +} + +func (s stateTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + sID, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderID(sID) + return &sender, nil + } else { + return nil, nil + } +} + +func (s stateTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + for uID, sID := range s.senderMapping { + if sID == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + return nil, nil +} + +func (s stateTestRoomserverAPI) QueryStateAfterEvents( + ctx context.Context, + req *rsapi.QueryStateAfterEventsRequest, + res *rsapi.QueryStateAfterEventsResponse, +) error { + return nil +} + +func Test_OnIncomingStateTypeRequest(t *testing.T) { + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + + userIDStr := "@testuser:domain" + eventType := "com.example.test" + stateKey := "testStateKey" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: userIDStr, + } + + t.Run("request simple state key", func(t *testing.T) { + ctx := context.Background() + + rsAPI := stateTestRoomserverAPI{ + roomVersion: defaultRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateKey, + }: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: pseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "foo": "bar", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "not": "thisone", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderRoomKey, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: nonPseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "not": "thisone", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderUserID, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) +} + +func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent { + t.Helper() + roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer) + + evBytes, err := json.Marshal(map[string]interface{}{ + "room_id": roomID, + "type": stateType, + "state_key": stateKey, + "content": stateContent, + }) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + return &types.HeaderedEvent{PDU: ev} +} diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go index a0b2807893..03c0230e67 100644 --- a/clientapi/routing/upgrade_room.go +++ b/clientapi/routing/upgrade_room.go @@ -59,7 +59,15 @@ func UpgradeRoom( } } - newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, device.UserID, gomatrixserverlib.RoomVersion(r.NewVersion)) + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("device UserID is invalid") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + newRoomID, err := rsAPI.PerformRoomUpgrade(req.Context(), roomID, *userID, gomatrixserverlib.RoomVersion(r.NewVersion)) switch e := err.(type) { case nil: case roomserverAPI.ErrNotAllowed: diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index c296939d57..365e9f8691 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -355,8 +355,22 @@ func emit3PIDInviteEvent( rsAPI api.ClientRoomserverAPI, evTime time.Time, ) error { + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return err + } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return err + } + sender, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID) + if err != nil { + return err + } else if sender == nil { + return fmt.Errorf("sender ID not found for %s in %s", *userID, *validRoomID) + } proto := &gomatrixserverlib.ProtoEvent{ - Sender: device.UserID, + SenderID: string(*sender), RoomID: roomID, Type: "m.room.third_party_invite", StateKey: &res.Token, @@ -370,7 +384,7 @@ func emit3PIDInviteEvent( PublicKeys: res.PublicKeys, } - if err := proto.SetContent(content); err != nil { + if err = proto.SetContent(content); err != nil { return err } @@ -380,7 +394,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.QueryAndBuildEvent(ctx, proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes) + event, err := eventutil.QueryAndBuildEvent(ctx, proto, identity, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 397473865f..41af568a61 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -98,7 +98,7 @@ func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir st cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix))) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix))) cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix))) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true @@ -126,7 +126,7 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) { } func (p *P2PMonolith) SetupDendrite( - processCtx *process.ProcessContext, cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, + processCtx *process.ProcessContext, cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) { p.port = port @@ -143,13 +143,12 @@ func (p *P2PMonolith) SetupDendrite( fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) + rsAPI.SetFederationAPI(fsAPI, keyRing) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetFederationAPI(fsAPI, keyRing) - userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation) roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation) @@ -222,8 +221,8 @@ func (p *P2PMonolith) closeAllResources() { p.httpServerMu.Lock() if p.httpServer != nil { _ = p.httpServer.Shutdown(context.Background()) - p.httpServerMu.Unlock() } + p.httpServerMu.Unlock() select { case p.stopHandlingEvents <- true: diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 14fc3a2db0..23304c214a 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 25c1475cbe..3ec5501139 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -134,7 +134,7 @@ func main() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName))) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index dcc45bdcc0..b78c5f6055 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -55,7 +54,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const DockerfilePostgreSQL = `FROM golang:1.18-buster as build +const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -74,16 +73,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/11/main/pg_hba.conf +RUN sed -i "s%127.0.0.1/32 scram-sha-256%127.0.0.1/32 trust%g" /etc/postgresql/15/main/pg_hba.conf # Bump up max conns for moar concurrency -RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf +RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ -pg_ctlcluster 11 main start \n\ +pg_ctlcluster 15 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -101,7 +100,7 @@ ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh` -const DockerfileSQLite = `FROM golang:1.18-buster as build +const DockerfileSQLite = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -119,7 +118,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Make sure the SQLite databases are in a persistent location, we're already mapping # the postgresql folder so let's just use that for simplicity -RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml +RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/15\/main\/%g" dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ @@ -402,7 +401,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI { Type: mount.TypeVolume, Source: volumeName, - Target: "/var/lib/postgresql/11/main", + Target: "/var/lib/postgresql/15/main", }, }, }, nil, nil, "dendrite_upgrade_test_"+branchName) @@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con } defer response.Close() - data, err := ioutil.ReadAll(response.Reader) + data, err := io.ReadAll(response.Reader) if err != nil { return err } @@ -557,8 +556,8 @@ func cleanup(dockerClient *client.Client) { }) for _, c := range containers { log.Printf("Removing container: %v %v\n", c.ID, c.Names) - s := time.Second - _ = dockerClient.ContainerStop(context.Background(), c.ID, &s) + timeout := 1 + _ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout}) _ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{ Force: true, }) @@ -592,7 +591,7 @@ func main() { branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions) // make a shared postgres volume - volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{ + volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{ Name: "dendrite_upgrade_test", Labels: map[string]string{ dendriteUpgradeTestLabel: "yes", diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go index 03438bd4d5..692ab34ef7 100644 --- a/cmd/dendrite-upgrade-tests/tests.go +++ b/cmd/dendrite-upgrade-tests/tests.go @@ -9,6 +9,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) const userPassword = "this_is_a_long_password" @@ -56,7 +57,7 @@ func runTests(baseURL string, v *semver.Version) error { // create DM room, join it and exchange messages createRoomResp, err := users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{ - Preset: "trusted_private_chat", + Preset: spec.PresetTrustedPrivateChat, Invite: []string{users[1].userID}, IsDirect: true, }) @@ -98,7 +99,7 @@ func runTests(baseURL string, v *semver.Version) error { publicRoomID := "" createRoomResp, err = users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{ RoomAliasName: "global", - Preset: "public_chat", + Preset: spec.PresetPublicChat, }) if err != nil { // this is okay and expected if the room already exists and the aliases clash // try to join it diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index 66eb88f875..f3140b4e22 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice" @@ -156,13 +157,14 @@ func main() { keyRing := fsAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) + + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) rsAPI.SetUserAPI(userAPI) @@ -187,6 +189,16 @@ func main() { } } + upCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "dendrite", + Name: "up", + ConstLabels: map[string]string{ + "version": internal.VersionString(), + }, + }) + upCounter.Add(1) + prometheus.MustRegister(upCounter) + // Expose the matrix APIs directly rather than putting them under a /api path. go func() { basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpAddr, nil, nil) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index cb57ed78fb..2379ce2bb9 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -74,7 +74,7 @@ func main() { // don't hit matrix.org when running tests!!! cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media")) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} + cfg.MSCs.MSCs = []string{"msc2836", "msc2444", "msc2753"} cfg.Logging[0].Level = "trace" cfg.Logging[0].Type = "std" cfg.UserAPI.BCryptCost = bcrypt.MinCost diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 3a4255bae0..5be4490979 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -18,6 +18,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) // This is a utility for inspecting state snapshots and running state resolution @@ -32,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve var filterType = flag.String("filtertype", "", "the event types to filter on") var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots") +// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms +// to avoid having to "start" a full roomserver API. +type dummyQuerier struct{} + +func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + s := spec.SenderIDFromUserID(userID) + return &s, nil +} + +func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return senderID.ToUserID(), nil +} + // nolint:gocyclo func main() { ctx := context.Background() @@ -53,22 +67,31 @@ func main() { } } - fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") - processCtx := process.NewProcessContext() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + dbOpts := cfg.RoomServer.Database + if dbOpts.ConnectionString == "" { + dbOpts = cfg.Global.DatabaseOptions + } + + fmt.Println("Opening database") roomserverDB, err := storage.Open( - processCtx.Context(), cm, &cfg.RoomServer.Database, - caching.NewRistrettoCache(128*1024*1024, time.Hour, true), + processCtx.Context(), cm, &dbOpts, + caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics), ) if err != nil { panic(err) } + rsAPI := dummyQuerier{} + roomInfo := &types.RoomInfo{ RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion), } - stateres := state.NewStateResolution(roomserverDB, roomInfo) + stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI) + + fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") if *difference { if len(snapshotNIDs) != 2 { @@ -182,7 +205,9 @@ func main() { fmt.Println("Resolving state") var resolved Events resolved, err = gomatrixserverlib.ResolveConflicts( - gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, + gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, ) if err != nil { panic(err) diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 6b3ea74f22..7affc25994 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -69,10 +69,13 @@ global: # e.g. localhost:443 well_known_server_name: "" - # The server name to delegate client-server communications to, with optional port - # e.g. localhost:443 + # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" + # The server name to delegate sliding sync communications to, with optional port. + # Requires `well_known_client_name` to also be configured. + well_known_sliding_sync_proxy: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: @@ -277,7 +280,6 @@ media_api: mscs: mscs: # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) # Configuration for the Sync API. sync_api: diff --git a/docs/FAQ.md b/docs/FAQ.md index 2000207265..570ba677e9 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](https://matrix-org.github.io/dendrite/development/contributing). +the development efforts through [contributing](../development/contributing). ## Is there a migration path from Synapse to Dendrite? @@ -64,16 +64,18 @@ Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially suppo ## Does Dendrite support Space Summaries? -Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration: +Yes + +## Does Dendrite support Threads? + +Yes, to enable them [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. ``` mscs: mscs: - - msc2946 + - msc2836 ``` -Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. - Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject. ## Does Dendrite support push notifications? @@ -103,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom 0.1) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index ccfc58107a..8e72da9715 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -6,8 +6,8 @@ or alternatively, in the [installation](installation/) folder: 1. [Planning your deployment](installation/1_planning.md) 2. [Setting up the domain](installation/2_domainname.md) -3. [Preparing database storage](installation/3_database.md) -4. [Generating signing keys](installation/4_signingkey.md) -5. [Installing as a monolith](installation/5_install_monolith.md) -6. [Populate the configuration](installation/7_configuration.md) -7. [Starting the monolith](installation/8_starting_monolith.md) +3. [Installing Dendrite](installation/manual/1_build.md) +4. [Preparing database storage](installation/manual/2_database.md) +5. [Populate the configuration](installation/manual/3_configuration.md) +6. [Generating signing keys](installation/manual/4_signingkey.md) +7. [Starting Dendrite](installation/manual/5_starting_dendrite.md) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index 24eba666dc..cbdccd18b7 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -11,10 +11,9 @@ User accounts can be created on a Dendrite instance in a number of ways. ## From the command line -The `create-account` tool is built in the `bin` folder when building Dendrite with -the `build.sh` script. +The `create-account` tool is built in the `bin` folder when [building](../installation/build) Dendrite. -It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires +It uses the `dendrite.yaml` configuration file to connect to a **running** Dendrite instance and requires shared secret registration to be enabled as explained below. An example of using `create-account` to create a **normal account**: diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index b11aeb1a60..40d02622bd 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -1,6 +1,7 @@ --- title: Supported admin APIs parent: Administration +nav_order: 4 permalink: /administration/adminapi --- @@ -49,13 +50,17 @@ the room IDs of all affected rooms. ## POST `/_dendrite/admin/resetPassword/{userID}` -Reset the password of a local user. +Reset the password of a local user. + +**If `logout_devices` is set to `true`, all `access_tokens` will be invalidated, resulting +in the potential loss of encrypted messages** Request body format: -``` +```json { - "password": "new_password_here" + "password": "new_password_here", + "logout_devices": false } ``` @@ -68,11 +73,14 @@ Indexing is done in the background, the server logs every 1000 events (or below) This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a federated server. An empty JSON body will be returned on success, updating all locally stored user devices/keys. This can be used to possibly resolve E2EE issues, where the remote user can't decrypt messages. +## POST `/_dendrite/admin/purgeRoom/{roomID}` + +This endpoint instructs Dendrite to remove the given room from its database. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room. ## POST `/_synapse/admin/v1/send_server_notice` Request body format: -``` +```json { "user_id": "@target_user:server_name", "content": { @@ -85,7 +93,7 @@ Request body format: Send a server notice to a specific user. See the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#server-notices) for additional details on server notice behaviour. If successfully sent, the API will return the following response: -``` +```json { "event_id": "" } diff --git a/docs/installation/11_optimisation.md b/docs/administration/5_optimisation.md similarity index 88% rename from docs/installation/11_optimisation.md rename to docs/administration/5_optimisation.md index 686ec2eb9b..57b7924d36 100644 --- a/docs/installation/11_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -1,9 +1,9 @@ --- title: Optimise your installation -parent: Installation +parent: Administration has_toc: true -nav_order: 11 -permalink: /installation/start/optimisation +nav_order: 5 +permalink: /administration/optimisation --- # Optimise your installation @@ -36,11 +36,6 @@ connections it will open to the database. **If you are using the `global` database pool** then you only need to configure the `max_open_conns` setting once in the `global` section. -**If you are defining a `database` config per component** then you will need to ensure that -the **sum total** of all configured `max_open_conns` to a given database server do not exceed -the connection limit. If you configure a total that adds up to more connections than are available -then this will cause database queries to fail. - You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate additional connections, in which case you should also update the `max_open_conns` in your Dendrite configuration accordingly. However be aware that this is only advisable on particularly @@ -100,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config ## Time synchronisation Matrix relies heavily on TLS which requires the system time to be correct. If the clock -drifts then you may find that federation no works reliably (or at all) and clients may +drifts then you may find that federation will not work reliably (or at all) and clients may struggle to connect to your Dendrite server. Ensure that the time is synchronised on your system by enabling NTP sync. diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/6_troubleshooting.md similarity index 88% rename from docs/administration/5_troubleshooting.md rename to docs/administration/6_troubleshooting.md index 8ba510ef61..5f11f99316 100644 --- a/docs/administration/5_troubleshooting.md +++ b/docs/administration/6_troubleshooting.md @@ -1,6 +1,7 @@ --- title: Troubleshooting parent: Administration +nav_order: 6 permalink: /administration/troubleshooting --- @@ -18,7 +19,7 @@ be clues in the logs. You can increase this log level to the more verbose `debug` level if necessary by adding this to the config and restarting Dendrite: -``` +```yaml logging: - type: std level: debug @@ -56,12 +57,7 @@ number of database connections does not exceed the maximum allowed by PostgreSQL Open your `postgresql.conf` configuration file and check the value of `max_connections` (which is typically `100` by default). Then open your `dendrite.yaml` configuration file -and ensure that: - -1. If you are using the `global.database` section, that `max_open_conns` does not exceed - that number; -2. If you are **not** using the `global.database` section, that the sum total of all - `max_open_conns` across all `database` blocks does not exceed that number. +and ensure that in the `global.database` section, `max_open_conns` does not exceed that number. ## 5. File descriptors @@ -77,7 +73,7 @@ If there aren't, you will see a log lines like this: level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues" ``` -Follow the [Optimisation](../installation/11_optimisation.md) instructions to correct the +Follow the [Optimisation](5_optimisation.md) instructions to correct the available number of file descriptors. ## 6. STUN/TURN Server tester diff --git a/docs/caddy/monolith/Caddyfile b/docs/caddy/Caddyfile similarity index 100% rename from docs/caddy/monolith/Caddyfile rename to docs/caddy/Caddyfile diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile deleted file mode 100644 index c2d81b49bf..0000000000 --- a/docs/caddy/polylith/Caddyfile +++ /dev/null @@ -1,85 +0,0 @@ -# Sample Caddyfile for using Caddy in front of Dendrite - -# - -# Customize email address and domain names - -# Optional settings commented out - -# - -# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST - -# Documentation: - -# - -# Bonus tip: If your IP address changes, use Caddy's - -# dynamic DNS plugin to update your DNS records to - -# point to your new IP automatically - -# - -# - -# Global options block - -{ - # In case there is a problem with your certificates. - # email example@example.com - - # Turn off the admin endpoint if you don't need graceful config - # changes and/or are running untrusted code on your machine. - # admin off - - # Enable this if your clients don't send ServerName in TLS handshakes. - # default_sni example.com - - # Enable debug mode for verbose logging. - # debug - - # Use Let's Encrypt's staging endpoint for testing. - # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory - - # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something - # else, enable these and put the alternate port numbers here. - # http_port 8080 - # https_port 8443 -} - -# The server name of your matrix homeserver. This example shows - -# "well-known delegation" from the registered domain to a subdomain - -# which is only needed if your server_name doesn't match your Matrix - -# homeserver URL (i.e. you can show users a vanity domain that looks - -# nice and is easy to remember but still have your Matrix server on - -# its own subdomain or hosted service) - -example.com { - header /.well-known/matrix/*Content-Type application/json - header /.well-known/matrix/* Access-Control-Allow-Origin * - respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` - respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` -} - -# The actual domain name whereby your Matrix server is accessed - -matrix.example.com { - # Change the end of each reverse_proxy line to the correct - # address for your various services. - @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ - } - reverse_proxy @sync_api sync_api:8073 - - reverse_proxy /_matrix/client* client_api:8071 - reverse_proxy /_matrix/federation* federation_api:8071 - reverse_proxy /_matrix/key* federation_api:8071 - reverse_proxy /_matrix/media* media_api:8071 -} diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index 2aec4c3631..71e7516a29 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -1,6 +1,7 @@ --- title: Contributing parent: Development +nav_order: 1 permalink: /development/contributing --- diff --git a/docs/development/PROFILING.md b/docs/development/PROFILING.md index 57c37a9006..dc4eca7b75 100644 --- a/docs/development/PROFILING.md +++ b/docs/development/PROFILING.md @@ -1,6 +1,7 @@ --- title: Profiling parent: Development +nav_order: 4 permalink: /development/profiling --- diff --git a/docs/development/coverage.md b/docs/development/coverage.md index c4a8a11743..1b15f71a2d 100644 --- a/docs/development/coverage.md +++ b/docs/development/coverage.md @@ -1,78 +1,130 @@ --- title: Coverage parent: Development +nav_order: 3 permalink: /development/coverage --- -To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary: -```patch -diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm -index 8f0e209c..ad057e52 100644 ---- a/lib/SyTest/Homeserver/Dendrite.pm -+++ b/lib/SyTest/Homeserver/Dendrite.pm -@@ -337,7 +337,7 @@ sub _start_monolith - - $output->diag( "Starting monolith server" ); - my @command = ( -- $self->{bindir} . '/dendrite', -+ $self->{bindir} . '/dendrite', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL", - '--config', $self->{paths}{config}, - '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port, - '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port, -diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh -index f009332b..7ea79869 100755 ---- a/scripts/dendrite_sytest.sh -+++ b/scripts/dendrite_sytest.sh -@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin - echo >&2 "--- Building dendrite from source" - cd /src - mkdir -p $GOBIN --go install -v ./cmd/dendrite -+# go install -v ./cmd/dendrite -+go test -c -cover -covermode=atomic -o $GOBIN/dendrite -coverpkg "github.com/matrix-org/..." ./cmd/dendrite - go install -v ./cmd/generate-keys - cd - - ``` +## Running unit tests with coverage enabled + +Running unit tests with coverage enabled can be done with the following commands, this will generate a `integrationcover.log` +```bash +go test -covermode=atomic -coverpkg=./... -coverprofile=integrationcover.log $(go list ./... | grep -v '/cmd/') +go tool cover -func=integrationcover.log +``` + +## Running Sytest with coverage enabled + +To run Sytest with coverage enabled: + +```bash +docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" \ + -v "/Users/kegan/github/dendrite:/src" -v "$(pwd)/sytest_logs:/logs" \ + -v "/Users/kegan/go/:/gopath" -e "POSTGRES=1" \ + -e "COVER=1" \ + matrixdotorg/sytest-dendrite:latest + +# to get a more accurate coverage you may also need to run Sytest using SQLite as the database: +docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" \ + -v "/Users/kegan/github/dendrite:/src" -v "$(pwd)/sytest_logs:/logs" \ + -v "/Users/kegan/go/:/gopath" \ + -e "COVER=1" \ + matrixdotorg/sytest-dendrite:latest +``` + +This will generate a folder `covdatafiles` in each server's directory, e.g `server-0/covdatafiles`. To parse them, +ensure your working directory is under the Dendrite repository then run: - Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it, - ensure your working directory is under the Dendrite repository then run: ```bash - go tool cover -func=/path/to/server-0/integrationcover.log + go tool covdata func -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" ``` which will produce an output like: ``` ... - github.com/matrix-org/util/json.go:83: NewJSONRequestHandler 100.0% -github.com/matrix-org/util/json.go:90: Protect 57.1% -github.com/matrix-org/util/json.go:110: RequestWithLogging 100.0% -github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0% -github.com/matrix-org/util/json.go:151: respond 61.5% -github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0% -github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0% -github.com/matrix-org/util/json.go:202: RandomString 100.0% -github.com/matrix-org/util/json.go:210: init 100.0% -github.com/matrix-org/util/unique.go:13: Unique 91.7% -github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0% -github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0% -total: (statements) 53.7% +github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0% +github.com/matrix-org/util/json.go:151: respond 84.6% +github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0% +github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0% +github.com/matrix-org/util/json.go:202: RandomString 100.0% +github.com/matrix-org/util/json.go:210: init 100.0% +github.com/matrix-org/util/unique.go:13: Unique 91.7% +github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0% +github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0% +total (statements) 64.0% +``` +(after running Sytest for Postgres _and_ SQLite) + +The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in different configurations, +which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, you'll need run Sytest for Postgres and SQLite (see commands above). +Additional processing is required also to remove packages which will never be tested and extension MSCs: + +```bash +# If you executed both commands from above, you can get the total coverage using the following commands +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov + +# If you only executed the one for Postgres: +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|sqlite|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov + +# If you only executed the one for SQLite: +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|postgres|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov ``` -The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations, -which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, additional processing is required -to remove packages which will never be tested and extension MSCs: + +## Getting coverage from Complement + +Getting the coverage for Complement runs is a bit more involved. + +First you'll need a docker image compatible with Complement, one can be built using +```bash +docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile . +``` +from within the Dendrite repository. + +Clone complement to a directory of your liking: ```bash -# These commands are all similar but change which package paths are _removed_ from the output. +git clone https://github.com/matrix-org/complement.git +cd complement +``` -# For Postgres -go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt +Next we'll need a script to execute after a test finishes, create a new file `posttest.sh`, make the file executable (`chmod +x posttest.sh`) +and add the following content: +```bash +#!/bin/bash -# For SQLite -go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt +mkdir -p /tmp/Complement/logs/$2/$1/ +docker cp $1:/tmp/covdatafiles/. /tmp/Complement/logs/$2/$1/ ``` +This will copy the `covdatafiles` files from each container to something like +`/tmp/Complement/logs/TestLogin/94f9c428de95779d2b62a3ccd8eab9d5ddcf65cc259a40ece06bdc61687ffed3/`. (`$1` is the containerID, `$2` the test name) -A total value can then be calculated using: +Now that we have set up everything we need, we can finally execute Complement: ```bash -cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}' +COMPLEMENT_BASE_IMAGE=complement-dendrite \ +COMPLEMENT_SHARE_ENV_PREFIX=COMPLEMENT_DENDRITE_ \ +COMPLEMENT_DENDRITE_COVER=1 \ +COMPLEMENT_POST_TEST_SCRIPT=$(pwd)/posttest.sh \ + go test -tags dendrite_blacklist ./tests/... -count=1 -v -timeout=30m -failfast=false ``` +Once this is done, you can copy the resulting `covdatafiles` files to your Dendrite repository for the next step. +```bash +cp -pr /tmp/Complement/logs PathToYourDendriteRepository +``` -We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report. \ No newline at end of file +You can also run the following to get the coverage for Complement runs alone: +```bash +go tool covdata func -i="$(find /tmp/Complement -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" +``` + +## Combining the results of (almost) all runs + +Now that we have all our `covdatafiles` files within the Dendrite repository, you can now execute the following command, to get the coverage +overall (excluding unit tests): +```bash +go tool covdata func -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" +``` \ No newline at end of file diff --git a/docs/development/sytest.md b/docs/development/sytest.md index 4fae2ea3d1..2f681f3e5b 100644 --- a/docs/development/sytest.md +++ b/docs/development/sytest.md @@ -1,6 +1,7 @@ --- title: SyTest parent: Development +nav_order: 2 permalink: /development/sytest --- @@ -23,7 +24,7 @@ After running the tests, a script will print the tests you need to add to You should proceed after you see no build problems for dendrite after running: ```sh -./build.sh +go build -o bin/ ./cmd/... ``` If you are fixing an issue marked with @@ -61,6 +62,8 @@ When debugging, the following Docker `run` options may also be useful: * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs. * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to server logs. +* `-e "COVER=1"`: Run Sytest with an instrumented binary, producing a Go coverage file per server. +* `-e "RACE_DETECTION=1"`: Build the binaries with the `-race` flag (Note: This will significantly slow down test runs) The docker command also supports a single positional argument for the test file to run, so you can run a single `.pl` file rather than the whole test suite. For example: @@ -71,68 +74,3 @@ docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" -v "/Users/kegan/go/:/gopath" -e "POSTGRES=1" -e "DENDRITE_TRACE_HTTP=1" matrixdotorg/sytest-dendrite:latest tests/50federation/40devicelists.pl ``` - -### Manually Setting up SyTest - -**We advise AGAINST using manual SyTest setups.** - -If you don't want to use the Docker image, you can also run SyTest by hand. Make -sure you have Perl 5 or above, and get SyTest with: - -(Note that this guide assumes your SyTest checkout is next to your -`dendrite` checkout.) - -```sh -git clone -b develop https://github.com/matrix-org/sytest -cd sytest -./install-deps.pl -``` - -Set up the database: - -```sh -sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'" -sudo -u postgres psql -c "ALTER USER dendrite CREATEDB" -for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done -mkdir -p "server-0" -cat > "server-0/database.yaml" << EOF -args: - user: dendrite - password: itsasecret - database: dendrite0 - host: 127.0.0.1 - sslmode: disable -type: pg -EOF -mkdir -p "server-1" -cat > "server-1/database.yaml" << EOF -args: - user: dendrite - password: itsasecret - database: dendrite1 - host: 127.0.0.1 - sslmode: disable -type: pg -EOF -``` - -Run the tests: - -```sh -POSTGRES=1 ./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/sytest-whitelist -O tap --all | tee results.tap -``` - -where `tee` lets you see the results while they're being piped to the file, and -`POSTGRES=1` enables testing with PostgeSQL. If the `POSTGRES` environment -variable is not set or is set to 0, SyTest will fall back to SQLite 3. For more -flags and options, see . - -Once the tests are complete, run the helper script to see if you need to add -any newly passing test names to `sytest-whitelist` in the project's root -directory: - -```sh -../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/sytest-whitelist ../dendrite/sytest-blacklist -``` - -If the script prints nothing/exits with 0, then you're good to go. diff --git a/docs/development/tracing/opentracing.md b/docs/development/tracing/opentracing.md deleted file mode 100644 index 8528c2ba3f..0000000000 --- a/docs/development/tracing/opentracing.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: OpenTracing -has_children: true -parent: Development -permalink: /development/opentracing ---- - -# OpenTracing - -Dendrite extensively uses the [opentracing.io](http://opentracing.io) framework -to trace work across the different logical components. - -At its most basic opentracing tracks "spans" of work; recording start and end -times as well as any parent span that caused the piece of work. - -A typical example would be a new span being created on an incoming request that -finishes when the response is sent. When the code needs to hit out to a -different component a new span is created with the initial span as its parent. -This would end up looking roughly like: - -``` -Received request Sent response - |<───────────────────────────────────────>| - |<────────────────────>| - RPC call RPC call returns -``` - -This is useful to see where the time is being spent processing a request on a -component. However, opentracing allows tracking of spans across components. This -makes it possible to see exactly what work goes into processing a request: - -``` -Component 1 |<─────────────────── HTTP ────────────────────>| - |<──────────────── RPC ─────────────────>| -Component 2 |<─ SQL ─>| |<── RPC ───>| -Component 3 |<─ SQL ─>| -``` - -This is achieved by serializing span information during all communication -between components. For HTTP requests, this is achieved by the sender -serializing the span into a HTTP header, and the receiver deserializing the span -on receipt. (Generally a new span is then immediately created with the -deserialized span as the parent). - -A collection of spans that are related is called a trace. - -Spans are passed through the code via contexts, rather than manually. It is -therefore important that all spans that are created are immediately added to the -current context. Thankfully the opentracing library gives helper functions for -doing this: - -```golang -span, ctx := opentracing.StartSpanFromContext(ctx, spanName) -defer span.Finish() -``` - -This will create a new span, adding any span already in `ctx` as a parent to the -new span. - -Adding Information ------------------- - -Opentracing allows adding information to a trace via three mechanisms: - -- "tags" ─ A span can be tagged with a key/value pair. This is typically - information that relates to the span, e.g. for spans created for incoming HTTP - requests could include the request path and response codes as tags, spans for - SQL could include the query being executed. -- "logs" ─ Key/value pairs can be looged at a particular instance in a trace. - This can be useful to log e.g. any errors that happen. -- "baggage" ─ Arbitrary key/value pairs can be added to a span to which all - child spans have access. Baggage isn't saved and so isn't available when - inspecting the traces, but can be used to add context to logs or tags in child - spans. - -See -[specification.md](https://github.com/opentracing/specification/blob/master/specification.md) -for some of the common tags and log fields used. - -Span Relationships ------------------- - -Spans can be related to each other. The most common relation is `childOf`, which -indicates the child span somehow depends on the parent span ─ typically the -parent span cannot complete until all child spans are completed. - -A second relation type is `followsFrom`, where the parent has no dependence on -the child span. This usually indicates some sort of fire and forget behaviour, -e.g. adding a message to a pipeline or inserting into a kafka topic. - -Jaeger ------- - -Opentracing is just a framework. We use -[jaeger](https://github.com/jaegertracing/jaeger) as the actual implementation. - -Jaeger is responsible for recording, sending and saving traces, as well as -giving a UI for viewing and interacting with traces. - -To enable jaeger a `Tracer` object must be instansiated from the config (as well -as having a jaeger server running somewhere, usually locally). A `Tracer` does -several things: - -- Decides which traces to save and send to the server. There are multiple - schemes for doing this, with a simple example being to save a certain fraction - of traces. -- Communicating with the jaeger backend. If not explicitly specified uses the - default port on localhost. -- Associates a service name to all spans created by the tracer. This service - name equates to a logical component, e.g. spans created by clientapi will have - a different service name than ones created by the syncapi. Database access - will also typically use a different service name. - - This means that there is a tracer per service name/component. diff --git a/docs/development/tracing/setup.md b/docs/development/tracing/setup.md deleted file mode 100644 index cef1089e46..0000000000 --- a/docs/development/tracing/setup.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Setup -parent: OpenTracing -grand_parent: Development -permalink: /development/opentracing/setup ---- - -# OpenTracing Setup - -Dendrite uses [Jaeger](https://www.jaegertracing.io/) for tracing between microservices. -Tracing shows the nesting of logical spans which provides visibility on how the microservices interact. -This document explains how to set up Jaeger locally on a single machine. - -## Set up the Jaeger backend - -The [easiest way](https://www.jaegertracing.io/docs/1.18/getting-started/) is to use the all-in-one Docker image: - -``` -$ docker run -d --name jaeger \ - -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ - -p 5775:5775/udp \ - -p 6831:6831/udp \ - -p 6832:6832/udp \ - -p 5778:5778 \ - -p 16686:16686 \ - -p 14268:14268 \ - -p 14250:14250 \ - -p 9411:9411 \ - jaegertracing/all-in-one:1.18 -``` - -## Configuring Dendrite to talk to Jaeger - -Modify your config to look like: (this will send every single span to Jaeger which will be slow on large instances, but for local testing it's fine) - -``` -tracing: - enabled: true - jaeger: - serviceName: "dendrite" - disabled: false - rpc_metrics: true - tags: [] - sampler: - type: const - param: 1 -``` - -then run the monolith server: - -``` -./dendrite --tls-cert server.crt --tls-key server.key --config dendrite.yaml -``` - -## Checking traces - -Visit to see traces under `DendriteMonolith`. diff --git a/docs/hiawatha/monolith-sample.conf b/docs/hiawatha/dendrite-sample.conf similarity index 100% rename from docs/hiawatha/monolith-sample.conf rename to docs/hiawatha/dendrite-sample.conf diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf deleted file mode 100644 index eb1dd4f9ae..0000000000 --- a/docs/hiawatha/polylith-sample.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Depending on which port is used for federation (.well-known/matrix/server or SRV record), -# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port -# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for -# each ip address, e.g. if you use both ipv4 and ipv6 addresses). - -Binding { - Port = FEDPORT - Interface = IPV4 - TLScertFile = /path/to/fullchainandprivkey.pem -} - - -VirtualHost { - ... - # route requests to: - # /_matrix/client/.*/sync - # /_matrix/client/.*/user/{userId}/filter - # /_matrix/client/.*/user/{userId}/filter/{filterID} - # /_matrix/client/.*/keys/changes - # /_matrix/client/.*/rooms/{roomId}/messages - # /_matrix/client/.*/rooms/{roomId}/context/{eventID} - # /_matrix/client/.*/rooms/{roomId}/event/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} - # /_matrix/client/.*/rooms/{roomId}/members - # /_matrix/client/.*/rooms/{roomId}/joined_members - # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 - ReverseProxy = /_matrix/client http://localhost:8071 600 - ReverseProxy = /_matrix/federation http://localhost:8072 600 - ReverseProxy = /_matrix/key http://localhost:8072 600 - ReverseProxy = /_matrix/media http://localhost:8074 600 - ... -} diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 36d90abdaa..37ca5702a3 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -7,23 +7,13 @@ permalink: /installation/planning # Planning your installation -## Modes - -Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol. -Users can run Dendrite in one of two modes which dictate how these components are executed and communicate. - -* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS - server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the - best balance between performance and resource usage for low-to-mid volume deployments. - - -## Databases +## Database Dendrite can run with either a PostgreSQL or a SQLite backend. There are considerable tradeoffs to consider: * **PostgreSQL**: Needs to run separately to Dendrite, needs to be installed and configured separately - and and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL + and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL has much better write concurrency which will allow Dendrite to process more tasks in parallel. This will be necessary for federated deployments to perform adequately. @@ -69,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.18 or later. We do not support building +At this time, Dendrite supports being built with Go 1.20 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. @@ -80,18 +70,17 @@ If using the PostgreSQL database engine, you should install PostgreSQL 12 or lat ### NATS Server Dendrite comes with a built-in [NATS Server](https://github.com/nats-io/nats-server) and -therefore does not need this to be manually installed. If you are planning a monolith installation, you -do not need to do anything. +therefore does not need this to be manually installed. ### Reverse proxy A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or -[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring those is not covered in this documentation, although sample configurations +[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring this is not covered in this documentation, although sample configurations for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided. ### Windows -Finally, if you want to build Dendrite on Windows, you will need need `gcc` in the path. The best +Finally, if you want to build Dendrite on Windows, you will need `gcc` in the path. The best way to achieve this is by installing and building Dendrite under [MinGW-w64](https://www.mingw-w64.org/). diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index 545a2daf6e..d86a664cb1 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -20,7 +20,7 @@ Matrix servers usually discover each other when federating using the following m well-known file to connect to the remote homeserver; 2. If a DNS SRV delegation exists on `example.com`, use the IP address and port from the DNS SRV record to connect to the remote homeserver; -3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote +3. If neither well-known nor DNS SRV delegation are configured, attempt to connect to the remote homeserver by connecting to `example.com` port TCP/8448 using HTTPS. The exact details of how server name resolution works can be found in diff --git a/docs/installation/5_install_monolith.md b/docs/installation/5_install_monolith.md deleted file mode 100644 index 901975a654..0000000000 --- a/docs/installation/5_install_monolith.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Installing as a monolith -parent: Installation -has_toc: true -nav_order: 5 -permalink: /installation/install/monolith ---- - -# Installing as a monolith - -You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: - -```sh -go install ./cmd/dendrite -``` - -Alternatively, you can specify a custom path for the binary to be written to using `go build`: - -```sh -go build -o /usr/local/bin/ ./cmd/dendrite -``` diff --git a/docs/installation/9_starting_monolith.md b/docs/installation/9_starting_monolith.md deleted file mode 100644 index d7e8c0b8ba..0000000000 --- a/docs/installation/9_starting_monolith.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Starting the monolith -parent: Installation -has_toc: true -nav_order: 9 -permalink: /installation/start/monolith ---- - -# Starting the monolith - -Once you have completed all of the preparation and installation steps, -you can start your Dendrite monolith deployment by starting `dendrite`: - -```bash -./dendrite -config /path/to/dendrite.yaml -``` - -By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses -or ports that Dendrite listens on, you can use the `-http-bind-address` and -`-https-bind-address` command line arguments: - -```bash -./dendrite -config /path/to/dendrite.yaml \ - -http-bind-address 1.2.3.4:12345 \ - -https-bind-address 1.2.3.4:54321 -``` - -## Running under systemd - -A common deployment pattern is to run the monolith under systemd. For this, you -will need to create a service unit file. An example service unit file is available -in the [GitHub repository](https://github.com/matrix-org/dendrite/blob/main/docs/systemd/monolith-example.service). - -Once you have installed the service unit, you can notify systemd, enable and start -the service: - -```bash -systemctl daemon-reload -systemctl enable dendrite -systemctl start dendrite -journalctl -fu dendrite -``` diff --git a/docs/installation/docker.md b/docs/installation/docker.md new file mode 100644 index 0000000000..1ecc7c6ee9 --- /dev/null +++ b/docs/installation/docker.md @@ -0,0 +1,11 @@ +--- +title: Docker +parent: Installation +has_children: true +nav_order: 4 +permalink: /docker +--- + +# Installation using Docker + +This section contains documentation how to install Dendrite using Docker diff --git a/docs/installation/docker/1_docker.md b/docs/installation/docker/1_docker.md new file mode 100644 index 0000000000..1fe7926366 --- /dev/null +++ b/docs/installation/docker/1_docker.md @@ -0,0 +1,57 @@ +--- +title: Installation +parent: Docker +grand_parent: Installation +has_toc: true +nav_order: 1 +permalink: /installation/docker/install +--- + +# Installing Dendrite using Docker Compose + +Dendrite provides an [example](https://github.com/matrix-org/dendrite/blob/main/build/docker/docker-compose.yml) +Docker compose file, which needs some preparation to start successfully. +Please note that this compose file only has Postgres as a dependency, and you need to configure +a [reverse proxy](../planning#reverse-proxy). + +## Preparations + +### Generate a private key + +First we'll generate private key, which is used to sign events, the following will create one in `./config`: + +```bash +mkdir -p ./config +docker run --rm --entrypoint="/usr/bin/generate-keys" \ + -v $(pwd)/config:/mnt \ + matrixdotorg/dendrite-monolith:latest \ + -private-key /mnt/matrix_key.pem +``` +(**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key) + +### Generate a config + +Similar to the command above, we can generate a config to be used, which will use the correct paths +as specified in the example docker-compose file. Change `server` to your domain and `db` according to your changes +to the docker-compose file (`services.postgres.environment` values): + +```bash +mkdir -p ./config +docker run --rm --entrypoint="/bin/sh" \ + -v $(pwd)/config:/mnt \ + matrixdotorg/dendrite-monolith:latest \ + -c "/usr/bin/generate-config \ + -dir /var/dendrite/ \ + -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \ + -server YourDomainHere > /mnt/dendrite.yaml" +``` + +You can then change `config/dendrite.yaml` to your liking. + +## Starting Dendrite + +Once you're done changing the config, you can now start up Dendrite with + +```bash +docker-compose -f docker-compose.yml up +``` diff --git a/docs/installation/helm.md b/docs/installation/helm.md new file mode 100644 index 0000000000..dd20e0261b --- /dev/null +++ b/docs/installation/helm.md @@ -0,0 +1,11 @@ +--- +title: Helm +parent: Installation +has_children: true +nav_order: 3 +permalink: /helm +--- + +# Helm + +This section contains documentation how to use [Helm](https://helm.sh/) to install Dendrite on a [Kubernetes](https://kubernetes.io/) cluster. diff --git a/docs/installation/helm/1_helm.md b/docs/installation/helm/1_helm.md new file mode 100644 index 0000000000..00fe4fdcaf --- /dev/null +++ b/docs/installation/helm/1_helm.md @@ -0,0 +1,58 @@ +--- +title: Installation +parent: Helm +grand_parent: Installation +has_toc: true +nav_order: 1 +permalink: /installation/helm/install +--- + +# Installing Dendrite using Helm + +To install Dendrite using the Helm chart, you first have to add the repository using the following commands: + +```bash +helm repo add dendrite https://matrix-org.github.io/dendrite/ +helm repo update +``` + +Next you'll need to create a `values.yaml` file and configure it to your liking. All possible values can be found +[here](https://github.com/matrix-org/dendrite/blob/main/helm/dendrite/values.yaml), but at least you need to configure +a `server_name`, otherwise the chart will complain about it: + +```yaml +dendrite_config: + global: + server_name: "localhost" +``` + +If you are going to use an existing Postgres database, you'll also need to configure this connection: + +```yaml +dendrite_config: + global: + database: + connection_string: "postgresql://PostgresUser:PostgresPassword@PostgresHostName/DendriteDatabaseName" + max_open_conns: 90 + max_idle_conns: 5 + conn_max_lifetime: -1 +``` + +## Installing with PostgreSQL + +The chart comes with a dependency on Postgres, which can be installed alongside Dendrite, this needs to be enabled in +the `values.yaml`: + +```yaml +postgresql: + enabled: true # this installs Postgres + primary: + persistence: + size: 1Gi # defines the size for $PGDATA + +dendrite_config: + global: + server_name: "localhost" +``` + +Using this option, the `database.connection_string` will be set for you automatically. \ No newline at end of file diff --git a/docs/installation/manual.md b/docs/installation/manual.md new file mode 100644 index 0000000000..3ab1fd6275 --- /dev/null +++ b/docs/installation/manual.md @@ -0,0 +1,11 @@ +--- +title: Manual +parent: Installation +has_children: true +nav_order: 5 +permalink: /manual +--- + +# Manual Installation + +This section contains documentation how to manually install Dendrite diff --git a/docs/installation/3_build.md b/docs/installation/manual/1_build.md similarity index 53% rename from docs/installation/3_build.md rename to docs/installation/manual/1_build.md index 824c81d37f..73a6268820 100644 --- a/docs/installation/3_build.md +++ b/docs/installation/manual/1_build.md @@ -1,31 +1,26 @@ --- -title: Building Dendrite -parent: Installation +title: Building/Installing Dendrite +parent: Manual +grand_parent: Installation has_toc: true -nav_order: 3 -permalink: /installation/build +nav_order: 1 +permalink: /installation/manual/build --- # Build all Dendrite commands Dendrite has numerous utility commands in addition to the actual server binaries. -Build them all from the root of the source repo with `build.sh` (Linux/Mac): +Build them all from the root of the source repo with: ```sh -./build.sh -``` - -or `build.cmd` (Windows): - -```powershell -build.cmd +go build -o bin/ ./cmd/... ``` The resulting binaries will be placed in the `bin` subfolder. -# Installing as a monolith +# Installing Dendrite -You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: +You can install the Dendrite binary into `$GOPATH/bin` by using `go install`: ```sh go install ./cmd/dendrite diff --git a/docs/installation/4_database.md b/docs/installation/manual/2_database.md similarity index 57% rename from docs/installation/4_database.md rename to docs/installation/manual/2_database.md index d64ee6615d..1be602c663 100644 --- a/docs/installation/4_database.md +++ b/docs/installation/manual/2_database.md @@ -1,8 +1,10 @@ --- title: Preparing database storage parent: Installation -nav_order: 3 -permalink: /installation/database +nav_order: 2 +parent: Manual +grand_parent: Installation +permalink: /installation/manual/database --- # Preparing database storage @@ -13,31 +15,22 @@ may need to perform some manual steps outlined below. ## PostgreSQL Dendrite can automatically populate the database with the relevant tables and indexes, but -it is not capable of creating the databases themselves. You will need to create the databases +it is not capable of creating the database itself. You will need to create the database manually. -The databases **must** be created with UTF-8 encoding configured or you will likely run into problems +The database **must** be created with UTF-8 encoding configured, or you will likely run into problems with your Dendrite deployment. -At this point, you can choose to either use a single database for all Dendrite components, -or you can run each component with its own separate database: +You will need to create a single PostgreSQL database. Deployments +can use a single global connection pool, which makes updating the configuration file much easier. +Only one database connection string to manage and likely simpler to back up the database. All +components will be sharing the same database resources (CPU, RAM, storage). -* **Single database**: You will need to create a single PostgreSQL database. Monolith deployments - can use a single global connection pool, which makes updating the configuration file much easier. - Only one database connection string to manage and likely simpler to back up the database. All - components will be sharing the same database resources (CPU, RAM, storage). - -* **Separate databases**: You will need to create a separate PostgreSQL database for each - component. You will need to configure each component that has storage in the Dendrite - configuration file with its own connection parameters. Allows running a different database engine - for each component on a different machine if needs be, each with their own CPU, RAM and storage — - almost certainly overkill unless you are running a very large Dendrite deployment. - -For either configuration, you will want to: +You will most likely want to: 1. Configure a role (with a username and password) which Dendrite can use to connect to the database; -2. Create the database(s) themselves, ensuring that the Dendrite role has privileges over them. +2. Create the database itself, ensuring that the Dendrite role has privileges over them. As Dendrite will create and manage the database tables, indexes and sequences by itself, the Dendrite role must have suitable privileges over the database. @@ -71,27 +64,6 @@ Create the database itself, using the `dendrite` role from above: sudo -u postgres createdb -O dendrite -E UTF-8 dendrite ``` -### Multiple database creation - -The following eight components require a database. In this example they will be named: - -| Appservice API | `dendrite_appservice` | -| Federation API | `dendrite_federationapi` | -| Media API | `dendrite_mediaapi` | -| MSCs | `dendrite_mscs` | -| Roomserver | `dendrite_roomserver` | -| Sync API | `dendrite_syncapi` | -| Key server | `dendrite_keyserver` | -| User API | `dendrite_userapi` | - -... therefore you will need to create eight different databases: - -```bash -for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do - sudo -u postgres createdb -O dendrite -E UTF-8 dendrite_$i -done -``` - ## SQLite **WARNING:** The Dendrite SQLite backend is slower, less reliable and not recommended for diff --git a/docs/installation/8_signingkey.md b/docs/installation/manual/3_signingkey.md similarity index 92% rename from docs/installation/8_signingkey.md rename to docs/installation/manual/3_signingkey.md index 323759a88f..91289fd6ac 100644 --- a/docs/installation/8_signingkey.md +++ b/docs/installation/manual/3_signingkey.md @@ -1,8 +1,9 @@ --- title: Generating signing keys -parent: Installation -nav_order: 8 -permalink: /installation/signingkeys +parent: Manual +grand_parent: Installation +nav_order: 3 +permalink: /installation/manual/signingkeys --- # Generating signing keys @@ -11,7 +12,7 @@ All Matrix homeservers require a signing private key, which will be used to auth federation requests and events. The `generate-keys` utility can be used to generate a private key. Assuming that Dendrite was -built using `build.sh`, you should find the `generate-keys` utility in the `bin` folder. +built using `go build -o bin/ ./cmd/...`, you should find the `generate-keys` utility in the `bin` folder. To generate a Matrix signing private key: diff --git a/docs/installation/7_configuration.md b/docs/installation/manual/4_configuration.md similarity index 66% rename from docs/installation/7_configuration.md rename to docs/installation/manual/4_configuration.md index 0cc67b1561..624cc4155f 100644 --- a/docs/installation/7_configuration.md +++ b/docs/installation/manual/4_configuration.md @@ -1,8 +1,9 @@ --- title: Configuring Dendrite -parent: Installation -nav_order: 7 -permalink: /installation/configuration +parent: Manual +grand_parent: Installation +nav_order: 4 +permalink: /installation/manual/configuration --- # Configuring Dendrite @@ -20,7 +21,7 @@ sections: First of all, you will need to configure the server name of your Matrix homeserver. This must match the domain name that you have selected whilst [configuring the domain -name delegation](domainname). +name delegation](../domainname#delegation). In the `global` section, set the `server_name` to your delegated domain name: @@ -44,7 +45,7 @@ global: ## JetStream configuration -Monolith deployments can use the built-in NATS Server rather than running a standalone +Dendrite deployments can use the built-in NATS Server rather than running a standalone server. If you want to use a standalone NATS Server anyway, you can also configure that too. ### Built-in NATS Server @@ -56,7 +57,6 @@ configured and set a `storage_path` to a persistent folder on the filesystem: global: # ... jetstream: - in_memory: false storage_path: /path/to/storage/folder topic_prefix: Dendrite ``` @@ -79,22 +79,17 @@ You do not need to configure the `storage_path` when using a standalone NATS Ser In the case that you are connecting to a multi-node NATS cluster, you can configure more than one address in the `addresses` field. -## Database connections +## Database connection using a global connection pool -Configuring database connections varies based on the [database configuration](database) -that you chose. - -### Global connection pool - -If you want to use a single connection pool to a single PostgreSQL database, then you must -uncomment and configure the `database` section within the `global` section: +If you want to use a single connection pool to a single PostgreSQL database, +then you must uncomment and configure the `database` section within the `global` section: ```yaml global: # ... database: connection_string: postgres://user:pass@hostname/database?sslmode=disable - max_open_conns: 100 + max_open_conns: 90 max_idle_conns: 5 conn_max_lifetime: -1 ``` @@ -104,42 +99,13 @@ configuration file, e.g. under the `app_service_api`, `federation_api`, `key_ser `media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will override the `global` database configuration. -### Per-component connections (all other configurations) - -If you are are using SQLite databases or separate PostgreSQL -databases per component, then you must instead configure the `database` sections under each -of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`, -`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks. - -For example, with PostgreSQL: - -```yaml -room_server: - # ... - database: - connection_string: postgres://user:pass@hostname/dendrite_component?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 -``` - -... or with SQLite: - -```yaml -room_server: - # ... - database: - connection_string: file:roomserver.db - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 -``` - ## Full-text search -Dendrite supports experimental full-text indexing using [Bleve](https://github.com/blevesearch/bleve). It is configured in the `sync_api` section as follows. +Dendrite supports full-text indexing using [Bleve](https://github.com/blevesearch/bleve). It is configured in the `sync_api` section as follows. -Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expectations. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang). +Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, +to ensure the returned results match the expectations. A full list of possible languages +can be found [here](https://github.com/matrix-org/dendrite/blob/5b73592f5a4dddf64184fcbe33f4c1835c656480/internal/fulltext/bleve.go#L25-L46). ```yaml sync_api: diff --git a/docs/installation/manual/5_starting_dendrite.md b/docs/installation/manual/5_starting_dendrite.md new file mode 100644 index 0000000000..d135043720 --- /dev/null +++ b/docs/installation/manual/5_starting_dendrite.md @@ -0,0 +1,26 @@ +--- +title: Starting Dendrite +parent: Manual +grand_parent: Installation +nav_order: 5 +permalink: /installation/manual/start +--- + +# Starting Dendrite + +Once you have completed all preparation and installation steps, +you can start your Dendrite deployment by executing the `dendrite` binary: + +```bash +./dendrite -config /path/to/dendrite.yaml +``` + +By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses +or ports that Dendrite listens on, you can use the `-http-bind-address` and +`-https-bind-address` command line arguments: + +```bash +./dendrite -config /path/to/dendrite.yaml \ + -http-bind-address 1.2.3.4:12345 \ + -https-bind-address 1.2.3.4:54321 +``` diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/dendrite-sample.conf similarity index 100% rename from docs/nginx/monolith-sample.conf rename to docs/nginx/dendrite-sample.conf diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf deleted file mode 100644 index 0ad24509a4..0000000000 --- a/docs/nginx/polylith-sample.conf +++ /dev/null @@ -1,58 +0,0 @@ -server { - listen 443 ssl; # IPv4 - listen [::]:443 ssl; # IPv6 - server_name my.hostname.com; - - ssl_certificate /path/to/fullchain.pem; - ssl_certificate_key /path/to/privkey.pem; - ssl_dhparam /path/to/ssl-dhparams.pem; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_read_timeout 600; - - location /.well-known/matrix/server { - return 200 '{ "m.server": "my.hostname.com:443" }'; - } - - location /.well-known/matrix/client { - # If your sever_name here doesn't match your matrix homeserver URL - # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL) - # add_header Access-Control-Allow-Origin '*'; - return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; - } - - # route requests to: - # /_matrix/client/.*/sync - # /_matrix/client/.*/user/{userId}/filter - # /_matrix/client/.*/user/{userId}/filter/{filterID} - # /_matrix/client/.*/keys/changes - # /_matrix/client/.*/rooms/{roomId}/messages - # /_matrix/client/.*/rooms/{roomId}/context/{eventID} - # /_matrix/client/.*/rooms/{roomId}/event/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} - # /_matrix/client/.*/rooms/{roomId}/members - # /_matrix/client/.*/rooms/{roomId}/joined_members - # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ { - proxy_pass http://sync_api:8073; - } - - location /_matrix/client { - proxy_pass http://client_api:8071; - } - - location /_matrix/federation { - proxy_pass http://federation_api:8072; - } - - location /_matrix/key { - proxy_pass http://federation_api:8072; - } - - location /_matrix/media { - proxy_pass http://media_api:8074; - } -} diff --git a/docs/systemd/monolith-example.service b/docs/systemd/monolith-example.service deleted file mode 100644 index 8a948a3faa..0000000000 --- a/docs/systemd/monolith-example.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Dendrite (Matrix Homeserver) -After=syslog.target -After=network.target -After=postgresql.service - -[Service] -Environment=GODEBUG=madvdontneed=1 -RestartSec=2s -Type=simple -User=dendrite -Group=dendrite -WorkingDirectory=/opt/dendrite/ -ExecStart=/opt/dendrite/bin/dendrite -Restart=always -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target diff --git a/federationapi/api/api.go b/federationapi/api/api.go index b53ec3dd8c..efe0547df0 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -27,7 +27,6 @@ type FederationInternalAPI interface { QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error LookupServerKeys(ctx context.Context, s spec.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp) ([]gomatrixserverlib.ServerKeys, error) MSC2836EventRelationships(ctx context.Context, origin, dst spec.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error) - MSC2946Spaces(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) // Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos. PerformBroadcastEDU( @@ -62,7 +61,9 @@ type RoomserverFederationAPI interface { // Handle an instruction to make_leave & send_leave with a remote server. PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error // Handle sending an invite to a remote server. - PerformInvite(ctx context.Context, request *PerformInviteRequest, response *PerformInviteResponse) error + SendInvite(ctx context.Context, event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error) + // Handle sending an invite to a remote server. + SendInviteV3(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error) // Handle an instruction to peek a room on a remote server. PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error // Query the server names of the joined hosts in a room. @@ -73,6 +74,8 @@ type RoomserverFederationAPI interface { GetEventAuth(ctx context.Context, origin, s spec.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error) GetEvent(ctx context.Context, origin, s spec.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) LookupMissingEvents(ctx context.Context, origin, s spec.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) + + RoomHierarchies(ctx context.Context, origin, dst spec.ServerName, roomID string, suggestedOnly bool) (res fclient.RoomHierarchyResponse, err error) } type P2PFederationAPI interface { @@ -190,9 +193,9 @@ type PerformLeaveResponse struct { } type PerformInviteRequest struct { - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - Event *rstypes.HeaderedEvent `json:"event"` - InviteRoomState []fclient.InviteV2StrippedState `json:"invite_room_state"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + Event *rstypes.HeaderedEvent `json:"event"` + InviteRoomState []gomatrixserverlib.InviteStrippedState `json:"invite_room_state"` } type PerformInviteResponse struct { diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 3fdc835bb1..6210bddb6f 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -117,19 +117,27 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { return true } - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: m.UserID, - WantMembership: "join", - }, &queryRes) + userID, err := spec.NewUserID(m.UserID, true) + if err != nil { + sentry.CaptureException(err) + logger.WithError(err).Error("invalid user ID") + return true + } + + roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *userID, "join") if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined rooms for user") return true } + roomIDStrs := make([]string, len(roomIDs)) + for i, room := range roomIDs { + roomIDStrs[i] = room.String() + } + // send this key change to all servers who share rooms with this user. - destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in") @@ -179,18 +187,27 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { } logger := logrus.WithField("user_id", output.UserID) - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: output.UserID, - WantMembership: "join", - }, &queryRes) + outputUserID, err := spec.NewUserID(output.UserID, true) + if err != nil { + sentry.CaptureException(err) + logrus.WithError(err).Errorf("invalid user ID") + return true + } + + rooms, err := t.rsAPI.QueryRoomsForUser(t.ctx, *outputUserID, "join") if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") return true } + + roomIDStrs := make([]string, len(rooms)) + for i, room := range rooms { + roomIDStrs[i] = room.String() + } + // send this key change to all servers who share rooms with this user. - destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go index e751b65d4b..dd100bc08d 100644 --- a/federationapi/consumers/presence.go +++ b/federationapi/consumers/presence.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" ) @@ -94,16 +95,23 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg return true } - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: userID, - WantMembership: "join", - }, &queryRes) + parsedUserID, err := spec.NewUserID(userID, true) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("user_id", userID).Error("invalid user ID") + return true + } + + roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *parsedUserID, "join") if err != nil { log.WithError(err).Error("failed to calculate joined rooms for user") return true } + roomIDStrs := make([]string, len(roomIDs)) + for i, roomID := range roomIDs { + roomIDStrs[i] = roomID.String() + } + presence := msg.Header.Get("presence") ts, err := strconv.Atoi(msg.Header.Get("last_active_ts")) @@ -112,7 +120,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg } // send this presence to all servers who share rooms with this user. - joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + joined, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { log.WithError(err).Error("failed to get joined hosts") return true diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index c6ad3f7485..f1dcb11759 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -16,7 +16,9 @@ package consumers import ( "context" + "encoding/base64" "encoding/json" + "errors" "fmt" "strconv" "time" @@ -174,7 +176,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew // Finally, work out if there are any more events missing. if len(missingEventIDs) > 0 { eventsReq := &api.QueryEventsByIDRequest{ - RoomID: ore.Event.RoomID(), + RoomID: ore.Event.RoomID().String(), EventIDs: missingEventIDs, } eventsRes := &api.QueryEventsByIDResponse{} @@ -192,7 +194,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew evs[i] = addsStateEvents[i].PDU } - addsJoinedHosts, err := JoinedHostsFromEvents(evs) + addsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, evs, s.rsAPI) if err != nil { return err } @@ -203,7 +205,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew // talking to the roomserver oldJoinedHosts, err := s.db.UpdateRoom( s.ctx, - ore.Event.RoomID(), + ore.Event.RoomID().String(), addsJoinedHosts, ore.RemovesStateEventIDs, rewritesState, // if we're re-writing state, nuke all joined hosts before adding @@ -216,7 +218,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == spec.MRoomMember && ore.Event.StateKey() != nil { membership, _ := ore.Event.Membership() if membership == spec.Join { - s.sendPresence(ore.Event.RoomID(), addsJoinedHosts) + s.sendPresence(ore.Event.RoomID().String(), addsJoinedHosts) } } @@ -345,7 +347,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( return nil, err } - combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents) + combinedAddsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, combinedAddsEvents, s.rsAPI) if err != nil { return nil, err } @@ -374,7 +376,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( } // handle peeking hosts - inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID()) + inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID().String()) if err != nil { return nil, err } @@ -394,7 +396,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( // JoinedHostsFromEvents turns a list of state events into a list of joined hosts. // This errors if one of the events was invalid. // It should be impossible for an invalid event to get this far in the pipeline. -func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, error) { +func JoinedHostsFromEvents(ctx context.Context, evs []gomatrixserverlib.PDU, rsAPI api.FederationRoomserverAPI) ([]types.JoinedHost, error) { var joinedHosts []types.JoinedHost for _, ev := range evs { if ev.Type() != "m.room.member" || ev.StateKey() == nil { @@ -407,12 +409,26 @@ func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, err if membership != spec.Join { continue } - _, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) + var domain spec.ServerName + userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey())) if err != nil { - return nil, err + if errors.As(err, new(base64.CorruptInputError)) { + // Fallback to using the "old" way of getting the user domain, avoids + // "illegal base64 data at input byte 0" errors + // FIXME: we should do this in QueryUserIDForSender instead + _, domain, err = gomatrixserverlib.SplitID('@', *ev.StateKey()) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } else { + domain = userID.Domain() } + joinedHosts = append(joinedHosts, types.JoinedHost{ - MemberEventID: ev.EventID(), ServerName: serverName, + MemberEventID: ev.EventID(), ServerName: domain, }) } return joinedHosts, nil @@ -490,7 +506,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // At this point the missing events are neither the event itself nor are // they present in our local database. Our only option is to fetch them // from the roomserver using the query API. - eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID()} + eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID().String()} var eventResp api.QueryEventsByIDResponse if err := s.rsAPI.QueryEventsByID(s.ctx, &eventReq, &eventResp); err != nil { return nil, err diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ee15a8a6e0..e148199fb1 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -95,7 +95,7 @@ func AddPublicRoutes( func NewInternalAPI( processContext *process.ProcessContext, dendriteCfg *config.Dendrite, - cm sqlutil.Connections, + cm *sqlutil.Connections, natsInstance *jetstream.NATSInstance, federation fclient.FederationClient, rsAPI roomserverAPI.FederationRoomserverAPI, diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index beb648a48f..1ea8c40ea1 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -33,7 +33,16 @@ import ( type fedRoomserverAPI struct { rsapi.FederationRoomserverAPI inputRoomEvents func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) - queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error + queryRoomsForUser func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) +} + +func (f *fedRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + +func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil } // PerformJoin will call this function @@ -45,11 +54,11 @@ func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.Input } // keychange consumer calls this -func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { +func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { if f.queryRoomsForUser == nil { - return nil + return nil, nil } - return f.queryRoomsForUser(ctx, req, res) + return f.queryRoomsForUser(ctx, userID, desiredMembership) } // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate @@ -111,12 +120,13 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s spec.ServerName, roo defer f.fedClientMutex.Unlock() for _, r := range f.allowJoins { if r.ID == roomID { + senderIDString := userID res.RoomVersion = r.Version res.JoinEvent = gomatrixserverlib.ProtoEvent{ - Sender: userID, + SenderID: senderIDString, RoomID: roomID, Type: "m.room.member", - StateKey: &userID, + StateKey: &senderIDString, Content: spec.RawJSON([]byte(`{"membership":"join"}`)), PrevEvents: r.ForwardExtremities(), } @@ -136,7 +146,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s spec.ServerName, eve f.fedClientMutex.Lock() defer f.fedClientMutex.Unlock() for _, r := range f.allowJoins { - if r.ID == event.RoomID() { + if r.ID == event.RoomID().String() { r.InsertEvent(f.t, &types.HeaderedEvent{PDU: event}) f.t.Logf("Join event: %v", event.EventID()) res.StateEvents = types.NewEventJSONsFromHeaderedEvents(r.CurrentState()) @@ -189,18 +199,22 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID) room := test.NewRoom(t, creator) + roomID, err := spec.NewRoomID(room.ID) + if err != nil { + t.Fatalf("Invalid room ID: %q", roomID) + } + rsapi := &fedRoomserverAPI{ inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { if req.Asynchronous { t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous") } }, - queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { - if req.UserID == joiningUser.ID && req.WantMembership == "join" { - res.RoomIDs = []string{room.ID} - return nil + queryRoomsForUser: func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + if userID.String() == joiningUser.ID && desiredMembership == "join" { + return []spec.RoomID{*roomID}, nil } - return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req) + return nil, fmt.Errorf("unexpected queryRoomsForUser: %v, %v", userID, desiredMembership) }, } fc := &fedClient{ diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index aa501f63c0..3e6f39566e 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -54,11 +54,14 @@ func NewFederationInternalAPI( KeyDatabase: serverKeyDB, } + pubKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey) addDirectFetcher := func() { keyRing.KeyFetchers = append( keyRing.KeyFetchers, &gomatrixserverlib.DirectKeyFetcher{ - Client: federation, + Client: federation, + IsLocalServerName: cfg.Matrix.IsLocalServerName, + LocalPublicKey: []byte(pubKey), }, ) } diff --git a/federationapi/internal/federationclient.go b/federationapi/internal/federationclient.go index d4d7269dbb..b6bc7a5eda 100644 --- a/federationapi/internal/federationclient.go +++ b/federationapi/internal/federationclient.go @@ -29,7 +29,7 @@ func (a *FederationInternalAPI) MakeJoin( func (a *FederationInternalAPI) SendJoin( ctx context.Context, origin, s spec.ServerName, event gomatrixserverlib.PDU, ) (res gomatrixserverlib.SendJoinResponse, err error) { - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + ctx, cancel := context.WithTimeout(ctx, time.Minute*5) defer cancel() ires, err := a.federation.SendJoin(ctx, origin, s, event) if err != nil { @@ -194,16 +194,16 @@ func (a *FederationInternalAPI) MSC2836EventRelationships( return ires.(fclient.MSC2836EventRelationshipsResponse), nil } -func (a *FederationInternalAPI) MSC2946Spaces( +func (a *FederationInternalAPI) RoomHierarchies( ctx context.Context, origin, s spec.ServerName, roomID string, suggestedOnly bool, -) (res fclient.MSC2946SpacesResponse, err error) { +) (res fclient.RoomHierarchyResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { - return a.federation.MSC2946Spaces(ctx, origin, s, roomID, suggestedOnly) + return a.federation.RoomHierarchy(ctx, origin, s, roomID, suggestedOnly) }) if err != nil { return res, err } - return ires.(fclient.MSC2946SpacesResponse), nil + return ires.(fclient.RoomHierarchyResponse), nil } diff --git a/federationapi/internal/keys.go b/federationapi/internal/keys.go index 00e78a1c17..a642f3a4bd 100644 --- a/federationapi/internal/keys.go +++ b/federationapi/internal/keys.go @@ -170,7 +170,7 @@ func (s *FederationInternalAPI) handleDatabaseKeys( // in that case. If the key isn't valid right now, then by // leaving it in the 'requests' map, we'll try to update the // key using the fetchers in handleFetcherKeys. - if res.WasValidAt(now, true) { + if res.WasValidAt(now, gomatrixserverlib.StrictValiditySignatureCheck) { delete(requests, req) } } diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 99943d86cd..0200cf69b3 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -2,6 +2,7 @@ package internal import ( "context" + "crypto/ed25519" "encoding/json" "errors" "fmt" @@ -156,15 +157,39 @@ func (r *FederationInternalAPI) performJoinUsingServer( } joinInput := gomatrixserverlib.PerformJoinInput{ - UserID: user, - RoomID: room, - ServerName: serverName, - Content: content, - Unsigned: unsigned, - PrivateKey: r.cfg.Matrix.PrivateKey, - KeyID: r.cfg.Matrix.KeyID, - KeyRing: r.keyRing, - EventProvider: federatedEventProvider(ctx, r.federation, r.keyRing, user.Domain(), serverName), + UserID: user, + RoomID: room, + ServerName: serverName, + Content: content, + Unsigned: unsigned, + PrivateKey: r.cfg.Matrix.PrivateKey, + KeyID: r.cfg.Matrix.KeyID, + KeyRing: r.keyRing, + EventProvider: federatedEventProvider(ctx, r.federation, r.keyRing, user.Domain(), serverName, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }), + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, + GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) { + // assign a roomNID, otherwise we can't create a private key for the user + _, nidErr := r.rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion)) + if nidErr != nil { + return "", nil, nidErr + } + key, keyErr := r.rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if keyErr != nil { + return "", nil, keyErr + } + return spec.SenderIDFromPseudoIDKey(key), key, nil + }, + StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error { + storeUserID, userErr := spec.NewUserID(userIDRaw, true) + if userErr != nil { + return userErr + } + return r.rsAPI.StoreUserRoomPublicKey(ctx, senderID, *storeUserID, roomID) + }, } response, joinErr := gomatrixserverlib.PerformJoin(ctx, r, joinInput) @@ -187,7 +212,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( // joining a room, waiting for 200 OK then changing device keys and have those keys not be sent // to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers") // The events are trusted now as we performed auth checks above. - joinedHosts, err := consumers.JoinedHostsFromEvents(response.StateSnapshot.GetStateEvents().TrustedEvents(response.JoinEvent.Version(), false)) + joinedHosts, err := consumers.JoinedHostsFromEvents(ctx, response.StateSnapshot.GetStateEvents().TrustedEvents(response.JoinEvent.Version(), false), r.rsAPI) if err != nil { return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err) } @@ -358,8 +383,11 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer( // authenticate the state returned (check its auth events etc) // the equivalent of CheckSendJoinResponse() + userIDProvider := func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + } authEvents, stateEvents, err := gomatrixserverlib.CheckStateResponse( - ctx, &respPeek, respPeek.RoomVersion, r.keyRing, federatedEventProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName), + ctx, &respPeek, respPeek.RoomVersion, r.keyRing, federatedEventProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName, userIDProvider), userIDProvider, ) if err != nil { return fmt.Errorf("error checking state returned from peeking: %w", err) @@ -406,7 +434,7 @@ func (r *FederationInternalAPI) PerformLeave( request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, ) (err error) { - _, origin, err := r.cfg.Matrix.SplitLocalID('@', request.UserID) + userID, err := spec.NewUserID(request.UserID, true) if err != nil { return err } @@ -425,7 +453,7 @@ func (r *FederationInternalAPI) PerformLeave( // request. respMakeLeave, err := r.federation.MakeLeave( ctx, - origin, + userID.Domain(), serverName, request.RoomID, request.UserID, @@ -446,9 +474,20 @@ func (r *FederationInternalAPI) PerformLeave( // Set all the fields to be what they should be, this should be a no-op // but it's possible that the remote server returned us something "odd" + roomID, err := spec.NewRoomID(request.RoomID) + if err != nil { + return err + } + senderID, err := r.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID) + if err != nil { + return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", *userID, *roomID) + } + senderIDString := string(*senderID) respMakeLeave.LeaveEvent.Type = spec.MRoomMember - respMakeLeave.LeaveEvent.Sender = request.UserID - respMakeLeave.LeaveEvent.StateKey = &request.UserID + respMakeLeave.LeaveEvent.SenderID = senderIDString + respMakeLeave.LeaveEvent.StateKey = &senderIDString respMakeLeave.LeaveEvent.RoomID = request.RoomID respMakeLeave.LeaveEvent.Redacts = "" leaveEB := verImpl.NewEventBuilderFromProtoEvent(&respMakeLeave.LeaveEvent) @@ -470,7 +509,7 @@ func (r *FederationInternalAPI) PerformLeave( // Build the leave event. event, err := leaveEB.Build( time.Now(), - origin, + userID.Domain(), r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey, ) @@ -482,7 +521,7 @@ func (r *FederationInternalAPI) PerformLeave( // Try to perform a send_leave using the newly built event. err = r.federation.SendLeave( ctx, - origin, + userID.Domain(), serverName, event, ) @@ -503,60 +542,111 @@ func (r *FederationInternalAPI) PerformLeave( ) } -// PerformLeaveRequest implements api.FederationInternalAPI -func (r *FederationInternalAPI) PerformInvite( +// SendInvite implements api.FederationInternalAPI +func (r *FederationInternalAPI) SendInvite( ctx context.Context, - request *api.PerformInviteRequest, - response *api.PerformInviteResponse, -) (err error) { - _, origin, err := r.cfg.Matrix.SplitLocalID('@', request.Event.Sender()) + event gomatrixserverlib.PDU, + strippedState []gomatrixserverlib.InviteStrippedState, +) (gomatrixserverlib.PDU, error) { + inviter, err := r.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err != nil { - return err + return nil, err } - if request.Event.StateKey() == nil { - return errors.New("invite must be a state event") + if event.StateKey() == nil { + return nil, errors.New("invite must be a state event") } - _, destination, err := gomatrixserverlib.SplitID('@', *request.Event.StateKey()) + _, destination, err := gomatrixserverlib.SplitID('@', *event.StateKey()) if err != nil { - return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) + return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } // TODO (devon): This should be allowed via a relay. Currently only transactions // can be sent to relays. Would need to extend relays to handle invites. if !r.shouldAttemptDirectFederation(destination) { - return fmt.Errorf("relay servers have no meaningful response for invite.") + return nil, fmt.Errorf("relay servers have no meaningful response for invite.") } logrus.WithFields(logrus.Fields{ - "event_id": request.Event.EventID(), - "user_id": *request.Event.StateKey(), - "room_id": request.Event.RoomID(), - "room_version": request.RoomVersion, + "event_id": event.EventID(), + "user_id": *event.StateKey(), + "room_id": event.RoomID().String(), + "room_version": event.Version(), "destination": destination, }).Info("Sending invite") - inviteReq, err := fclient.NewInviteV2Request(request.Event.PDU, request.InviteRoomState) + inviteReq, err := fclient.NewInviteV2Request(event, strippedState) if err != nil { - return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err) + return nil, fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err) } - inviteRes, err := r.federation.SendInviteV2(ctx, origin, destination, inviteReq) + inviteRes, err := r.federation.SendInviteV2(ctx, inviter.Domain(), destination, inviteReq) if err != nil { - return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err) + return nil, fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err) } - verImpl, err := gomatrixserverlib.GetRoomVersion(request.RoomVersion) + verImpl, err := gomatrixserverlib.GetRoomVersion(event.Version()) if err != nil { - return err + return nil, err } inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event) if err != nil { - return fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err) + return nil, fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err) } - response.Event = &types.HeaderedEvent{PDU: inviteEvent} - return nil + return inviteEvent, nil +} + +// SendInviteV3 implements api.FederationInternalAPI +func (r *FederationInternalAPI) SendInviteV3( + ctx context.Context, + event gomatrixserverlib.ProtoEvent, + invitee spec.UserID, + version gomatrixserverlib.RoomVersion, + strippedState []gomatrixserverlib.InviteStrippedState, +) (gomatrixserverlib.PDU, error) { + validRoomID, err := spec.NewRoomID(event.RoomID) + if err != nil { + return nil, err + } + verImpl, err := gomatrixserverlib.GetRoomVersion(version) + if err != nil { + return nil, err + } + + inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(event.SenderID)) + if err != nil { + return nil, err + } + + // TODO (devon): This should be allowed via a relay. Currently only transactions + // can be sent to relays. Would need to extend relays to handle invites. + if !r.shouldAttemptDirectFederation(invitee.Domain()) { + return nil, fmt.Errorf("relay servers have no meaningful response for invite.") + } + + logrus.WithFields(logrus.Fields{ + "user_id": invitee.String(), + "room_id": event.RoomID, + "room_version": version, + "destination": invitee.Domain(), + }).Info("Sending invite") + + inviteReq, err := fclient.NewInviteV3Request(event, version, strippedState) + if err != nil { + return nil, fmt.Errorf("gomatrixserverlib.NewInviteV3Request: %w", err) + } + + inviteRes, err := r.federation.SendInviteV3(ctx, inviter.Domain(), invitee.Domain(), inviteReq, invitee) + if err != nil { + return nil, fmt.Errorf("r.federation.SendInviteV3: failed to send invite: %w", err) + } + + inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event) + if err != nil { + return nil, fmt.Errorf("r.federation.SendInviteV3 failed to decode event response: %w", err) + } + return inviteEvent, nil } // PerformServersAlive implements api.FederationInternalAPI @@ -636,6 +726,7 @@ func checkEventsContainCreateEvent(events []gomatrixserverlib.PDU) error { func federatedEventProvider( ctx context.Context, federation fclient.FederationClient, keyRing gomatrixserverlib.JSONVerifier, origin, server spec.ServerName, + userIDForSender spec.UserIDForSender, ) gomatrixserverlib.EventProvider { // A list of events that we have retried, if they were not included in // the auth events supplied in the send_join. @@ -685,7 +776,7 @@ func federatedEventProvider( } // Check the signatures of the event. - if err := gomatrixserverlib.VerifyEventSignatures(ctx, ev, keyRing); err != nil { + if err := gomatrixserverlib.VerifyEventSignatures(ctx, ev, keyRing, userIDForSender); err != nil { return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) } diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index 2f61235aec..656755f968 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -16,6 +16,7 @@ package internal import ( "context" + "crypto/ed25519" "testing" "github.com/matrix-org/dendrite/federationapi/api" @@ -53,10 +54,14 @@ func TestPerformWakeupServers(t *testing.T) { assert.NoError(t, err) assert.True(t, offline) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -95,10 +100,14 @@ func TestQueryRelayServers(t *testing.T) { err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers) assert.NoError(t, err) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -132,10 +141,14 @@ func TestRemoveRelayServers(t *testing.T) { err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers) assert.NoError(t, err) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -168,10 +181,14 @@ func TestRemoveRelayServers(t *testing.T) { func TestPerformDirectoryLookup(t *testing.T) { testDB := test.NewInMemoryFederationDatabase() + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -192,7 +209,7 @@ func TestPerformDirectoryLookup(t *testing.T) { ServerName: "server", } res := api.PerformDirectoryLookupResponse{} - err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) + err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) assert.NoError(t, err) } @@ -203,10 +220,14 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { testDB.SetServerAssumedOffline(context.Background(), server) testDB.P2PAddRelayServersForServer(context.Background(), server, []spec.ServerName{"relay"}) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ - ServerName: server, + ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -227,6 +248,6 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { ServerName: server, } res := api.PerformDirectoryLookupResponse{} - err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) + err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) assert.Error(t, err) } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 26305ed7aa..24b3efd2d4 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -218,7 +218,7 @@ func (oqs *OutgoingQueues) SendEvent( if api.IsServerBannedFromRoom( oqs.process.Context(), oqs.rsAPI, - ev.RoomID(), + ev.RoomID().String(), destination, ) { delete(destmap, destination) diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index cc38e136ff..e75615e05c 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -104,7 +104,7 @@ func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u func mustCreatePDU(t *testing.T) *types.HeaderedEvent { t.Helper() - content := `{"type":"m.room.message"}` + content := `{"type":"m.room.message", "room_id":"!room:a"}` ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(content), false) if err != nil { t.Fatalf("failed to create event: %v", err) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 9e15950533..75a0072656 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -95,7 +95,7 @@ func Backfill( } } - // Query the roomserver. + // Query the Roomserver. if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed") return util.JSONResponse{ @@ -109,7 +109,7 @@ func Backfill( var ev *types.HeaderedEvent for _, ev = range res.Events { - if ev.RoomID() == roomID { + if ev.RoomID().String() == roomID { evs = append(evs, ev.PDU) } } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index ca279ac22c..2be3ecdb1d 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -42,10 +42,10 @@ func GetEventAuth( return *resErr } - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")} } - resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID) + resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) if resErr != nil { return *resErr } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 196a54db1b..f4659f5285 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -35,10 +35,6 @@ func GetEvent( eventID string, origin spec.ServerName, ) util.JSONResponse { - err := allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID) - if err != nil { - return *err - } // /_matrix/federation/v1/event/{eventId} doesn't have a roomID, we use an empty string, // which results in `QueryEventsByID` to first get the event and use that to determine the roomID. event, err := fetchEvent(ctx, rsAPI, "", eventID) @@ -46,6 +42,11 @@ func GetEvent( return *err } + err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) + if err != nil { + return *err + } + return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ Origin: origin, OriginServerTS: spec.AsTimestamp(time.Now()), @@ -62,8 +63,9 @@ func allowedToSeeEvent( origin spec.ServerName, rsAPI api.FederationRoomserverAPI, eventID string, + roomID string, ) *util.JSONResponse { - allowed, err := rsAPI.QueryServerAllowedToSeeEvent(ctx, origin, eventID) + allowed, err := rsAPI.QueryServerAllowedToSeeEvent(ctx, origin, eventID, roomID) if err != nil { resErr := util.ErrorResponse(err) return &resErr diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 993d404666..76fadaa93e 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -16,11 +16,11 @@ package routing import ( "context" + "crypto/ed25519" "encoding/json" "fmt" "net/http" - "github.com/getsentry/sentry-go" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" @@ -30,11 +30,78 @@ import ( "github.com/matrix-org/util" ) +// InviteV3 implements /_matrix/federation/v2/invite/{roomID}/{userID} +func InviteV3( + httpReq *http.Request, + request *fclient.FederationRequest, + roomID spec.RoomID, + invitedUser spec.UserID, + cfg *config.FederationAPI, + rsAPI api.FederationRoomserverAPI, + keys gomatrixserverlib.JSONVerifier, +) util.JSONResponse { + inviteReq := fclient.InviteV3Request{} + err := json.Unmarshal(request.Content(), &inviteReq) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON(err.Error()), + } + } + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), + } + } + + input := gomatrixserverlib.HandleInviteV3Input{ + HandleInviteInput: gomatrixserverlib.HandleInviteInput{ + RoomVersion: inviteReq.RoomVersion(), + RoomID: roomID, + InvitedUser: invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: nil, + StrippedState: inviteReq.InviteRoomState(), + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, + }, + InviteProtoEvent: inviteReq.Event(), + GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) { + // assign a roomNID, otherwise we can't create a private key for the user + _, nidErr := rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion)) + if nidErr != nil { + return "", nil, nidErr + } + key, keyErr := rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if keyErr != nil { + return "", nil, keyErr + } + + return spec.SenderIDFromPseudoIDKey(key), key, nil + }, + } + event, jsonErr := handleInviteV3(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInviteV2{Event: event.JSON()}, + } +} + // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID} func InviteV2( httpReq *http.Request, request *fclient.FederationRequest, - roomID string, + roomID spec.RoomID, eventID string, cfg *config.FederationAPI, rsAPI api.FederationRoomserverAPI, @@ -56,9 +123,58 @@ func InviteV2( JSON: spec.BadJSON(err.Error()), } case nil: - return processInvite( - httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys, - ) + if inviteReq.Event().StateKey() == nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("The invite event has no state key"), + } + } + + invitedUser, userErr := spec.NewUserID(*inviteReq.Event().StateKey(), true) + if userErr != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The user ID is invalid"), + } + } + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), + } + } + + if inviteReq.Event().EventID() != eventID { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), + } + } + + input := gomatrixserverlib.HandleInviteInput{ + RoomVersion: inviteReq.RoomVersion(), + RoomID: roomID, + InvitedUser: *invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: inviteReq.Event(), + StrippedState: inviteReq.InviteRoomState(), + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, + } + event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInviteV2{Event: event.JSON()}, + } default: return util.JSONResponse{ Code: http.StatusBadRequest, @@ -71,7 +187,7 @@ func InviteV2( func InviteV1( httpReq *http.Request, request *fclient.FederationRequest, - roomID string, + roomID spec.RoomID, eventID string, cfg *config.FederationAPI, rsAPI api.FederationRoomserverAPI, @@ -94,55 +210,11 @@ func InviteV1( JSON: spec.NotJSON("The request body could not be decoded into an invite v1 request. " + err.Error()), } } - var strippedState []fclient.InviteV2StrippedState - if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil { + var strippedState []gomatrixserverlib.InviteStrippedState + if jsonErr := json.Unmarshal(event.Unsigned(), &strippedState); jsonErr != nil { // just warn, they may not have added any. util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event") } - return processInvite( - httpReq.Context(), false, event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys, - ) -} - -func processInvite( - ctx context.Context, - isInviteV2 bool, - event gomatrixserverlib.PDU, - roomVer gomatrixserverlib.RoomVersion, - strippedState []fclient.InviteV2StrippedState, - roomID string, - eventID string, - cfg *config.FederationAPI, - rsAPI api.FederationRoomserverAPI, - keys gomatrixserverlib.JSONVerifier, -) util.JSONResponse { - - // Check that we can accept invites for this room version. - verImpl, err := gomatrixserverlib.GetRoomVersion(roomVer) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.UnsupportedRoomVersion( - fmt.Sprintf("Room version %q is not supported by this server.", roomVer), - ), - } - } - - // Check that the room ID is correct. - if event.RoomID() != roomID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("The room ID in the request path must match the room ID in the invite event JSON"), - } - } - - // Check that the event ID is correct. - if event.EventID() != eventID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), - } - } if event.StateKey() == nil { return util.JSONResponse{ @@ -151,105 +223,104 @@ func processInvite( } } - _, domain, err := cfg.Matrix.SplitLocalID('@', *event.StateKey()) + invitedUser, err := spec.NewUserID(*event.StateKey(), true) if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.InvalidParam(fmt.Sprintf("The user ID is invalid or domain %q does not belong to this server", domain)), + JSON: spec.InvalidParam("The user ID is invalid"), } } - - // Check that the event is signed by the server sending the request. - redacted, err := verImpl.RedactEventJSON(event.JSON()) - if err != nil { + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event JSON could not be redacted"), + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), } } - _, serverName, err := gomatrixserverlib.SplitID('@', event.Sender()) - if err != nil { + + if event.EventID() != eventID { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event JSON contains an invalid sender"), + JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), } } - verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: serverName, - Message: redacted, - AtTS: event.OriginServerTS(), - StrictValidityChecking: true, - }} - verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } + + input := gomatrixserverlib.HandleInviteInput{ + RoomVersion: roomVer, + RoomID: roomID, + InvitedUser: *invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: event, + StrippedState: strippedState, + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, } - if verifyResults[0].Error != nil { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("The invite must be signed by the server it originated on"), - } + event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInvite{Event: event.JSON()}, } +} - // Sign the event so that other servers will know that we have received the invite. - signedEvent := event.Sign( - string(domain), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, - ) +func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { + inviteEvent, err := gomatrixserverlib.HandleInvite(ctx, input) + return handleInviteResult(ctx, inviteEvent, err, rsAPI) +} - // Add the invite event to the roomserver. - inviteEvent := &types.HeaderedEvent{PDU: signedEvent} - request := &api.PerformInviteRequest{ - Event: inviteEvent, - InviteRoomState: strippedState, - RoomVersion: inviteEvent.Version(), - SendAsServer: string(api.DoNotSendToOtherServers), - TransactionID: nil, - } +func handleInviteV3(ctx context.Context, input gomatrixserverlib.HandleInviteV3Input, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { + inviteEvent, err := gomatrixserverlib.HandleInviteV3(ctx, input) + return handleInviteResult(ctx, inviteEvent, err, rsAPI) +} - if err = rsAPI.PerformInvite(ctx, request); err != nil { - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - return util.JSONResponse{ +func handleInviteResult(ctx context.Context, inviteEvent gomatrixserverlib.PDU, err error, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { + switch e := err.(type) { + case nil: + case spec.InternalServerError: + util.GetLogger(ctx).WithError(err) + return nil, &util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, } - } - - switch e := err.(type) { - case api.ErrInvalidID: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.Unknown(e.Error()), + case spec.MatrixError: + util.GetLogger(ctx).WithError(err) + code := http.StatusInternalServerError + switch e.ErrCode { + case spec.ErrorForbidden: + code = http.StatusForbidden + case spec.ErrorUnsupportedRoomVersion: + fallthrough // http.StatusBadRequest + case spec.ErrorBadJSON: + code = http.StatusBadRequest } - case api.ErrNotAllowed: - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden(e.Error()), + + return nil, &util.JSONResponse{ + Code: code, + JSON: e, } - case nil: default: - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - sentry.CaptureException(err) - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, + util.GetLogger(ctx).WithError(err) + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("unknown error"), } } - // Return the signed event to the originating server, it should then tell - // the other servers in the room that we have been invited. - if isInviteV2 { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: fclient.RespInviteV2{Event: signedEvent.JSON()}, - } - } else { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: fclient.RespInvite{Event: signedEvent.JSON()}, + headeredInvite := &types.HeaderedEvent{PDU: inviteEvent} + if err = rsAPI.HandleInvite(ctx, headeredInvite); err != nil { + util.GetLogger(ctx).WithError(err).Error("HandleInvite failed") + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, } } + return inviteEvent, nil + } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 4cbfc5e87f..ce7ad30ff7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,53 +33,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" ) -type JoinRoomQuerier struct { - roomserver api.FederationRoomserverAPI -} - -func (rq *JoinRoomQuerier) CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) { - return rq.roomserver.CurrentStateEvent(ctx, roomID, eventType, stateKey) -} - -func (rq *JoinRoomQuerier) InvitePending(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (bool, error) { - return rq.roomserver.InvitePending(ctx, roomID, userID) -} - -func (rq *JoinRoomQuerier) RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, userID spec.UserID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) { - roomInfo, err := rq.roomserver.QueryRoomInfo(ctx, roomID) - if err != nil || roomInfo == nil || roomInfo.IsStub() { - return nil, err - } - - req := api.QueryServerJoinedToRoomRequest{ - ServerName: localServerName, - RoomID: roomID.String(), - } - res := api.QueryServerJoinedToRoomResponse{} - if err = rq.roomserver.QueryServerJoinedToRoom(ctx, &req, &res); err != nil { - util.GetLogger(ctx).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed") - return nil, fmt.Errorf("InternalServerError: Failed to query room: %w", err) - } - - userJoinedToRoom, err := rq.roomserver.UserJoinedToRoom(ctx, types.RoomNID(roomInfo.RoomNID), userID) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("rsAPI.UserJoinedToRoom failed") - return nil, fmt.Errorf("InternalServerError: %w", err) - } - - locallyJoinedUsers, err := rq.roomserver.LocallyJoinedUsers(ctx, roomInfo.RoomVersion, types.RoomNID(roomInfo.RoomNID)) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("rsAPI.GetLocallyJoinedUsers failed") - return nil, fmt.Errorf("InternalServerError: %w", err) - } - - return &gomatrixserverlib.RestrictedRoomJoinInfo{ - LocalServerInRoom: res.RoomExists && res.IsInRoom, - UserJoinedToRoom: userJoinedToRoom, - JoinedUsers: locallyJoinedUsers, - }, nil -} - // MakeJoin implements the /make_join API func MakeJoin( httpReq *http.Request, @@ -103,7 +56,7 @@ func MakeJoin( RoomID: roomID.String(), } res := api.QueryServerJoinedToRoomResponse{} - if err := rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -112,26 +65,26 @@ func MakeJoin( } createJoinTemplate := func(proto *gomatrixserverlib.ProtoEvent) (gomatrixserverlib.PDU, []gomatrixserverlib.PDU, error) { - identity, err := cfg.Matrix.SigningIdentityFor(request.Destination()) - if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Errorf("obtaining signing identity for %s failed", request.Destination()) + identity, signErr := cfg.Matrix.SigningIdentityFor(request.Destination()) + if signErr != nil { + util.GetLogger(httpReq.Context()).WithError(signErr).Errorf("obtaining signing identity for %s failed", request.Destination()) return nil, nil, spec.NotFound(fmt.Sprintf("Server name %q does not exist", request.Destination())) } queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: roomVersion, } - event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) - switch e := err.(type) { + event, signErr := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes) + switch e := signErr.(type) { case nil: case eventutil.ErrRoomNoExists: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.NotFound("Room does not exist") case gomatrixserverlib.BadJSONError: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.BadJSON(e.Error()) default: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(signErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.InternalServerError{} } @@ -142,20 +95,40 @@ func MakeJoin( return event, stateEvents, nil } - roomQuerier := JoinRoomQuerier{ - roomserver: rsAPI, + roomQuerier := api.JoinRoomQuerier{ + Roomserver: rsAPI, + } + + senderIDPtr, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID) + if err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + var senderID spec.SenderID + if senderIDPtr == nil { + senderID = spec.SenderID(userID.String()) + } else { + senderID = *senderIDPtr } input := gomatrixserverlib.HandleMakeJoinInput{ - Context: httpReq.Context(), - UserID: userID, - RoomID: roomID, - RoomVersion: roomVersion, - RemoteVersions: remoteVersions, - RequestOrigin: request.Origin(), - LocalServerName: cfg.Matrix.ServerName, - LocalServerInRoom: res.RoomExists && res.IsInRoom, - RoomQuerier: &roomQuerier, + Context: httpReq.Context(), + UserID: userID, + SenderID: senderID, + RoomID: roomID, + RoomVersion: roomVersion, + RemoteVersions: remoteVersions, + RequestOrigin: request.Origin(), + LocalServerName: cfg.Matrix.ServerName, + LocalServerInRoom: res.RoomExists && res.IsInRoom, + RoomQuerier: &roomQuerier, + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, BuildEventTemplate: createJoinTemplate, } response, internalErr := gomatrixserverlib.HandleMakeJoin(input) @@ -216,29 +189,7 @@ func MakeJoin( } } -type MembershipQuerier struct { - roomserver api.FederationRoomserverAPI -} - -func (mq *MembershipQuerier) CurrentMembership(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (string, error) { - req := api.QueryMembershipForUserRequest{ - RoomID: roomID.String(), - UserID: userID.String(), - } - res := api.QueryMembershipForUserResponse{} - err := mq.roomserver.QueryMembershipForUser(ctx, &req, &res) - - membership := "" - if err == nil { - membership = res.Membership - } - return membership, err -} - // SendJoin implements the /send_join API -// The make-join send-join dance makes much more sense as a single -// flow so the cyclomatic complexity is high: -// nolint:gocyclo func SendJoin( httpReq *http.Request, request *fclient.FederationRequest, @@ -268,7 +219,17 @@ func SendJoin( KeyID: cfg.Matrix.KeyID, PrivateKey: cfg.Matrix.PrivateKey, Verifier: keys, - MembershipQuerier: &MembershipQuerier{roomserver: rsAPI}, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, + StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error { + userID, userErr := spec.NewUserID(userIDRaw, true) + if userErr != nil { + return userErr + } + return rsAPI.StoreUserRoomPublicKey(ctx, senderID, *userID, roomID) + }, } response, joinErr := gomatrixserverlib.HandleSendJoin(input) switch e := joinErr.(type) { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3e576e09c6..7c86ba69bd 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -50,7 +50,7 @@ func MakeLeave( RoomID: roomID.String(), } res := api.QueryServerJoinedToRoomResponse{} - if err := rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryServerJoinedToRoom(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -59,24 +59,24 @@ func MakeLeave( } createLeaveTemplate := func(proto *gomatrixserverlib.ProtoEvent) (gomatrixserverlib.PDU, []gomatrixserverlib.PDU, error) { - identity, err := cfg.Matrix.SigningIdentityFor(request.Destination()) - if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Errorf("obtaining signing identity for %s failed", request.Destination()) + identity, signErr := cfg.Matrix.SigningIdentityFor(request.Destination()) + if signErr != nil { + util.GetLogger(httpReq.Context()).WithError(signErr).Errorf("obtaining signing identity for %s failed", request.Destination()) return nil, nil, spec.NotFound(fmt.Sprintf("Server name %q does not exist", request.Destination())) } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) - switch e := err.(type) { + event, buildErr := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes) + switch e := buildErr.(type) { case nil: case eventutil.ErrRoomNoExists: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.NotFound("Room does not exist") case gomatrixserverlib.BadJSONError: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.BadJSON(e.Error()) default: - util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(buildErr).Error("eventutil.BuildEvent failed") return nil, nil, spec.InternalServerError{} } @@ -87,14 +87,33 @@ func MakeLeave( return event, stateEvents, nil } + senderID, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID) + if err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } else if senderID == nil { + util.GetLogger(httpReq.Context()).WithField("roomID", roomID).WithField("userID", userID).Error("rsAPI.QuerySenderIDForUser returned nil sender ID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + input := gomatrixserverlib.HandleMakeLeaveInput{ UserID: userID, + SenderID: *senderID, RoomID: roomID, RoomVersion: roomVersion, RequestOrigin: request.Origin(), LocalServerName: cfg.Matrix.ServerName, LocalServerInRoom: res.RoomExists && res.IsInRoom, BuildEventTemplate: createLeaveTemplate, + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, } response, internalErr := gomatrixserverlib.HandleMakeLeave(input) @@ -192,7 +211,7 @@ func SendLeave( } // Check that the room ID is correct. - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("The room ID in the request path must match the room ID in the leave event JSON"), @@ -213,7 +232,7 @@ func SendLeave( JSON: spec.BadJSON("No state key was provided in the leave event."), } } - if !event.StateKeyEquals(event.Sender()) { + if !event.StateKeyEquals(string(event.SenderID())) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("Event state key must match the event sender."), @@ -223,13 +242,13 @@ func SendLeave( // Check that the sender belongs to the server that is sending us // the request. By this point we've already asserted that the sender // and the state key are equal so we don't need to check both. - var serverName spec.ServerName - if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil { + sender, err := rsAPI.QueryUserIDForSender(httpReq.Context(), event.RoomID(), event.SenderID()) + if err != nil { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("The sender of the join is invalid"), } - } else if serverName != request.Origin() { + } else if sender.Domain() != request.Origin() { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("The sender does not match the server that originated the request"), @@ -291,10 +310,10 @@ func SendLeave( } } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: serverName, - Message: redacted, - AtTS: event.OriginServerTS(), - StrictValidityChecking: true, + ServerName: sender.Domain(), + Message: redacted, + AtTS: event.OriginServerTS(), + ValidityCheckingFunc: gomatrixserverlib.StrictValiditySignatureCheck, }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index f57d302041..b1cefe7b4c 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -87,7 +87,7 @@ func filterEvents( ) []*types.HeaderedEvent { ref := events[:0] for _, ev := range events { - if ev.RoomID() == roomID { + if ev.RoomID().String() == roomID { ref = append(ref, ev) } } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index e6a488ba33..e8c7ff793d 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -15,9 +15,11 @@ package routing import ( + "errors" "fmt" "net/http" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -52,6 +54,12 @@ func GetProfile( profile, err := userAPI.QueryProfile(httpReq.Context(), userID) if err != nil { + if errors.Is(err, appserviceAPI.ErrProfileNotExists) { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The user does not exist or does not have a profile."), + } + } util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed") return util.JSONResponse{ Code: http.StatusInternalServerError, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 2e845f32ca..327ba9b08a 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -20,12 +20,14 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // RoomAliasToID converts the queried alias into a room ID and returns it @@ -116,3 +118,65 @@ func RoomAliasToID( JSON: resp, } } + +// Query the immediate children of a room/space +// +// Implements /_matrix/federation/v1/hierarchy/{roomID} +func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationRequest, roomIDStr string, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch httpReq.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) + discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1) + + if err != nil { + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(httpReq.Context()).WithError(err).Debugln("room unknown/forbidden when handling SS room hierarchy request") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (SS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + } + + if len(discoveredRooms) == 0 { + util.GetLogger(httpReq.Context()).Debugln("no rooms found when handling SS room hierarchy request") + return util.JSONResponse{ + Code: 404, + JSON: spec.NotFound("room is unknown/forbidden"), + } + } + return util.JSONResponse{ + Code: 200, + JSON: fclient.RoomHierarchyResponse{ + Room: discoveredRooms[0], + Children: discoveredRooms[1:], + }, + } +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index fad06c1cf1..dc7a363e72 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -78,6 +78,7 @@ func Setup( v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter() v2fedmux := fedMux.PathPrefix("/v2").Subrouter() + v3fedmux := fedMux.PathPrefix("/v3").Subrouter() wakeup := &FederationWakeups{ FsAPI: fsAPI, @@ -152,8 +153,16 @@ func Setup( JSON: spec.Forbidden("Forbidden by server ACLs"), } } + + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } return InviteV1( - httpReq, request, vars["roomID"], vars["eventID"], + httpReq, request, *roomID, vars["eventID"], cfg, rsAPI, keys, ) }, @@ -168,8 +177,47 @@ func Setup( JSON: spec.Forbidden("Forbidden by server ACLs"), } } + + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } return InviteV2( - httpReq, request, vars["roomID"], vars["eventID"], + httpReq, request, *roomID, vars["eventID"], + cfg, rsAPI, keys, + ) + }, + )).Methods(http.MethodPut, http.MethodOptions) + + v3fedmux.Handle("/invite/{roomID}/{userID}", MakeFedAPI( + "federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, + func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse { + if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("Forbidden by server ACLs"), + } + } + + userID, err := spec.NewUserID(vars["userID"], true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid UserID"), + } + } + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } + return InviteV3( + httpReq, request, *roomID, *userID, cfg, rsAPI, keys, ) }, @@ -548,6 +596,13 @@ func Setup( return GetOpenIDUserInfo(req, userAPI) }), ).Methods(http.MethodGet) + + v1fedmux.Handle("/hierarchy/{roomID}", MakeFedAPI( + "federation_room_hierarchy", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, + func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse { + return QueryRoomHierarchy(httpReq, request, vars["roomID"], rsAPI) + }, + )).Methods(http.MethodGet) } func ErrorIfLocalServerNotInRoom( diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 3c8e0cbeff..9666945414 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ import ( ) const ( - // Event was passed to the roomserver + // Event was passed to the Roomserver MetricsOutcomeOK = "ok" // Event failed to be processed MetricsOutcomeFail = "fail" diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index fa0e9351ea..d109105733 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -113,10 +113,10 @@ func getState( return nil, nil, resErr } - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")} } - resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID) + resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) if resErr != nil { return nil, nil, resErr } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index beeb52495a..42ba8bfe52 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -140,22 +140,31 @@ func ExchangeThirdPartyInvite( } } - _, senderDomain, err := cfg.Matrix.SplitLocalID('@', proto.Sender) + validRoomID, err := spec.NewRoomID(roomID) if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("Invalid sender ID: " + err.Error()), + JSON: spec.BadJSON("Invalid room ID"), } } + userID, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, spec.SenderID(proto.SenderID)) + if err != nil || userID == nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("Invalid sender ID"), + } + } + senderDomain := userID.Domain() // Check that the state key is correct. - _, targetDomain, err := gomatrixserverlib.SplitID('@', *proto.StateKey) - if err != nil { + targetUserID, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, spec.SenderID(*proto.StateKey)) + if err != nil || targetUserID == nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("The event's state key isn't a Matrix user ID"), } } + targetDomain := targetUserID.Domain() // Check that the target user is from the requesting homeserver. if targetDomain != request.Origin() { @@ -223,7 +232,7 @@ func ExchangeThirdPartyInvite( } } - // Send the event to the roomserver + // Send the event to the Roomserver if err = api.SendEvents( httpReq.Context(), rsAPI, api.KindNew, @@ -271,7 +280,7 @@ func createInviteFrom3PIDInvite( // Build the event proto := &gomatrixserverlib.ProtoEvent{ Type: "m.room.member", - Sender: inv.Sender, + SenderID: inv.Sender, RoomID: inv.RoomID, StateKey: &inv.MXID, } @@ -324,7 +333,7 @@ func buildMembershipEvent( return nil, errors.New("expecting state tuples for event builder, got none") } - // Ask the roomserver for information about this room + // Ask the Roomserver for information about this room queryReq := api.QueryLatestEventsAndStateRequest{ RoomID: protoEvent.RoomID, StateToFetch: eventsNeeded.Tuples(), diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go index 30665bc56b..2caa7a0552 100644 --- a/federationapi/storage/postgres/storage.go +++ b/federationapi/storage/postgres/storage.go @@ -36,7 +36,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) { +func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) { var d Database var err error if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go index 00c8afa059..524bf1d5b4 100644 --- a/federationapi/storage/sqlite3/storage.go +++ b/federationapi/storage/sqlite3/storage.go @@ -34,7 +34,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) { +func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (*Database, error) { var d Database var err error if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { diff --git a/federationapi/storage/storage.go b/federationapi/storage/storage.go index 322a6c75bb..f926b62e73 100644 --- a/federationapi/storage/storage.go +++ b/federationapi/storage/storage.go @@ -30,7 +30,7 @@ import ( ) // NewDatabase opens a new database -func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (Database, error) { +func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName) diff --git a/go.mod b/go.mod index 16e5adc8c8..c2ab105b13 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 - github.com/blevesearch/bleve/v2 v2.3.6 + github.com/blevesearch/bleve/v2 v2.3.8 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v20.10.24+incompatible + github.com/docker/docker v24.0.5+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 @@ -18,39 +18,41 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/kardianos/minwinsvc v1.0.2 - github.com/lib/pq v1.10.8 + github.com/lib/pq v1.10.9 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6 - github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a + github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 + github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 - github.com/mattn/go-sqlite3 v1.14.16 - github.com/nats-io/nats-server/v2 v2.9.15 - github.com/nats-io/nats.go v1.24.0 + github.com/mattn/go-sqlite3 v1.14.17 + github.com/nats-io/nats-server/v2 v2.9.19 + github.com/nats-io/nats.go v1.27.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.13.0 - github.com/sirupsen/logrus v1.9.2 + github.com/prometheus/client_golang v1.16.0 + github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.2 - github.com/tidwall/gjson v1.14.4 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.13.0 + golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 golang.org/x/image v0.5.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e - golang.org/x/term v0.8.0 + golang.org/x/sync v0.3.0 + golang.org/x/term v0.12.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 maunium.net/go/mautrix v0.15.1 - modernc.org/sqlite v1.19.3 + modernc.org/sqlite v1.23.1 nhooyr.io/websocket v1.8.7 ) @@ -74,75 +76,71 @@ require ( github.com/blevesearch/zapx/v12 v12.3.7 // indirect github.com/blevesearch/zapx/v13 v13.3.7 // indirect github.com/blevesearch/zapx/v14 v14.3.7 // indirect - github.com/blevesearch/zapx/v15 v15.3.8 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/blevesearch/zapx/v15 v15.3.10 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nats-io/jwt/v2 v2.3.0 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/jwt/v2 v2.4.1 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/onsi/ginkgo/v2 v2.3.0 // indirect - github.com/onsi/gomega v1.22.1 // indirect + github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/quic-go/qtls-go1-18 v0.2.0 // indirect - github.com/quic-go/qtls-go1-19 v0.2.0 // indirect - github.com/quic-go/qtls-go1-20 v0.1.0 // indirect - github.com/quic-go/quic-go v0.32.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/quic-go/qtls-go1-20 v0.3.2 // indirect + github.com/quic-go/quic-go v0.37.4 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/zerolog v1.29.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.6.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + golang.org/x/tools v0.12.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect maunium.net/go/maulogger/v2 v2.4.1 // indirect modernc.org/cc/v3 v3.40.0 // indirect - modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 // indirect - modernc.org/libc v1.21.4 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.4.0 // indirect + modernc.org/memory v1.5.0 // indirect modernc.org/opt v0.1.3 // indirect modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.1 // indirect ) -go 1.18 +go 1.20 diff --git a/go.sum b/go.sum index 98e7e839df..7f0e988382 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 h1:Usab30pNT2i/vZvpXcN9uOr5IO1RZPcUqoGH0DIAPnU= github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= @@ -37,7 +5,6 @@ github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -53,11 +20,6 @@ github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.1.1 h1:sHQCyj7HtiSfaZAzL2rJrQdyS7odLqlwO6nhk/tG/j8= @@ -69,15 +31,13 @@ github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQy github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4= -github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4= +github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ= +github.com/blevesearch/bleve/v2 v2.3.8/go.mod h1:Lh9aZEHrLKxwPnW4z4lsBEGnflZQ1V/aWP/t+htsiDw= github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= @@ -106,21 +66,15 @@ github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ub github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s= github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20= github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w= -github.com/blevesearch/zapx/v15 v15.3.8 h1:q4uMngBHzL1IIhRc8AJUEkj6dGOE3u1l3phLu7hq8uk= -github.com/blevesearch/zapx/v15 v15.3.8/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= +github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ= +github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -135,20 +89,17 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= -github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= +github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -162,16 +113,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -179,9 +121,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -190,7 +131,6 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -199,71 +139,32 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgR github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -273,26 +174,14 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= @@ -301,32 +190,28 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= -github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6 h1:FQpdh/KGCCQJytz4GAdG6pbx3DJ1HNzdKFc/BCZ0hP0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= -github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= -github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 h1:UuXfC7b29RBDfMdLmggeF3opu3XuGi8bNT9SKZtZc3I= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= +github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -334,13 +219,13 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= @@ -350,7 +235,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -358,16 +242,14 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= -github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/nats-server/v2 v2.9.15 h1:MuwEJheIwpvFgqvbs20W8Ish2azcygjf4Z0liVu2I4c= -github.com/nats-io/nats-server/v2 v2.9.15/go.mod h1:QlCTy115fqpx4KSOPFIxSV7DdI6OxtZsGOL1JLdeRlE= -github.com/nats-io/nats.go v1.24.0 h1:CRiD8L5GOQu/DcfkmgBcTTIQORMwizF+rPk6T0RaHVQ= -github.com/nats-io/nats.go v1.24.0/go.mod h1:dVQF+BK3SzUZpwyzHedXsvH3EO38aVKuOPkkHlv5hXA= -github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= -github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= +github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= +github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw= +github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00= +github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= +github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= @@ -375,10 +257,9 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHG github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= @@ -390,67 +271,39 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= -github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= -github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= -github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= -github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= -github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= +github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4= +github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -458,15 +311,15 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -486,259 +339,114 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/yggdrasil-network/yggdrasil-go v0.4.6 h1:GALUDV9QPz/5FVkbazpkTc9EABHufA556JwUJZr41j4= github.com/yggdrasil-network/yggdrasil-go v0.4.6/go.mod h1:PBMoAOvQjA9geNEeGyMXA9QgCS6Bu+9V+1VkWM84wpw= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -747,101 +455,21 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM= gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -850,13 +478,6 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= @@ -865,29 +486,26 @@ maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0= maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 h1:0+dsXf0zeLx9ixj4nilg6jKe5Bg1ilzBwSFq4kJmIUc= -modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/libc v1.21.4 h1:CzTlumWeIbPV5/HVIMzYHNPCRP8uiU/CWiN2gtd/Qu8= -modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.19.3 h1:dIoagx6yIQT3V/zOSeAyZ8OqQyEr17YTgETOXTZNJMA= -modernc.org/sqlite v1.19.3/go.mod h1:xiyJD7FY8mTZXnQwE/gEL1STtFrrnDx03V8KhVQmcr8= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= +modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= +modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/helm/cr.yaml b/helm/cr.yaml index 884c2b46bf..d39e8bdce2 100644 --- a/helm/cr.yaml +++ b/helm/cr.yaml @@ -1,2 +1,3 @@ release-name-template: "helm-{{ .Name }}-{{ .Version }}" -pages-index-path: docs/index.yaml \ No newline at end of file +pages-index-path: docs/index.yaml +make-release-latest: false \ No newline at end of file diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 182f53c391..6741520515 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.12.3" -appVersion: "0.12.0" +version: "0.13.4" +appVersion: "0.13.3" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index ca5705c036..cb78a54f21 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.12.0](https://img.shields.io/badge/AppVersion-0.12.0-informational?style=flat-square) +![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -48,13 +48,16 @@ Create a folder `appservices` and place your configurations in there. The confi | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | +| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | @@ -97,7 +100,7 @@ Create a folder `appservices` and place your configurations in there. The confi | dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | | dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | | dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | -| dendrite_config.mscs | object | `{"mscs":["msc2946"]}` | Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) | +| dendrite_config.mscs | object | `{"mscs":[]}` | Configuration for experimental MSC's. (Valid values are: msc2836) | | dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | | dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | | dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | @@ -144,12 +147,11 @@ Create a folder `appservices` and place your configurations in there. The confi | postgresql.auth.password | string | `"changeme"` | | | postgresql.auth.database | string | `"dendrite"` | | | postgresql.persistence.enabled | bool | `false` | | -| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | -| ingress.hosts | list | `[]` | | -| ingress.className | string | `""` | | -| ingress.hostName | string | `""` | | +| ingress.enabled | bool | `false` | Create an ingress for the deployment | +| ingress.className | string | `""` | The ingressClass to use. Will be converted to annotation if not yet supported. | | ingress.annotations | object | `{}` | Extra, custom annotations | -| ingress.tls | list | `[]` | | +| ingress.hostName | string | `""` | The ingress hostname for your matrix server. Should align with the server_name and well_known_* hosts. If not set, generated from the dendrite_config values. | +| ingress.tls | list | `[]` | TLS configuration. Should contain information for the server_name and well-known hosts. Alternatively, set tls.generate=true to generate defaults based on the dendrite_config. | | service.type | string | `"ClusterIP"` | | | service.port | int | `8008` | | | prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index df7dbbdc3d..e3f84cdaec 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -109,4 +109,4 @@ spec: failureThreshold: 10 httpGet: path: /_dendrite/monitor/up - port: http \ No newline at end of file + port: http diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml index 8f86ad7239..4bcaee12da 100644 --- a/helm/dendrite/templates/ingress.yaml +++ b/helm/dendrite/templates/ingress.yaml @@ -1,24 +1,25 @@ {{- if .Values.ingress.enabled -}} - {{- $fullName := include "dendrite.fullname" . -}} - {{- $svcPort := .Values.service.port -}} - {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} - {{- end }} - {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +{{- $fullName := include "dendrite.fullname" . -}} +{{- $serverNameHost := .Values.dendrite_config.global.server_name -}} +{{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}} +{{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}} +{{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 - {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 - {{- else -}} +{{- else -}} apiVersion: extensions/v1beta1 - {{- end }} +{{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "dendrite.labels" . | nindent 4 }} annotations: + {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + kubernetes.io/ingress.class: {{ .Values.ingress.className }} + {{- end }} {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} @@ -26,7 +27,7 @@ spec: {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ .Values.ingress.className }} {{- end }} - {{- if .Values.ingress.tls }} + {{- if kindIs "slice" .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} - hosts: @@ -35,8 +36,16 @@ spec: {{- end }} secretName: {{ .secretName }} {{- end }} + {{- else if .Values.ingress.tls.generate }} + tls: + - hosts: + {{- range $allHosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ $fullName }}-ingress-tls {{- end }} rules: + {{- if .Values.ingress.hostName }} - host: {{ .Values.ingress.hostName | quote }} http: paths: @@ -47,9 +56,60 @@ spec: service: name: {{ $fullName }} port: - number: {{ $svcPort }} + name: http + {{- else }} + serviceName: {{ $fullName }} + servicePort: http + {{- end }} + {{- else }} + - host: {{ $serverNameHost | quote }} + http: + paths: + - path: /.well-known/matrix + pathType: Prefix + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + name: http {{- else }} serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} + servicePort: http {{- end }} - {{- end }} \ No newline at end of file + - host: {{ $wellKnownServerHost | quote }} + http: + paths: + {{- range list "/_matrix/key" "/_matrix/federation" }} + - path: {{ . | quote }} + pathType: Prefix + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + name: http + {{- else }} + serviceName: {{ $fullName }} + servicePort: http + {{- end }} + {{- end }} + - host: {{ $wellKnownClientHost | quote }} + http: + paths: + {{- range list "/_matrix/client" "/_matrix/media" }} + - path: {{ . | quote }} + pathType: Prefix + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + name: http + {{- else }} + serviceName: {{ $fullName }} + servicePort: http + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/dendrite/templates/prometheus-rules.yaml b/helm/dendrite/templates/prometheus-rules.yaml index 6693a4ed9d..dc6c12cf9d 100644 --- a/helm/dendrite/templates/prometheus-rules.yaml +++ b/helm/dendrite/templates/prometheus-rules.yaml @@ -6,7 +6,9 @@ metadata: name: {{ include "dendrite.fullname" . }} labels: {{- include "dendrite.labels" . | nindent 4 }} - {{- toYaml .Values.prometheus.rules.labels | nindent 4 }} + {{- with .Values.prometheus.rules.labels }} + {{- . | toYaml | nindent 4 }} + {{- end }} spec: groups: {{- if .Values.prometheus.rules.additionalRules }} diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml index 897957e600..88eff3bede 100644 --- a/helm/dendrite/templates/pvc.yaml +++ b/helm/dendrite/templates/pvc.yaml @@ -12,7 +12,7 @@ spec: resources: requests: storage: {{ .Values.persistence.media.capacity }} - storageClassName: {{ .Values.persistence.storageClass }} + storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }} {{ end }} {{ if not .Values.persistence.jetstream.existingClaim }} --- @@ -28,7 +28,7 @@ spec: resources: requests: storage: {{ .Values.persistence.jetstream.capacity }} - storageClassName: {{ .Values.persistence.storageClass }} + storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }} {{ end }} {{ if not .Values.persistence.search.existingClaim }} --- @@ -44,5 +44,5 @@ spec: resources: requests: storage: {{ .Values.persistence.search.capacity }} - storageClassName: {{ .Values.persistence.storageClass }} -{{ end }} \ No newline at end of file + storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }} +{{ end }} diff --git a/helm/dendrite/templates/servicemonitor.yaml b/helm/dendrite/templates/servicemonitor.yaml index 3819c7d020..4602140f87 100644 --- a/helm/dendrite/templates/servicemonitor.yaml +++ b/helm/dendrite/templates/servicemonitor.yaml @@ -9,7 +9,9 @@ metadata: name: {{ include "dendrite.fullname" . }} labels: {{- include "dendrite.labels" . | nindent 4 }} - {{- toYaml .Values.prometheus.servicemonitor.labels | nindent 4 }} + {{- with .Values.prometheus.servicemonitor.labels }} + {{- . | toYaml | nindent 4 }} + {{- end }} spec: endpoints: - port: http diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 41ec1c3906..8a72f66930 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -19,29 +19,38 @@ signing_key: resources: requests: memory: "512Mi" - limits: memory: "4096Mi" persistence: - # -- The storage class to use for volume claims. Defaults to the - # cluster default storage class. + # -- The storage class to use for volume claims. + # Used unless specified at the specific component. + # Defaults to the cluster default storage class. storageClass: "" jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" # -- PVC Storage Request for the jetstream volume capacity: "1Gi" + # -- The storage class to use for volume claims. + # Defaults to persistence.storageClass + storageClass: "" media: # -- Use an existing volume claim for media files existingClaim: "" # -- PVC Storage Request for the media volume capacity: "1Gi" + # -- The storage class to use for volume claims. + # Defaults to persistence.storageClass + storageClass: "" search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" # -- PVC Storage Request for the search volume capacity: "1Gi" + # -- The storage class to use for volume claims. + # Defaults to persistence.storageClass + storageClass: "" # -- Add additional volumes to the Dendrite Pod extraVolumes: [] @@ -50,7 +59,6 @@ extraVolumes: [] # secret: # secretName: extra-config - # -- Configure additional mount points volumes in the Dendrite Pod extraVolumeMounts: [] # ex. @@ -203,15 +211,12 @@ dendrite_config: # -- pprof port, if enabled port: 65432 - # -- Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) + # -- Configuration for experimental MSC's. (Valid values are: msc2836) mscs: - mscs: - - msc2946 + mscs: [] # A list of enabled MSC's # Currently valid values are: # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) - app_service_api: # -- Disable the validation of TLS certificates of appservices. This is @@ -359,14 +364,18 @@ postgresql: enabled: false ingress: - # -- Create an ingress for a monolith deployment + # -- Create an ingress for the deployment enabled: false - hosts: [] + # -- The ingressClass to use. Will be converted to annotation if not yet supported. className: "" - hostName: "" # -- Extra, custom annotations annotations: {} - + # -- The ingress hostname for your matrix server. + # Should align with the server_name and well_known_* hosts. + # If not set, generated from the dendrite_config values. + hostName: "" + # -- TLS configuration. Should contain information for the server_name and well-known hosts. + # Alternatively, set tls.generate=true to generate defaults based on the dendrite_config. tls: [] service: diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go index 734a3a04fa..fa0781ef3a 100644 --- a/internal/caching/cache_roomservernids.go +++ b/internal/caching/cache_roomservernids.go @@ -8,6 +8,7 @@ type RoomServerCaches interface { RoomServerNIDsCache RoomVersionCache RoomServerEventsCache + RoomHierarchyCache EventStateKeyCache EventTypeCache } diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go index 37e331ab07..7400b868cf 100644 --- a/internal/caching/cache_serverkeys.go +++ b/internal/caching/cache_serverkeys.go @@ -28,7 +28,7 @@ func (c Caches) GetServerKey( ) (gomatrixserverlib.PublicKeyLookupResult, bool) { key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) val, found := c.ServerKeys.Get(key) - if found && !val.WasValidAt(timestamp, true) { + if found && !val.WasValidAt(timestamp, gomatrixserverlib.StrictValiditySignatureCheck) { // The key wasn't valid at the requested timestamp so don't // return it. The caller will have to work out what to do. c.ServerKeys.Unset(key) diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go index 100ab9023e..90eeb7861c 100644 --- a/internal/caching/cache_space_rooms.go +++ b/internal/caching/cache_space_rooms.go @@ -2,15 +2,16 @@ package caching import "github.com/matrix-org/gomatrixserverlib/fclient" -type SpaceSummaryRoomsCache interface { - GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) - StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) +// RoomHierarchy cache caches responses to federated room hierarchy requests (A.K.A. 'space summaries') +type RoomHierarchyCache interface { + GetRoomHierarchy(roomID string) (r fclient.RoomHierarchyResponse, ok bool) + StoreRoomHierarchy(roomID string, r fclient.RoomHierarchyResponse) } -func (c Caches) GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) { - return c.SpaceSummaryRooms.Get(roomID) +func (c Caches) GetRoomHierarchy(roomID string) (r fclient.RoomHierarchyResponse, ok bool) { + return c.RoomHierarchies.Get(roomID) } -func (c Caches) StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) { - c.SpaceSummaryRooms.Set(roomID, r) +func (c Caches) StoreRoomHierarchy(roomID string, r fclient.RoomHierarchyResponse) { + c.RoomHierarchies.Set(roomID, r) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 6bae60d59f..16e547578f 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -35,7 +35,7 @@ type Caches struct { RoomServerEventTypes Cache[types.EventTypeNID, string] // eventType NID -> eventType FederationPDUs Cache[int64, *types.HeaderedEvent] // queue NID -> PDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU - SpaceSummaryRooms Cache[string, fclient.MSC2946SpacesResponse] // room ID -> space response + RoomHierarchies Cache[string, fclient.RoomHierarchyResponse] // room ID -> space response LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 00989b7601..97ea9548f6 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -147,7 +147,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: lesserOf(time.Hour/2, maxAge), }, }, - SpaceSummaryRooms: &RistrettoCachePartition[string, fclient.MSC2946SpacesResponse]{ // room ID -> space response + RoomHierarchies: &RistrettoCachePartition[string, fclient.RoomHierarchyResponse]{ // room ID -> space response cache: cache, Prefix: spaceSummaryRoomsCache, Mutable: true, diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index ca052c3105..40d62fd68e 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -22,7 +22,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -51,7 +51,7 @@ func (e ErrRoomNoExists) Unwrap() error { // Returns an error if something else went wrong func QueryAndBuildEvent( ctx context.Context, - proto *gomatrixserverlib.ProtoEvent, cfg *config.Global, + proto *gomatrixserverlib.ProtoEvent, identity *fclient.SigningIdentity, evTime time.Time, rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*types.HeaderedEvent, error) { @@ -64,14 +64,14 @@ func QueryAndBuildEvent( // This can pass through a ErrRoomNoExists to the caller return nil, err } - return BuildEvent(ctx, proto, cfg, identity, evTime, eventsNeeded, queryRes) + return BuildEvent(ctx, proto, identity, evTime, eventsNeeded, queryRes) } // BuildEvent builds a Matrix event from the builder and QueryLatestEventsAndStateResponse // provided. func BuildEvent( ctx context.Context, - proto *gomatrixserverlib.ProtoEvent, cfg *config.Global, + proto *gomatrixserverlib.ProtoEvent, identity *fclient.SigningIdentity, evTime time.Time, eventsNeeded *gomatrixserverlib.StateNeeded, queryRes *api.QueryLatestEventsAndStateResponse, ) (*types.HeaderedEvent, error) { @@ -170,13 +170,20 @@ func truncateAuthAndPrevEvents(auth, prev []string) ( // RedactEvent redacts the given event and sets the unsigned field appropriately. This should be used by // downstream components to the roomserver when an OutputTypeRedactedEvent occurs. -func RedactEvent(redactionEvent, redactedEvent gomatrixserverlib.PDU) error { +func RedactEvent(ctx context.Context, redactionEvent, redactedEvent gomatrixserverlib.PDU, querier api.QuerySenderIDAPI) error { // sanity check if redactionEvent.Type() != spec.MRoomRedaction { return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type()) } redactedEvent.Redact() - if err := redactedEvent.SetUnsignedField("redacted_because", redactionEvent); err != nil { + clientEvent, err := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return querier.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + return err + } + redactedBecause := clientEvent + if err := redactedEvent.SetUnsignedField("redacted_because", redactedBecause); err != nil { return err } // NOTSPEC: sytest relies on this unspecced field existing :( diff --git a/internal/pushrules/evaluate.go b/internal/pushrules/evaluate.go index 7c98efd30f..6baef4347b 100644 --- a/internal/pushrules/evaluate.go +++ b/internal/pushrules/evaluate.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) // A RuleSetEvaluator encapsulates context to evaluate an event @@ -27,7 +28,7 @@ type EvaluationContext interface { // HasPowerLevel returns whether the user has at least the given // power in the room of the current event. - HasPowerLevel(userID, levelKey string) (bool, error) + HasPowerLevel(senderID spec.SenderID, levelKey string) (bool, error) } // A kindAndRules is just here to simplify iteration of the (ordered) @@ -53,7 +54,7 @@ func NewRuleSetEvaluator(ec EvaluationContext, ruleSet *RuleSet) *RuleSetEvaluat // MatchEvent returns the first matching rule. Returns nil if there // was no match rule. -func (rse *RuleSetEvaluator) MatchEvent(event gomatrixserverlib.PDU) (*Rule, error) { +func (rse *RuleSetEvaluator) MatchEvent(event gomatrixserverlib.PDU, userIDForSender spec.UserIDForSender) (*Rule, error) { // TODO: server-default rules have lower priority than user rules, // but they are stored together with the user rules. It's a bit // unclear what the specification (11.14.1.4 Predefined rules) @@ -68,7 +69,7 @@ func (rse *RuleSetEvaluator) MatchEvent(event gomatrixserverlib.PDU) (*Rule, err if rule.Default != defRules { continue } - ok, err := ruleMatches(rule, rsat.Kind, event, rse.ec) + ok, err := ruleMatches(rule, rsat.Kind, event, rse.ec, userIDForSender) if err != nil { return nil, err } @@ -83,7 +84,7 @@ func (rse *RuleSetEvaluator) MatchEvent(event gomatrixserverlib.PDU) (*Rule, err return nil, nil } -func ruleMatches(rule *Rule, kind Kind, event gomatrixserverlib.PDU, ec EvaluationContext) (bool, error) { +func ruleMatches(rule *Rule, kind Kind, event gomatrixserverlib.PDU, ec EvaluationContext, userIDForSender spec.UserIDForSender) (bool, error) { if !rule.Enabled { return false, nil } @@ -110,10 +111,15 @@ func ruleMatches(rule *Rule, kind Kind, event gomatrixserverlib.PDU, ec Evaluati return patternMatches("content.body", *rule.Pattern, event) case RoomKind: - return rule.RuleID == event.RoomID(), nil + return rule.RuleID == event.RoomID().String(), nil case SenderKind: - return rule.RuleID == event.Sender(), nil + userID := "" + sender, err := userIDForSender(event.RoomID(), event.SenderID()) + if err == nil { + userID = sender.String() + } + return rule.RuleID == userID, nil default: return false, nil @@ -143,7 +149,7 @@ func conditionMatches(cond *Condition, event gomatrixserverlib.PDU, ec Evaluatio return cmp(n), nil case SenderNotificationPermissionCondition: - return ec.HasPowerLevel(event.Sender(), cond.Key) + return ec.HasPowerLevel(event.SenderID(), cond.Key) default: return false, nil diff --git a/internal/pushrules/evaluate_test.go b/internal/pushrules/evaluate_test.go index 5045a864eb..fbc88b2e75 100644 --- a/internal/pushrules/evaluate_test.go +++ b/internal/pushrules/evaluate_test.go @@ -5,10 +5,15 @@ import ( "github.com/google/go-cmp/cmp" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) +func UserIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func TestRuleSetEvaluatorMatchEvent(t *testing.T) { - ev := mustEventFromJSON(t, `{}`) + ev := mustEventFromJSON(t, `{"room_id":"!room:a"}`) defaultEnabled := &Rule{ RuleID: ".default.enabled", Default: true, @@ -39,13 +44,13 @@ func TestRuleSetEvaluatorMatchEvent(t *testing.T) { {"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled, ev}, - {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"type":"m.reaction"}`)}, - {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"type":"m.receipt"}`)}, + {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.reaction"}`)}, + {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.receipt"}`)}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { rse := NewRuleSetEvaluator(fakeEvaluationContext{3}, &tst.RuleSet) - got, err := rse.MatchEvent(tst.Event) + got, err := rse.MatchEvent(tst.Event, UserIDForSender) if err != nil { t.Fatalf("MatchEvent failed: %v", err) } @@ -65,32 +70,31 @@ func TestRuleMatches(t *testing.T) { EventJSON string Want bool }{ - {"emptyOverride", OverrideKind, emptyRule, `{}`, true}, - {"emptyContent", ContentKind, emptyRule, `{}`, false}, - {"emptyRoom", RoomKind, emptyRule, `{}`, true}, - {"emptySender", SenderKind, emptyRule, `{}`, true}, - {"emptyUnderride", UnderrideKind, emptyRule, `{}`, true}, + {"emptyOverride", OverrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, + {"emptyContent", ContentKind, emptyRule, `{"room_id":"!room:example.com"}`, false}, + {"emptySender", SenderKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, + {"emptyUnderride", UnderrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, - {"disabled", OverrideKind, Rule{}, `{}`, false}, + {"disabled", OverrideKind, Rule{}, `{"room_id":"!room:example.com"}`, false}, - {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{}`, true}, - {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{}`, true}, - {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"content":{"body":"abc"}}`, true}, - {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"content":{"body":"abc"}}`, false}, + {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, true}, + {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, false}, - {"roomMatch", RoomKind, Rule{Enabled: true, RuleID: "!room@example.com"}, `{"room_id":"!room@example.com"}`, true}, - {"roomNoMatch", RoomKind, Rule{Enabled: true, RuleID: "!room@example.com"}, `{"room_id":"!otherroom@example.com"}`, false}, + {"roomMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!room:example.com"}`, true}, + {"roomNoMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!otherroom:example.com"}`, false}, - {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user@example.com"}, `{"sender":"@user@example.com"}`, true}, - {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user@example.com"}, `{"sender":"@otheruser@example.com"}`, false}, + {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@user:example.com","room_id":"!room:example.com"}`, true}, + {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { - got, err := ruleMatches(&tst.Rule, tst.Kind, mustEventFromJSON(t, tst.EventJSON), nil) + got, err := ruleMatches(&tst.Rule, tst.Kind, mustEventFromJSON(t, tst.EventJSON), nil, UserIDForSender) if err != nil { t.Fatalf("ruleMatches failed: %v", err) } @@ -109,32 +113,32 @@ func TestConditionMatches(t *testing.T) { WantMatch bool WantErr bool }{ - {Name: "empty", Cond: Condition{}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, // Neither of these should match because `content` is not a full string match, // and `content.body` is not a string value. - {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"content":{}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"content":{"body": "3"}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, - {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, - - {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, - {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, - - {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - - {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, - {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "3"}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, + {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, + + {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, + {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, + + {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + + {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -153,8 +157,8 @@ type fakeEvaluationContext struct{ memberCount int } func (fakeEvaluationContext) UserDisplayName() string { return "Dear User" } func (f fakeEvaluationContext) RoomMemberCount() (int, error) { return f.memberCount, nil } -func (fakeEvaluationContext) HasPowerLevel(userID, levelKey string) (bool, error) { - return userID == "@poweruser:example.com" && levelKey == "powerlevel", nil +func (fakeEvaluationContext) HasPowerLevel(senderID spec.SenderID, levelKey string) (bool, error) { + return senderID == "@poweruser:example.com" && levelKey == "powerlevel", nil } func TestPatternMatches(t *testing.T) { @@ -165,15 +169,15 @@ func TestPatternMatches(t *testing.T) { EventJSON string Want bool }{ - {"empty", "", "", `{}`, false}, + {"empty", "", "", `{"room_id":"!room:a"}`, false}, - {"patternEmpty", "content", "", `{"content":{}}`, false}, + {"patternEmpty", "content", "", `{"room_id":"!room:a","content":{}}`, false}, - {"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true}, - {"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true}, - {"singlePattern", "content.creator", "acr?ator", `{"content":{"creator":"acreator"}}`, true}, - {"multiPattern", "content.creator", "a*ea*r", `{"content":{"creator":"acreator"}}`, true}, - {"patternNoSubstring", "content.creator", "r*t", `{"content":{"creator":"acreator"}}`, false}, + {"literal", "content.creator", "acreator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"substring", "content.creator", "reat", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"singlePattern", "content.creator", "acr?ator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"multiPattern", "content.creator", "a*ea*r", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"patternNoSubstring", "content.creator", "r*t", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { diff --git a/internal/sqlutil/connection_manager.go b/internal/sqlutil/connection_manager.go index 934a2954a1..437da6c807 100644 --- a/internal/sqlutil/connection_manager.go +++ b/internal/sqlutil/connection_manager.go @@ -17,59 +17,70 @@ package sqlutil import ( "database/sql" "fmt" + "sync" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" ) type Connections struct { - db *sql.DB - writer Writer - globalConfig config.DatabaseOptions - processContext *process.ProcessContext + globalConfig config.DatabaseOptions + processContext *process.ProcessContext + existingConnections sync.Map } -func NewConnectionManager(processCtx *process.ProcessContext, globalConfig config.DatabaseOptions) Connections { - return Connections{ +type con struct { + db *sql.DB + writer Writer +} + +func NewConnectionManager(processCtx *process.ProcessContext, globalConfig config.DatabaseOptions) *Connections { + return &Connections{ globalConfig: globalConfig, processContext: processCtx, } } func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, Writer, error) { - writer := NewDummyWriter() - if dbProperties.ConnectionString.IsSQLite() { - writer = NewExclusiveWriter() - } var err error + // If no connectionString was provided, try the global one if dbProperties.ConnectionString == "" { - // if no connectionString was provided, try the global one dbProperties = &c.globalConfig - } - if dbProperties.ConnectionString != "" || c.db == nil { - // Open a new database connection using the supplied config. - c.db, err = Open(dbProperties, writer) - if err != nil { - return nil, nil, err + // If we still don't have a connection string, that's a problem + if dbProperties.ConnectionString == "" { + return nil, nil, fmt.Errorf("no database connections configured") } - c.writer = writer - go func() { - if c.processContext == nil { - return - } - // If we have a ProcessContext, start a component and wait for - // Dendrite to shut down to cleanly close the database connection. - c.processContext.ComponentStarted() - <-c.processContext.WaitForShutdown() - _ = c.db.Close() - c.processContext.ComponentFinished() - }() - return c.db, c.writer, nil } - if c.db != nil && c.writer != nil { - // Ignore the supplied config and return the global pool and - // writer. - return c.db, c.writer, nil + + writer := NewDummyWriter() + if dbProperties.ConnectionString.IsSQLite() { + writer = NewExclusiveWriter() + } + + existing, loaded := c.existingConnections.LoadOrStore(dbProperties.ConnectionString, &con{}) + if loaded { + // We found an existing connection + ex := existing.(*con) + return ex.db, ex.writer, nil } - return nil, nil, fmt.Errorf("no database connections configured") + + // Open a new database connection using the supplied config. + db, err := Open(dbProperties, writer) + if err != nil { + return nil, nil, err + } + c.existingConnections.Store(dbProperties.ConnectionString, &con{db: db, writer: writer}) + go func() { + if c.processContext == nil { + return + } + // If we have a ProcessContext, start a component and wait for + // Dendrite to shut down to cleanly close the database connection. + c.processContext.ComponentStarted() + <-c.processContext.WaitForShutdown() + _ = db.Close() + c.processContext.ComponentFinished() + }() + return db, writer, nil + } diff --git a/internal/sqlutil/connection_manager_test.go b/internal/sqlutil/connection_manager_test.go index a9ac8d57f4..5086684b5c 100644 --- a/internal/sqlutil/connection_manager_test.go +++ b/internal/sqlutil/connection_manager_test.go @@ -6,51 +6,135 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" ) func TestConnectionManager(t *testing.T) { - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - conStr, close := test.PrepareDBConnectionString(t, dbType) - t.Cleanup(close) - cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) - - dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)} - db, writer, err := cm.Connection(dbProps) - if err != nil { - t.Fatal(err) - } - - switch dbType { - case test.DBTypeSQLite: - _, ok := writer.(*sqlutil.ExclusiveWriter) - if !ok { - t.Fatalf("expected exclusive writer") - } - case test.DBTypePostgres: - _, ok := writer.(*sqlutil.DummyWriter) - if !ok { - t.Fatalf("expected dummy writer") - } - } - - // test global db pool - dbGlobal, writerGlobal, err := cm.Connection(&config.DatabaseOptions{}) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(db, dbGlobal) { - t.Fatalf("expected database connection to be reused") - } - if !reflect.DeepEqual(writer, writerGlobal) { - t.Fatalf("expected database writer to be reused") - } - - // test invalid connection string configured - cm2 := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) - _, _, err = cm2.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) - if err == nil { - t.Fatal("expected an error but got none") - } + + t.Run("component defined connection string", func(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + t.Cleanup(close) + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) + + dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)} + db, writer, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + + switch dbType { + case test.DBTypeSQLite: + _, ok := writer.(*sqlutil.ExclusiveWriter) + if !ok { + t.Fatalf("expected exclusive writer") + } + case test.DBTypePostgres: + _, ok := writer.(*sqlutil.DummyWriter) + if !ok { + t.Fatalf("expected dummy writer") + } + } + + // reuse existing connection + db2, writer2, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(db, db2) { + t.Fatalf("expected database connection to be reused") + } + if !reflect.DeepEqual(writer, writer2) { + t.Fatalf("expected database writer to be reused") + } + + // This test does not work with Postgres, because we can't just simply append + // "x" or replace the database to use. + if dbType == test.DBTypePostgres { + return + } + + // Test different connection string + dbProps = &config.DatabaseOptions{ConnectionString: config.DataSource(conStr + "x")} + db3, _, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(db, db3) { + t.Fatalf("expected different database connection") + } + }) + }) + + t.Run("global connection pool", func(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + t.Cleanup(close) + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}) + + dbProps := &config.DatabaseOptions{} + db, writer, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + + switch dbType { + case test.DBTypeSQLite: + _, ok := writer.(*sqlutil.ExclusiveWriter) + if !ok { + t.Fatalf("expected exclusive writer") + } + case test.DBTypePostgres: + _, ok := writer.(*sqlutil.DummyWriter) + if !ok { + t.Fatalf("expected dummy writer") + } + } + + // reuse existing connection + db2, writer2, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(db, db2) { + t.Fatalf("expected database connection to be reused") + } + if !reflect.DeepEqual(writer, writer2) { + t.Fatalf("expected database writer to be reused") + } + }) }) + + t.Run("shutdown", func(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + t.Cleanup(close) + + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}) + + dbProps := &config.DatabaseOptions{} + _, _, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + + processCtx.ShutdownDendrite() + processCtx.WaitForComponentsToFinish() + }) + }) + + // test invalid connection string configured + cm2 := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) + _, _, err := cm2.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) + if err == nil { + t.Fatal("expected an error but got none") + } + + // empty connection string is not allowed + _, _, err = cm2.Connection(&config.DatabaseOptions{}) + if err == nil { + t.Fatal("expected an error but got none") + } } diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go index a66a75826e..735fb4927c 100644 --- a/internal/sqlutil/migrate.go +++ b/internal/sqlutil/migrate.go @@ -112,7 +112,13 @@ func (m *Migrator) Up(ctx context.Context) error { func (m *Migrator) insertMigration(ctx context.Context, txn *sql.Tx, migrationName string) error { if m.insertStmt == nil { - stmt, err := m.db.Prepare(insertVersionSQL) + var stmt *sql.Stmt + var err error + if txn == nil { + stmt, err = m.db.PrepareContext(ctx, insertVersionSQL) + } else { + stmt, err = txn.PrepareContext(ctx, insertVersionSQL) + } if err != nil { return fmt.Errorf("unable to prepare insert statement: %w", err) } diff --git a/internal/transactionrequest.go b/internal/transactionrequest.go index c9d321f259..0663c2dcbe 100644 --- a/internal/transactionrequest.go +++ b/internal/transactionrequest.go @@ -161,13 +161,15 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*fclient.RespSend, *ut if event.Type() == spec.MRoomCreate && event.StateKeyEquals("") { continue } - if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) { + if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID().String(), t.Origin) { results[event.EventID()] = fclient.PDUResult{ Error: "Forbidden by server ACLs", } continue } - if err = gomatrixserverlib.VerifyEventSignatures(ctx, event, t.keys); err != nil { + if err = gomatrixserverlib.VerifyEventSignatures(ctx, event, t.keys, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { util.GetLogger(ctx).WithError(err).Debugf("Transaction: Couldn't validate signature of event %q", event.EventID()) results[event.EventID()] = fclient.PDUResult{ Error: err.Error(), diff --git a/internal/transactionrequest_test.go b/internal/transactionrequest_test.go index fb30d410e5..ffc1cd89ab 100644 --- a/internal/transactionrequest_test.go +++ b/internal/transactionrequest_test.go @@ -70,6 +70,10 @@ type FakeRsAPI struct { bannedFromRoom bool } +func (r *FakeRsAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func (r *FakeRsAPI) QueryRoomVersionForRoom( ctx context.Context, roomID string, @@ -638,6 +642,10 @@ type testRoomserverAPI struct { queryLatestEventsAndState func(*rsAPI.QueryLatestEventsAndStateRequest) rsAPI.QueryLatestEventsAndStateResponse } +func (t *testRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func (t *testRoomserverAPI) InputRoomEvents( ctx context.Context, request *rsAPI.InputRoomEventsRequest, diff --git a/internal/version.go b/internal/version.go index 9075475891..1f8a62bc2f 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,6 +2,7 @@ package internal import ( "fmt" + "runtime/debug" "strings" ) @@ -16,9 +17,11 @@ var build string const ( VersionMajor = 0 - VersionMinor = 12 - VersionPatch = 0 + VersionMinor = 13 + VersionPatch = 3 VersionTag = "" // example: "rc1" + + gitRevLen = 7 // 7 matches the displayed characters on github.com ) func VersionString() string { @@ -37,7 +40,30 @@ func init() { if branch != "" { parts = append(parts, branch) } - if len(parts) > 0 { - version += "+" + strings.Join(parts, ".") + + defer func() { + if len(parts) > 0 { + version += "+" + strings.Join(parts, ".") + } + }() + + // Try to get the revision Dendrite was build from. + // If we can't, e.g. Dendrite wasn't built (go run) or no VCS version is present, + // we just use the provided version above. + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + revLen := len(setting.Value) + if revLen >= gitRevLen { + parts = append(parts, setting.Value[:gitRevLen]) + } else { + parts = append(parts, setting.Value[:revLen]) + } + break + } } } diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 284071a536..3425fbce65 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -28,7 +28,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( mediaRouter *mux.Router, - cm sqlutil.Connections, + cm *sqlutil.Connections, cfg *config.Dendrite, userAPI userapi.MediaUserAPI, client *fclient.Client, diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index e9f161a3c2..51afa1f9f3 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "mime" "net/http" "net/url" @@ -126,6 +127,17 @@ func Download( activeRemoteRequests, activeThumbnailGeneration, ) if err != nil { + // If we bubbled up a os.PathError, e.g. no such file or directory, don't send + // it to the client, be more generic. + var perr *fs.PathError + if errors.As(err, &perr) { + dReq.Logger.WithError(err).Error("failed to open file") + dReq.jsonErrorResponse(w, util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("File not found"), + }) + return + } // TODO: Handle the fact we might have started writing the response dReq.jsonErrorResponse(w, util.JSONResponse{ Code: http.StatusNotFound, @@ -341,6 +353,7 @@ func (r *downloadRequest) addDownloadFilenameToHeaders( } if len(filename) == 0 { + w.Header().Set("Content-Disposition", "attachment") return nil } @@ -376,13 +389,13 @@ func (r *downloadRequest) addDownloadFilenameToHeaders( // that would otherwise be parsed as a control character in the // Content-Disposition header w.Header().Set("Content-Disposition", fmt.Sprintf( - `inline; filename=%s%s%s`, + `attachment; filename=%s%s%s`, quote, unescaped, quote, )) } else { // For UTF-8 filenames, we quote always, as that's the standard w.Header().Set("Content-Disposition", fmt.Sprintf( - `inline; filename*=utf-8''%s`, + `attachment; filename*=utf-8''%s`, url.QueryEscape(unescaped), )) } diff --git a/mediaapi/storage/postgres/mediaapi.go b/mediaapi/storage/postgres/mediaapi.go index 5b66877435..e2a2b25ce2 100644 --- a/mediaapi/storage/postgres/mediaapi.go +++ b/mediaapi/storage/postgres/mediaapi.go @@ -24,7 +24,7 @@ import ( ) // NewDatabase opens a postgres database. -func NewDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { +func NewDatabase(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err diff --git a/mediaapi/storage/sqlite3/mediaapi.go b/mediaapi/storage/sqlite3/mediaapi.go index 4d484f326c..086beb8e2e 100644 --- a/mediaapi/storage/sqlite3/mediaapi.go +++ b/mediaapi/storage/sqlite3/mediaapi.go @@ -23,7 +23,7 @@ import ( ) // NewDatabase opens a SQLIte database. -func NewDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { +func NewDatabase(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 8e67af9f9d..71ab72077b 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -27,7 +27,7 @@ import ( ) // NewMediaAPIDatasource opens a database connection. -func NewMediaAPIDatasource(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { +func NewMediaAPIDatasource(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(conMan, dbProperties) diff --git a/relayapi/relayapi.go b/relayapi/relayapi.go index acabb4b207..4402274954 100644 --- a/relayapi/relayapi.go +++ b/relayapi/relayapi.go @@ -53,7 +53,7 @@ func AddPublicRoutes( func NewRelayInternalAPI( dendriteCfg *config.Dendrite, - cm sqlutil.Connections, + cm *sqlutil.Connections, fedClient fclient.FederationClient, rsAPI rsAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, diff --git a/relayapi/storage/postgres/storage.go b/relayapi/storage/postgres/storage.go index 35c08c283d..dd30c1b563 100644 --- a/relayapi/storage/postgres/storage.go +++ b/relayapi/storage/postgres/storage.go @@ -33,7 +33,7 @@ type Database struct { // NewDatabase opens a new database func NewDatabase( - conMan sqlutil.Connections, + conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool, diff --git a/relayapi/storage/sqlite3/storage.go b/relayapi/storage/sqlite3/storage.go index 7b46396fdb..69df401e61 100644 --- a/relayapi/storage/sqlite3/storage.go +++ b/relayapi/storage/sqlite3/storage.go @@ -33,7 +33,7 @@ type Database struct { // NewDatabase opens a new database func NewDatabase( - conMan sqlutil.Connections, + conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool, diff --git a/relayapi/storage/storage.go b/relayapi/storage/storage.go index 6fce1efe3c..4eccd002d0 100644 --- a/relayapi/storage/storage.go +++ b/relayapi/storage/storage.go @@ -30,7 +30,7 @@ import ( // NewDatabase opens a new database func NewDatabase( - conMan sqlutil.Connections, + conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(spec.ServerName) bool, diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go index b04828b692..601ce90632 100644 --- a/roomserver/acls/acls.go +++ b/roomserver/acls/acls.go @@ -119,7 +119,7 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) { }).Debugf("Updating server ACLs for %q", state.RoomID()) s.aclsMutex.Lock() defer s.aclsMutex.Unlock() - s.acls[state.RoomID()] = acls + s.acls[state.RoomID().String()] = acls } func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool { diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 37892a44a1..6269d0b046 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -14,23 +14,9 @@ package api -import "regexp" - -// SetRoomAliasRequest is a request to SetRoomAlias -type SetRoomAliasRequest struct { - // ID of the user setting the alias - UserID string `json:"user_id"` - // New alias for the room - Alias string `json:"alias"` - // The room ID the alias is referring to - RoomID string `json:"room_id"` -} - -// SetRoomAliasResponse is a response to SetRoomAlias -type SetRoomAliasResponse struct { - // Does the alias already refer to a room? - AliasExists bool `json:"alias_exists"` -} +import ( + "regexp" +) // GetRoomIDForAliasRequest is a request to GetRoomIDForAlias type GetRoomIDForAliasRequest struct { @@ -59,22 +45,6 @@ type GetAliasesForRoomIDResponse struct { Aliases []string `json:"aliases"` } -// RemoveRoomAliasRequest is a request to RemoveRoomAlias -type RemoveRoomAliasRequest struct { - // ID of the user removing the alias - UserID string `json:"user_id"` - // The room alias to remove - Alias string `json:"alias"` -} - -// RemoveRoomAliasResponse is a response to RemoveRoomAlias -type RemoveRoomAliasResponse struct { - // Did the alias exist before? - Found bool `json:"found"` - // Did we remove it? - Removed bool `json:"removed"` -} - type AliasEvent struct { Alias string `json:"alias"` AltAliases []string `json:"alt_aliases"` diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 213e16e5df..ef5bc3d17f 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -2,9 +2,12 @@ package api import ( "context" + "crypto/ed25519" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" asAPI "github.com/matrix-org/dendrite/appservice/api" fsAPI "github.com/matrix-org/dendrite/federationapi/api" @@ -31,6 +34,32 @@ func (e ErrNotAllowed) Error() string { return e.Err.Error() } +// ErrRoomUnknownOrNotAllowed is an error return if either the provided +// room ID does not exist, or points to a room that the requester does +// not have access to. +type ErrRoomUnknownOrNotAllowed struct { + Err error +} + +func (e ErrRoomUnknownOrNotAllowed) Error() string { + return e.Err.Error() +} + +type RestrictedJoinAPI interface { + CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) + InvitePending(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (bool, error) + RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) + QueryRoomInfo(ctx context.Context, roomID spec.RoomID) (*types.RoomInfo, error) + QueryServerJoinedToRoom(ctx context.Context, req *QueryServerJoinedToRoomRequest, res *QueryServerJoinedToRoomResponse) error + UserJoinedToRoom(ctx context.Context, roomID types.RoomNID, senderID spec.SenderID) (bool, error) + LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomNID types.RoomNID) ([]gomatrixserverlib.PDU, error) +} + +type DefaultRoomVersionAPI interface { + // Returns the default room version used. + DefaultRoomVersion() gomatrixserverlib.RoomVersion +} + // RoomserverInputAPI is used to write events to the room server. type RoomserverInternalAPI interface { SyncRoomserverAPI @@ -38,6 +67,9 @@ type RoomserverInternalAPI interface { ClientRoomserverAPI UserRoomserverAPI FederationRoomserverAPI + QuerySenderIDAPI + UserRoomPrivateKeyCreator + DefaultRoomVersionAPI // needed to avoid chicken and egg scenario when setting up the // interdependencies between the roomserver and other input APIs @@ -56,6 +88,12 @@ type RoomserverInternalAPI interface { ) error } +type UserRoomPrivateKeyCreator interface { + // GetOrCreateUserRoomPrivateKey gets the user room key for the specified user. If no key exists yet, a new one is created. + GetOrCreateUserRoomPrivateKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (ed25519.PrivateKey, error) + StoreUserRoomPublicKey(ctx context.Context, senderID spec.SenderID, userID spec.UserID, roomID spec.RoomID) error +} + type InputRoomEventsAPI interface { InputRoomEvents( ctx context.Context, @@ -64,6 +102,11 @@ type InputRoomEventsAPI interface { ) } +type QuerySenderIDAPI interface { + QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) + QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) +} + // Query the latest events and state for a room from the room server. type QueryLatestEventsAndStateAPI interface { QueryLatestEventsAndState(ctx context.Context, req *QueryLatestEventsAndStateRequest, res *QueryLatestEventsAndStateResponse) error @@ -87,10 +130,39 @@ type QueryEventsAPI interface { QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error } +type QueryRoomHierarchyAPI interface { + // Traverse the room hierarchy using the provided walker up to the provided limit, + // returning a new walker which can be used to fetch the next page. + // + // If limit is -1, this is treated as no limit, and the entire hierarchy will be traversed. + // + // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it + // can be cached. + QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error) +} + +type QueryMembershipAPI interface { + QueryMembershipForSenderID(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, res *QueryMembershipForUserResponse) error + QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error + QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error + QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) + + // QueryMembershipAtEvent queries the memberships at the given events. + // Returns a map from eventID to *types.HeaderedEvent of membership events. + QueryMembershipAtEvent( + ctx context.Context, + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, + ) (map[string]*types.HeaderedEvent, error) +} + // API functions required by the syncapi type SyncRoomserverAPI interface { QueryLatestEventsAndStateAPI QueryBulkStateContentAPI + QuerySenderIDAPI + QueryMembershipAPI // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error // QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine @@ -100,12 +172,6 @@ type SyncRoomserverAPI interface { req *QueryEventsByIDRequest, res *QueryEventsByIDResponse, ) error - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - req *QueryMembershipForUserRequest, - res *QueryMembershipForUserResponse, - ) error // Query the state after a list of events in a room from the room server. QueryStateAfterEvents( @@ -120,17 +186,10 @@ type SyncRoomserverAPI interface { req *PerformBackfillRequest, res *PerformBackfillResponse, ) error - - // QueryMembershipAtEvent queries the memberships at the given events. - // Returns a map from eventID to a slice of types.HeaderedEvent. - QueryMembershipAtEvent( - ctx context.Context, - request *QueryMembershipAtEventRequest, - response *QueryMembershipAtEventResponse, - ) error } type AppserviceRoomserverAPI interface { + QuerySenderIDAPI // QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine // which room to use by querying the first events roomID. QueryEventsByID( @@ -157,9 +216,13 @@ type ClientRoomserverAPI interface { QueryLatestEventsAndStateAPI QueryBulkStateContentAPI QueryEventsAPI + QuerySenderIDAPI + UserRoomPrivateKeyCreator + QueryRoomHierarchyAPI + DefaultRoomVersionAPI QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error - QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error + QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) QueryStateAfterEvents(ctx context.Context, req *QueryStateAfterEventsRequest, res *QueryStateAfterEventsResponse) error // QueryKnownUsers returns a list of users that we know about from our joined rooms. QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error @@ -169,8 +232,9 @@ type ClientRoomserverAPI interface { GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error GetAliasesForRoomID(ctx context.Context, req *GetAliasesForRoomIDRequest, res *GetAliasesForRoomIDResponse) error + PerformCreateRoom(ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *PerformCreateRoomRequest) (string, *util.JSONResponse) // PerformRoomUpgrade upgrades a room to a newer version - PerformRoomUpgrade(ctx context.Context, roomID, userID string, roomVersion gomatrixserverlib.RoomVersion) (newRoomID string, err error) + PerformRoomUpgrade(ctx context.Context, roomID string, userID spec.UserID, roomVersion gomatrixserverlib.RoomVersion) (newRoomID string, err error) PerformAdminEvacuateRoom(ctx context.Context, roomID string) (affected []string, err error) PerformAdminEvacuateUser(ctx context.Context, userID string) (affected []string, err error) PerformAdminPurgeRoom(ctx context.Context, roomID string) error @@ -183,28 +247,46 @@ type ClientRoomserverAPI interface { PerformPublish(ctx context.Context, req *PerformPublishRequest) error // PerformForget forgets a rooms history for a specific user PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error - SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error - RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error + + // Sets a room alias, as provided sender, pointing to the provided room ID. + // + // If err is nil, then the returned boolean indicates if the alias is already in use. + // If true, then the alias has not been set to the provided room, as it already in use. + SetRoomAlias(ctx context.Context, senderID spec.SenderID, roomID spec.RoomID, alias string) (aliasAlreadyExists bool, err error) + + //RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error + // Removes a room alias, as provided sender. + // + // Returns whether the alias was found, whether it was removed, and an error (if any occurred) + RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) + + SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) } type UserRoomserverAPI interface { + QuerySenderIDAPI QueryLatestEventsAndStateAPI KeyserverRoomserverAPI QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error PerformAdminEvacuateUser(ctx context.Context, userID string) (affected []string, err error) PerformJoin(ctx context.Context, req *PerformJoinRequest) (roomID string, joinedVia spec.ServerName, err error) + JoinedUserCount(ctx context.Context, roomID string) (int, error) } type FederationRoomserverAPI interface { + RestrictedJoinAPI InputRoomEventsAPI QueryLatestEventsAndStateAPI QueryBulkStateContentAPI + QuerySenderIDAPI + QueryRoomHierarchyAPI + QueryMembershipAPI + UserRoomPrivateKeyCreator + AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) + SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) // QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs. QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error - QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error - QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error - QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error // QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine // which room to use by querying the first events roomID. @@ -213,25 +295,22 @@ type FederationRoomserverAPI interface { // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate // the state and auth chain to return. QueryStateAndAuthChain(ctx context.Context, req *QueryStateAndAuthChainRequest, res *QueryStateAndAuthChainResponse) error - // Query if we think we're still in a room. - QueryServerJoinedToRoom(ctx context.Context, req *QueryServerJoinedToRoomRequest, res *QueryServerJoinedToRoomResponse) error QueryPublishedRooms(ctx context.Context, req *QueryPublishedRoomsRequest, res *QueryPublishedRoomsResponse) error // Query missing events for a room from roomserver QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error // Query whether a server is allowed to see an event - QueryServerAllowedToSeeEvent(ctx context.Context, serverName spec.ServerName, eventID string) (allowed bool, err error) - QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error - QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error + QueryServerAllowedToSeeEvent(ctx context.Context, serverName spec.ServerName, eventID string, roomID string) (allowed bool, err error) + QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) + QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (string, error) PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error + HandleInvite(ctx context.Context, event *types.HeaderedEvent) error + PerformInvite(ctx context.Context, req *PerformInviteRequest) error // Query a given amount (or less) of events prior to a given set of events. PerformBackfill(ctx context.Context, req *PerformBackfillRequest, res *PerformBackfillResponse) error - CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) - InvitePending(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (bool, error) - QueryRoomInfo(ctx context.Context, roomID spec.RoomID) (*types.RoomInfo, error) - UserJoinedToRoom(ctx context.Context, roomID types.RoomNID, userID spec.UserID) (bool, error) - LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomNID types.RoomNID) ([]gomatrixserverlib.PDU, error) + IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) + StateQuerier() gomatrixserverlib.StateQuerier } type KeyserverRoomserverAPI interface { diff --git a/roomserver/api/output.go b/roomserver/api/output.go index 16b5049570..852b64206d 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -215,8 +215,10 @@ type OutputNewInviteEvent struct { type OutputRetireInviteEvent struct { // The ID of the "m.room.member" invite event. EventID string - // The target user ID of the "m.room.member" invite event that was retired. - TargetUserID string + // The room ID of the "m.room.member" invite event. + RoomID string + // The target sender ID of the "m.room.member" invite event that was retired. + TargetSenderID spec.SenderID // Optional event ID of the event that replaced the invite. // This can be empty if the invite was rejected locally and we were unable // to reach the server that originally sent the invite. diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index c6e5f5a1c0..2818efaa33 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -1,13 +1,36 @@ package api import ( + "crypto/ed25519" + "encoding/json" + "time" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) +type PerformCreateRoomRequest struct { + InvitedUsers []string + RoomName string + Visibility string + Topic string + StatePreset string + CreationContent json.RawMessage + InitialState []gomatrixserverlib.FledglingEvent + RoomAliasName string + RoomVersion gomatrixserverlib.RoomVersion + PowerLevelContentOverride json.RawMessage + IsDirect bool + + UserDisplayName string + UserAvatarURL string + KeyID gomatrixserverlib.KeyID + PrivateKey ed25519.PrivateKey + EventTime time.Time +} + type PerformJoinRequest struct { RoomIDOrAlias string `json:"room_id_or_alias"` UserID string `json:"user_id"` @@ -18,8 +41,8 @@ type PerformJoinRequest struct { } type PerformLeaveRequest struct { - RoomID string `json:"room_id"` - UserID string `json:"user_id"` + RoomID string + Leaver spec.UserID } type PerformLeaveResponse struct { @@ -27,12 +50,24 @@ type PerformLeaveResponse struct { Message interface{} `json:"message,omitempty"` } +type InviteInput struct { + RoomID spec.RoomID + Inviter spec.UserID + Invitee spec.UserID + DisplayName string + AvatarURL string + Reason string + IsDirect bool + KeyID gomatrixserverlib.KeyID + PrivateKey ed25519.PrivateKey + EventTime time.Time +} + type PerformInviteRequest struct { - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - Event *types.HeaderedEvent `json:"event"` - InviteRoomState []fclient.InviteV2StrippedState `json:"invite_room_state"` - SendAsServer string `json:"send_as_server"` - TransactionID *TransactionID `json:"transaction_id"` + InviteInput InviteInput + InviteRoomState []gomatrixserverlib.InviteStrippedState `json:"invite_room_state"` + SendAsServer string `json:"send_as_server"` + TransactionID *TransactionID `json:"transaction_id"` } type PerformPeekRequest struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 1726bfe1f3..893d5dccfa 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -17,12 +17,14 @@ package api import ( + "context" "encoding/json" "fmt" "strings" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/types" @@ -111,9 +113,9 @@ type QueryEventsByIDResponse struct { // QueryMembershipForUserRequest is a request to QueryMembership type QueryMembershipForUserRequest struct { // ID of the room to fetch membership from - RoomID string `json:"room_id"` + RoomID string // ID of the user for whom membership is requested - UserID string `json:"user_id"` + UserID spec.UserID } // QueryMembershipForUserResponse is a response to QueryMembership @@ -130,6 +132,8 @@ type QueryMembershipForUserResponse struct { // True if the user asked to forget this room. IsRoomForgotten bool `json:"is_room_forgotten"` RoomExists bool `json:"room_exists"` + // The sender ID of the user in the room, if it exists + SenderID *spec.SenderID } // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom @@ -143,7 +147,7 @@ type QueryMembershipsForRoomRequest struct { // Optional - ID of the user sending the request, for checking if the // user is allowed to see the memberships. If not specified then all // room memberships will be returned. - Sender string `json:"sender"` + SenderID spec.SenderID `json:"sender"` } // QueryMembershipsForRoomResponse is a response to QueryMembershipsForRoom @@ -172,6 +176,8 @@ type QueryServerJoinedToRoomResponse struct { RoomExists bool `json:"room_exists"` // True if we still believe that the server is participating in the room IsInRoom bool `json:"is_in_room"` + // The roomversion if joined to room + RoomVersion gomatrixserverlib.RoomVersion } // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent @@ -285,16 +291,6 @@ type QuerySharedUsersResponse struct { UserIDsToCount map[string]int } -type QueryRoomsForUserRequest struct { - UserID string - // The desired membership of the user. If this is the empty string then no rooms are returned. - WantMembership string -} - -type QueryRoomsForUserResponse struct { - RoomIDs []string -} - type QueryBulkStateContentRequest struct { // Returns state events in these rooms RoomIDs []string @@ -350,26 +346,6 @@ type QueryServerBannedFromRoomResponse struct { Banned bool `json:"banned"` } -type QueryRestrictedJoinAllowedRequest struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` -} - -type QueryRestrictedJoinAllowedResponse struct { - // True if the room membership is restricted by the join rule being set to "restricted" - Restricted bool `json:"restricted"` - // True if our local server is joined to all of the allowed rooms specified in the "allow" - // key of the join rule, false if we are missing from some of them and therefore can't - // reliably decide whether or not we can satisfy the join - Resident bool `json:"resident"` - // True if the restricted join is allowed because we found the membership in one of the - // allowed rooms from the join rule, false if not - Allowed bool `json:"allowed"` - // Contains the user ID of the selected user ID that has power to issue invites, this will - // get populated into the "join_authorised_via_users_server" content in the membership - AuthorisedVia string `json:"authorised_via,omitempty"` -} - // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode. func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) { se := make(map[string]string) @@ -430,22 +406,6 @@ func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error { return nil } -// QueryMembershipAtEventRequest requests the membership event for a user -// for a list of eventIDs. -type QueryMembershipAtEventRequest struct { - RoomID string - EventIDs []string - UserID string -} - -// QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest. -type QueryMembershipAtEventResponse struct { - // Membership is a map from eventID to membership event. Events that - // do not have known state will return a nil event, resulting in a "leave" membership - // when calculating history visibility. - Membership map[string]*types.HeaderedEvent `json:"membership"` -} - // QueryLeftUsersRequest is a request to calculate users that we (the server) don't share a // a room with anymore. This is used to cleanup stale device list entries, where we would // otherwise keep on trying to get device lists. @@ -457,3 +417,141 @@ type QueryLeftUsersRequest struct { type QueryLeftUsersResponse struct { LeftUsers []string `json:"user_ids"` } + +type JoinRoomQuerier struct { + Roomserver RestrictedJoinAPI +} + +func (rq *JoinRoomQuerier) CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) { + return rq.Roomserver.CurrentStateEvent(ctx, roomID, eventType, stateKey) +} + +func (rq *JoinRoomQuerier) InvitePending(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (bool, error) { + return rq.Roomserver.InvitePending(ctx, roomID, senderID) +} + +func (rq *JoinRoomQuerier) RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) { + roomInfo, err := rq.Roomserver.QueryRoomInfo(ctx, roomID) + if err != nil || roomInfo == nil || roomInfo.IsStub() { + return nil, err + } + + req := QueryServerJoinedToRoomRequest{ + ServerName: localServerName, + RoomID: roomID.String(), + } + res := QueryServerJoinedToRoomResponse{} + if err = rq.Roomserver.QueryServerJoinedToRoom(ctx, &req, &res); err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed") + return nil, fmt.Errorf("InternalServerError: Failed to query room: %w", err) + } + + userJoinedToRoom, err := rq.Roomserver.UserJoinedToRoom(ctx, types.RoomNID(roomInfo.RoomNID), senderID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.UserJoinedToRoom failed") + return nil, fmt.Errorf("InternalServerError: %w", err) + } + + locallyJoinedUsers, err := rq.Roomserver.LocallyJoinedUsers(ctx, roomInfo.RoomVersion, types.RoomNID(roomInfo.RoomNID)) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.GetLocallyJoinedUsers failed") + return nil, fmt.Errorf("InternalServerError: %w", err) + } + + return &gomatrixserverlib.RestrictedRoomJoinInfo{ + LocalServerInRoom: res.RoomExists && res.IsInRoom, + UserJoinedToRoom: userJoinedToRoom, + JoinedUsers: locallyJoinedUsers, + }, nil +} + +type MembershipQuerier struct { + Roomserver FederationRoomserverAPI +} + +func (mq *MembershipQuerier) CurrentMembership(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (string, error) { + res := QueryMembershipForUserResponse{} + err := mq.Roomserver.QueryMembershipForSenderID(ctx, roomID, senderID, &res) + + membership := "" + if err == nil { + membership = res.Membership + } + return membership, err +} + +type QueryRoomHierarchyRequest struct { + SuggestedOnly bool `json:"suggested_only"` + Limit int `json:"limit"` + MaxDepth int `json:"max_depth"` + From int `json:"json"` +} + +// A struct storing the intermediate state of a room hierarchy query for pagination purposes. +// +// Used for implementing space summaries / room hierarchies +// +// Use NewRoomHierarchyWalker to construct this, and QueryNextRoomHierarchyPage on the roomserver API +// to traverse the room hierarchy. +type RoomHierarchyWalker struct { + RootRoomID spec.RoomID + Caller types.DeviceOrServerName + SuggestedOnly bool + MaxDepth int + Processed RoomSet + Unvisited []RoomHierarchyWalkerQueuedRoom +} + +type RoomHierarchyWalkerQueuedRoom struct { + RoomID spec.RoomID + ParentRoomID *spec.RoomID + Depth int + Vias []string // vias to query this room by +} + +// Create a new room hierarchy walker, starting from the provided root room ID. +// +// Use the resulting struct with QueryNextRoomHierarchyPage on the roomserver API to traverse the room hierarchy. +func NewRoomHierarchyWalker(caller types.DeviceOrServerName, roomID spec.RoomID, suggestedOnly bool, maxDepth int) RoomHierarchyWalker { + walker := RoomHierarchyWalker{ + RootRoomID: roomID, + Caller: caller, + SuggestedOnly: suggestedOnly, + MaxDepth: maxDepth, + Unvisited: []RoomHierarchyWalkerQueuedRoom{{ + RoomID: roomID, + ParentRoomID: nil, + Depth: 0, + }}, + Processed: NewRoomSet(), + } + + return walker +} + +// A set of room IDs. +type RoomSet map[spec.RoomID]struct{} + +// Create a new empty room set. +func NewRoomSet() RoomSet { + return RoomSet{} +} + +// Check if a room ID is in a room set. +func (s RoomSet) Contains(val spec.RoomID) bool { + _, ok := s[val] + return ok +} + +// Add a room ID to a room set. +func (s RoomSet) Add(val spec.RoomID) { + s[val] = struct{}{} +} + +func (s RoomSet) Copy() RoomSet { + copied := make(RoomSet, len(s)) + for k := range s { + copied.Add(k) + } + return copied +} diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 2505a993b9..0ad5d20130 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -75,7 +75,7 @@ func SendEventWithState( } logrus.WithContext(ctx).WithFields(logrus.Fields{ - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "event_id": event.EventID(), "outliers": len(ires), "state_ids": len(stateEventIDs), diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index b6168d38b8..d5172dab92 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -13,6 +13,9 @@ package auth import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" ) @@ -22,14 +25,11 @@ import ( // IsServerAllowed returns true if the server is allowed to see events in the room // at this particular state. This function implements https://matrix.org/docs/spec/client_server/r0.6.0#id87 func IsServerAllowed( + ctx context.Context, querier api.QuerySenderIDAPI, serverName spec.ServerName, serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.PDU, ) bool { - // In practice should not happen, but avoids unneeded CPU cycles - if serverName == "" || len(authEvents) == 0 { - return false - } historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. @@ -37,7 +37,7 @@ func IsServerAllowed( return true } // 2. If the user's membership was join, allow. - joinedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, spec.Join) + joinedUserExists := IsAnyUserOnServerWithMembership(ctx, querier, serverName, authEvents, spec.Join) if joinedUserExists { return true } @@ -46,7 +46,7 @@ func IsServerAllowed( return true } // 4. If the user's membership was invite, and the history_visibility was set to invited, allow. - invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, spec.Invite) + invitedUserExists := IsAnyUserOnServerWithMembership(ctx, querier, serverName, authEvents, spec.Invite) if invitedUserExists && historyVisibility == gomatrixserverlib.HistoryVisibilityInvited { return true } @@ -70,7 +70,7 @@ func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.PDU) gomatrixserver return visibility } -func IsAnyUserOnServerWithMembership(serverName spec.ServerName, authEvents []gomatrixserverlib.PDU, wantMembership string) bool { +func IsAnyUserOnServerWithMembership(ctx context.Context, querier api.QuerySenderIDAPI, serverName spec.ServerName, authEvents []gomatrixserverlib.PDU, wantMembership string) bool { for _, ev := range authEvents { if ev.Type() != spec.MRoomMember { continue @@ -85,12 +85,12 @@ func IsAnyUserOnServerWithMembership(serverName spec.ServerName, authEvents []go continue } - _, domain, err := gomatrixserverlib.SplitID('@', *stateKey) + userID, err := querier.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*stateKey)) if err != nil { continue } - if domain == serverName { + if userID.Domain() == serverName { return true } } diff --git a/roomserver/auth/auth_test.go b/roomserver/auth/auth_test.go index e3eea5d8b2..058361e6ed 100644 --- a/roomserver/auth/auth_test.go +++ b/roomserver/auth/auth_test.go @@ -1,13 +1,23 @@ package auth import ( + "context" "testing" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" ) +type FakeQuerier struct { + api.QuerySenderIDAPI +} + +func (f *FakeQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func TestIsServerAllowed(t *testing.T) { alice := test.NewUser(t) @@ -77,7 +87,7 @@ func TestIsServerAllowed(t *testing.T) { authEvents = append(authEvents, ev.PDU) } - if got := IsServerAllowed(tt.serverName, tt.serverCurrentlyInRoom, authEvents); got != tt.want { + if got := IsServerAllowed(context.Background(), &FakeQuerier{}, tt.serverName, tt.serverCurrentlyInRoom, authEvents); got != tt.want { t.Errorf("IsServerAllowed() = %v, want %v", got, tt.want) } }) diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index 4d2de9a5a6..5ceda7e010 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -35,27 +35,27 @@ import ( // SetRoomAlias implements alias.RoomserverInternalAPI func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *api.SetRoomAliasRequest, - response *api.SetRoomAliasResponse, -) error { + senderID spec.SenderID, + roomID spec.RoomID, + alias string, +) (aliasAlreadyUsed bool, err error) { // Check if the alias isn't already referring to a room - roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) + existingRoomID, err := r.DB.GetRoomIDForAlias(ctx, alias) if err != nil { - return err + return false, err } - if len(roomID) > 0 { + + if len(existingRoomID) > 0 { // If the alias already exists, stop the process - response.AliasExists = true - return nil + return true, nil } - response.AliasExists = false // Save the new alias - if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID, request.UserID); err != nil { - return err + if err := r.DB.SetRoomAlias(ctx, alias, roomID.String(), string(senderID)); err != nil { + return false, err } - return nil + return false, nil } // GetRoomIDForAlias implements alias.RoomserverInternalAPI @@ -113,83 +113,83 @@ func (r *RoomserverInternalAPI) GetAliasesForRoomID( return nil } +// nolint:gocyclo // RemoveRoomAlias implements alias.RoomserverInternalAPI -func (r *RoomserverInternalAPI) RemoveRoomAlias( - ctx context.Context, - request *api.RemoveRoomAliasRequest, - response *api.RemoveRoomAliasResponse, -) error { - _, virtualHost, err := r.Cfg.Global.SplitLocalID('@', request.UserID) +// nolint: gocyclo +func (r *RoomserverInternalAPI) RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) { + roomID, err := r.DB.GetRoomIDForAlias(ctx, alias) if err != nil { - return err + return false, false, fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err) + } + if roomID == "" { + return false, false, nil } - roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) + validRoomID, err := spec.NewRoomID(roomID) if err != nil { - return fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err) + return true, false, err } - if roomID == "" { - response.Found = false - response.Removed = false - return nil + + sender, err := r.QueryUserIDForSender(ctx, *validRoomID, senderID) + if err != nil || sender == nil { + return true, false, fmt.Errorf("r.QueryUserIDForSender: %w", err) } + virtualHost := sender.Domain() - response.Found = true - creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) + creatorID, err := r.DB.GetCreatorIDForAlias(ctx, alias) if err != nil { - return fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err) + return true, false, fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err) } - if creatorID != request.UserID { + if spec.SenderID(creatorID) != senderID { var plEvent *types.HeaderedEvent var pls *gomatrixserverlib.PowerLevelContent plEvent, err = r.DB.GetStateEvent(ctx, roomID, spec.MRoomPowerLevels, "") if err != nil { - return fmt.Errorf("r.DB.GetStateEvent: %w", err) + return true, false, fmt.Errorf("r.DB.GetStateEvent: %w", err) } pls, err = plEvent.PowerLevels() if err != nil { - return fmt.Errorf("plEvent.PowerLevels: %w", err) + return true, false, fmt.Errorf("plEvent.PowerLevels: %w", err) } - if pls.UserLevel(request.UserID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) { - response.Removed = false - return nil + if pls.UserLevel(senderID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) { + return true, false, nil } } ev, err := r.DB.GetStateEvent(ctx, roomID, spec.MRoomCanonicalAlias, "") if err != nil && err != sql.ErrNoRows { - return err + return true, false, err } else if ev != nil { stateAlias := gjson.GetBytes(ev.Content(), "alias").Str // the alias to remove is currently set as the canonical alias, remove it - if stateAlias == request.Alias { + if stateAlias == alias { res, err := sjson.DeleteBytes(ev.Content(), "alias") if err != nil { - return err + return true, false, err } - sender := request.UserID - if request.UserID != ev.Sender() { - sender = ev.Sender() + canonicalSenderID := ev.SenderID() + canonicalSender, err := r.QueryUserIDForSender(ctx, *validRoomID, canonicalSenderID) + if err != nil || canonicalSender == nil { + return true, false, err } - _, senderDomain, err := r.Cfg.Global.SplitLocalID('@', sender) + validRoomID, err := spec.NewRoomID(roomID) if err != nil { - return err + return true, false, err } - - identity, err := r.Cfg.Global.SigningIdentityFor(senderDomain) + identity, err := r.SigningIdentityFor(ctx, *validRoomID, *canonicalSender) if err != nil { - return err + return true, false, err } proto := &gomatrixserverlib.ProtoEvent{ - Sender: sender, - RoomID: ev.RoomID(), + SenderID: string(canonicalSenderID), + RoomID: ev.RoomID().String(), Type: ev.Type(), StateKey: ev.StateKey(), Content: res, @@ -197,34 +197,33 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( eventsNeeded, err := gomatrixserverlib.StateNeededForProtoEvent(proto) if err != nil { - return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) + return true, false, fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) } if len(eventsNeeded.Tuples()) == 0 { - return errors.New("expecting state tuples for event builder, got none") + return true, false, errors.New("expecting state tuples for event builder, got none") } stateRes := &api.QueryLatestEventsAndStateResponse{} - if err = helpers.QueryLatestEventsAndState(ctx, r.DB, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil { - return err + if err = helpers.QueryLatestEventsAndState(ctx, r.DB, r, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil { + return true, false, err } - newEvent, err := eventutil.BuildEvent(ctx, proto, &r.Cfg.Global, identity, time.Now(), &eventsNeeded, stateRes) + newEvent, err := eventutil.BuildEvent(ctx, proto, &identity, time.Now(), &eventsNeeded, stateRes) if err != nil { - return err + return true, false, err } err = api.SendEvents(ctx, r, api.KindNew, []*types.HeaderedEvent{newEvent}, virtualHost, r.ServerName, r.ServerName, nil, false) if err != nil { - return err + return true, false, err } } } // Remove the alias from the database - if err := r.DB.RemoveRoomAlias(ctx, request.Alias); err != nil { - return err + if err := r.DB.RemoveRoomAlias(ctx, alias); err != nil { + return true, false, err } - response.Removed = true - return nil + return true, true, nil } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 81904c8b89..1e08f6a3a2 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -2,10 +2,13 @@ package internal import ( "context" + "crypto/ed25519" "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" @@ -19,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/producers" "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -40,6 +44,7 @@ type RoomserverInternalAPI struct { *perform.Forgetter *perform.Upgrader *perform.Admin + *perform.Creator ProcessContext *process.ProcessContext DB storage.Database Cfg *config.Dendrite @@ -56,6 +61,7 @@ type RoomserverInternalAPI struct { OutputProducer *producers.RoomEventProducer PerspectiveServerNames []spec.ServerName enableMetrics bool + defaultRoomVersion gomatrixserverlib.RoomVersion } func NewRoomserverAPI( @@ -86,14 +92,9 @@ func NewRoomserverAPI( NATSClient: nc, Durable: dendriteCfg.Global.JetStream.Durable("RoomserverInputConsumer"), ServerACLs: serverACLs, - Queryer: &query.Queryer{ - DB: roomserverDB, - Cache: caches, - IsLocalServerName: dendriteCfg.Global.IsLocalServerName, - ServerACLs: serverACLs, - }, - enableMetrics: enableMetrics, - // perform-er structs get initialised when we have a federation sender to use + enableMetrics: enableMetrics, + defaultRoomVersion: dendriteCfg.RoomServer.DefaultRoomVersion, + // perform-er structs + queryer struct get initialised when we have a federation sender to use } return a } @@ -105,9 +106,13 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio r.fsAPI = fsAPI r.KeyRing = keyRing - identity, err := r.Cfg.Global.SigningIdentityFor(r.ServerName) - if err != nil { - logrus.Panic(err) + r.Queryer = &query.Queryer{ + DB: r.DB, + Cache: r.Cache, + IsLocalServerName: r.Cfg.Global.IsLocalServerName, + ServerACLs: r.ServerACLs, + Cfg: r.Cfg, + FSAPI: fsAPI, } r.Inputer = &input.Inputer{ @@ -120,16 +125,19 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio NATSClient: r.NATSClient, Durable: nats.Durable(r.Durable), ServerName: r.ServerName, - SigningIdentity: identity, + SigningIdentity: r.SigningIdentityFor, FSAPI: fsAPI, + RSAPI: r, KeyRing: keyRing, ACLs: r.ServerACLs, Queryer: r.Queryer, + EnableMetrics: r.enableMetrics, } r.Inviter = &perform.Inviter{ DB: r.DB, Cfg: &r.Cfg.RoomServer, FSAPI: r.fsAPI, + RSAPI: r, Inputer: r.Inputer, } r.Joiner = &perform.Joiner{ @@ -171,6 +179,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio IsLocalServerName: r.Cfg.Global.IsLocalServerName, DB: r.DB, FSAPI: r.fsAPI, + Querier: r.Queryer, KeyRing: r.KeyRing, // Perspective servers are trusted to not lie about server keys, so we will also // prefer these servers when backfilling (assuming they are in the room) rather @@ -191,6 +200,11 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio Queryer: r.Queryer, Leaver: r.Leaver, } + r.Creator = &perform.Creator{ + DB: r.DB, + Cfg: &r.Cfg.RoomServer, + RSAPI: r, + } if err := r.Inputer.Start(); err != nil { logrus.WithError(err).Panic("failed to start roomserver input API") @@ -206,18 +220,39 @@ func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalA r.asAPI = asAPI } -func (r *RoomserverInternalAPI) PerformInvite( - ctx context.Context, - req *api.PerformInviteRequest, +func (r *RoomserverInternalAPI) DefaultRoomVersion() gomatrixserverlib.RoomVersion { + return r.defaultRoomVersion +} + +func (r *RoomserverInternalAPI) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) { + return r.Inviter.IsKnownRoom(ctx, roomID) +} + +func (r *RoomserverInternalAPI) StateQuerier() gomatrixserverlib.StateQuerier { + return r.Inviter.StateQuerier() +} + +func (r *RoomserverInternalAPI) HandleInvite( + ctx context.Context, inviteEvent *types.HeaderedEvent, ) error { - outputEvents, err := r.Inviter.PerformInvite(ctx, req) + outputEvents, err := r.Inviter.ProcessInviteMembership(ctx, inviteEvent) if err != nil { return err } - if len(outputEvents) == 0 { - return nil - } - return r.OutputProducer.ProduceRoomEvents(req.Event.RoomID(), outputEvents) + return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID().String(), outputEvents) +} + +func (r *RoomserverInternalAPI) PerformCreateRoom( + ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *api.PerformCreateRoomRequest, +) (string, *util.JSONResponse) { + return r.Creator.PerformCreateRoom(ctx, userID, roomID, createRequest) +} + +func (r *RoomserverInternalAPI) PerformInvite( + ctx context.Context, + req *api.PerformInviteRequest, +) error { + return r.Inviter.PerformInvite(ctx, req) } func (r *RoomserverInternalAPI) PerformLeave( @@ -243,3 +278,65 @@ func (r *RoomserverInternalAPI) PerformForget( ) error { return r.Forgetter.PerformForget(ctx, req, resp) } + +// GetOrCreateUserRoomPrivateKey gets the user room key for the specified user. If no key exists yet, a new one is created. +func (r *RoomserverInternalAPI) GetOrCreateUserRoomPrivateKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (ed25519.PrivateKey, error) { + key, err := r.DB.SelectUserRoomPrivateKey(ctx, userID, roomID) + if err != nil { + return nil, err + } + // no key found, create one + if len(key) == 0 { + _, key, err = ed25519.GenerateKey(nil) + if err != nil { + return nil, err + } + key, err = r.DB.InsertUserRoomPrivatePublicKey(ctx, userID, roomID, key) + if err != nil { + return nil, err + } + } + return key, nil +} + +func (r *RoomserverInternalAPI) StoreUserRoomPublicKey(ctx context.Context, senderID spec.SenderID, userID spec.UserID, roomID spec.RoomID) error { + pubKeyBytes, err := senderID.RawBytes() + if err != nil { + return err + } + _, err = r.DB.InsertUserRoomPublicKey(ctx, userID, roomID, ed25519.PublicKey(pubKeyBytes)) + return err +} + +func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) { + roomVersion, ok := r.Cache.GetRoomVersion(roomID.String()) + if !ok { + roomInfo, err := r.DB.RoomInfo(ctx, roomID.String()) + if err != nil { + return fclient.SigningIdentity{}, err + } + if roomInfo != nil { + roomVersion = roomInfo.RoomVersion + } + } + if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + privKey, err := r.GetOrCreateUserRoomPrivateKey(ctx, senderID, roomID) + if err != nil { + return fclient.SigningIdentity{}, err + } + return fclient.SigningIdentity{ + PrivateKey: privKey, + KeyID: "ed25519:1", + ServerName: spec.ServerName(spec.SenderIDFromPseudoIDKey(privKey)), + }, nil + } + identity, err := r.Cfg.Global.SigningIdentityFor(senderID.Domain()) + if err != nil { + return fclient.SigningIdentity{}, err + } + return *identity, err +} + +func (r *RoomserverInternalAPI) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) { + return r.DB.AssignRoomNID(ctx, roomID, roomVersion) +} diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 24958091b7..9da751b1a2 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" @@ -36,6 +37,7 @@ func CheckForSoftFail( roomInfo *types.RoomInfo, event *types.HeaderedEvent, stateEventIDs []string, + querier api.QuerySenderIDAPI, ) (bool, error) { rewritesState := len(stateEventIDs) > 1 @@ -49,7 +51,7 @@ func CheckForSoftFail( } else { // Then get the state entries for the current state snapshot. // We'll use this to check if the event is allowed right now. - roomState := state.NewStateResolution(db, roomInfo) + roomState := state.NewStateResolution(db, roomInfo, querier) authStateEntries, err = roomState.LoadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID()) if err != nil { return true, fmt.Errorf("roomState.LoadStateAtSnapshot: %w", err) @@ -70,28 +72,29 @@ func CheckForSoftFail( ) // Load the actual auth events from the database. - authEvents, err := loadAuthEvents(ctx, db, roomInfo, stateNeeded, authStateEntries) + authEvents, err := loadAuthEvents(ctx, db, roomInfo.RoomVersion, stateNeeded, authStateEntries) if err != nil { return true, fmt.Errorf("loadAuthEvents: %w", err) } // Check if the event is allowed. - if err = gomatrixserverlib.Allowed(event.PDU, &authEvents); err != nil { + if err = gomatrixserverlib.Allowed(event.PDU, &authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return querier.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { // return true, nil return true, err } return false, nil } -// CheckAuthEvents checks that the event passes authentication checks -// Returns the numeric IDs for the auth events. -func CheckAuthEvents( +// GetAuthEvents returns the numeric IDs for the auth events. +func GetAuthEvents( ctx context.Context, db storage.RoomDatabase, - roomInfo *types.RoomInfo, - event *types.HeaderedEvent, + roomVersion gomatrixserverlib.RoomVersion, + event gomatrixserverlib.PDU, authEventIDs []string, -) ([]types.EventNID, error) { +) (gomatrixserverlib.AuthEventProvider, error) { // Grab the numeric IDs for the supplied auth state events from the database. authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs, true) if err != nil { @@ -100,25 +103,14 @@ func CheckAuthEvents( authStateEntries = types.DeduplicateStateEntries(authStateEntries) // Work out which of the state events we actually need. - stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.PDU{event.PDU}) + stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.PDU{event}) // Load the actual auth events from the database. - authEvents, err := loadAuthEvents(ctx, db, roomInfo, stateNeeded, authStateEntries) + authEvents, err := loadAuthEvents(ctx, db, roomVersion, stateNeeded, authStateEntries) if err != nil { return nil, fmt.Errorf("loadAuthEvents: %w", err) } - - // Check if the event is allowed. - if err = gomatrixserverlib.Allowed(event.PDU, &authEvents); err != nil { - return nil, err - } - - // Return the numeric IDs for the auth events. - result := make([]types.EventNID, len(authStateEntries)) - for i := range authStateEntries { - result[i] = authStateEntries[i].EventNID - } - return result, nil + return &authEvents, nil } type authEvents struct { @@ -149,8 +141,8 @@ func (ae *authEvents) JoinRules() (gomatrixserverlib.PDU, error) { } // Memmber implements gomatrixserverlib.AuthEventProvider -func (ae *authEvents) Member(stateKey string) (gomatrixserverlib.PDU, error) { - return ae.lookupEvent(types.MRoomMemberNID, stateKey), nil +func (ae *authEvents) Member(stateKey spec.SenderID) (gomatrixserverlib.PDU, error) { + return ae.lookupEvent(types.MRoomMemberNID, string(stateKey)), nil } // ThirdPartyInvite implements gomatrixserverlib.AuthEventProvider @@ -196,7 +188,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) g func loadAuthEvents( ctx context.Context, db state.StateResolutionStorage, - roomInfo *types.RoomInfo, + roomVersion gomatrixserverlib.RoomVersion, needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { @@ -220,19 +212,15 @@ func loadAuthEvents( } } - if roomInfo == nil { - err = types.ErrorInvalidRoomInfo - return - } - if result.events, err = db.Events(ctx, roomInfo.RoomVersion, eventNIDs); err != nil { + if result.events, err = db.Events(ctx, roomVersion, eventNIDs); err != nil { return } roomID := "" for _, ev := range result.events { if roomID == "" { - roomID = ev.RoomID() + roomID = ev.RoomID().String() } - if ev.RoomID() != roomID { + if ev.RoomID().String() != roomID { result.valid = false break } diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index 95397cd5e2..b2e21bf546 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "sort" - "strings" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -55,9 +54,10 @@ func UpdateToInviteMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, + RoomID: add.RoomID().String(), Membership: spec.Join, RetiredByEventID: add.EventID(), - TargetUserID: *add.StateKey(), + TargetSenderID: spec.SenderID(*add.StateKey()), }, }) } @@ -68,7 +68,7 @@ func UpdateToInviteMembership( // memberships. If the servername is not supplied then the local server will be // checked instead using a faster code path. // TODO: This should probably be replaced by an API call. -func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverName spec.ServerName, roomID string) (bool, error) { +func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, querier api.QuerySenderIDAPI, serverName spec.ServerName, roomID string) (bool, error) { info, err := db.RoomInfo(ctx, roomID) if err != nil { return false, err @@ -94,13 +94,13 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam for i := range events { gmslEvents[i] = events[i].PDU } - return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, spec.Join), nil + return auth.IsAnyUserOnServerWithMembership(ctx, querier, serverName, gmslEvents, spec.Join), nil } func IsInvitePending( ctx context.Context, db storage.Database, - roomID, userID string, -) (bool, string, string, gomatrixserverlib.PDU, error) { + roomID string, senderID spec.SenderID, +) (bool, spec.SenderID, string, gomatrixserverlib.PDU, error) { // Look up the room NID for the supplied room ID. info, err := db.RoomInfo(ctx, roomID) if err != nil { @@ -111,13 +111,13 @@ func IsInvitePending( } // Look up the state key NID for the supplied user ID. - targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{userID}) + targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{string(senderID)}) if err != nil { return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) } - targetUserNID, targetUserFound := targetUserNIDs[userID] + targetUserNID, targetUserFound := targetUserNIDs[string(senderID)] if !targetUserFound { - return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) + return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", senderID, targetUserNIDs) } // Let's see if we have an event active for the user in the room. If @@ -156,7 +156,7 @@ func IsInvitePending( event, err := verImpl.NewEventFromTrustedJSON(eventJSON, false) - return true, senderUser, userNIDToEventID[senderUserNIDs[0]], event, err + return true, spec.SenderID(senderUser), userNIDToEventID[senderUserNIDs[0]], event, err } // GetMembershipsAtState filters the state events to @@ -211,8 +211,8 @@ func GetMembershipsAtState( return events, nil } -func StateBeforeEvent(ctx context.Context, db storage.Database, info *types.RoomInfo, eventNID types.EventNID) ([]types.StateEntry, error) { - roomState := state.NewStateResolution(db, info) +func StateBeforeEvent(ctx context.Context, db storage.Database, info *types.RoomInfo, eventNID types.EventNID, querier api.QuerySenderIDAPI) ([]types.StateEntry, error) { + roomState := state.NewStateResolution(db, info, querier) // Lookup the event NID eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { @@ -229,8 +229,8 @@ func StateBeforeEvent(ctx context.Context, db storage.Database, info *types.Room return roomState.LoadCombinedStateAfterEvents(ctx, prevState) } -func MembershipAtEvent(ctx context.Context, db storage.RoomDatabase, info *types.RoomInfo, eventIDs []string, stateKeyNID types.EventStateKeyNID) (map[string][]types.StateEntry, error) { - roomState := state.NewStateResolution(db, info) +func MembershipAtEvent(ctx context.Context, db storage.RoomDatabase, info *types.RoomInfo, eventIDs []string, stateKeyNID types.EventStateKeyNID, querier api.QuerySenderIDAPI) (map[string][]types.StateEntry, error) { + roomState := state.NewStateResolution(db, info, querier) // Fetch the state as it was when this event was fired return roomState.LoadMembershipAtEvent(ctx, eventIDs, stateKeyNID) } @@ -264,7 +264,7 @@ func LoadStateEvents( } func CheckServerAllowedToSeeEvent( - ctx context.Context, db storage.Database, info *types.RoomInfo, eventID string, serverName spec.ServerName, isServerInRoom bool, + ctx context.Context, db storage.Database, info *types.RoomInfo, roomID string, eventID string, serverName spec.ServerName, isServerInRoom bool, querier api.QuerySenderIDAPI, ) (bool, error) { stateAtEvent, err := db.GetHistoryVisibilityState(ctx, info, eventID, string(serverName)) switch err { @@ -273,7 +273,7 @@ func CheckServerAllowedToSeeEvent( case tables.OptimisationNotSupportedError: // The database engine didn't support this optimisation, so fall back to using // the old and slow method - stateAtEvent, err = slowGetHistoryVisibilityState(ctx, db, info, eventID, serverName) + stateAtEvent, err = slowGetHistoryVisibilityState(ctx, db, info, roomID, eventID, serverName, querier) if err != nil { return false, err } @@ -288,13 +288,13 @@ func CheckServerAllowedToSeeEvent( return false, err } } - return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil + return auth.IsServerAllowed(ctx, querier, serverName, isServerInRoom, stateAtEvent), nil } func slowGetHistoryVisibilityState( - ctx context.Context, db storage.Database, info *types.RoomInfo, eventID string, serverName spec.ServerName, + ctx context.Context, db storage.Database, info *types.RoomInfo, roomID, eventID string, serverName spec.ServerName, querier api.QuerySenderIDAPI, ) ([]gomatrixserverlib.PDU, error) { - roomState := state.NewStateResolution(db, info) + roomState := state.NewStateResolution(db, info, querier) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -318,9 +318,18 @@ func slowGetHistoryVisibilityState( // If the event state key doesn't match the given servername // then we'll filter it out. This does preserve state keys that // are "" since these will contain history visibility etc. + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, err + } for nid, key := range stateKeys { - if key != "" && !strings.HasSuffix(key, ":"+string(serverName)) { - delete(stateKeys, nid) + if key != "" { + userID, err := querier.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(key)) + if err == nil && userID != nil { + if userID.Domain() != serverName { + delete(stateKeys, nid) + } + } } } @@ -344,7 +353,7 @@ func slowGetHistoryVisibilityState( // TODO: Remove this when we have tests to assert correctness of this function func ScanEventTree( ctx context.Context, db storage.Database, info *types.RoomInfo, front []string, visited map[string]bool, limit int, - serverName spec.ServerName, + serverName spec.ServerName, querier api.QuerySenderIDAPI, ) ([]types.EventNID, map[string]struct{}, error) { var resultNIDs []types.EventNID var err error @@ -387,7 +396,7 @@ BFSLoop: // It's nasty that we have to extract the room ID from an event, but many federation requests // only talk in event IDs, no room IDs at all (!!!) ev := events[0] - isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, serverName, ev.RoomID()) + isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, querier, serverName, ev.RoomID().String()) if err != nil { util.GetLogger(ctx).WithError(err).Error("Failed to check if server is currently in room, assuming not.") } @@ -410,7 +419,7 @@ BFSLoop: // hasn't been seen before. if !visited[pre] { visited[pre] = true - allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, pre, serverName, isServerInRoom) + allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, ev.RoomID().String(), pre, serverName, isServerInRoom, querier) if err != nil { util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).WithError(err).Error( "Error checking if allowed to see event", @@ -439,7 +448,7 @@ BFSLoop: } func QueryLatestEventsAndState( - ctx context.Context, db storage.Database, + ctx context.Context, db storage.Database, querier api.QuerySenderIDAPI, request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { @@ -452,7 +461,7 @@ func QueryLatestEventsAndState( return nil } - roomState := state.NewStateResolution(db, roomInfo) + roomState := state.NewStateResolution(db, roomInfo, querier) response.RoomExists = true response.RoomVersion = roomInfo.RoomVersion diff --git a/roomserver/internal/helpers/helpers_test.go b/roomserver/internal/helpers/helpers_test.go index f1896277e9..1cef83df75 100644 --- a/roomserver/internal/helpers/helpers_test.go +++ b/roomserver/internal/helpers/helpers_test.go @@ -8,6 +8,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/roomserver/types" @@ -58,12 +59,12 @@ func TestIsInvitePendingWithoutNID(t *testing.T) { } // Alice should have no pending invites and should have a NID - pendingInvite, _, _, _, err := IsInvitePending(context.Background(), db, room.ID, alice.ID) + pendingInvite, _, _, _, err := IsInvitePending(context.Background(), db, room.ID, spec.SenderID(alice.ID)) assert.NoError(t, err, "failed to get pending invites") assert.False(t, pendingInvite, "unexpected pending invite") // Bob should have no pending invites and receive a new NID - pendingInvite, _, _, _, err = IsInvitePending(context.Background(), db, room.ID, bob.ID) + pendingInvite, _, _, _, err = IsInvitePending(context.Background(), db, room.ID, spec.SenderID(bob.ID)) assert.NoError(t, err, "failed to get pending invites") assert.False(t, pendingInvite, "unexpected pending invite") }) diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 3db2d0a67a..4047515327 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -81,8 +81,9 @@ type Inputer struct { JetStream nats.JetStreamContext Durable nats.SubOpt ServerName spec.ServerName - SigningIdentity *fclient.SigningIdentity + SigningIdentity func(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) FSAPI fedapi.RoomserverFederationAPI + RSAPI api.RoomserverInternalAPI KeyRing gomatrixserverlib.JSONVerifier ACLs *acls.ServerACLs InputRoomEventTopic string @@ -91,7 +92,7 @@ type Inputer struct { Queryer *query.Queryer UserAPI userapi.RoomserverUserAPI - enableMetrics bool + EnableMetrics bool } // If a room consumer is inactive for a while then we will allow NATS @@ -178,7 +179,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { // will look to see if we have a worker for that room which has its // own consumer. If we don't, we'll start one. func (r *Inputer) Start() error { - if r.enableMetrics { + if r.EnableMetrics { prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration) } _, err := r.JetStream.Subscribe( @@ -357,7 +358,7 @@ func (r *Inputer) queueInputRoomEvents( // For each event, marshal the input room event and then // send it into the input queue. for _, e := range request.InputRoomEvents { - roomID := e.Event.RoomID() + roomID := e.Event.RoomID().String() subj := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(roomID)) msg := &nats.Msg{ Subject: subj, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 02a1a2802a..77b50d0e21 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -87,7 +87,7 @@ func (r *Inputer) processRoomEvent( } trace, ctx := internal.StartRegion(ctx, "processRoomEvent") - trace.SetTag("room_id", input.Event.RoomID()) + trace.SetTag("room_id", input.Event.RoomID().String()) trace.SetTag("event_id", input.Event.EventID()) defer trace.EndRegion() @@ -96,7 +96,7 @@ func (r *Inputer) processRoomEvent( defer func() { timetaken := time.Since(started) processRoomEventDuration.With(prometheus.Labels{ - "room_id": input.Event.RoomID(), + "room_id": input.Event.RoomID().String(), }).Observe(float64(timetaken.Milliseconds())) }() @@ -105,7 +105,7 @@ func (r *Inputer) processRoomEvent( event := headered.PDU logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "kind": input.Kind, "origin": input.Origin, "type": event.Type(), @@ -120,17 +120,21 @@ func (r *Inputer) processRoomEvent( // Don't waste time processing the event if the room doesn't exist. // A room entry locally will only be created in response to a create // event. - roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID()) + roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID().String()) if rerr != nil { return fmt.Errorf("r.DB.RoomInfo: %w", rerr) } isCreateEvent := event.Type() == spec.MRoomCreate && event.StateKeyEquals("") if roomInfo == nil && !isCreateEvent { - return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID()) + return fmt.Errorf("room %s does not exist for event %s", event.RoomID().String(), event.EventID()) } - _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()) + sender, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err != nil { - return fmt.Errorf("event has invalid sender %q", input.Event.Sender()) + return fmt.Errorf("failed getting userID for sender %q. %w", event.SenderID(), err) + } + senderDomain := spec.ServerName("") + if sender != nil { + senderDomain = sender.Domain() } // If we already know about this outlier and it hasn't been rejected @@ -171,7 +175,7 @@ func (r *Inputer) processRoomEvent( // If we have missing events (auth or prev), we build a list of servers to ask if missingAuth || missingPrev { serverReq := &fedapi.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), ExcludeSelf: true, ExcludeBlacklisted: true, } @@ -193,7 +197,9 @@ func (r *Inputer) processRoomEvent( serverRes.ServerNames = append(serverRes.ServerNames, input.Origin) delete(servers, input.Origin) } - if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName { + // Only perform this check if the sender mxid_mapping can be resolved. + // Don't fail processing the event if we have no mxid_maping. + if sender != nil && senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, senderDomain) delete(servers, senderDomain) } @@ -240,6 +246,21 @@ func (r *Inputer) processRoomEvent( // really do anything with the event other than reject it at this point. isRejected = true rejectionErr = fmt.Errorf("missingState.processEventWithMissingState: %w", err) + switch e := err.(type) { + case gomatrixserverlib.EventValidationError: + if e.Persistable && stateSnapshot != nil { + // We retrieved some state and we ended up having to call /state_ids for + // the new event in question (probably because closing the gap by using + // /get_missing_events didn't do what we hoped) so we'll instead overwrite + // the state snapshot with the newly resolved state. + missingPrev = false + input.HasState = true + input.StateEventIDs = make([]string, 0, len(stateSnapshot.StateEvents)) + for _, se := range stateSnapshot.StateEvents { + input.StateEventIDs = append(input.StateEventIDs, se.EventID()) + } + } + } } else if stateSnapshot != nil { // We retrieved some state and we ended up having to call /state_ids for // the new event in question (probably because closing the gap by using @@ -276,7 +297,9 @@ func (r *Inputer) processRoomEvent( // Check if the event is allowed by its auth events. If it isn't then // we consider the event to be "rejected" — it will still be persisted. - if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { + if err = gomatrixserverlib.Allowed(event, &authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { isRejected = true rejectionErr = err logger.WithError(rejectionErr).Warnf("Event %s not allowed by auth events", event.EventID()) @@ -313,7 +336,7 @@ func (r *Inputer) processRoomEvent( if input.Kind == api.KindNew && !isCreateEvent { // Check that the event passes authentication checks based on the // current room state. - softfail, err = helpers.CheckForSoftFail(ctx, r.DB, roomInfo, headered, input.StateEventIDs) + softfail, err = helpers.CheckForSoftFail(ctx, r.DB, roomInfo, headered, input.StateEventIDs, r.Queryer) if err != nil { logger.WithError(err).Warn("Error authing soft-failed event") } @@ -368,12 +391,12 @@ func (r *Inputer) processRoomEvent( // Request the room info again — it's possible that the room has been // created by now if it didn't exist already. - roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID()) + roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID().String()) if err != nil { return fmt.Errorf("updater.RoomInfo: %w", err) } if roomInfo == nil { - return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID()) + return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID().String()) } if input.HasState || (!missingPrev && stateAtEvent.BeforeStateSnapshotNID == 0) { @@ -393,8 +416,8 @@ func (r *Inputer) processRoomEvent( redactedEvent gomatrixserverlib.PDU ) if !isRejected && !isCreateEvent { - resolver := state.NewStateResolution(r.DB, roomInfo) - redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver) + resolver := state.NewStateResolution(r.DB, roomInfo, r.Queryer) + redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver, r.Queryer) if err != nil { return err } @@ -421,6 +444,24 @@ func (r *Inputer) processRoomEvent( return nil } + // TODO: Revist this to ensure we don't replace a current state mxid_mapping with an older one. + if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && event.Type() == spec.MRoomMember { + mapping := gomatrixserverlib.MemberContent{} + if err = json.Unmarshal(event.Content(), &mapping); err != nil { + return err + } + if mapping.MXIDMapping != nil { + storeUserID, userErr := spec.NewUserID(mapping.MXIDMapping.UserID, true) + if userErr != nil { + return userErr + } + err = r.RSAPI.StoreUserRoomPublicKey(ctx, mapping.MXIDMapping.UserRoomKey, *storeUserID, event.RoomID()) + if err != nil { + return fmt.Errorf("failed storing user room public key: %w", err) + } + } + } + switch input.Kind { case api.KindNew: if err = r.updateLatestEvents( @@ -436,7 +477,7 @@ func (r *Inputer) processRoomEvent( return fmt.Errorf("r.updateLatestEvents: %w", err) } case api.KindOld: - err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{ + err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{ { Type: api.OutputTypeOldRoomEvent, OldRoomEvent: &api.OutputOldRoomEvent{ @@ -462,7 +503,7 @@ func (r *Inputer) processRoomEvent( // so notify downstream components to redact this event - they should have it if they've // been tracking our output log. if redactedEventID != "" { - err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{ + err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{ { Type: api.OutputTypeRedactedEvent, RedactedEvent: &api.OutputRedactedEvent{ @@ -491,9 +532,9 @@ func (r *Inputer) processRoomEvent( // handleRemoteRoomUpgrade updates published rooms and room aliases func (r *Inputer) handleRemoteRoomUpgrade(ctx context.Context, event gomatrixserverlib.PDU) error { - oldRoomID := event.RoomID() + oldRoomID := event.RoomID().String() newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str - return r.DB.UpgradeRoom(ctx, oldRoomID, newRoomID, event.Sender()) + return r.DB.UpgradeRoom(ctx, oldRoomID, newRoomID, string(event.SenderID())) } // processStateBefore works out what the state is before the event and @@ -551,7 +592,7 @@ func (r *Inputer) processStateBefore( StateKey: "", }) stateBeforeReq := &api.QueryStateAfterEventsRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), PrevEventIDs: event.PrevEventIDs(), StateToFetch: tuplesNeeded, } @@ -561,7 +602,7 @@ func (r *Inputer) processStateBefore( } switch { case !stateBeforeRes.RoomExists: - rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID()) + rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID().String()) return case !stateBeforeRes.PrevEventsExist: rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID()) @@ -579,7 +620,9 @@ func (r *Inputer) processStateBefore( stateBeforeAuth := gomatrixserverlib.NewAuthEvents( gomatrixserverlib.ToPDUs(stateBeforeEvent), ) - if rejectionErr = gomatrixserverlib.Allowed(event, &stateBeforeAuth); rejectionErr != nil { + if rejectionErr = gomatrixserverlib.Allowed(event, &stateBeforeAuth, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); rejectionErr != nil { rejectionErr = fmt.Errorf("Allowed() failed for stateBeforeEvent: %w", rejectionErr) return } @@ -660,7 +703,7 @@ func (r *Inputer) fetchAuthEvents( // Request the entire auth chain for the event in question. This should // contain all of the auth events — including ones that we already know — // so we'll need to filter through those in the next section. - res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID(), event.EventID()) + res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID().String(), event.EventID()) if err != nil { logger.WithError(err).Warnf("Failed to get event auth from federation for %q: %s", event.EventID(), err) continue @@ -690,7 +733,9 @@ nextAuthEvent: // Check the signatures of the event. If this fails then we'll simply // skip it, because gomatrixserverlib.Allowed() will notice a problem // if a critical event is missing anyway. - if err := gomatrixserverlib.VerifyEventSignatures(ctx, authEvent, r.FSAPI.KeyRing()); err != nil { + if err := gomatrixserverlib.VerifyEventSignatures(ctx, authEvent, r.FSAPI.KeyRing(), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { continue nextAuthEvent } @@ -706,7 +751,9 @@ nextAuthEvent: } // Check if the auth event should be rejected. - err := gomatrixserverlib.Allowed(authEvent, auth) + err := gomatrixserverlib.Allowed(authEvent, auth, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }) if isRejected = err != nil; isRejected { logger.WithError(err).Warnf("Auth event %s rejected", authEvent.EventID()) } @@ -769,7 +816,7 @@ func (r *Inputer) calculateAndSetState( return fmt.Errorf("r.DB.GetRoomUpdater: %w", err) } defer sqlutil.EndTransactionWithCheck(updater, &succeeded, &err) - roomState := state.NewStateResolution(updater, roomInfo) + roomState := state.NewStateResolution(updater, roomInfo, r.Queryer) if input.HasState { // We've been told what the state at the event is so we don't need to calculate it. @@ -815,7 +862,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents)) latestReq := &api.QueryLatestEventsAndStateRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), } latestRes := &api.QueryLatestEventsAndStateResponse{} if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil { @@ -828,15 +875,15 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r continue } - localpart, senderDomain, err := gomatrixserverlib.SplitID('@', *memberEvent.StateKey()) + memberUserID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*memberEvent.StateKey())) if err != nil { continue } accountRes := &userAPI.QueryAccountByLocalpartResponse{} if err = r.UserAPI.QueryAccountByLocalpart(ctx, &userAPI.QueryAccountByLocalpartRequest{ - Localpart: localpart, - ServerName: senderDomain, + Localpart: memberUserID.Local(), + ServerName: memberUserID.Domain(), }, accountRes); err != nil { return err } @@ -856,10 +903,10 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r stateKey := *memberEvent.StateKey() fledglingEvent := &gomatrixserverlib.ProtoEvent{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), Type: spec.MRoomMember, StateKey: &stateKey, - Sender: stateKey, + SenderID: stateKey, PrevEvents: prevEvents, } @@ -872,7 +919,12 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r return err } - event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, r.SigningIdentity, time.Now(), &eventsNeeded, latestRes) + signingIdentity, err := r.SigningIdentity(ctx, event.RoomID(), *memberUserID) + if err != nil { + return err + } + + event, err := eventutil.BuildEvent(ctx, fledglingEvent, &signingIdentity, time.Now(), &eventsNeeded, latestRes) if err != nil { return err } @@ -880,8 +932,8 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r inputEvents = append(inputEvents, api.InputRoomEvent{ Kind: api.KindNew, Event: event, - Origin: senderDomain, - SendAsServer: string(senderDomain), + Origin: memberUserID.Domain(), + SendAsServer: string(memberUserID.Domain()), }) prevEvents = []string{event.EventID()} } diff --git a/roomserver/internal/input/input_events_test.go b/roomserver/internal/input/input_events_test.go index 568038132f..4ee6d21106 100644 --- a/roomserver/internal/input/input_events_test.go +++ b/roomserver/internal/input/input_events_test.go @@ -58,7 +58,9 @@ func Test_EventAuth(t *testing.T) { } // Finally check that the event is NOT allowed - if err := gomatrixserverlib.Allowed(ev.PDU, &allower); err == nil { + if err := gomatrixserverlib.Allowed(ev.PDU, &allower, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) + }); err == nil { t.Fatalf("event should not be allowed, but it was") } } diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 7a7a021a37..ec03d6f137 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -197,7 +197,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // send the event asynchronously but we would need to ensure that 1) the events are written to the log in // the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the // necessary bookkeeping we'll keep the event sending synchronous for now. - if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID(), updates); err != nil { + if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID().String(), updates); err != nil { return fmt.Errorf("u.api.WriteOutputEvents: %w", err) } @@ -213,7 +213,7 @@ func (u *latestEventsUpdater) latestState() error { defer trace.EndRegion() var err error - roomState := state.NewStateResolution(u.updater, u.roomInfo) + roomState := state.NewStateResolution(u.updater, u.roomInfo, u.api.Queryer) // Work out if the state at the extremities has actually changed // or not. If they haven't then we won't bother doing all of the @@ -290,7 +290,7 @@ func (u *latestEventsUpdater) latestState() error { if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 { logrus.WithFields(logrus.Fields{ "event_id": u.event.EventID(), - "room_id": u.event.RoomID(), + "room_id": u.event.RoomID().String(), "old_state_nid": u.oldStateNID, "new_state_nid": u.newStateNID, "old_latest": u.oldLatest.EventIDs(), diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go index 98d7d13b13..4cfc2cda9c 100644 --- a/roomserver/internal/input/input_membership.go +++ b/roomserver/internal/input/input_membership.go @@ -18,7 +18,6 @@ import ( "context" "fmt" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/dendrite/internal" @@ -72,7 +71,7 @@ func (r *Inputer) updateMemberships( if change.addedEventNID != 0 { ae, _ = helpers.EventMap(events).Lookup(change.addedEventNID) } - if updates, err = r.updateMembership(updater, targetUserNID, re, ae, updates); err != nil { + if updates, err = r.updateMembership(ctx, updater, targetUserNID, re, ae, updates); err != nil { return nil, err } } @@ -80,6 +79,7 @@ func (r *Inputer) updateMemberships( } func (r *Inputer) updateMembership( + ctx context.Context, updater *shared.RoomUpdater, targetUserNID types.EventStateKeyNID, remove, add *types.Event, @@ -97,7 +97,7 @@ func (r *Inputer) updateMembership( var targetLocal bool if add != nil { - targetLocal = r.isLocalTarget(add) + targetLocal = r.isLocalTarget(ctx, add) } mu, err := updater.MembershipUpdater(targetUserNID, targetLocal) @@ -136,11 +136,14 @@ func (r *Inputer) updateMembership( } } -func (r *Inputer) isLocalTarget(event *types.Event) bool { +func (r *Inputer) isLocalTarget(ctx context.Context, event *types.Event) bool { isTargetLocalUser := false if statekey := event.StateKey(); statekey != nil { - _, domain, _ := gomatrixserverlib.SplitID('@', *statekey) - isTargetLocalUser = domain == r.ServerName + userID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*statekey)) + if err != nil || userID == nil { + return isTargetLocalUser + } + isTargetLocalUser = userID.Domain() == r.ServerName } return isTargetLocalUser } @@ -161,9 +164,10 @@ func updateToJoinMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, + RoomID: add.RoomID().String(), Membership: spec.Join, RetiredByEventID: add.EventID(), - TargetUserID: *add.StateKey(), + TargetSenderID: spec.SenderID(*add.StateKey()), }, }) } @@ -187,9 +191,10 @@ func updateToLeaveMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, + RoomID: add.RoomID().String(), Membership: newMembership, RetiredByEventID: add.EventID(), - TargetUserID: *add.StateKey(), + TargetSenderID: spec.SenderID(*add.StateKey()), }, }) } diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index 10486138df..d9ab291e94 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -84,7 +84,7 @@ func (t *missingStateReq) processEventWithMissingState( // need to fallback to /state. t.log = util.GetLogger(ctx).WithFields(map[string]interface{}{ "txn_event": e.EventID(), - "room_id": e.RoomID(), + "room_id": e.RoomID().String(), "txn_prev_events": e.PrevEventIDs(), }) @@ -259,12 +259,20 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query // the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event. var states []*respState + var validationError error for _, prevEventID := range e.PrevEventIDs() { // Look up what the state is after the backward extremity. This will either // come from the roomserver, if we know all the required events, or it will // come from a remote server via /state_ids if not. - prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID(), prevEventID) - if err != nil { + prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID().String(), prevEventID) + switch err2 := err.(type) { + case gomatrixserverlib.EventValidationError: + if !err2.Persistable { + return nil, err2 + } + validationError = err2 + case nil: + default: return nil, fmt.Errorf("t.lookupStateAfterEvent: %w", err) } // Append the state onto the collected state. We'll run this through the @@ -308,15 +316,22 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e } // There's more than one previous state - run them all through state res var err error - t.roomsMu.Lock(e.RoomID()) + t.roomsMu.Lock(e.RoomID().String()) resolvedState, err = t.resolveStatesAndCheck(ctx, roomVersion, respStates, e) - t.roomsMu.Unlock(e.RoomID()) - if err != nil { + t.roomsMu.Unlock(e.RoomID().String()) + switch err2 := err.(type) { + case gomatrixserverlib.EventValidationError: + if !err2.Persistable { + return nil, err2 + } + validationError = err2 + case nil: + default: return nil, fmt.Errorf("t.resolveStatesAndCheck: %w", err) } } - return resolvedState, nil + return resolvedState, validationError } // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) @@ -339,8 +354,15 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion } // fetch the event we're missing and add it to the pile + var validationError error h, err := t.lookupEvent(ctx, roomVersion, roomID, eventID, false) - switch err.(type) { + switch e := err.(type) { + case gomatrixserverlib.EventValidationError: + if !e.Persistable { + logrus.WithContext(ctx).WithError(err).Errorf("Failed to look up event %s", eventID) + return nil, false, e + } + validationError = e case verifySigError: return respState, false, nil case nil: @@ -365,7 +387,7 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion } } - return respState, false, nil + return respState, false, validationError } func (t *missingStateReq) cacheAndReturn(ev gomatrixserverlib.PDU) gomatrixserverlib.PDU { @@ -383,7 +405,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, even defer trace.EndRegion() var res parsedRespState - roomState := state.NewStateResolution(t.db, t.roomInfo) + roomState := state.NewStateResolution(t.db, t.roomInfo, t.inputer.Queryer) stateAtEvents, err := t.db.StateAtEventIDs(ctx, []string{eventID}) if err != nil { t.log.WithError(err).Warnf("failed to get state after %s locally", eventID) @@ -473,18 +495,28 @@ func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion stateEventList = append(stateEventList, state.StateEvents...) } resolvedStateEvents, err := gomatrixserverlib.ResolveConflicts( - roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), + roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }, ) if err != nil { return nil, err } // apply the current event + var validationError error retryAllowedState: - if err = checkAllowedByState(backwardsExtremity, resolvedStateEvents); err != nil { + if err = checkAllowedByState(backwardsExtremity, resolvedStateEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { switch missing := err.(type) { case gomatrixserverlib.MissingAuthEventError: - h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID(), missing.AuthEventID, true) - switch err2.(type) { + h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID().String(), missing.AuthEventID, true) + switch e := err2.(type) { + case gomatrixserverlib.EventValidationError: + if !e.Persistable { + return nil, e + } + validationError = e case verifySigError: return &parsedRespState{ AuthEvents: authEventList, @@ -505,7 +537,7 @@ retryAllowedState: return &parsedRespState{ AuthEvents: authEventList, StateEvents: resolvedStateEvents, - }, nil + }, validationError } // get missing events for `e`. If `isGapFilled`=true then `newEvents` contains all the events to inject, @@ -514,7 +546,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver trace, ctx := internal.StartRegion(ctx, "getMissingEvents") defer trace.EndRegion() - logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) + logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID().String()) latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID) if err != nil { return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err) @@ -528,7 +560,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver var missingResp *fclient.RespMissingEvents for _, server := range t.servers { var m fclient.RespMissingEvents - if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID(), fclient.MissingEvents{ + if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID().String(), fclient.MissingEvents{ Limit: 20, // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. EarliestEvents: latestEvents, @@ -565,7 +597,9 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver // will be added and duplicates will be removed. missingEvents := make([]gomatrixserverlib.PDU, 0, len(missingResp.Events)) for _, ev := range missingResp.Events.UntrustedEvents(roomVersion) { - if err = gomatrixserverlib.VerifyEventSignatures(ctx, ev, t.keys); err != nil { + if err = gomatrixserverlib.VerifyEventSignatures(ctx, ev, t.keys, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { continue } missingEvents = append(missingEvents, t.cacheAndReturn(ev)) @@ -654,7 +688,9 @@ func (t *missingStateReq) lookupMissingStateViaState( authEvents, stateEvents, err := gomatrixserverlib.CheckStateResponse(ctx, &fclient.RespState{ StateEvents: state.GetStateEvents(), AuthEvents: state.GetAuthEvents(), - }, roomVersion, t.keys, nil) + }, roomVersion, t.keys, nil, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { return nil, err } @@ -771,7 +807,11 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo // Define what we'll do in order to fetch the missing event ID. fetch := func(missingEventID string) { h, herr := t.lookupEvent(ctx, roomVersion, roomID, missingEventID, false) - switch herr.(type) { + switch e := herr.(type) { + case gomatrixserverlib.EventValidationError: + if !e.Persistable { + return + } case verifySigError: return case nil: @@ -861,6 +901,8 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs } var event gomatrixserverlib.PDU found := false + var validationError error +serverLoop: for _, serverName := range t.servers { reqctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() @@ -878,25 +920,40 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs continue } event, err = verImpl.NewEventFromUntrustedJSON(txn.PDUs[0]) - if err != nil { + switch e := err.(type) { + case gomatrixserverlib.EventValidationError: + // If the event is persistable, e.g. failed validation for exceeding + // byte sizes, we can "accept" the event. + if e.Persistable { + validationError = e + found = true + break serverLoop + } + // If we can't persist the event, we probably can't do so with results + // from other servers, so also break the loop. + break serverLoop + case nil: + found = true + break serverLoop + default: t.log.WithError(err).WithField("missing_event_id", missingEventID).Warnf("Failed to parse event JSON of event returned from /event") continue } - found = true - break } if !found { t.log.WithField("missing_event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(t.servers)) return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(t.servers)) } - if err := gomatrixserverlib.VerifyEventSignatures(ctx, event, t.keys); err != nil { + if err := gomatrixserverlib.VerifyEventSignatures(ctx, event, t.keys, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { t.log.WithError(err).Warnf("Couldn't validate signature of event %q from /event", event.EventID()) return nil, verifySigError{event.EventID(), err} } - return t.cacheAndReturn(event), nil + return t.cacheAndReturn(event), validationError } -func checkAllowedByState(e gomatrixserverlib.PDU, stateEvents []gomatrixserverlib.PDU) error { +func checkAllowedByState(e gomatrixserverlib.PDU, stateEvents []gomatrixserverlib.PDU, userIDForSender spec.UserIDForSender) error { authUsingState := gomatrixserverlib.NewAuthEvents(nil) for i := range stateEvents { err := authUsingState.AddEvent(stateEvents[i]) @@ -904,7 +961,7 @@ func checkAllowedByState(e gomatrixserverlib.PDU, stateEvents []gomatrixserverli return err } } - return gomatrixserverlib.Allowed(e, &authUsingState) + return gomatrixserverlib.Allowed(e, &authUsingState, userIDForSender) } func (t *missingStateReq) hadEvent(eventID string) { diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 17296febce..ae203854b6 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -74,6 +74,10 @@ func (r *Admin) PerformAdminEvacuateRoom( if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil { return nil, err } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, err + } prevEvents := latestRes.LatestEvents var senderDomain spec.ServerName @@ -96,14 +100,15 @@ func (r *Admin) PerformAdminEvacuateRoom( RoomID: roomID, Type: spec.MRoomMember, StateKey: &stateKey, - Sender: stateKey, + SenderID: stateKey, PrevEvents: prevEvents, } - _, senderDomain, err = gomatrixserverlib.SplitID('@', fledglingEvent.Sender) - if err != nil { + userID, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(fledglingEvent.SenderID)) + if err != nil || userID == nil { continue } + senderDomain = userID.Domain() if fledglingEvent.Content, err = json.Marshal(memberContent); err != nil { return nil, err @@ -119,7 +124,7 @@ func (r *Admin) PerformAdminEvacuateRoom( continue } - event, err = eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, identity, time.Now(), &eventsNeeded, latestRes) + event, err = eventutil.BuildEvent(ctx, fledglingEvent, identity, time.Now(), &eventsNeeded, latestRes) if err != nil { return nil, err } @@ -136,7 +141,7 @@ func (r *Admin) PerformAdminEvacuateRoom( inputReq := &api.InputRoomEventsRequest{ InputRoomEvents: inputEvents, - Asynchronous: true, + Asynchronous: false, } inputRes := &api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(ctx, inputReq, inputRes) @@ -148,20 +153,20 @@ func (r *Admin) PerformAdminEvacuateUser( ctx context.Context, userID string, ) (affected []string, err error) { - _, domain, err := gomatrixserverlib.SplitID('@', userID) + fullUserID, err := spec.NewUserID(userID, true) if err != nil { return nil, err } - if !r.Cfg.Matrix.IsLocalServerName(domain) { + if !r.Cfg.Matrix.IsLocalServerName(fullUserID.Domain()) { return nil, fmt.Errorf("can only evacuate local users using this endpoint") } - roomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Join) + roomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Join) if err != nil { return nil, err } - inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Invite) + inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Invite) if err != nil && err != sql.ErrNoRows { return nil, err } @@ -171,7 +176,7 @@ func (r *Admin) PerformAdminEvacuateUser( for _, roomID := range allRooms { leaveReq := &api.PerformLeaveRequest{ RoomID: roomID, - UserID: userID, + Leaver: *fullUserID, } leaveRes := &api.PerformLeaveResponse{} outputEvents, err := r.Leaver.PerformLeave(ctx, leaveReq, leaveRes) @@ -199,19 +204,13 @@ func (r *Admin) PerformAdminPurgeRoom( return err } - // Evacuate the room before purging it from the database - if _, err := r.PerformAdminEvacuateRoom(ctx, roomID); err != nil { - logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to evacuate room before purging") - return err - } - logrus.WithField("room_id", roomID).Warn("Purging room from roomserver") if err := r.DB.PurgeRoom(ctx, roomID); err != nil { logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to purge room from roomserver") return err } - logrus.WithField("room_id", roomID).Warn("Room purged from roomserver") + logrus.WithField("room_id", roomID).Warn("Room purged from roomserver, informing other components") return r.Inputer.OutputProducer.ProduceRoomEvents(roomID, []api.OutputEvent{ { @@ -227,10 +226,11 @@ func (r *Admin) PerformAdminDownloadState( ctx context.Context, roomID, userID string, serverName spec.ServerName, ) error { - _, senderDomain, err := r.Cfg.Matrix.SplitLocalID('@', userID) + fullUserID, err := spec.NewUserID(userID, true) if err != nil { return err } + senderDomain := fullUserID.Domain() roomInfo, err := r.DB.RoomInfo(ctx, roomID) if err != nil { @@ -256,13 +256,17 @@ func (r *Admin) PerformAdminDownloadState( return fmt.Errorf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity, err) } for _, authEvent := range state.GetAuthEvents().UntrustedEvents(roomInfo.RoomVersion) { - if err = gomatrixserverlib.VerifyEventSignatures(ctx, authEvent, r.Inputer.KeyRing); err != nil { + if err = gomatrixserverlib.VerifyEventSignatures(ctx, authEvent, r.Inputer.KeyRing, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { continue } authEventMap[authEvent.EventID()] = authEvent } for _, stateEvent := range state.GetStateEvents().UntrustedEvents(roomInfo.RoomVersion) { - if err = gomatrixserverlib.VerifyEventSignatures(ctx, stateEvent, r.Inputer.KeyRing); err != nil { + if err = gomatrixserverlib.VerifyEventSignatures(ctx, stateEvent, r.Inputer.KeyRing, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Queryer.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { continue } stateEventMap[stateEvent.EventID()] = stateEvent @@ -281,11 +285,21 @@ func (r *Admin) PerformAdminDownloadState( stateIDs = append(stateIDs, stateEvent.EventID()) } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return err + } + senderID, err := r.Queryer.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) + if err != nil { + return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID) + } proto := &gomatrixserverlib.ProtoEvent{ - Type: "org.matrix.dendrite.state_download", - Sender: userID, - RoomID: roomID, - Content: spec.RawJSON("{}"), + Type: "org.matrix.dendrite.state_download", + SenderID: string(*senderID), + RoomID: roomID, + Content: spec.RawJSON("{}"), } eventsNeeded, err := gomatrixserverlib.StateNeededForProtoEvent(proto) @@ -306,7 +320,7 @@ func (r *Admin) PerformAdminDownloadState( return err } - ev, err := eventutil.BuildEvent(ctx, proto, r.Cfg.Matrix, identity, time.Now(), &eventsNeeded, queryRes) + ev, err := eventutil.BuildEvent(ctx, proto, identity, time.Now(), &eventsNeeded, queryRes) if err != nil { return fmt.Errorf("eventutil.BuildEvent: %w", err) } diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index fb579f03a0..dafa58736a 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -42,6 +42,7 @@ type Backfiller struct { DB storage.Database FSAPI federationAPI.RoomserverFederationAPI KeyRing gomatrixserverlib.JSONVerifier + Querier api.QuerySenderIDAPI // The servers which should be preferred above other servers when backfilling PreferServers []spec.ServerName @@ -79,7 +80,7 @@ func (r *Backfiller) PerformBackfill( } // Scan the event tree for events to send back. - resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName, r.Querier) if err != nil { return err } @@ -113,7 +114,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform if info == nil || info.IsStub() { return fmt.Errorf("backfillViaFederation: missing room info for room %s", req.RoomID) } - requester := newBackfillRequester(r.DB, r.FSAPI, req.VirtualHost, r.IsLocalServerName, req.BackwardsExtremities, r.PreferServers) + requester := newBackfillRequester(r.DB, r.FSAPI, r.Querier, req.VirtualHost, r.IsLocalServerName, req.BackwardsExtremities, r.PreferServers, info.RoomVersion) // Request 100 items regardless of what the query asks for. // We don't want to go much higher than this. // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass @@ -121,7 +122,9 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform // Specifically the test "Outbound federation can backfill events" events, err := gomatrixserverlib.RequestBackfill( ctx, req.VirtualHost, requester, - r.KeyRing, req.RoomID, info.RoomVersion, req.PrevEventIDs(), 100, + r.KeyRing, req.RoomID, info.RoomVersion, req.PrevEventIDs(), 100, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Querier.QueryUserIDForSender(ctx, roomID, senderID) + }, ) // Only return an error if we really couldn't get any events. if err != nil && len(events) == 0 { @@ -133,7 +136,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform logrus.WithError(err).WithField("room_id", req.RoomID).Infof("backfilled %d events", len(events)) // persist these new events - auth checks have already been done - roomNID, backfilledEventMap := persistEvents(ctx, r.DB, events) + roomNID, backfilledEventMap := persistEvents(ctx, r.DB, r.Querier, events) for _, ev := range backfilledEventMap { // now add state for these events @@ -210,7 +213,9 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom continue } loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false) - result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents) + result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.Querier.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { logger.WithError(err).Warn("failed to load and verify event") continue @@ -242,13 +247,14 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom } } util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents)) - persistEvents(ctx, r.DB, newEvents) + persistEvents(ctx, r.DB, r.Querier, newEvents) } // backfillRequester implements gomatrixserverlib.BackfillRequester type backfillRequester struct { db storage.Database fsAPI federationAPI.RoomserverFederationAPI + querier api.QuerySenderIDAPI virtualHost spec.ServerName isLocalServerName func(spec.ServerName) bool preferServer map[spec.ServerName]bool @@ -259,14 +265,16 @@ type backfillRequester struct { eventIDToBeforeStateIDs map[string][]string eventIDMap map[string]gomatrixserverlib.PDU historyVisiblity gomatrixserverlib.HistoryVisibility - roomInfo types.RoomInfo + roomVersion gomatrixserverlib.RoomVersion } func newBackfillRequester( db storage.Database, fsAPI federationAPI.RoomserverFederationAPI, + querier api.QuerySenderIDAPI, virtualHost spec.ServerName, isLocalServerName func(spec.ServerName) bool, bwExtrems map[string][]string, preferServers []spec.ServerName, + roomVersion gomatrixserverlib.RoomVersion, ) *backfillRequester { preferServer := make(map[spec.ServerName]bool) for _, p := range preferServers { @@ -275,6 +283,7 @@ func newBackfillRequester( return &backfillRequester{ db: db, fsAPI: fsAPI, + querier: querier, virtualHost: virtualHost, isLocalServerName: isLocalServerName, eventIDToBeforeStateIDs: make(map[string][]string), @@ -282,6 +291,7 @@ func newBackfillRequester( bwExtrems: bwExtrems, preferServer: preferServer, historyVisiblity: gomatrixserverlib.HistoryVisibilityShared, + roomVersion: roomVersion, } } @@ -291,7 +301,7 @@ func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent return ids, nil } if len(targetEvent.PrevEventIDs()) == 0 && targetEvent.Type() == "m.room.create" && targetEvent.StateKeyEquals("") { - util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID()).Info("Backfilled to the beginning of the room") + util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID().String()).Info("Backfilled to the beginning of the room") b.eventIDToBeforeStateIDs[targetEvent.EventID()] = []string{} return nil, nil } @@ -456,14 +466,14 @@ FindSuccessor: return nil } - stateEntries, err := helpers.StateBeforeEvent(ctx, b.db, info, NIDs[eventID].EventNID) + stateEntries, err := helpers.StateBeforeEvent(ctx, b.db, info, NIDs[eventID].EventNID, b.querier) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") return nil } // possibly return all joined servers depending on history visiblity - memberEventsFromVis, visibility, err := joinEventsFromHistoryVisibility(ctx, b.db, info, stateEntries, b.virtualHost) + memberEventsFromVis, visibility, err := joinEventsFromHistoryVisibility(ctx, b.db, b.querier, info, stateEntries, b.virtualHost) b.historyVisiblity = visibility if err != nil { logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") @@ -484,8 +494,8 @@ FindSuccessor: // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[spec.ServerName]bool) for _, event := range memberEvents { - if _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()); err == nil { - serverSet[senderDomain] = true + if sender, err := b.querier.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()); err == nil { + serverSet[sender.Domain()] = true } } var servers []spec.ServerName @@ -525,15 +535,11 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, } eventNIDs := make([]types.EventNID, len(nidMap)) i := 0 - roomNID := b.roomInfo.RoomNID for _, nid := range nidMap { eventNIDs[i] = nid.EventNID i++ - if roomNID == 0 { - roomNID = nid.RoomNID - } } - eventsWithNids, err := b.db.Events(ctx, b.roomInfo.RoomVersion, eventNIDs) + eventsWithNids, err := b.db.Events(ctx, b.roomVersion, eventNIDs) if err != nil { logrus.WithError(err).WithField("event_nids", eventNIDs).Error("Failed to load events") return nil, err @@ -550,7 +556,7 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, // TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just // pull all events and then filter by that table. func joinEventsFromHistoryVisibility( - ctx context.Context, db storage.RoomDatabase, roomInfo *types.RoomInfo, stateEntries []types.StateEntry, + ctx context.Context, db storage.RoomDatabase, querier api.QuerySenderIDAPI, roomInfo *types.RoomInfo, stateEntries []types.StateEntry, thisServer spec.ServerName) ([]types.Event, gomatrixserverlib.HistoryVisibility, error) { var eventNIDs []types.EventNID @@ -578,7 +584,7 @@ func joinEventsFromHistoryVisibility( } // Can we see events in the room? - canSeeEvents := auth.IsServerAllowed(thisServer, true, events) + canSeeEvents := auth.IsServerAllowed(ctx, querier, thisServer, true, events) visibility := auth.HistoryVisibilityForRoom(events) if !canSeeEvents { logrus.Infof("ServersAtEvent history not visible to us: %s", visibility) @@ -593,7 +599,7 @@ func joinEventsFromHistoryVisibility( return evs, visibility, err } -func persistEvents(ctx context.Context, db storage.Database, events []gomatrixserverlib.PDU) (types.RoomNID, map[string]types.Event) { +func persistEvents(ctx context.Context, db storage.Database, querier api.QuerySenderIDAPI, events []gomatrixserverlib.PDU) (types.RoomNID, map[string]types.Event) { var roomNID types.RoomNID var eventNID types.EventNID backfilledEventMap := make(map[string]types.Event) @@ -635,9 +641,9 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse continue } - resolver := state.NewStateResolution(db, roomInfo) + resolver := state.NewStateResolution(db, roomInfo, querier) - _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev, &resolver) + _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev, &resolver, querier) if err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event") continue diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go new file mode 100644 index 0000000000..eb8de78110 --- /dev/null +++ b/roomserver/internal/perform/perform_create_room.go @@ -0,0 +1,562 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "net/http" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +const ( + historyVisibilityShared = "shared" +) + +type Creator struct { + DB storage.Database + Cfg *config.RoomServer + RSAPI api.RoomserverInternalAPI +} + +// PerformCreateRoom handles all the steps necessary to create a new room. +// nolint: gocyclo +func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *api.PerformCreateRoomRequest) (string, *util.JSONResponse) { + verImpl, err := gomatrixserverlib.GetRoomVersion(createRequest.RoomVersion) + if err != nil { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("unknown room version"), + } + } + + createContent := map[string]interface{}{} + if len(createRequest.CreationContent) > 0 { + if err = json.Unmarshal(createRequest.CreationContent, &createContent); err != nil { + util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed") + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("invalid create content"), + } + } + } + + _, err = c.DB.AssignRoomNID(ctx, roomID, createRequest.RoomVersion) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to assign roomNID") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + var senderID spec.SenderID + if createRequest.RoomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + // create user room key if needed + key, keyErr := c.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if keyErr != nil { + util.GetLogger(ctx).WithError(keyErr).Error("GetOrCreateUserRoomPrivateKey failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + senderID = spec.SenderIDFromPseudoIDKey(key) + } else { + senderID = spec.SenderID(userID.String()) + } + + // TODO: Maybe, at some point, GMSL should return the events to create, so we can define the version + // entirely there. + switch createRequest.RoomVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + default: + createContent["creator"] = senderID + } + + createContent["room_version"] = createRequest.RoomVersion + powerLevelContent := eventutil.InitialPowerLevelsContent(string(senderID)) + joinRuleContent := gomatrixserverlib.JoinRuleContent{ + JoinRule: spec.Invite, + } + historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{ + HistoryVisibility: historyVisibilityShared, + } + + if createRequest.PowerLevelContentOverride != nil { + // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults + err = json.Unmarshal(createRequest.PowerLevelContentOverride, &powerLevelContent) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed") + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("malformed power_level_content_override"), + } + } + } + + var guestsCanJoin bool + switch createRequest.StatePreset { + case spec.PresetPrivateChat: + joinRuleContent.JoinRule = spec.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + guestsCanJoin = true + case spec.PresetTrustedPrivateChat: + joinRuleContent.JoinRule = spec.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + for _, invitee := range createRequest.InvitedUsers { + powerLevelContent.Users[invitee] = 100 + } + guestsCanJoin = true + case spec.PresetPublicChat: + joinRuleContent.JoinRule = spec.Public + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + } + + createEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomCreate, + Content: createContent, + } + powerLevelEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomPowerLevels, + Content: powerLevelContent, + } + joinRuleEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomJoinRules, + Content: joinRuleContent, + } + historyVisibilityEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomHistoryVisibility, + Content: historyVisibilityContent, + } + membershipEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomMember, + StateKey: string(senderID), + } + + memberContent := gomatrixserverlib.MemberContent{ + Membership: spec.Join, + DisplayName: createRequest.UserDisplayName, + AvatarURL: createRequest.UserAvatarURL, + } + + // get the signing identity + identity, err := c.Cfg.Matrix.SigningIdentityFor(userID.Domain()) // we MUST use the server signing mxid_mapping + if err != nil { + logrus.WithError(err).WithField("domain", userID.Domain()).Error("unable to find signing identity for domain") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // If we are creating a room with pseudo IDs, create and sign the MXIDMapping + if createRequest.RoomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + var pseudoIDKey ed25519.PrivateKey + pseudoIDKey, err = c.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("GetOrCreateUserRoomPrivateKey failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + mapping := &gomatrixserverlib.MXIDMapping{ + UserRoomKey: spec.SenderIDFromPseudoIDKey(pseudoIDKey), + UserID: userID.String(), + } + + // Sign the mapping with the server identity + if err = mapping.Sign(identity.ServerName, identity.KeyID, identity.PrivateKey); err != nil { + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + memberContent.MXIDMapping = mapping + + // sign all events with the pseudo ID key + identity = &fclient.SigningIdentity{ + ServerName: spec.ServerName(spec.SenderIDFromPseudoIDKey(pseudoIDKey)), + KeyID: "ed25519:1", + PrivateKey: pseudoIDKey, + } + } + membershipEvent.Content = memberContent + + var nameEvent *gomatrixserverlib.FledglingEvent + var topicEvent *gomatrixserverlib.FledglingEvent + var guestAccessEvent *gomatrixserverlib.FledglingEvent + var aliasEvent *gomatrixserverlib.FledglingEvent + + if createRequest.RoomName != "" { + nameEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomName, + Content: eventutil.NameContent{ + Name: createRequest.RoomName, + }, + } + } + + if createRequest.Topic != "" { + topicEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomTopic, + Content: eventutil.TopicContent{ + Topic: createRequest.Topic, + }, + } + } + + if guestsCanJoin { + guestAccessEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomGuestAccess, + Content: eventutil.GuestAccessContent{ + GuestAccess: "can_join", + }, + } + } + + var roomAlias string + if createRequest.RoomAliasName != "" { + roomAlias = fmt.Sprintf("#%s:%s", createRequest.RoomAliasName, userID.Domain()) + // check it's free + // TODO: This races but is better than nothing + hasAliasReq := api.GetRoomIDForAliasRequest{ + Alias: roomAlias, + IncludeAppservices: false, + } + + var aliasResp api.GetRoomIDForAliasResponse + err = c.RSAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + if aliasResp.RoomID != "" { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.RoomInUse("Room ID already exists."), + } + } + + aliasEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomCanonicalAlias, + Content: eventutil.CanonicalAlias{ + Alias: roomAlias, + }, + } + } + + var initialStateEvents []gomatrixserverlib.FledglingEvent + for i := range createRequest.InitialState { + if createRequest.InitialState[i].StateKey != "" { + initialStateEvents = append(initialStateEvents, createRequest.InitialState[i]) + continue + } + + switch createRequest.InitialState[i].Type { + case spec.MRoomCreate: + continue + + case spec.MRoomPowerLevels: + powerLevelEvent = createRequest.InitialState[i] + + case spec.MRoomJoinRules: + joinRuleEvent = createRequest.InitialState[i] + + case spec.MRoomHistoryVisibility: + historyVisibilityEvent = createRequest.InitialState[i] + + case spec.MRoomGuestAccess: + guestAccessEvent = &createRequest.InitialState[i] + + case spec.MRoomName: + nameEvent = &createRequest.InitialState[i] + + case spec.MRoomTopic: + topicEvent = &createRequest.InitialState[i] + + default: + initialStateEvents = append(initialStateEvents, createRequest.InitialState[i]) + } + } + + // send events into the room in order of: + // 1- m.room.create + // 2- room creator join member + // 3- m.room.power_levels + // 4- m.room.join_rules + // 5- m.room.history_visibility + // 6- m.room.canonical_alias (opt) + // 7- m.room.guest_access (opt) + // 8- other initial state items + // 9- m.room.name (opt) + // 10- m.room.topic (opt) + // 11- invite events (opt) - with is_direct flag if applicable TODO + // 12- 3pid invite events (opt) TODO + // This differs from Synapse slightly. Synapse would vary the ordering of 3-7 + // depending on if those events were in "initial_state" or not. This made it + // harder to reason about, hence sticking to a strict static ordering. + // TODO: Synapse has txn/token ID on each event. Do we need to do this here? + eventsToMake := []gomatrixserverlib.FledglingEvent{ + createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent, + } + if guestAccessEvent != nil { + eventsToMake = append(eventsToMake, *guestAccessEvent) + } + eventsToMake = append(eventsToMake, initialStateEvents...) + if nameEvent != nil { + eventsToMake = append(eventsToMake, *nameEvent) + } + if topicEvent != nil { + eventsToMake = append(eventsToMake, *topicEvent) + } + if aliasEvent != nil { + // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. + // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. + eventsToMake = append(eventsToMake, *aliasEvent) + } + + // TODO: invite events + // TODO: 3pid invite events + + var builtEvents []*types.HeaderedEvent + authEvents := gomatrixserverlib.NewAuthEvents(nil) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("rsapi.QuerySenderIDForUser failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + for i, e := range eventsToMake { + depth := i + 1 // depth starts at 1 + + builder := verImpl.NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ + SenderID: string(senderID), + RoomID: roomID.String(), + Type: e.Type, + StateKey: &e.StateKey, + Depth: int64(depth), + }) + err = builder.SetContent(e.Content) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + if i > 0 { + builder.PrevEvents = []string{builtEvents[i-1].EventID()} + } + var ev gomatrixserverlib.PDU + if err = builder.AddAuthEvents(&authEvents); err != nil { + util.GetLogger(ctx).WithError(err).Error("AddAuthEvents failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + ev, err = builder.Build(createRequest.EventTime, identity.ServerName, identity.KeyID, identity.PrivateKey) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("buildEvent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + if err = gomatrixserverlib.Allowed(ev, &authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return c.RSAPI.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { + util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // Add the event to the list of auth events + builtEvents = append(builtEvents, &types.HeaderedEvent{PDU: ev}) + err = authEvents.AddEvent(ev) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + + inputs := make([]api.InputRoomEvent, 0, len(builtEvents)) + for _, event := range builtEvents { + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + Origin: userID.Domain(), + SendAsServer: api.DoNotSendToOtherServers, + }) + } + + // send the events to the roomserver + if err = api.SendInputRoomEvents(ctx, c.RSAPI, userID.Domain(), inputs, false); err != nil { + util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // TODO(#269): Reserve room alias while we create the room. This stops us + // from creating the room but still failing due to the alias having already + // been taken. + if roomAlias != "" { + aliasAlreadyExists, aliasErr := c.RSAPI.SetRoomAlias(ctx, senderID, roomID, roomAlias) + if aliasErr != nil { + util.GetLogger(ctx).WithError(aliasErr).Error("aliasAPI.SetRoomAlias failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + if aliasAlreadyExists { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.RoomInUse("Room alias already exists."), + } + } + } + + // If this is a direct message then we should invite the participants. + if len(createRequest.InvitedUsers) > 0 { + // Build some stripped state for the invite. + var globalStrippedState []gomatrixserverlib.InviteStrippedState + for _, event := range builtEvents { + // Chosen events from the spec: + // https://spec.matrix.org/v1.3/client-server-api/#stripped-state + switch event.Type() { + case spec.MRoomCreate: + fallthrough + case spec.MRoomName: + fallthrough + case spec.MRoomAvatar: + fallthrough + case spec.MRoomTopic: + fallthrough + case spec.MRoomCanonicalAlias: + fallthrough + case spec.MRoomEncryption: + fallthrough + case spec.MRoomMember: + fallthrough + case spec.MRoomJoinRules: + ev := event.PDU + globalStrippedState = append( + globalStrippedState, + gomatrixserverlib.NewInviteStrippedState(ev), + ) + } + } + + // Process the invites. + for _, invitee := range createRequest.InvitedUsers { + inviteeUserID, userIDErr := spec.NewUserID(invitee, true) + if userIDErr != nil { + util.GetLogger(ctx).WithError(userIDErr).Error("invalid UserID") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + err = c.RSAPI.PerformInvite(ctx, &api.PerformInviteRequest{ + InviteInput: api.InviteInput{ + RoomID: roomID, + Inviter: userID, + Invitee: *inviteeUserID, + DisplayName: createRequest.UserDisplayName, + AvatarURL: createRequest.UserAvatarURL, + Reason: "", + IsDirect: createRequest.IsDirect, + KeyID: createRequest.KeyID, + PrivateKey: createRequest.PrivateKey, + EventTime: createRequest.EventTime, + }, + InviteRoomState: globalStrippedState, + SendAsServer: string(userID.Domain()), + }) + switch e := err.(type) { + case api.ErrInvalidID: + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown(e.Error()), + } + case api.ErrNotAllowed: + return "", &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden(e.Error()), + } + case nil: + default: + util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") + sentry.CaptureException(err) + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + } + + if createRequest.Visibility == spec.Public { + // expose this room in the published room list + if err = c.RSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: roomID.String(), + Visibility: spec.Public, + }); err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to publish room") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + + // TODO: visibility/presets/raw initial state + // TODO: Create room alias association + // Make sure this doesn't fall into an application service's namespace though! + + return roomAlias, nil +} diff --git a/roomserver/internal/perform/perform_inbound_peek.go b/roomserver/internal/perform/perform_inbound_peek.go index 3ac0f6f4df..7fbec37107 100644 --- a/roomserver/internal/perform/perform_inbound_peek.go +++ b/roomserver/internal/perform/perform_inbound_peek.go @@ -79,7 +79,7 @@ func (r *InboundPeeker) PerformInboundPeek( response.LatestEvent = &types.HeaderedEvent{PDU: sortedLatestEvents[0]} // XXX: do we actually need to do a state resolution here? - roomState := state.NewStateResolution(r.DB, info) + roomState := state.NewStateResolution(r.DB, info, r.Inputer.Queryer) var stateEntries []types.StateEntry stateEntries, err = roomState.LoadStateAtSnapshot( diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index a3fa2e011e..3abb69cb92 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -16,6 +16,7 @@ package perform import ( "context" + "crypto/ed25519" "fmt" federationAPI "github.com/matrix-org/dendrite/federationapi/api" @@ -28,186 +29,210 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" - log "github.com/sirupsen/logrus" ) +type QueryState struct { + storage.Database + querier api.QuerySenderIDAPI +} + +func (q *QueryState) GetAuthEvents(ctx context.Context, event gomatrixserverlib.PDU) (gomatrixserverlib.AuthEventProvider, error) { + return helpers.GetAuthEvents(ctx, q.Database, event.Version(), event, event.AuthEventIDs()) +} + +func (q *QueryState) GetState(ctx context.Context, roomID spec.RoomID, stateWanted []gomatrixserverlib.StateKeyTuple) ([]gomatrixserverlib.PDU, error) { + info, err := q.Database.RoomInfo(ctx, roomID.String()) + if err != nil { + return nil, fmt.Errorf("failed to load RoomInfo: %w", err) + } + if info != nil { + roomState := state.NewStateResolution(q.Database, info, q.querier) + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( + ctx, info.StateSnapshotNID(), stateWanted, + ) + if err != nil { + return nil, nil + } + stateNIDs := []types.EventNID{} + for _, stateNID := range stateEntries { + stateNIDs = append(stateNIDs, stateNID.EventNID) + } + stateEvents, err := q.Database.Events(ctx, info.RoomVersion, stateNIDs) + if err != nil { + return nil, fmt.Errorf("failed to obtain required events: %w", err) + } + + events := []gomatrixserverlib.PDU{} + for _, event := range stateEvents { + events = append(events, event.PDU) + } + return events, nil + } + + return nil, nil +} + type Inviter struct { DB storage.Database Cfg *config.RoomServer FSAPI federationAPI.RoomserverFederationAPI + RSAPI api.RoomserverInternalAPI Inputer *input.Inputer } -// nolint:gocyclo -func (r *Inviter) PerformInvite( - ctx context.Context, - req *api.PerformInviteRequest, +func (r *Inviter) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) { + info, err := r.DB.RoomInfo(ctx, roomID.String()) + if err != nil { + return false, fmt.Errorf("failed to load RoomInfo: %w", err) + } + return (info != nil && !info.IsStub()), nil +} + +func (r *Inviter) StateQuerier() gomatrixserverlib.StateQuerier { + return &QueryState{Database: r.DB} +} + +func (r *Inviter) ProcessInviteMembership( + ctx context.Context, inviteEvent *types.HeaderedEvent, ) ([]api.OutputEvent, error) { var outputUpdates []api.OutputEvent - event := req.Event - if event.StateKey() == nil { - return nil, fmt.Errorf("invite must be a state event") - } - _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()) + var updater *shared.MembershipUpdater + + userID, err := r.RSAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), spec.SenderID(*inviteEvent.StateKey())) if err != nil { - return nil, fmt.Errorf("sender %q is invalid", event.Sender()) + return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", *inviteEvent.StateKey())} } - - roomID := event.RoomID() - targetUserID := *event.StateKey() - info, err := r.DB.RoomInfo(ctx, roomID) + isTargetLocal := r.Cfg.Matrix.IsLocalServerName(userID.Domain()) + if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID().String(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil { + return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err) + } + outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{ + EventNID: 0, + PDU: inviteEvent.PDU, + }, outputUpdates, inviteEvent.Version()) if err != nil { - return nil, fmt.Errorf("failed to load RoomInfo: %w", err) + return nil, fmt.Errorf("updateToInviteMembership: %w", err) } + if err = updater.Commit(); err != nil { + return nil, fmt.Errorf("updater.Commit: %w", err) + } + return outputUpdates, nil +} - _, domain, err := gomatrixserverlib.SplitID('@', targetUserID) +// nolint:gocyclo +func (r *Inviter) PerformInvite( + ctx context.Context, + req *api.PerformInviteRequest, +) error { + senderID, err := r.RSAPI.QuerySenderIDForUser(ctx, req.InviteInput.RoomID, req.InviteInput.Inviter) if err != nil { - return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", targetUserID)} + return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", req.InviteInput.Inviter, req.InviteInput.RoomID) } - isTargetLocal := r.Cfg.Matrix.IsLocalServerName(domain) - isOriginLocal := r.Cfg.Matrix.IsLocalServerName(senderDomain) - if !isOriginLocal && !isTargetLocal { - return nil, api.ErrInvalidID{Err: fmt.Errorf("the invite must be either from or to a local user")} + info, err := r.DB.RoomInfo(ctx, req.InviteInput.RoomID.String()) + if err != nil { + return err } - logger := util.GetLogger(ctx).WithFields(map[string]interface{}{ - "inviter": event.Sender(), - "invitee": *event.StateKey(), - "room_id": roomID, - "event_id": event.EventID(), - }) - logger.WithFields(log.Fields{ - "room_version": req.RoomVersion, - "room_info_exists": info != nil, - "target_local": isTargetLocal, - "origin_local": isOriginLocal, - }).Debug("processing invite event") - - inviteState := req.InviteRoomState - if len(inviteState) == 0 && info != nil { - var is []fclient.InviteV2StrippedState - if is, err = buildInviteStrippedState(ctx, r.DB, info, req); err == nil { - inviteState = is - } + proto := gomatrixserverlib.ProtoEvent{ + SenderID: string(*senderID), + RoomID: req.InviteInput.RoomID.String(), + Type: "m.room.member", } - if len(inviteState) == 0 { - if err = event.SetUnsignedField("invite_room_state", struct{}{}); err != nil { - return nil, fmt.Errorf("event.SetUnsignedField: %w", err) - } - } else { - if err = event.SetUnsignedField("invite_room_state", inviteState); err != nil { - return nil, fmt.Errorf("event.SetUnsignedField: %w", err) - } + + content := gomatrixserverlib.MemberContent{ + Membership: spec.Invite, + DisplayName: req.InviteInput.DisplayName, + AvatarURL: req.InviteInput.AvatarURL, + Reason: req.InviteInput.Reason, + IsDirect: req.InviteInput.IsDirect, } - updateMembershipTableManually := func() ([]api.OutputEvent, error) { - var updater *shared.MembershipUpdater - if updater, err = r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocal, req.RoomVersion); err != nil { - return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err) - } - outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{ - EventNID: 0, - PDU: event.PDU, - }, outputUpdates, req.Event.Version()) - if err != nil { - return nil, fmt.Errorf("updateToInviteMembership: %w", err) - } - if err = updater.Commit(); err != nil { - return nil, fmt.Errorf("updater.Commit: %w", err) - } - logger.Debugf("updated membership to invite and sending invite OutputEvent") - return outputUpdates, nil + if err = proto.SetContent(content); err != nil { + return err } - if (info == nil || info.IsStub()) && !isOriginLocal && isTargetLocal { - // The invite came in over federation for a room that we don't know about - // yet. We need to handle this a bit differently to most invites because - // we don't know the room state, therefore the roomserver can't process - // an input event. Instead we will update the membership table with the - // new invite and generate an output event. - return updateMembershipTableManually() + if !r.Cfg.Matrix.IsLocalServerName(req.InviteInput.Inviter.Domain()) { + return api.ErrInvalidID{Err: fmt.Errorf("the invite must be from a local user")} } - var isAlreadyJoined bool - if info != nil { - _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey()) + isTargetLocal := r.Cfg.Matrix.IsLocalServerName(req.InviteInput.Invitee.Domain()) + + signingKey := req.InviteInput.PrivateKey + if info.RoomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + signingKey, err = r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, req.InviteInput.Inviter, req.InviteInput.RoomID) if err != nil { - return nil, fmt.Errorf("r.DB.GetMembership: %w", err) + return err } } - if isAlreadyJoined { - // If the user is joined to the room then that takes precedence over this - // invite event. It makes little sense to move a user that is already - // joined to the room into the invite state. - // This could plausibly happen if an invite request raced with a join - // request for a user. For example if a user was invited to a public - // room and they joined the room at the same time as the invite was sent. - // The other way this could plausibly happen is if an invite raced with - // a kick. For example if a user was kicked from a room in error and in - // response someone else in the room re-invited them then it is possible - // for the invite request to race with the leave event so that the - // target receives invite before it learns that it has been kicked. - // There are a few ways this could be plausibly handled in the roomserver. - // 1) Store the invite, but mark it as retired. That will result in the - // permanent rejection of that invite event. So even if the target - // user leaves the room and the invite is retransmitted it will be - // ignored. However a new invite with a new event ID would still be - // accepted. - // 2) Silently discard the invite event. This means that if the event - // was retransmitted at a later date after the target user had left - // the room we would accept the invite. However since we hadn't told - // the sending server that the invite had been discarded it would - // have no reason to attempt to retry. - // 3) Signal the sending server that the user is already joined to the - // room. - // For now we will implement option 2. Since in the abesence of a retry - // mechanism it will be equivalent to option 1, and we don't have a - // signalling mechanism to implement option 3. - logger.Debugf("user already joined") - return nil, api.ErrNotAllowed{Err: fmt.Errorf("user is already joined to room")} - } - // If the invite originated remotely then we can't send an - // InputRoomEvent for the invite as it will never pass auth checks - // due to lacking room state, but we still need to tell the client - // about the invite so we can accept it, hence we return an output - // event to send to the Sync API. - if !isOriginLocal { - return updateMembershipTableManually() - } + input := gomatrixserverlib.PerformInviteInput{ + RoomID: req.InviteInput.RoomID, + RoomVersion: info.RoomVersion, + Inviter: req.InviteInput.Inviter, + Invitee: req.InviteInput.Invitee, + IsTargetLocal: isTargetLocal, + EventTemplate: proto, + StrippedState: req.InviteRoomState, + KeyID: req.InviteInput.KeyID, + SigningKey: signingKey, + EventTime: req.InviteInput.EventTime, + MembershipQuerier: &api.MembershipQuerier{Roomserver: r.RSAPI}, + StateQuerier: &QueryState{r.DB, r.RSAPI}, + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.RSAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, + SenderIDQuerier: func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + return r.RSAPI.QuerySenderIDForUser(ctx, roomID, userID) + }, + SenderIDCreator: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) { + key, keyErr := r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if keyErr != nil { + return "", nil, keyErr + } - // The invite originated locally. Therefore we have a responsibility to - // try and see if the user is allowed to make this invite. We can't do - // this for invites coming in over federation - we have to take those on - // trust. - _, err = helpers.CheckAuthEvents(ctx, r.DB, info, event, event.AuthEventIDs()) - if err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error( - "processInviteEvent.checkAuthEvents failed for event", - ) - return nil, api.ErrNotAllowed{Err: err} + return spec.SenderIDFromPseudoIDKey(key), key, nil + }, + EventQuerier: func(ctx context.Context, roomID spec.RoomID, eventsNeeded []gomatrixserverlib.StateKeyTuple) (gomatrixserverlib.LatestEvents, error) { + req := api.QueryLatestEventsAndStateRequest{RoomID: roomID.String(), StateToFetch: eventsNeeded} + res := api.QueryLatestEventsAndStateResponse{} + err = r.RSAPI.QueryLatestEventsAndState(ctx, &req, &res) + if err != nil { + return gomatrixserverlib.LatestEvents{}, nil + } + + stateEvents := []gomatrixserverlib.PDU{} + for _, event := range res.StateEvents { + stateEvents = append(stateEvents, event.PDU) + } + return gomatrixserverlib.LatestEvents{ + RoomExists: res.RoomExists, + StateEvents: stateEvents, + PrevEventIDs: res.LatestEvents, + Depth: res.Depth, + }, nil + }, + StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error { + storeUserID, userErr := spec.NewUserID(userIDRaw, true) + if userErr != nil { + return userErr + } + return r.RSAPI.StoreUserRoomPublicKey(ctx, senderID, *storeUserID, roomID) + }, } - // If the invite originated from us and the target isn't local then we - // should try and send the invite over federation first. It might be - // that the remote user doesn't exist, in which case we can give up - // processing here. - if req.SendAsServer != api.DoNotSendToOtherServers && !isTargetLocal { - fsReq := &federationAPI.PerformInviteRequest{ - RoomVersion: req.RoomVersion, - Event: event, - InviteRoomState: inviteState, - } - fsRes := &federationAPI.PerformInviteResponse{} - if err = r.FSAPI.PerformInvite(ctx, fsReq, fsRes); err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).Error("r.FSAPI.PerformInvite failed") - return nil, api.ErrNotAllowed{Err: err} + inviteEvent, err := gomatrixserverlib.PerformInvite(ctx, input, r.FSAPI) + if err != nil { + switch e := err.(type) { + case spec.MatrixError: + if e.ErrCode == spec.ErrorForbidden { + return api.ErrNotAllowed{Err: fmt.Errorf("%s", e.Err)} + } } - event = fsRes.Event - logger.Debugf("Federated PerformInvite success with event ID %s", event.EventID()) + return err } // Send the invite event to the roomserver input stream. This will @@ -219,67 +244,18 @@ func (r *Inviter) PerformInvite( InputRoomEvents: []api.InputRoomEvent{ { Kind: api.KindNew, - Event: event, - Origin: senderDomain, + Event: &types.HeaderedEvent{PDU: inviteEvent}, + Origin: req.InviteInput.Inviter.Domain(), SendAsServer: req.SendAsServer, }, }, } inputRes := &api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes) - if err = inputRes.Err(); err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed") - return nil, api.ErrNotAllowed{Err: err} + if err := inputRes.Err(); err != nil { + util.GetLogger(ctx).WithField("event_id", inviteEvent.EventID()).Error("r.InputRoomEvents failed") + return api.ErrNotAllowed{Err: err} } - // Don't notify the sync api of this event in the same way as a federated invite so the invitee - // gets the invite, as the roomserver will do this when it processes the m.room.member invite. - return outputUpdates, nil -} - -func buildInviteStrippedState( - ctx context.Context, - db storage.Database, - info *types.RoomInfo, - input *api.PerformInviteRequest, -) ([]fclient.InviteV2StrippedState, error) { - stateWanted := []gomatrixserverlib.StateKeyTuple{} - // "If they are set on the room, at least the state for m.room.avatar, m.room.canonical_alias, m.room.join_rules, and m.room.name SHOULD be included." - // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member - for _, t := range []string{ - spec.MRoomName, spec.MRoomCanonicalAlias, - spec.MRoomJoinRules, spec.MRoomAvatar, - spec.MRoomEncryption, spec.MRoomCreate, - } { - stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ - EventType: t, - StateKey: "", - }) - } - roomState := state.NewStateResolution(db, info) - stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( - ctx, info.StateSnapshotNID(), stateWanted, - ) - if err != nil { - return nil, err - } - stateNIDs := []types.EventNID{} - for _, stateNID := range stateEntries { - stateNIDs = append(stateNIDs, stateNID.EventNID) - } - if info == nil { - return nil, types.ErrorInvalidRoomInfo - } - stateEvents, err := db.Events(ctx, info.RoomVersion, stateNIDs) - if err != nil { - return nil, err - } - inviteState := []fclient.InviteV2StrippedState{ - fclient.NewInviteV2StrippedState(input.Event.PDU), - } - stateEvents = append(stateEvents, types.Event{PDU: input.Event.PDU}) - for _, event := range stateEvents { - inviteState = append(inviteState, fclient.NewInviteV2StrippedState(event.PDU)) - } - return inviteState, nil + return nil } diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 5f4ad1861c..23988e467d 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -16,6 +16,7 @@ package perform import ( "context" + "crypto/ed25519" "database/sql" "errors" "fmt" @@ -24,7 +25,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" "github.com/tidwall/gjson" @@ -162,7 +165,7 @@ func (r *Joiner) performJoinRoomByID( } // Get the domain part of the room ID. - _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) + roomID, err := spec.NewRoomID(req.RoomIDOrAlias) if err != nil { return "", "", rsAPI.ErrInvalidID{Err: fmt.Errorf("room ID %q is invalid: %w", req.RoomIDOrAlias, err)} } @@ -170,41 +173,8 @@ func (r *Joiner) performJoinRoomByID( // If the server name in the room ID isn't ours then it's a // possible candidate for finding the room via federation. Add // it to the list of servers to try. - if !r.Cfg.Matrix.IsLocalServerName(domain) { - req.ServerNames = append(req.ServerNames, domain) - } - - // Prepare the template for the join event. - userID := req.UserID - _, userDomain, err := r.Cfg.Matrix.SplitLocalID('@', userID) - if err != nil { - return "", "", rsAPI.ErrInvalidID{Err: fmt.Errorf("user ID %q is invalid: %w", userID, err)} - } - proto := gomatrixserverlib.ProtoEvent{ - Type: spec.MRoomMember, - Sender: userID, - StateKey: &userID, - RoomID: req.RoomIDOrAlias, - Redacts: "", - } - if err = proto.SetUnsigned(struct{}{}); err != nil { - return "", "", fmt.Errorf("eb.SetUnsigned: %w", err) - } - - // It is possible for the request to include some "content" for the - // event. We'll always overwrite the "membership" key, but the rest, - // like "display_name" or "avatar_url", will be kept if supplied. - if req.Content == nil { - req.Content = map[string]interface{}{} - } - req.Content["membership"] = spec.Join - if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req); aerr != nil { - return "", "", aerr - } else if authorisedVia != "" { - req.Content["join_authorised_via_users_server"] = authorisedVia - } - if err = proto.SetContent(req.Content); err != nil { - return "", "", fmt.Errorf("eb.SetContent: %w", err) + if !r.Cfg.Matrix.IsLocalServerName(roomID.Domain()) { + req.ServerNames = append(req.ServerNames, roomID.Domain()) } // Force a federated join if we aren't in the room and we've been @@ -219,29 +189,66 @@ func (r *Joiner) performJoinRoomByID( serverInRoom := inRoomRes.IsInRoom forceFederatedJoin := len(req.ServerNames) > 0 && !serverInRoom + userID, err := spec.NewUserID(req.UserID, true) + if err != nil { + return "", "", rsAPI.ErrInvalidID{Err: fmt.Errorf("user ID %q is invalid: %w", req.UserID, err)} + } + + // Look up the room NID for the supplied room ID. + var senderID spec.SenderID + checkInvitePending := false + info, err := r.DB.RoomInfo(ctx, req.RoomIDOrAlias) + if err == nil && info != nil { + switch info.RoomVersion { + case gomatrixserverlib.RoomVersionPseudoIDs: + senderIDPtr, queryErr := r.Queryer.QuerySenderIDForUser(ctx, *roomID, *userID) + if queryErr == nil { + checkInvitePending = true + } + if senderIDPtr == nil { + // create user room key if needed + key, keyErr := r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, *userID, *roomID) + if keyErr != nil { + util.GetLogger(ctx).WithError(keyErr).Error("GetOrCreateUserRoomPrivateKey failed") + return "", "", fmt.Errorf("GetOrCreateUserRoomPrivateKey failed: %w", keyErr) + } + senderID = spec.SenderIDFromPseudoIDKey(key) + } else { + senderID = *senderIDPtr + } + default: + checkInvitePending = true + senderID = spec.SenderID(userID.String()) + } + } + + userDomain := userID.Domain() + // Force a federated join if we're dealing with a pending invite // and we aren't in the room. - isInvitePending, inviteSender, _, inviteEvent, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) - if err == nil && !serverInRoom && isInvitePending { - _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) - if ierr != nil { - return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err) - } + if checkInvitePending { + isInvitePending, inviteSender, _, inviteEvent, inviteErr := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, senderID) + if inviteErr == nil && !serverInRoom && isInvitePending { + inviter, queryErr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, inviteSender) + if queryErr != nil { + return "", "", fmt.Errorf("r.RSAPI.QueryUserIDForSender: %w", queryErr) + } - // If we were invited by someone from another server then we can - // assume they are in the room so we can join via them. - if !r.Cfg.Matrix.IsLocalServerName(inviterDomain) { - req.ServerNames = append(req.ServerNames, inviterDomain) - forceFederatedJoin = true - memberEvent := gjson.Parse(string(inviteEvent.JSON())) - // only set unsigned if we've got a content.membership, which we _should_ - if memberEvent.Get("content.membership").Exists() { - req.Unsigned = map[string]interface{}{ - "prev_sender": memberEvent.Get("sender").Str, - "prev_content": map[string]interface{}{ - "is_direct": memberEvent.Get("content.is_direct").Bool(), - "membership": memberEvent.Get("content.membership").Str, - }, + // If we were invited by someone from another server then we can + // assume they are in the room so we can join via them. + if inviter != nil && !r.Cfg.Matrix.IsLocalServerName(inviter.Domain()) { + req.ServerNames = append(req.ServerNames, inviter.Domain()) + forceFederatedJoin = true + memberEvent := gjson.Parse(string(inviteEvent.JSON())) + // only set unsigned if we've got a content.membership, which we _should_ + if memberEvent.Get("content.membership").Exists() { + req.Unsigned = map[string]interface{}{ + "prev_sender": memberEvent.Get("sender").Str, + "prev_content": map[string]interface{}{ + "is_direct": memberEvent.Get("content.is_direct").Bool(), + "membership": memberEvent.Get("content.membership").Str, + }, + } } } } @@ -280,11 +287,66 @@ func (r *Joiner) performJoinRoomByID( // but everyone has since left. I suspect it does the wrong thing. var buildRes rsAPI.QueryLatestEventsAndStateResponse - identity, err := r.Cfg.Matrix.SigningIdentityFor(userDomain) - if err != nil { - return "", "", fmt.Errorf("error joining local room: %q", err) + identity := r.Cfg.Matrix.SigningIdentity + + // at this point we know we have an existing room + if inRoomRes.RoomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + var pseudoIDKey ed25519.PrivateKey + pseudoIDKey, err = r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, *userID, *roomID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("GetOrCreateUserRoomPrivateKey failed") + return "", "", err + } + + mapping := &gomatrixserverlib.MXIDMapping{ + UserRoomKey: spec.SenderIDFromPseudoIDKey(pseudoIDKey), + UserID: userID.String(), + } + + // Sign the mapping with the server identity + if err = mapping.Sign(identity.ServerName, identity.KeyID, identity.PrivateKey); err != nil { + return "", "", err + } + req.Content["mxid_mapping"] = mapping + + // sign the event with the pseudo ID key + identity = fclient.SigningIdentity{ + ServerName: spec.ServerName(spec.SenderIDFromPseudoIDKey(pseudoIDKey)), + KeyID: "ed25519:1", + PrivateKey: pseudoIDKey, + } + } + + senderIDString := string(senderID) + + // Prepare the template for the join event. + proto := gomatrixserverlib.ProtoEvent{ + Type: spec.MRoomMember, + SenderID: senderIDString, + StateKey: &senderIDString, + RoomID: req.RoomIDOrAlias, + Redacts: "", + } + if err = proto.SetUnsigned(struct{}{}); err != nil { + return "", "", fmt.Errorf("eb.SetUnsigned: %w", err) + } + + // It is possible for the request to include some "content" for the + // event. We'll always overwrite the "membership" key, but the rest, + // like "display_name" or "avatar_url", will be kept if supplied. + if req.Content == nil { + req.Content = map[string]interface{}{} + } + req.Content["membership"] = spec.Join + if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req, senderID); aerr != nil { + return "", "", aerr + } else if authorisedVia != "" { + req.Content["join_authorised_via_users_server"] = authorisedVia } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, time.Now(), r.RSAPI, &buildRes) + if err = proto.SetContent(req.Content); err != nil { + return "", "", fmt.Errorf("eb.SetContent: %w", err) + } + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, time.Now(), r.RSAPI, &buildRes) switch err.(type) { case nil: @@ -293,12 +355,8 @@ func (r *Joiner) performJoinRoomByID( // a member of the room. This is best-effort (as in we won't // fail if we can't find the existing membership) because there // is really no harm in just sending another membership event. - membershipReq := &api.QueryMembershipForUserRequest{ - RoomID: req.RoomIDOrAlias, - UserID: userID, - } membershipRes := &api.QueryMembershipForUserResponse{} - _ = r.Queryer.QueryMembershipForUser(ctx, membershipReq, membershipRes) + _ = r.Queryer.QueryMembershipForSenderID(ctx, *roomID, senderID, membershipRes) // If we haven't already joined the room then send an event // into the room changing our membership status. @@ -323,7 +381,7 @@ func (r *Joiner) performJoinRoomByID( // The room doesn't exist locally. If the room ID looks like it should // be ours then this probably means that we've nuked our database at // some point. - if r.Cfg.Matrix.IsLocalServerName(domain) { + if r.Cfg.Matrix.IsLocalServerName(roomID.Domain()) { // If there are no more server names to try then give up here. // Otherwise we'll try a federated join as normal, since it's quite // possible that the room still exists on other servers. @@ -371,23 +429,12 @@ func (r *Joiner) performFederatedJoinRoomByID( func (r *Joiner) populateAuthorisedViaUserForRestrictedJoin( ctx context.Context, joinReq *rsAPI.PerformJoinRequest, + senderID spec.SenderID, ) (string, error) { - req := &api.QueryRestrictedJoinAllowedRequest{ - UserID: joinReq.UserID, - RoomID: joinReq.RoomIDOrAlias, - } - res := &api.QueryRestrictedJoinAllowedResponse{} - if err := r.Queryer.QueryRestrictedJoinAllowed(ctx, req, res); err != nil { - return "", fmt.Errorf("r.Queryer.QueryRestrictedJoinAllowed: %w", err) - } - if !res.Restricted { - return "", nil - } - if !res.Resident { - return "", nil - } - if !res.Allowed { - return "", rsAPI.ErrNotAllowed{Err: fmt.Errorf("the join to room %s was not allowed", joinReq.RoomIDOrAlias)} + roomID, err := spec.NewRoomID(joinReq.RoomIDOrAlias) + if err != nil { + return "", err } - return res.AuthorisedVia, nil + + return r.Queryer.QueryRestrictedJoinAllowed(ctx, *roomID, senderID) } diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index e71b3e9086..5bea004458 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -53,16 +53,12 @@ func (r *Leaver) PerformLeave( req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, ) ([]api.OutputEvent, error) { - _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) - if err != nil { - return nil, fmt.Errorf("supplied user ID %q in incorrect format", req.UserID) - } - if !r.Cfg.Matrix.IsLocalServerName(domain) { - return nil, fmt.Errorf("user %q does not belong to this homeserver", req.UserID) + if !r.Cfg.Matrix.IsLocalServerName(req.Leaver.Domain()) { + return nil, fmt.Errorf("user %q does not belong to this homeserver", req.Leaver.String()) } logger := logrus.WithContext(ctx).WithFields(logrus.Fields{ "room_id": req.RoomID, - "user_id": req.UserID, + "user_id": req.Leaver.String(), }) logger.Info("User requested to leave join") if strings.HasPrefix(req.RoomID, "!") { @@ -77,26 +73,46 @@ func (r *Leaver) PerformLeave( return nil, fmt.Errorf("room ID %q is invalid", req.RoomID) } +// nolint:gocyclo func (r *Leaver) performLeaveRoomByID( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam ) ([]api.OutputEvent, error) { + roomID, err := spec.NewRoomID(req.RoomID) + if err != nil { + return nil, err + } + leaver, err := r.RSAPI.QuerySenderIDForUser(ctx, *roomID, req.Leaver) + if err != nil || leaver == nil { + return nil, fmt.Errorf("leaver %s has no matching senderID in this room", req.Leaver.String()) + } + // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) + isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver) if err == nil && isInvitePending { - _, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser) + sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser) if serr != nil { - return nil, fmt.Errorf("sender %q is invalid", senderUser) + return nil, fmt.Errorf("failed looking up userID for sender %q: %w", senderUser, serr) + } + + var domain spec.ServerName + if sender == nil { + // TODO: Currently a federated invite has no way of knowing the mxid_mapping of the inviter. + // Should we add the inviter's m.room.member event (with mxid_mapping) to invite_room_state to allow + // the invited user to leave via the inviter's server? + domain = roomID.Domain() + } else { + domain = sender.Domain() } - if !r.Cfg.Matrix.IsLocalServerName(senderDomain) { - return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID) + if !r.Cfg.Matrix.IsLocalServerName(domain) { + return r.performFederatedRejectInvite(ctx, req, res, domain, eventID, *leaver) } // check that this is not a "server notice room" accData := &userapi.QueryAccountDataResponse{} if err = r.UserAPI.QueryAccountData(ctx, &userapi.QueryAccountDataRequest{ - UserID: req.UserID, + UserID: req.Leaver.String(), RoomID: req.RoomID, DataType: "m.tag", }, accData); err != nil { @@ -127,12 +143,12 @@ func (r *Leaver) performLeaveRoomByID( StateToFetch: []gomatrixserverlib.StateKeyTuple{ { EventType: spec.MRoomMember, - StateKey: req.UserID, + StateKey: string(*leaver), }, }, } latestRes := api.QueryLatestEventsAndStateResponse{} - if err = helpers.QueryLatestEventsAndState(ctx, r.DB, &latestReq, &latestRes); err != nil { + if err = helpers.QueryLatestEventsAndState(ctx, r.DB, r.RSAPI, &latestReq, &latestRes); err != nil { return nil, err } if !latestRes.RoomExists { @@ -141,22 +157,22 @@ func (r *Leaver) performLeaveRoomByID( // Now let's see if the user is in the room. if len(latestRes.StateEvents) == 0 { - return nil, fmt.Errorf("user %q is not a member of room %q", req.UserID, req.RoomID) + return nil, fmt.Errorf("user %q is not a member of room %q", req.Leaver.String(), req.RoomID) } membership, err := latestRes.StateEvents[0].Membership() if err != nil { return nil, fmt.Errorf("error getting membership: %w", err) } if membership != spec.Join && membership != spec.Invite { - return nil, fmt.Errorf("user %q is not joined to the room (membership is %q)", req.UserID, membership) + return nil, fmt.Errorf("user %q is not joined to the room (membership is %q)", req.Leaver.String(), membership) } // Prepare the template for the leave event. - userID := req.UserID + senderIDString := string(*leaver) proto := gomatrixserverlib.ProtoEvent{ Type: spec.MRoomMember, - Sender: userID, - StateKey: &userID, + SenderID: senderIDString, + StateKey: &senderIDString, RoomID: req.RoomID, Redacts: "", } @@ -167,23 +183,22 @@ func (r *Leaver) performLeaveRoomByID( return nil, fmt.Errorf("eb.SetUnsigned: %w", err) } - // Get the sender domain. - _, senderDomain, serr := r.Cfg.Matrix.SplitLocalID('@', proto.Sender) - if serr != nil { - return nil, fmt.Errorf("sender %q is invalid", proto.Sender) - } - // We know that the user is in the room at this point so let's build // a leave event. // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. + validRoomID, err := spec.NewRoomID(req.RoomID) + if err != nil { + return nil, err + } + var buildRes rsAPI.QueryLatestEventsAndStateResponse - identity, err := r.Cfg.Matrix.SigningIdentityFor(senderDomain) + identity, err := r.RSAPI.SigningIdentityFor(ctx, *validRoomID, req.Leaver) if err != nil { return nil, fmt.Errorf("SigningIdentityFor: %w", err) } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, time.Now(), r.RSAPI, &buildRes) + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, time.Now(), r.RSAPI, &buildRes) if err != nil { return nil, fmt.Errorf("eventutil.QueryAndBuildEvent: %w", err) } @@ -196,8 +211,8 @@ func (r *Leaver) performLeaveRoomByID( { Kind: api.KindNew, Event: event, - Origin: senderDomain, - SendAsServer: string(senderDomain), + Origin: req.Leaver.Domain(), + SendAsServer: string(req.Leaver.Domain()), }, }, } @@ -214,21 +229,17 @@ func (r *Leaver) performFederatedRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam - senderUser, eventID string, + inviteDomain spec.ServerName, eventID string, + leaver spec.SenderID, ) ([]api.OutputEvent, error) { - _, domain, err := gomatrixserverlib.SplitID('@', senderUser) - if err != nil { - return nil, fmt.Errorf("user ID %q invalid: %w", senderUser, err) - } - // Ask the federation sender to perform a federated leave for us. leaveReq := fsAPI.PerformLeaveRequest{ RoomID: req.RoomID, - UserID: req.UserID, - ServerNames: []spec.ServerName{domain}, + UserID: req.Leaver.String(), + ServerNames: []spec.ServerName{inviteDomain}, } leaveRes := fsAPI.PerformLeaveResponse{} - if err = r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { + if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { // failures in PerformLeave should NEVER stop us from telling other components like the // sync API that the invite was withdrawn. Otherwise we can end up with stuck invites. util.GetLogger(ctx).WithError(err).Errorf("failed to PerformLeave, still retiring invite event") @@ -239,7 +250,7 @@ func (r *Leaver) performFederatedRejectInvite( util.GetLogger(ctx).WithError(err).Errorf("failed to get RoomInfo, still retiring invite event") } - updater, err := r.DB.MembershipUpdater(ctx, req.RoomID, req.UserID, true, info.RoomVersion) + updater, err := r.DB.MembershipUpdater(ctx, req.RoomID, string(leaver), true, info.RoomVersion) if err != nil { util.GetLogger(ctx).WithError(err).Errorf("failed to get MembershipUpdater, still retiring invite event") } @@ -262,9 +273,10 @@ func (r *Leaver) performFederatedRejectInvite( { Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ - EventID: eventID, - Membership: "leave", - TargetUserID: req.UserID, + EventID: eventID, + RoomID: req.RoomID, + Membership: "leave", + TargetSenderID: leaver, }, }, }, nil diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go index 661fe20a81..88fa2a431f 100644 --- a/roomserver/internal/perform/perform_peek.go +++ b/roomserver/internal/perform/perform_peek.go @@ -157,7 +157,7 @@ func (r *Peeker) performPeekRoomByID( content := map[string]string{} if err = json.Unmarshal(ev.Content(), &content); err != nil { util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed") - return + return "", err } if visibility, ok := content["history_visibility"]; ok { worldReadable = visibility == "world_readable" @@ -185,7 +185,7 @@ func (r *Peeker) performPeekRoomByID( }, }) if err != nil { - return + return "", err } // By this point, if req.RoomIDOrAlias contained an alias, then diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index 60085cb6d6..9d7c7b5671 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -35,29 +35,18 @@ type Upgrader struct { URSAPI api.RoomserverInternalAPI } -// fledglingEvent is a helper representation of an event used when creating many events in succession. -type fledglingEvent struct { - Type string `json:"type"` - StateKey string `json:"state_key"` - Content interface{} `json:"content"` -} - // PerformRoomUpgrade upgrades a room from one version to another func (r *Upgrader) PerformRoomUpgrade( ctx context.Context, - roomID, userID string, roomVersion gomatrixserverlib.RoomVersion, + roomID string, userID spec.UserID, roomVersion gomatrixserverlib.RoomVersion, ) (newRoomID string, err error) { return r.performRoomUpgrade(ctx, roomID, userID, roomVersion) } func (r *Upgrader) performRoomUpgrade( ctx context.Context, - roomID, userID string, roomVersion gomatrixserverlib.RoomVersion, + roomID string, userID spec.UserID, roomVersion gomatrixserverlib.RoomVersion, ) (string, error) { - _, userDomain, err := r.Cfg.Matrix.SplitLocalID('@', userID) - if err != nil { - return "", api.ErrNotAllowed{Err: fmt.Errorf("error validating the user ID")} - } evTime := time.Now() // Return an immediate error if the room does not exist @@ -65,14 +54,27 @@ func (r *Upgrader) performRoomUpgrade( return "", err } + fullRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return "", err + } + senderID, err := r.URSAPI.QuerySenderIDForUser(ctx, *fullRoomID, userID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed getting senderID for user") + return "", err + } else if senderID == nil { + util.GetLogger(ctx).WithField("userID", userID).WithField("roomID", *fullRoomID).Error("No senderID for user") + return "", fmt.Errorf("No sender ID for %s in %s", userID, *fullRoomID) + } + // 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone) - if !r.userIsAuthorized(ctx, userID, roomID) { + if !r.userIsAuthorized(ctx, *senderID, roomID) { return "", api.ErrNotAllowed{Err: fmt.Errorf("You don't have permission to upgrade the room, power level too low.")} } // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? - newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain) + newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userID.Domain()) // Get the existing room state for the old room. oldRoomReq := &api.QueryLatestEventsAndStateRequest{ @@ -84,25 +86,25 @@ func (r *Upgrader) performRoomUpgrade( } // Make the tombstone event - tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, userID, roomID, newRoomID) + tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, *senderID, userID.Domain(), roomID, newRoomID) if pErr != nil { return "", pErr } // Generate the initial events we need to send into the new room. This includes copied state events and bans // as well as the power level events needed to set up the room - eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, userID, roomID, roomVersion, tombstoneEvent) + eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, *senderID, roomID, roomVersion, tombstoneEvent) if pErr != nil { return "", pErr } // Send the setup events to the new room - if pErr = r.sendInitialEvents(ctx, evTime, userID, userDomain, newRoomID, roomVersion, eventsToMake); pErr != nil { + if pErr = r.sendInitialEvents(ctx, evTime, *senderID, userID.Domain(), newRoomID, roomVersion, eventsToMake); pErr != nil { return "", pErr } // 5. Send the tombstone event to the old room - if pErr = r.sendHeaderedEvent(ctx, userDomain, tombstoneEvent, string(userDomain)); pErr != nil { + if pErr = r.sendHeaderedEvent(ctx, userID.Domain(), tombstoneEvent, string(userID.Domain())); pErr != nil { return "", pErr } @@ -112,17 +114,17 @@ func (r *Upgrader) performRoomUpgrade( } // If the old room had a canonical alias event, it should be deleted in the old room - if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, userDomain, roomID); pErr != nil { + if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, *senderID, userID.Domain(), roomID); pErr != nil { return "", pErr } // 4. Move local aliases to the new room - if pErr = moveLocalAliases(ctx, roomID, newRoomID, userID, r.URSAPI); pErr != nil { + if pErr = moveLocalAliases(ctx, roomID, newRoomID, *senderID, r.URSAPI); pErr != nil { return "", pErr } // 6. Restrict power levels in the old room - if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, userDomain, roomID); pErr != nil { + if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, *senderID, userID.Domain(), roomID); pErr != nil { return "", pErr } @@ -137,7 +139,7 @@ func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*goma return oldPowerLevelsEvent.PowerLevels() } -func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID string, userDomain spec.ServerName, roomID string) error { +func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, senderID spec.SenderID, userDomain spec.ServerName, roomID string) error { restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID) if pErr != nil { return pErr @@ -154,7 +156,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel - restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, senderID, userDomain, roomID, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, StateKey: "", Content: restrictedPowerLevelContent, @@ -172,7 +174,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T } func moveLocalAliases(ctx context.Context, - roomID, newRoomID, userID string, + roomID, newRoomID string, senderID spec.SenderID, URSAPI api.RoomserverInternalAPI, ) (err error) { @@ -182,23 +184,33 @@ func moveLocalAliases(ctx context.Context, return fmt.Errorf("Failed to get old room aliases: %w", err) } + // TODO: this should be spec.RoomID further up the call stack + parsedNewRoomID, err := spec.NewRoomID(newRoomID) + if err != nil { + return err + } + for _, alias := range aliasRes.Aliases { - removeAliasReq := api.RemoveRoomAliasRequest{UserID: userID, Alias: alias} - removeAliasRes := api.RemoveRoomAliasResponse{} - if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil { + aliasFound, aliasRemoved, err := URSAPI.RemoveRoomAlias(ctx, senderID, alias) + if err != nil { return fmt.Errorf("Failed to remove old room alias: %w", err) + } else if !aliasFound { + return fmt.Errorf("Failed to remove old room alias: alias not found, possible race") + } else if !aliasRemoved { + return fmt.Errorf("Failed to remove old alias") } - setAliasReq := api.SetRoomAliasRequest{UserID: userID, Alias: alias, RoomID: newRoomID} - setAliasRes := api.SetRoomAliasResponse{} - if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil { + aliasAlreadyExists, err := URSAPI.SetRoomAlias(ctx, senderID, *parsedNewRoomID, alias) + if err != nil { return fmt.Errorf("Failed to set new room alias: %w", err) + } else if aliasAlreadyExists { + return fmt.Errorf("Failed to set new room alias: alias exists when it should have just been removed") } } return nil } -func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID string, userDomain spec.ServerName, roomID string) error { +func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, senderID spec.SenderID, userDomain spec.ServerName, roomID string) error { for _, event := range oldRoom.StateEvents { if event.Type() != spec.MRoomCanonicalAlias || !event.StateKeyEquals("") { continue @@ -216,7 +228,7 @@ func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api } } - emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, senderID, userDomain, roomID, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomCanonicalAlias, Content: map[string]interface{}{}, }) @@ -279,7 +291,7 @@ func (r *Upgrader) validateRoomExists(ctx context.Context, roomID string) error return nil } -func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string, +func (r *Upgrader) userIsAuthorized(ctx context.Context, senderID spec.SenderID, roomID string, ) bool { plEvent := api.GetStateEvent(ctx, r.URSAPI, roomID, gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomPowerLevels, @@ -294,18 +306,18 @@ func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string, } // Check for power level required to send tombstone event (marks the current room as obsolete), // if not found, use the StateDefault power level - return pl.UserLevel(userID) >= pl.EventLevel("m.room.tombstone", true) + return pl.UserLevel(senderID) >= pl.EventLevel("m.room.tombstone", true) } // nolint:gocyclo -func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID string, newVersion gomatrixserverlib.RoomVersion, tombstoneEvent *types.HeaderedEvent) ([]fledglingEvent, error) { +func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, senderID spec.SenderID, roomID string, newVersion gomatrixserverlib.RoomVersion, tombstoneEvent *types.HeaderedEvent) ([]gomatrixserverlib.FledglingEvent, error) { state := make(map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent, len(oldRoom.StateEvents)) for _, event := range oldRoom.StateEvents { if event.StateKey() == nil { // This shouldn't ever happen, but better to be safe than sorry. continue } - if event.Type() == spec.MRoomMember && !event.StateKeyEquals(userID) { + if event.Type() == spec.MRoomMember && !event.StateKeyEquals(string(senderID)) { // With the exception of bans which we do want to copy, we // should ignore membership events that aren't our own, as event auth will // prevent us from being able to create membership events on behalf of other @@ -321,6 +333,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query } } // skip events that rely on a specific user being present + // TODO: What to do here for pseudoIDs? It's checking non-member events for state keys with userIDs. sKey := *event.StateKey() if event.Type() != spec.MRoomMember && len(sKey) > 0 && sKey[:1] == "@" { continue @@ -331,10 +344,10 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // The following events are ones that we are going to override manually // in the following section. override := map[gomatrixserverlib.StateKeyTuple]struct{}{ - {EventType: spec.MRoomCreate, StateKey: ""}: {}, - {EventType: spec.MRoomMember, StateKey: userID}: {}, - {EventType: spec.MRoomPowerLevels, StateKey: ""}: {}, - {EventType: spec.MRoomJoinRules, StateKey: ""}: {}, + {EventType: spec.MRoomCreate, StateKey: ""}: {}, + {EventType: spec.MRoomMember, StateKey: string(senderID)}: {}, + {EventType: spec.MRoomPowerLevels, StateKey: ""}: {}, + {EventType: spec.MRoomJoinRules, StateKey: ""}: {}, } // The overridden events are essential events that must be present in the @@ -346,7 +359,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query } oldCreateEvent := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomCreate, StateKey: ""}] - oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomMember, StateKey: userID}] + oldMembershipEvent := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomMember, StateKey: string(senderID)}] oldPowerLevelsEvent := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomPowerLevels, StateKey: ""}] oldJoinRulesEvent := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomJoinRules, StateKey: ""}] @@ -355,13 +368,22 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // in the create event (such as for the room types MSC). newCreateContent := map[string]interface{}{} _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) - newCreateContent["creator"] = userID + + switch newVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + // So if we are upgrading from pre v11, we need to remove the field. + delete(newCreateContent, "creator") + default: + newCreateContent["creator"] = senderID + } + newCreateContent["room_version"] = newVersion newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ EventID: tombstoneEvent.EventID(), RoomID: roomID, } - newCreateEvent := fledglingEvent{ + newCreateEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomCreate, StateKey: "", Content: newCreateContent, @@ -374,9 +396,9 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query newMembershipContent := map[string]interface{}{} _ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent) newMembershipContent["membership"] = spec.Join - newMembershipEvent := fledglingEvent{ + newMembershipEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomMember, - StateKey: userID, + StateKey: string(senderID), Content: newMembershipContent, } @@ -390,7 +412,8 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query util.GetLogger(ctx).WithError(err).Error() return nil, fmt.Errorf("Power level event content was invalid") } - tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, userID) + + tempPowerLevelsEvent, powerLevelsOverridden := createTemporaryPowerLevels(powerLevelContent, senderID) // Now do the join rules event, same as the create and membership // events. We'll set a sane default of "invite" so that if the @@ -400,13 +423,13 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query "join_rule": spec.Invite, // sane default } _ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent) - newJoinRulesEvent := fledglingEvent{ + newJoinRulesEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomJoinRules, StateKey: "", Content: newJoinRulesContent, } - eventsToMake := make([]fledglingEvent, 0, len(state)) + eventsToMake := make([]gomatrixserverlib.FledglingEvent, 0, len(state)) eventsToMake = append( eventsToMake, newCreateEvent, newMembershipEvent, tempPowerLevelsEvent, newJoinRulesEvent, @@ -415,7 +438,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // For some reason Sytest expects there to be a guest access event. // Create one if it doesn't exist. if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomGuestAccess, StateKey: ""}]; !ok { - eventsToMake = append(eventsToMake, fledglingEvent{ + eventsToMake = append(eventsToMake, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomGuestAccess, Content: map[string]string{ "guest_access": "forbidden", @@ -430,7 +453,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // are already in `eventsToMake`. continue } - newEvent := fledglingEvent{ + newEvent := gomatrixserverlib.FledglingEvent{ Type: tuple.EventType, StateKey: tuple.StateKey, } @@ -444,7 +467,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // If we sent a temporary power level event into the room before, // override that now by restoring the original power levels. if powerLevelsOverridden { - eventsToMake = append(eventsToMake, fledglingEvent{ + eventsToMake = append(eventsToMake, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, Content: powerLevelContent, }) @@ -452,7 +475,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query return eventsToMake, nil } -func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID string, userDomain spec.ServerName, newRoomID string, newVersion gomatrixserverlib.RoomVersion, eventsToMake []fledglingEvent) error { +func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, senderID spec.SenderID, userDomain spec.ServerName, newRoomID string, newVersion gomatrixserverlib.RoomVersion, eventsToMake []gomatrixserverlib.FledglingEvent) error { var err error var builtEvents []*types.HeaderedEvent authEvents := gomatrixserverlib.NewAuthEvents(nil) @@ -460,7 +483,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user depth := i + 1 // depth starts at 1 proto := gomatrixserverlib.ProtoEvent{ - Sender: userID, + SenderID: string(senderID), RoomID: newRoomID, Type: e.Type, StateKey: &e.StateKey, @@ -491,7 +514,9 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user } - if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { + if err = gomatrixserverlib.Allowed(event, &authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.URSAPI.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { return fmt.Errorf("Failed to auth new %q event: %w", builder.Type, err) } @@ -521,22 +546,22 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user func (r *Upgrader) makeTombstoneEvent( ctx context.Context, evTime time.Time, - userID, roomID, newRoomID string, + senderID spec.SenderID, senderDomain spec.ServerName, roomID, newRoomID string, ) (*types.HeaderedEvent, error) { content := map[string]interface{}{ "body": "This room has been replaced", "replacement_room": newRoomID, } - event := fledglingEvent{ + event := gomatrixserverlib.FledglingEvent{ Type: "m.room.tombstone", Content: content, } - return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event) + return r.makeHeaderedEvent(ctx, evTime, senderID, senderDomain, roomID, event) } -func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*types.HeaderedEvent, error) { +func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, senderID spec.SenderID, senderDomain spec.ServerName, roomID string, event gomatrixserverlib.FledglingEvent) (*types.HeaderedEvent, error) { proto := gomatrixserverlib.ProtoEvent{ - Sender: userID, + SenderID: string(senderID), RoomID: roomID, Type: event.Type, StateKey: &event.StateKey, @@ -546,16 +571,12 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user return nil, fmt.Errorf("failed to set new %q event content: %w", proto.Type, err) } // Get the sender domain. - _, senderDomain, serr := r.Cfg.Matrix.SplitLocalID('@', proto.Sender) - if serr != nil { - return nil, fmt.Errorf("Failed to split user ID %q: %w", proto.Sender, err) - } identity, err := r.Cfg.Matrix.SigningIdentityFor(senderDomain) if err != nil { return nil, fmt.Errorf("failed to get signing identity for %q: %w", senderDomain, err) } var queryRes api.QueryLatestEventsAndStateResponse - headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, evTime, r.URSAPI, &queryRes) + headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, r.URSAPI, &queryRes) switch e := err.(type) { case nil: case eventutil.ErrRoomNoExists: @@ -574,14 +595,16 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user stateEvents[i] = queryRes.StateEvents[i].PDU } provider := gomatrixserverlib.NewAuthEvents(stateEvents) - if err = gomatrixserverlib.Allowed(headeredEvent.PDU, &provider); err != nil { + if err = gomatrixserverlib.Allowed(headeredEvent.PDU, &provider, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.URSAPI.QueryUserIDForSender(ctx, roomID, senderID) + }); err != nil { return nil, api.ErrNotAllowed{Err: fmt.Errorf("failed to auth new %q event: %w", proto.Type, err)} // TODO: Is this error string comprehensible to the client? } return headeredEvent, nil } -func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) { +func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, senderID spec.SenderID) (gomatrixserverlib.FledglingEvent, bool) { // Work out what power level we need in order to be able to send events // of all types into the room. neededPowerLevel := powerLevelContent.StateDefault @@ -606,13 +629,13 @@ func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelC // If the user who is upgrading the room doesn't already have sufficient // power, then elevate their power levels. - if tempPowerLevelContent.UserLevel(userID) < neededPowerLevel { - tempPowerLevelContent.Users[userID] = neededPowerLevel + if tempPowerLevelContent.UserLevel(senderID) < neededPowerLevel { + tempPowerLevelContent.Users[string(senderID)] = neededPowerLevel powerLevelsOverridden = true } // Then return the temporary power levels event. - return fledglingEvent{ + return gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, Content: tempPowerLevelContent, }, powerLevelsOverridden diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index effcc90d7e..f87a3f7ed7 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -16,11 +16,13 @@ package query import ( "context" + "crypto/ed25519" "database/sql" - "encoding/json" "errors" "fmt" + //"github.com/matrix-org/dendrite/roomserver/internal" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" @@ -30,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/api" @@ -44,6 +47,43 @@ type Queryer struct { Cache caching.RoomServerCaches IsLocalServerName func(spec.ServerName) bool ServerACLs *acls.ServerACLs + Cfg *config.Dendrite + FSAPI fsAPI.RoomserverFederationAPI +} + +func (r *Queryer) RestrictedRoomJoinInfo(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, localServerName spec.ServerName) (*gomatrixserverlib.RestrictedRoomJoinInfo, error) { + roomInfo, err := r.QueryRoomInfo(ctx, roomID) + if err != nil || roomInfo == nil || roomInfo.IsStub() { + return nil, err + } + + req := api.QueryServerJoinedToRoomRequest{ + ServerName: localServerName, + RoomID: roomID.String(), + } + res := api.QueryServerJoinedToRoomResponse{} + if err = r.QueryServerJoinedToRoom(ctx, &req, &res); err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.QueryServerJoinedToRoom failed") + return nil, fmt.Errorf("InternalServerError: Failed to query room: %w", err) + } + + userJoinedToRoom, err := r.UserJoinedToRoom(ctx, types.RoomNID(roomInfo.RoomNID), senderID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.UserJoinedToRoom failed") + return nil, fmt.Errorf("InternalServerError: %w", err) + } + + locallyJoinedUsers, err := r.LocallyJoinedUsers(ctx, roomInfo.RoomVersion, types.RoomNID(roomInfo.RoomNID)) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("rsAPI.GetLocallyJoinedUsers failed") + return nil, fmt.Errorf("InternalServerError: %w", err) + } + + return &gomatrixserverlib.RestrictedRoomJoinInfo{ + LocalServerInRoom: res.RoomExists && res.IsInRoom, + UserJoinedToRoom: userJoinedToRoom, + JoinedUsers: locallyJoinedUsers, + }, nil } // QueryLatestEventsAndState implements api.RoomserverInternalAPI @@ -52,7 +92,7 @@ func (r *Queryer) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { - return helpers.QueryLatestEventsAndState(ctx, r.DB, request, response) + return helpers.QueryLatestEventsAndState(ctx, r.DB, r, request, response) } // QueryStateAfterEvents implements api.RoomserverInternalAPI @@ -69,7 +109,7 @@ func (r *Queryer) QueryStateAfterEvents( return nil } - roomState := state.NewStateResolution(r.DB, info) + roomState := state.NewStateResolution(r.DB, info, r) response.RoomExists = true response.RoomVersion = info.RoomVersion @@ -122,7 +162,9 @@ func (r *Queryer) QueryStateAfterEvents( } stateEvents, err = gomatrixserverlib.ResolveConflicts( - info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), + info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.QueryUserIDForSender(ctx, roomID, senderID) + }, ) if err != nil { return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err) @@ -181,13 +223,41 @@ func (r *Queryer) QueryEventsByID( return nil } +// QueryMembershipForSenderID implements api.RoomserverInternalAPI +func (r *Queryer) QueryMembershipForSenderID( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, + response *api.QueryMembershipForUserResponse, +) error { + return r.queryMembershipForOptionalSenderID(ctx, roomID, &senderID, response) +} + // QueryMembershipForUser implements api.RoomserverInternalAPI func (r *Queryer) QueryMembershipForUser( ctx context.Context, request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, ) error { - info, err := r.DB.RoomInfo(ctx, request.RoomID) + roomID, err := spec.NewRoomID(request.RoomID) + if err != nil { + return err + } + senderID, err := r.QuerySenderIDForUser(ctx, *roomID, request.UserID) + if err != nil { + return err + } + + return r.queryMembershipForOptionalSenderID(ctx, *roomID, senderID, response) +} + +// Query membership information for provided sender ID and room ID +// +// If sender ID is nil, then act as if the provided sender is not a member of the room. +func (r *Queryer) queryMembershipForOptionalSenderID(ctx context.Context, roomID spec.RoomID, senderID *spec.SenderID, response *api.QueryMembershipForUserResponse) error { + response.SenderID = senderID + + info, err := r.DB.RoomInfo(ctx, roomID.String()) if err != nil { return err } @@ -197,7 +267,11 @@ func (r *Queryer) QueryMembershipForUser( } response.RoomExists = true - membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID) + if senderID == nil { + return nil + } + + membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, *senderID) if err != nil { return err } @@ -228,49 +302,52 @@ func (r *Queryer) QueryMembershipForUser( // QueryMembershipAtEvent returns the known memberships at a given event. // If the state before an event is not known, an empty list will be returned // for that event instead. +// +// Returned map from eventID to membership event. Events that +// do not have known state will return a nil event, resulting in a "leave" membership +// when calculating history visibility. func (r *Queryer) QueryMembershipAtEvent( ctx context.Context, - request *api.QueryMembershipAtEventRequest, - response *api.QueryMembershipAtEventResponse, -) error { - response.Membership = make(map[string]*types.HeaderedEvent) - - info, err := r.DB.RoomInfo(ctx, request.RoomID) + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, +) (map[string]*types.HeaderedEvent, error) { + info, err := r.DB.RoomInfo(ctx, roomID.String()) if err != nil { - return fmt.Errorf("unable to get roomInfo: %w", err) + return nil, fmt.Errorf("unable to get roomInfo: %w", err) } if info == nil { - return fmt.Errorf("no roomInfo found") + return nil, fmt.Errorf("no roomInfo found") } // get the users stateKeyNID - stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.UserID}) + stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{string(senderID)}) if err != nil { - return fmt.Errorf("unable to get stateKeyNIDs for %s: %w", request.UserID, err) + return nil, fmt.Errorf("unable to get stateKeyNIDs for %s: %w", senderID, err) } - if _, ok := stateKeyNIDs[request.UserID]; !ok { - return fmt.Errorf("requested stateKeyNID for %s was not found", request.UserID) + if _, ok := stateKeyNIDs[string(senderID)]; !ok { + return nil, fmt.Errorf("requested stateKeyNID for %s was not found", senderID) } - response.Membership, err = r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[request.UserID], info, request.EventIDs...) + eventIDMembershipMap, err := r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[string(senderID)], info, eventIDs...) switch err { case nil: - return nil + return eventIDMembershipMap, nil case tables.OptimisationNotSupportedError: // fallthrough, slow way of getting the membership events for each event default: - return err + return eventIDMembershipMap, err } - response.Membership = make(map[string]*types.HeaderedEvent) - stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, request.EventIDs, stateKeyNIDs[request.UserID]) + eventIDMembershipMap = make(map[string]*types.HeaderedEvent) + stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, eventIDs, stateKeyNIDs[string(senderID)], r) if err != nil { - return fmt.Errorf("unable to get state before event: %w", err) + return eventIDMembershipMap, fmt.Errorf("unable to get state before event: %w", err) } // If we only have one or less state entries, we can short circuit the below // loop and avoid hitting the database allStateEventNIDs := make(map[types.EventNID]types.StateEntry) - for _, eventID := range request.EventIDs { + for _, eventID := range eventIDs { stateEntry := stateEntries[eventID] for _, s := range stateEntry { allStateEventNIDs[s.EventNID] = s @@ -283,10 +360,10 @@ func (r *Queryer) QueryMembershipAtEvent( } var memberships []types.Event - for _, eventID := range request.EventIDs { + for _, eventID := range eventIDs { stateEntry, ok := stateEntries[eventID] if !ok || len(stateEntry) == 0 { - response.Membership[eventID] = nil + eventIDMembershipMap[eventID] = nil continue } @@ -300,7 +377,7 @@ func (r *Queryer) QueryMembershipAtEvent( memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, info, stateEntry, false) } if err != nil { - return fmt.Errorf("unable to get memberships at state: %w", err) + return eventIDMembershipMap, fmt.Errorf("unable to get memberships at state: %w", err) } // Iterate over all membership events we got. Given we only query the membership for @@ -308,13 +385,13 @@ func (r *Queryer) QueryMembershipAtEvent( // a given event, overwrite any other existing membership events. for i := range memberships { ev := memberships[i] - if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(request.UserID) { - response.Membership[eventID] = &types.HeaderedEvent{PDU: ev.PDU} + if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(senderID)) { + eventIDMembershipMap[eventID] = &types.HeaderedEvent{PDU: ev.PDU} } } } - return nil + return eventIDMembershipMap, nil } // QueryMembershipsForRoom implements api.RoomserverInternalAPI @@ -334,7 +411,7 @@ func (r *Queryer) QueryMembershipsForRoom( // If no sender is specified then we will just return the entire // set of memberships for the room, regardless of whether a specific // user is allowed to see them or not. - if request.Sender == "" { + if request.SenderID == "" { var events []types.Event var eventNIDs []types.EventNID eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, request.LocalOnly) @@ -349,13 +426,15 @@ func (r *Queryer) QueryMembershipsForRoom( return fmt.Errorf("r.DB.Events: %w", err) } for _, event := range events { - clientEvent := synctypes.ToClientEvent(event.PDU, synctypes.FormatAll) + clientEvent := synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.QueryUserIDForSender(ctx, roomID, senderID) + }, event) response.JoinEvents = append(response.JoinEvents, clientEvent) } return nil } - membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender) + membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.SenderID) if err != nil { return err } @@ -385,7 +464,7 @@ func (r *Queryer) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, info.RoomVersion, eventNIDs) } else { - stateEntries, err = helpers.StateBeforeEvent(ctx, r.DB, info, membershipEventNID) + stateEntries, err = helpers.StateBeforeEvent(ctx, r.DB, info, membershipEventNID, r) if err != nil { logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event") return err @@ -398,7 +477,9 @@ func (r *Queryer) QueryMembershipsForRoom( } for _, event := range events { - clientEvent := synctypes.ToClientEvent(event.PDU, synctypes.FormatAll) + clientEvent := synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.QueryUserIDForSender(ctx, roomID, senderID) + }, event) response.JoinEvents = append(response.JoinEvents, clientEvent) } @@ -415,6 +496,9 @@ func (r *Queryer) QueryServerJoinedToRoom( if err != nil { return fmt.Errorf("r.DB.RoomInfo: %w", err) } + if info != nil { + response.RoomVersion = info.RoomVersion + } if info == nil || info.IsStub() { return nil } @@ -440,6 +524,7 @@ func (r *Queryer) QueryServerAllowedToSeeEvent( ctx context.Context, serverName spec.ServerName, eventID string, + roomID string, ) (allowed bool, err error) { events, err := r.DB.EventNIDs(ctx, []string{eventID}) if err != nil { @@ -469,7 +554,7 @@ func (r *Queryer) QueryServerAllowedToSeeEvent( } return helpers.CheckServerAllowedToSeeEvent( - ctx, r.DB, info, eventID, serverName, isInRoom, + ctx, r.DB, info, roomID, eventID, serverName, isInRoom, r, ) } @@ -510,7 +595,7 @@ func (r *Queryer) QueryMissingEvents( return fmt.Errorf("missing RoomInfo for room %d", events[front[0]].RoomNID) } - resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName, r) if err != nil { return err } @@ -588,7 +673,9 @@ func (r *Queryer) QueryStateAndAuthChain( if request.ResolveState { stateEvents, err = gomatrixserverlib.ResolveConflicts( - info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), + info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return r.QueryUserIDForSender(ctx, roomID, senderID) + }, ) if err != nil { return err @@ -608,7 +695,7 @@ func (r *Queryer) QueryStateAndAuthChain( // first bool: is rejected, second bool: state missing func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]gomatrixserverlib.PDU, bool, bool, error) { - roomState := state.NewStateResolution(r.DB, roomInfo) + roomState := state.NewStateResolution(r.DB, roomInfo, r) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { switch err.(type) { @@ -759,13 +846,20 @@ func (r *Queryer) QueryCurrentState(ctx context.Context, req *api.QueryCurrentSt return nil } -func (r *Queryer) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership) +func (r *Queryer) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + roomIDStrs, err := r.DB.GetRoomsByMembership(ctx, userID, desiredMembership) if err != nil { - return err + return nil, err } - res.RoomIDs = roomIDs - return nil + roomIDs := make([]spec.RoomID, len(roomIDStrs)) + for i, roomIDStr := range roomIDStrs { + roomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return nil, err + } + roomIDs[i] = *roomID + } + return roomIDs, nil } func (r *Queryer) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error { @@ -808,7 +902,12 @@ func (r *Queryer) QueryLeftUsers(ctx context.Context, req *api.QueryLeftUsersReq } func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { - roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, "join") + parsedUserID, err := spec.NewUserID(req.UserID, true) + if err != nil { + return err + } + + roomIDs, err := r.DB.GetRoomsByMembership(ctx, *parsedUserID, "join") if err != nil { return err } @@ -858,8 +957,8 @@ func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainReq return nil } -func (r *Queryer) InvitePending(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (bool, error) { - pending, _, _, _, err := helpers.IsInvitePending(ctx, r.DB, roomID.String(), userID.String()) +func (r *Queryer) InvitePending(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (bool, error) { + pending, _, _, _, err := helpers.IsInvitePending(ctx, r.DB, roomID.String(), senderID) return pending, err } @@ -868,15 +967,15 @@ func (r *Queryer) QueryRoomInfo(ctx context.Context, roomID spec.RoomID) (*types } func (r *Queryer) CurrentStateEvent(ctx context.Context, roomID spec.RoomID, eventType string, stateKey string) (gomatrixserverlib.PDU, error) { - res, err := r.DB.GetStateEvent(ctx, roomID.String(), eventType, "") + res, err := r.DB.GetStateEvent(ctx, roomID.String(), eventType, stateKey) if res == nil { return nil, err } return res, err } -func (r *Queryer) UserJoinedToRoom(ctx context.Context, roomNID types.RoomNID, userID spec.UserID) (bool, error) { - _, isIn, _, err := r.DB.GetMembership(ctx, roomNID, userID.String()) +func (r *Queryer) UserJoinedToRoom(ctx context.Context, roomNID types.RoomNID, senderID spec.SenderID) (bool, error) { + _, isIn, _, err := r.DB.GetMembership(ctx, roomNID, senderID) return isIn, err } @@ -905,132 +1004,84 @@ func (r *Queryer) LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixse return joinedUsers, nil } +func (r *Queryer) JoinedUserCount(ctx context.Context, roomID string) (int, error) { + info, err := r.DB.RoomInfo(ctx, roomID) + if err != nil { + return 0, err + } + if info == nil { + return 0, nil + } + + // TODO: this can be further optimised by just using a SELECT COUNT query + nids, err := r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false) + return len(nids), err +} + // nolint:gocyclo -func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse) error { +func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (string, error) { // Look up if we know anything about the room. If it doesn't exist // or is a stub entry then we can't do anything. - roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID) + roomInfo, err := r.DB.RoomInfo(ctx, roomID.String()) if err != nil { - return fmt.Errorf("r.DB.RoomInfo: %w", err) + return "", fmt.Errorf("r.DB.RoomInfo: %w", err) } if roomInfo == nil || roomInfo.IsStub() { - return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID) + return "", nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID) } verImpl, err := gomatrixserverlib.GetRoomVersion(roomInfo.RoomVersion) if err != nil { - return err - } - // If the room version doesn't allow restricted joins then don't - // try to process any further. - allowRestrictedJoins := verImpl.MayAllowRestrictedJoinsInEventAuth() - if !allowRestrictedJoins { - return nil + return "", err } - // Start off by populating the "resident" flag in the response. If we - // come across any rooms in the request that are missing, we will unset - // the flag. - res.Resident = true - // Get the join rules to work out if the join rule is "restricted". - joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, spec.MRoomJoinRules, "") + + return verImpl.CheckRestrictedJoin(ctx, r.Cfg.Global.ServerName, &api.JoinRoomQuerier{Roomserver: r}, roomID, senderID) +} + +func (r *Queryer) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + version, err := r.DB.GetRoomVersion(ctx, roomID.String()) if err != nil { - return fmt.Errorf("r.DB.GetStateEvent: %w", err) - } - if joinRulesEvent == nil { - return nil - } - var joinRules gomatrixserverlib.JoinRuleContent - if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil { - return fmt.Errorf("json.Unmarshal: %w", err) + return nil, err } - // If the join rule isn't "restricted" or "knock_restricted" then there's nothing more to do. - res.Restricted = joinRules.JoinRule == spec.Restricted || joinRules.JoinRule == spec.KnockRestricted - if !res.Restricted { - return nil + + switch version { + case gomatrixserverlib.RoomVersionPseudoIDs: + key, err := r.DB.SelectUserRoomPublicKey(ctx, userID, roomID) + if err != nil { + return nil, err + } else if key == nil { + return nil, nil + } else { + senderID := spec.SenderID(spec.Base64Bytes(key).Encode()) + return &senderID, nil + } + default: + senderID := spec.SenderID(userID.String()) + return &senderID, nil } - // If the user is already invited to the room then the join is allowed - // but we don't specify an authorised via user, since the event auth - // will allow the join anyway. - var pending bool - if pending, _, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil { - return fmt.Errorf("helpers.IsInvitePending: %w", err) - } else if pending { - res.Allowed = true - return nil +} + +func (r *Queryer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + userID, err := spec.NewUserID(string(senderID), true) + if err == nil { + return userID, nil } - // We need to get the power levels content so that we can determine which - // users in the room are entitled to issue invites. We need to use one of - // these users as the authorising user. - powerLevelsEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, spec.MRoomPowerLevels, "") + + bytes := spec.Base64Bytes{} + err = bytes.Decode(string(senderID)) if err != nil { - return fmt.Errorf("r.DB.GetStateEvent: %w", err) + return nil, err } - powerLevels, err := powerLevelsEvent.PowerLevels() + queryMap := map[spec.RoomID][]ed25519.PublicKey{roomID: {ed25519.PublicKey(bytes)}} + result, err := r.DB.SelectUserIDsForPublicKeys(ctx, queryMap) if err != nil { - return fmt.Errorf("unable to get powerlevels: %w", err) + return nil, err } - // Step through the join rules and see if the user matches any of them. - for _, rule := range joinRules.Allow { - // We only understand "m.room_membership" rules at this point in - // time, so skip any rule that doesn't match those. - if rule.Type != spec.MRoomMembership { - continue - } - // See if the room exists. If it doesn't exist or if it's a stub - // room entry then we can't check memberships. - targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID) - if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub() { - res.Resident = false - continue - } - // First of all work out if *we* are still in the room, otherwise - // it's possible that the memberships will be out of date. - isIn, err := r.DB.GetLocalServerInRoom(ctx, targetRoomInfo.RoomNID) - if err != nil || !isIn { - // If we aren't in the room, we can no longer tell if the room - // memberships are up-to-date. - res.Resident = false - continue - } - // At this point we're happy that we are in the room, so now let's - // see if the target user is in the room. - _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID) - if err != nil { - continue - } - // If the user is not in the room then we will skip them. - if !isIn { - continue - } - // The user is in the room, so now we will need to authorise the - // join using the user ID of one of our own users in the room. Pick - // one. - joinNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, targetRoomInfo.RoomNID, true, true) - if err != nil || len(joinNIDs) == 0 { - // There should always be more than one join NID at this point - // because we are gated behind GetLocalServerInRoom, but y'know, - // sometimes strange things happen. - continue - } - // For each of the joined users, let's see if we can get a valid - // membership event. - for _, joinNID := range joinNIDs { - events, err := r.DB.Events(ctx, roomInfo.RoomVersion, []types.EventNID{joinNID}) - if err != nil || len(events) != 1 { - continue - } - event := events[0] - if event.Type() != spec.MRoomMember || event.StateKey() == nil { - continue // shouldn't happen - } - // Only users that have the power to invite should be chosen. - if powerLevels.UserLevel(*event.StateKey()) < powerLevels.Invite { - continue - } - res.Resident = true - res.Allowed = true - res.AuthorisedVia = *event.StateKey() - return nil + + if userKeys, ok := result[roomID]; ok { + if userID, ok := userKeys[string(senderID)]; ok { + return spec.NewUserID(userID, true) } } - return nil + + return nil, nil } diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go new file mode 100644 index 0000000000..76eba12bee --- /dev/null +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -0,0 +1,530 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + fs "github.com/matrix-org/dendrite/federationapi/api" + roomserver "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" +) + +// Traverse the room hierarchy using the provided walker up to the provided limit, +// returning a new walker which can be used to fetch the next page. +// +// If limit is -1, this is treated as no limit, and the entire hierarchy will be traversed. +// +// If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it +// can be cached. +func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *roomserver.RoomHierarchyWalker, error) { + if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { + return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} + } + + discoveredRooms := []fclient.RoomHierarchyRoom{} + + // Copy unvisited and processed to avoid modifying original walker (which is typically in cache) + unvisited := make([]roomserver.RoomHierarchyWalkerQueuedRoom, len(walker.Unvisited)) + copy(unvisited, walker.Unvisited) + processed := walker.Processed.Copy() + + // Depth first -> stack data structure + for len(unvisited) > 0 { + if len(discoveredRooms) >= limit && limit != -1 { + break + } + + // pop the stack + queuedRoom := unvisited[len(unvisited)-1] + unvisited = unvisited[:len(unvisited)-1] + // If this room has already been processed, skip. + // If this room exceeds the specified depth, skip. + if processed.Contains(queuedRoom.RoomID) || (walker.MaxDepth > 0 && queuedRoom.Depth > walker.MaxDepth) { + continue + } + + // Mark this room as processed. + processed.Add(queuedRoom.RoomID) + + // if this room is not a space room, skip. + var roomType string + create := stateEvent(ctx, querier, queuedRoom.RoomID, spec.MRoomCreate, "") + if create != nil { + var createContent gomatrixserverlib.CreateContent + err := json.Unmarshal(create.Content(), &createContent) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("create_content", create.Content()).Warn("failed to unmarshal m.room.create event") + } + roomType = createContent.RoomType + } + + // Collect rooms/events to send back (either locally or fetched via federation) + var discoveredChildEvents []fclient.RoomHierarchyStrippedEvent + + // If we know about this room and the caller is authorised (joined/world_readable) then pull + // events locally + roomExists := roomExists(ctx, querier, queuedRoom.RoomID) + if !roomExists { + // attempt to query this room over federation, as either we've never heard of it before + // or we've left it and hence are not authorised (but info may be exposed regardless) + fedRes := federatedRoomInfo(ctx, querier, walker.Caller, walker.SuggestedOnly, queuedRoom.RoomID, queuedRoom.Vias) + if fedRes != nil { + discoveredChildEvents = fedRes.Room.ChildrenState + discoveredRooms = append(discoveredRooms, fedRes.Room) + if len(fedRes.Children) > 0 { + discoveredRooms = append(discoveredRooms, fedRes.Children...) + } + // mark this room as a space room as the federated server responded. + // we need to do this so we add the children of this room to the unvisited stack + // as these children may be rooms we do know about. + roomType = spec.MSpace + } + } else if authorised, isJoinedOrInvited := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised { + // Get all `m.space.child` state events for this room + events, err := childReferences(ctx, querier, walker.SuggestedOnly, queuedRoom.RoomID) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("room_id", queuedRoom.RoomID).Error("failed to extract references for room") + continue + } + discoveredChildEvents = events + + pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) + + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ + PublicRoom: *pubRoom, + RoomType: roomType, + ChildrenState: events, + }) + // don't walk children if the user is not joined/invited to the space + if !isJoinedOrInvited { + continue + } + } else { + // room exists but user is not authorised + continue + } + + // don't walk the children + // if the parent is not a space room + if roomType != spec.MSpace { + continue + } + + // For each referenced room ID in the child events being returned to the caller + // add the room ID to the queue of unvisited rooms. Loop from the beginning. + // We need to invert the order here because the child events are lo->hi on the timestamp, + // so we need to ensure we pop in the same lo->hi order, which won't be the case if we + // insert the highest timestamp last in a stack. + for i := len(discoveredChildEvents) - 1; i >= 0; i-- { + spaceContent := struct { + Via []string `json:"via"` + }{} + ev := discoveredChildEvents[i] + _ = json.Unmarshal(ev.Content, &spaceContent) + + childRoomID, err := spec.NewRoomID(ev.StateKey) + + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("invalid_room_id", ev.StateKey).WithField("parent_room_id", queuedRoom.RoomID).Warn("Invalid room ID in m.space.child state event") + } else { + unvisited = append(unvisited, roomserver.RoomHierarchyWalkerQueuedRoom{ + RoomID: *childRoomID, + ParentRoomID: &queuedRoom.RoomID, + Depth: queuedRoom.Depth + 1, + Vias: spaceContent.Via, + }) + } + } + } + + if len(unvisited) == 0 { + // If no more rooms to walk, then don't return a walker for future pages + return discoveredRooms, nil, nil + } else { + // If there are more rooms to walk, then return a new walker to resume walking from (for querying more pages) + newWalker := roomserver.RoomHierarchyWalker{ + RootRoomID: walker.RootRoomID, + Caller: walker.Caller, + SuggestedOnly: walker.SuggestedOnly, + MaxDepth: walker.MaxDepth, + Unvisited: unvisited, + Processed: processed, + } + + return discoveredRooms, &newWalker, nil + } + +} + +// authorised returns true iff the user is joined this room or the room is world_readable +func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool) { + if clientCaller := caller.Device(); clientCaller != nil { + return authorisedUser(ctx, querier, clientCaller, roomID, parentRoomID) + } else { + return authorisedServer(ctx, querier, roomID, *caller.ServerName()), false + } +} + +// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable +func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) bool { + // Check history visibility / join rules first + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomHistoryVisibility, + StateKey: "", + } + joinRuleTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomJoinRules, + StateKey: "", + } + var queryRoomRes roomserver.QueryCurrentStateResponse + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, joinRuleTuple, + }, + }, &queryRoomRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") + return false + } + hisVisEv := queryRoomRes.StateEvents[hisVisTuple] + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true + } + } + + // check if this room is a restricted room and if so, we need to check if the server is joined to an allowed room ID + // in addition to the actual room ID (but always do the actual one first as it's quicker in the common case) + allowJoinedToRoomIDs := []spec.RoomID{roomID} + joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple] + + if joinRuleEv != nil { + rule, ruleErr := joinRuleEv.JoinRule() + if ruleErr != nil { + util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") + return false + } + + if rule == spec.Public || rule == spec.Knock { + return true + } + + if rule == spec.Restricted { + allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv)...) + } + } + + // check if server is joined to any allowed room + for _, allowedRoomID := range allowJoinedToRoomIDs { + var queryRes fs.QueryJoinedHostServerNamesInRoomResponse + err = querier.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: allowedRoomID.String(), + }, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") + continue + } + for _, srv := range queryRes.ServerNames { + if srv == callerServerName { + return true + } + } + } + + return false +} + +// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable +// or if the room has a public or knock join rule. +// Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. +func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool) { + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomHistoryVisibility, + StateKey: "", + } + joinRuleTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomJoinRules, + StateKey: "", + } + roomMemberTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomMember, + StateKey: clientCaller.UserID, + } + var queryRes roomserver.QueryCurrentStateResponse + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, joinRuleTuple, roomMemberTuple, + }, + }, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") + return false, false + } + memberEv := queryRes.StateEvents[roomMemberTuple] + if memberEv != nil { + membership, _ := memberEv.Membership() + if membership == spec.Join || membership == spec.Invite { + return true, true + } + } + hisVisEv := queryRes.StateEvents[hisVisTuple] + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true, false + } + } + joinRuleEv := queryRes.StateEvents[joinRuleTuple] + if parentRoomID != nil && joinRuleEv != nil { + var allowed bool + rule, ruleErr := joinRuleEv.JoinRule() + if ruleErr != nil { + util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") + } else if rule == spec.Public || rule == spec.Knock { + allowed = true + } else if rule == spec.Restricted { + allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv) + // check parent is in the allowed set + for _, a := range allowedRoomIDs { + if *parentRoomID == a { + allowed = true + break + } + } + } + if allowed { + // ensure caller is joined to the parent room + var queryRes2 roomserver.QueryCurrentStateResponse + err = querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: parentRoomID.String(), + StateTuples: []gomatrixserverlib.StateKeyTuple{ + roomMemberTuple, + }, + }, &queryRes2) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") + } else { + memberEv = queryRes2.StateEvents[roomMemberTuple] + if memberEv != nil { + membership, _ := memberEv.Membership() + if membership == spec.Join { + return true, false + } + } + } + } + } + return false, false +} + +// helper function to fetch a state event +func stateEvent(ctx context.Context, querier *Queryer, roomID spec.RoomID, evType, stateKey string) *types.HeaderedEvent { + var queryRes roomserver.QueryCurrentStateResponse + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: evType, + StateKey: stateKey, + } + err := querier.QueryCurrentState(ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &queryRes) + if err != nil { + return nil + } + return queryRes.StateEvents[tuple] +} + +// returns true if the current server is participating in the provided room +func roomExists(ctx context.Context, querier *Queryer, roomID spec.RoomID) bool { + var queryRes roomserver.QueryServerJoinedToRoomResponse + err := querier.QueryServerJoinedToRoom(ctx, &roomserver.QueryServerJoinedToRoomRequest{ + RoomID: roomID.String(), + ServerName: querier.Cfg.Global.ServerName, + }, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") + return false + } + // if the room exists but we aren't in the room then we might have stale data so we want to fetch + // it fresh via federation + return queryRes.RoomExists && queryRes.IsInRoom +} + +// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was +// unsuccessful. +func federatedRoomInfo(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, suggestedOnly bool, roomID spec.RoomID, vias []string) *fclient.RoomHierarchyResponse { + // only do federated requests for client requests + if caller.Device() == nil { + return nil + } + resp, ok := querier.Cache.GetRoomHierarchy(roomID.String()) + if ok { + util.GetLogger(ctx).Debugf("Returning cached response for %s", roomID) + return &resp + } + util.GetLogger(ctx).Debugf("Querying %s via %+v", roomID, vias) + innerCtx := context.Background() + // query more of the spaces graph using these servers + for _, serverName := range vias { + if serverName == string(querier.Cfg.Global.ServerName) { + continue + } + res, err := querier.FSAPI.RoomHierarchies(innerCtx, querier.Cfg.Global.ServerName, spec.ServerName(serverName), roomID.String(), suggestedOnly) + if err != nil { + util.GetLogger(ctx).WithError(err).Warnf("failed to call RoomHierarchies on server %s", serverName) + continue + } + // ensure nil slices are empty as we send this to the client sometimes + if res.Room.ChildrenState == nil { + res.Room.ChildrenState = []fclient.RoomHierarchyStrippedEvent{} + } + for i := 0; i < len(res.Children); i++ { + child := res.Children[i] + if child.ChildrenState == nil { + child.ChildrenState = []fclient.RoomHierarchyStrippedEvent{} + } + res.Children[i] = child + } + querier.Cache.StoreRoomHierarchy(roomID.String(), res) + + return &res + } + return nil +} + +// references returns all child references pointing to or from this room. +func childReferences(ctx context.Context, querier *Queryer, suggestedOnly bool, roomID spec.RoomID) ([]fclient.RoomHierarchyStrippedEvent, error) { + createTuple := gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomCreate, + StateKey: "", + } + var res roomserver.QueryCurrentStateResponse + err := querier.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ + RoomID: roomID.String(), + AllowWildcards: true, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + createTuple, { + EventType: spec.MSpaceChild, + StateKey: "*", + }, + }, + }, &res) + if err != nil { + return nil, err + } + + // don't return any child refs if the room is not a space room + if create := res.StateEvents[createTuple]; create != nil { + var createContent gomatrixserverlib.CreateContent + err := json.Unmarshal(create.Content(), &createContent) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("create_content", create.Content()).Warn("failed to unmarshal m.room.create event") + } + roomType := createContent.RoomType + if roomType != spec.MSpace { + return []fclient.RoomHierarchyStrippedEvent{}, nil + } + } + delete(res.StateEvents, createTuple) + + el := make([]fclient.RoomHierarchyStrippedEvent, 0, len(res.StateEvents)) + for _, ev := range res.StateEvents { + content := gjson.ParseBytes(ev.Content()) + // only return events that have a `via` key as per MSC1772 + // else we'll incorrectly walk redacted events (as the link + // is in the state_key) + if content.Get("via").Exists() { + strip := stripped(ev.PDU) + if strip == nil { + continue + } + // if suggested only and this child isn't suggested, skip it. + // if suggested only = false we include everything so don't need to check the content. + if suggestedOnly && !content.Get("suggested").Bool() { + continue + } + el = append(el, *strip) + } + } + // sort by origin_server_ts as per MSC2946 + sort.Slice(el, func(i, j int) bool { + return el[i].OriginServerTS < el[j].OriginServerTS + }) + + return el, nil +} + +// fetch public room information for provided room +func publicRoomsChunk(ctx context.Context, querier *Queryer, roomID spec.RoomID) *fclient.PublicRoom { + pubRooms, err := roomserver.PopulatePublicRooms(ctx, []string{roomID.String()}, querier) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to PopulatePublicRooms") + return nil + } + if len(pubRooms) == 0 { + return nil + } + return &pubRooms[0] +} + +func stripped(ev gomatrixserverlib.PDU) *fclient.RoomHierarchyStrippedEvent { + if ev.StateKey() == nil { + return nil + } + return &fclient.RoomHierarchyStrippedEvent{ + Type: ev.Type(), + StateKey: *ev.StateKey(), + Content: ev.Content(), + Sender: string(ev.SenderID()), + OriginServerTS: ev.OriginServerTS(), + } +} + +// given join_rule event, return list of rooms where membership of that room allows joining. +func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.HeaderedEvent) (allows []spec.RoomID) { + rule, _ := joinRuleEv.JoinRule() + if rule != spec.Restricted { + return nil + } + var jrContent gomatrixserverlib.JoinRuleContent + if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { + util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID().String(), err) + return nil + } + for _, allow := range jrContent.Allow { + if allow.Type == spec.MRoomMembership { + allowedRoomID, err := spec.NewRoomID(allow.RoomID) + if err != nil { + util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID().String(), err) + } else { + allows = append(allows, *allowedRoomID) + } + } + } + return +} diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go index 619d930306..296960b2ff 100644 --- a/roomserver/internal/query/query_test.go +++ b/roomserver/internal/query/query_test.go @@ -49,6 +49,7 @@ func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error { } builder := map[string]interface{}{ "event_id": eventID, + "room_id": "!room:a", "auth_events": authEvents, } diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index febe8ddf41..165304d49c 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -60,7 +60,7 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu "adds_state": len(update.NewRoomEvent.AddsStateEventIDs), "removes_state": len(update.NewRoomEvent.RemovesStateEventIDs), "send_as_server": update.NewRoomEvent.SendAsServer, - "sender": update.NewRoomEvent.Event.Sender(), + "sender": update.NewRoomEvent.Event.SenderID(), }) if update.NewRoomEvent.Event.StateKey() != nil { logger = logger.WithField("state_key", *update.NewRoomEvent.Event.StateKey()) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 4685f474f3..07c5d65619 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -28,10 +28,13 @@ import ( ) // NewInternalAPI returns a concrete implementation of the internal API. +// +// Many of the methods provided by this API depend on access to a federation API, and so +// you may wish to call `SetFederationAPI` on the returned struct to avoid nil-dereference errors. func NewInternalAPI( processContext *process.ProcessContext, cfg *config.Dendrite, - cm sqlutil.Connections, + cm *sqlutil.Connections, natsInstance *jetstream.NATSInstance, caches caching.RoomServerCaches, enableMetrics bool, diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index d19ebebe43..47626b30ac 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -11,7 +11,6 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" @@ -35,6 +34,14 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +type FakeQuerier struct { + api.QuerySenderIDAPI +} + +func (f *FakeQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func TestUsers(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { cfg, processCtx, close := testrig.CreateConfig(t, dbType) @@ -219,6 +226,11 @@ func TestPurgeRoom(t *testing.T) { bob := test.NewUser(t) room := test.NewRoom(t, alice, test.RoomPreset(test.PresetTrustedPrivateChat)) + roomID, err := spec.NewRoomID(room.ID) + if err != nil { + t.Fatal(err) + } + // Invite Bob inviteEvent := room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ "membership": "invite", @@ -241,13 +253,14 @@ func TestPurgeRoom(t *testing.T) { defer jetstream.DeleteAllStreams(jsCtx, &cfg.Global.JetStream) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err = api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -265,9 +278,7 @@ func TestPurgeRoom(t *testing.T) { if !isPublished { t.Fatalf("room should be published before purging") } - - aliasResp := &api.SetRoomAliasResponse{} - if err = rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{RoomID: room.ID, Alias: "myalias", UserID: alice.ID}, aliasResp); err != nil { + if _, err = rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), *roomID, "myalias"); err != nil { t.Fatal(err) } // check the alias is actually there @@ -392,7 +403,7 @@ func TestPurgeRoom(t *testing.T) { type fledglingEvent struct { Type string StateKey *string - Sender string + SenderID string RoomID string Redacts string Depth int64 @@ -405,7 +416,7 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve seed := make([]byte, ed25519.SeedSize) // zero seed key := ed25519.NewKeyFromSeed(seed) eb := gomatrixserverlib.MustGetRoomVersion(roomVer).NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ - Sender: ev.Sender, + SenderID: ev.SenderID, Type: ev.Type, StateKey: ev.StateKey, RoomID: ev.RoomID, @@ -444,7 +455,7 @@ func TestRedaction(t *testing.T) { builderEv := mustCreateEvent(t, fledglingEvent{ Type: spec.MRoomRedaction, - Sender: alice.ID, + SenderID: alice.ID, RoomID: room.ID, Redacts: redactedEvent.EventID(), Depth: redactedEvent.Depth() + 1, @@ -461,7 +472,7 @@ func TestRedaction(t *testing.T) { builderEv := mustCreateEvent(t, fledglingEvent{ Type: spec.MRoomRedaction, - Sender: alice.ID, + SenderID: alice.ID, RoomID: room.ID, Redacts: redactedEvent.EventID(), Depth: redactedEvent.Depth() + 1, @@ -478,7 +489,7 @@ func TestRedaction(t *testing.T) { builderEv := mustCreateEvent(t, fledglingEvent{ Type: spec.MRoomRedaction, - Sender: bob.ID, + SenderID: bob.ID, RoomID: room.ID, Redacts: redactedEvent.EventID(), Depth: redactedEvent.Depth() + 1, @@ -494,7 +505,7 @@ func TestRedaction(t *testing.T) { builderEv := mustCreateEvent(t, fledglingEvent{ Type: spec.MRoomRedaction, - Sender: charlie.ID, + SenderID: charlie.ID, RoomID: room.ID, Redacts: redactedEvent.EventID(), Depth: redactedEvent.Depth() + 1, @@ -516,6 +527,9 @@ func TestRedaction(t *testing.T) { t.Fatal(err) } + natsInstance := &jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { authEvents := []types.EventNID{} @@ -551,7 +565,7 @@ func TestRedaction(t *testing.T) { } // Calculate the snapshotNID etc. - plResolver := state.NewStateResolution(db, roomInfo) + plResolver := state.NewStateResolution(db, roomInfo, rsAPI) stateAtEvent.BeforeStateSnapshotNID, err = plResolver.CalculateAndStoreStateBeforeEvent(ctx, ev.PDU, false) assert.NoError(t, err) @@ -563,7 +577,7 @@ func TestRedaction(t *testing.T) { err = updater.Commit() assert.NoError(t, err) - _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.PDU, &plResolver) + _, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.PDU, &plResolver, &FakeQuerier{}) assert.NoError(t, err) if redactedEvent != nil { assert.Equal(t, ev.Redacts(), redactedEvent.EventID()) @@ -598,16 +612,15 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { testCases := []struct { name string prepareRoomFunc func(t *testing.T) *test.Room - wantResponse api.QueryRestrictedJoinAllowedResponse + wantResponse string + wantError bool }{ { name: "public room unrestricted", prepareRoomFunc: func(t *testing.T) *test.Room { return test.NewRoom(t, alice) }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Resident: true, - }, + wantResponse: "", }, { name: "room version without restrictions", @@ -624,10 +637,7 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { }, test.WithStateKey("")) return r }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Resident: true, - Restricted: true, - }, + wantError: true, }, { name: "knock_restricted", @@ -638,10 +648,7 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { }, test.WithStateKey("")) return r }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Resident: true, - Restricted: true, - }, + wantError: true, }, { name: "restricted with pending invite", // bob should be allowed to join @@ -655,11 +662,7 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { }, test.WithStateKey(bob.ID)) return r }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Resident: true, - Restricted: true, - Allowed: true, - }, + wantResponse: "", }, { name: "restricted with allowed room_id, but missing room", // bob should not be allowed to join, as we don't know about the room @@ -680,9 +683,7 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { }, test.WithStateKey(bob.ID)) return r }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Restricted: true, - }, + wantError: true, }, { name: "restricted with allowed room_id", // bob should be allowed to join, as we know about the room @@ -703,12 +704,7 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { }, test.WithStateKey(bob.ID)) return r }, - wantResponse: api.QueryRestrictedJoinAllowedResponse{ - Resident: true, - Restricted: true, - Allowed: true, - AuthorisedVia: alice.ID, - }, + wantResponse: alice.ID, }, } @@ -738,16 +734,17 @@ func TestQueryRestrictedJoinAllowed(t *testing.T) { t.Errorf("failed to send events: %v", err) } - req := api.QueryRestrictedJoinAllowedRequest{ - UserID: bob.ID, - RoomID: testRoom.ID, + roomID, _ := spec.NewRoomID(testRoom.ID) + userID, _ := spec.NewUserID(bob.ID, true) + got, err := rsAPI.QueryRestrictedJoinAllowed(processCtx.Context(), *roomID, spec.SenderID(userID.String())) + if tc.wantError && err == nil { + t.Fatal("expected error, got none") } - res := api.QueryRestrictedJoinAllowedResponse{} - if err := rsAPI.QueryRestrictedJoinAllowed(processCtx.Context(), &req, &res); err != nil { + if !tc.wantError && err != nil { t.Fatal(err) } - if !reflect.DeepEqual(tc.wantResponse, res) { - t.Fatalf("unexpected response, want %#v - got %#v", tc.wantResponse, res) + if !reflect.DeepEqual(tc.wantResponse, got) { + t.Fatalf("unexpected response, want %#v - got %#v", tc.wantResponse, got) } }) } @@ -838,17 +835,6 @@ func TestUpgrade(t *testing.T) { validateFunc func(t *testing.T, oldRoomID, newRoomID string, rsAPI api.RoomserverInternalAPI) wantNewRoom bool }{ - { - name: "invalid userID", - upgradeUser: "!notvalid:test", - roomFunc: func(rsAPI api.RoomserverInternalAPI) string { - room := test.NewRoom(t, alice) - if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { - t.Errorf("failed to send events: %v", err) - } - return room.ID - }, - }, { name: "invalid roomID", upgradeUser: alice.ID, @@ -946,14 +932,17 @@ func TestUpgrade(t *testing.T) { upgradeUser: alice.ID, roomFunc: func(rsAPI api.RoomserverInternalAPI) string { r := test.NewRoom(t, alice) + roomID, err := spec.NewRoomID(r.ID) + if err != nil { + t.Fatal(err) + } if err := api.SendEvents(ctx, rsAPI, api.KindNew, r.Events(), "test", "test", "test", nil, false); err != nil { t.Errorf("failed to send events: %v", err) } - if err := rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{ - RoomID: r.ID, - Alias: "#myroomalias:test", - }, &api.SetRoomAliasResponse{}); err != nil { + if _, err := rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), + *roomID, + "#myroomalias:test"); err != nil { t.Fatal(err) } @@ -1052,8 +1041,8 @@ func TestUpgrade(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { @@ -1066,7 +1055,11 @@ func TestUpgrade(t *testing.T) { } roomID := tc.roomFunc(rsAPI) - newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, tc.upgradeUser, version.DefaultRoomVersion()) + userID, err := spec.NewUserID(tc.upgradeUser, true) + if err != nil { + t.Fatalf("upgrade userID is invalid") + } + newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, rsAPI.DefaultRoomVersion()) if err != nil && tc.wantNewRoom { t.Fatal(err) } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index f38d8f96a3..1e776ff6c8 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -24,10 +24,12 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -49,13 +51,15 @@ type StateResolution struct { db StateResolutionStorage roomInfo *types.RoomInfo events map[types.EventNID]gomatrixserverlib.PDU + Querier api.QuerySenderIDAPI } -func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo) StateResolution { +func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo, querier api.QuerySenderIDAPI) StateResolution { return StateResolution{ db: db, roomInfo: roomInfo, events: make(map[types.EventNID]gomatrixserverlib.PDU), + Querier: querier, } } @@ -945,7 +949,9 @@ func (v *StateResolution) resolveConflictsV1( } // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) + resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return v.Querier.QueryUserIDForSender(ctx, roomID, senderID) + }) // Map from the full events back to numeric state entries. for _, resolvedEvent := range resolvedEvents { @@ -1057,6 +1063,9 @@ func (v *StateResolution) resolveConflictsV2( conflictedEvents, nonConflictedEvents, authEvents, + func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return v.Querier.QueryUserIDForSender(ctx, roomID, senderID) + }, ) }() diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 7d22df0084..0638252b2e 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -16,7 +16,9 @@ package storage import ( "context" + "crypto/ed25519" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -27,8 +29,10 @@ import ( ) type Database interface { + UserRoomKeys // Do we support processing input events for more than one room at a time? SupportsConcurrentRoomInputs() bool + AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) // RoomInfo returns room information for the given room ID, or nil if there is no room. RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo, error) RoomInfoByNID(ctx context.Context, roomNID types.RoomNID) (*types.RoomInfo, error) @@ -129,7 +133,7 @@ type Database interface { // in this room, along a boolean set to true if the user is still in this room, // false if not. // Returns an error if there was a problem talking to the database. - GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error) + GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderID spec.SenderID) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error) // Lookup the membership event numeric IDs for all user that are or have // been members of a given room. Only lookup events of "join" membership if // joinOnly is set to true. @@ -154,7 +158,7 @@ type Database interface { GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error) GetStateEventsWithEventType(ctx context.Context, roomID, evType string) ([]*types.HeaderedEvent, error) // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key). - GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) + GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error) // GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match. // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) @@ -183,15 +187,35 @@ type Database interface { ctx context.Context, userNID types.EventStateKeyNID, info *types.RoomInfo, eventIDs ...string, ) (map[string]*types.HeaderedEvent, error) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserverlib.PDU) (*types.RoomInfo, error) + GetRoomVersion(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error) GetOrCreateEventStateKeyNID(ctx context.Context, eventStateKey *string) (types.EventStateKeyNID, error) MaybeRedactEvent( - ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, + ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI, ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) } +type UserRoomKeys interface { + // InsertUserRoomPrivatePublicKey inserts the given private key as well as the public key for it. This should be used + // when creating keys locally. + InsertUserRoomPrivatePublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID, key ed25519.PrivateKey) (result ed25519.PrivateKey, err error) + // InsertUserRoomPublicKey inserts the given public key, this should be used for users NOT local to this server + InsertUserRoomPublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID, key ed25519.PublicKey) (result ed25519.PublicKey, err error) + // SelectUserRoomPrivateKey selects the private key for the given user and room combination + SelectUserRoomPrivateKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (key ed25519.PrivateKey, err error) + // SelectUserRoomPublicKey selects the public key for the given user and room combination + SelectUserRoomPublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (key ed25519.PublicKey, err error) + // SelectUserIDsForPublicKeys selects all userIDs for the requested senderKeys. Returns a map from roomID -> map from publicKey to userID. + // If a senderKey can't be found, it is omitted in the result. + // TODO: Why is the result map indexed by string not public key? + // TODO: Shouldn't the input & result map be changed to be indexed by string instead of the RoomID struct? + SelectUserIDsForPublicKeys(ctx context.Context, publicKeys map[spec.RoomID][]ed25519.PublicKey) (map[spec.RoomID]map[string]string, error) +} + type RoomDatabase interface { EventDatabase + UserRoomKeys + AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) // RoomInfo returns room information for the given room ID, or nil if there is no room. RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo, error) RoomInfoByNID(ctx context.Context, roomNID types.RoomNID) (*types.RoomInfo, error) @@ -228,7 +252,7 @@ type EventDatabase interface { // MaybeRedactEvent returns the redaction event and the redacted event if this call resulted in a redaction, else an error // (nil if there was nothing to do) MaybeRedactEvent( - ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, + ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI, ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) StoreEvent(ctx context.Context, event gomatrixserverlib.PDU, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error) } diff --git a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go index c195777138..1b1dd44d3c 100644 --- a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go +++ b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go @@ -18,19 +18,14 @@ import ( "context" "database/sql" "fmt" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" ) func UpDropEventReferenceSHAEvents(ctx context.Context, tx *sql.Tx) error { - var count int - err := tx.QueryRowContext(ctx, `SELECT count(*) FROM roomserver_events GROUP BY event_id HAVING count(event_id) > 1`). - Scan(&count) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to query duplicate event ids") - } - if count > 0 { - return fmt.Errorf("unable to drop column, as there are duplicate event ids") - } - _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_events DROP COLUMN IF EXISTS reference_sha256;`) + _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_events DROP COLUMN IF EXISTS reference_sha256;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -46,9 +41,80 @@ func UpDropEventReferenceSHAPrevEvents(ctx context.Context, tx *sql.Tx) error { if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } + + // figure out if there are duplicates + dupeRows, err := tx.QueryContext(ctx, `SELECT previous_event_id FROM roomserver_previous_events GROUP BY previous_event_id HAVING count(previous_event_id) > 1`) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeRows, "failed to close rows") + + var prevEvents []string + var prevEventID string + for dupeRows.Next() { + if err = dupeRows.Scan(&prevEventID); err != nil { + return err + } + prevEvents = append(prevEvents, prevEventID) + } + if dupeRows.Err() != nil { + return dupeRows.Err() + } + + // if we found duplicates, check if we can combine them, e.g. they are in the same room + for _, dupeID := range prevEvents { + var dupeNIDsRows *sql.Rows + dupeNIDsRows, err = tx.QueryContext(ctx, `SELECT event_nids FROM roomserver_previous_events WHERE previous_event_id = $1`, dupeID) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeNIDsRows, "failed to close rows") + var dupeNIDs []int64 + for dupeNIDsRows.Next() { + var nids pq.Int64Array + if err = dupeNIDsRows.Scan(&nids); err != nil { + return err + } + dupeNIDs = append(dupeNIDs, nids...) + } + + if dupeNIDsRows.Err() != nil { + return dupeNIDsRows.Err() + } + // dedupe NIDs + dupeNIDs = dupeNIDs[:util.SortAndUnique(nids(dupeNIDs))] + // now that we have all NIDs, check which room they belong to + var roomCount int + err = tx.QueryRowContext(ctx, `SELECT count(distinct room_nid) FROM roomserver_events WHERE event_nid = ANY($1)`, pq.Array(dupeNIDs)).Scan(&roomCount) + if err != nil { + return err + } + // if the events are from different rooms, that's bad and we can't continue + if roomCount > 1 { + return fmt.Errorf("detected events (%v) referenced for different rooms (%v)", dupeNIDs, roomCount) + } + // otherwise delete the dupes + _, err = tx.ExecContext(ctx, "DELETE FROM roomserver_previous_events WHERE previous_event_id = $1", dupeID) + if err != nil { + return fmt.Errorf("unable to delete duplicates: %w", err) + } + + // insert combined values + _, err = tx.ExecContext(ctx, "INSERT INTO roomserver_previous_events (previous_event_id, event_nids) VALUES ($1, $2)", dupeID, pq.Array(dupeNIDs)) + if err != nil { + return fmt.Errorf("unable to insert new event NIDs: %w", err) + } + } + _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_previous_events ADD CONSTRAINT roomserver_previous_event_id_unique UNIQUE (previous_event_id);`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } return nil } + +type nids []int64 + +func (s nids) Len() int { return len(s) } +func (s nids) Less(i, j int) bool { return s[i] < s[j] } +func (s nids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go new file mode 100644 index 0000000000..c79daac5fc --- /dev/null +++ b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go @@ -0,0 +1,60 @@ +package deltas + +import ( + "testing" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/stretchr/testify/assert" +) + +func TestUpDropEventReferenceSHAPrevEvents(t *testing.T) { + + cfg, ctx, close := testrig.CreateConfig(t, test.DBTypePostgres) + defer close() + + db, err := sqlutil.Open(&cfg.Global.DatabaseOptions, sqlutil.NewDummyWriter()) + assert.Nil(t, err) + assert.NotNil(t, db) + defer db.Close() + + // create the table in the old layout + _, err = db.ExecContext(ctx.Context(), ` +CREATE TABLE IF NOT EXISTS roomserver_previous_events ( + previous_event_id TEXT NOT NULL, + event_nids BIGINT[] NOT NULL, + previous_reference_sha256 BYTEA NOT NULL, + CONSTRAINT roomserver_previous_event_id_unique UNIQUE (previous_event_id, previous_reference_sha256) +);`) + assert.Nil(t, err) + + // create the events table as well, slimmed down with one eventNID + _, err = db.ExecContext(ctx.Context(), ` +CREATE SEQUENCE IF NOT EXISTS roomserver_event_nid_seq; +CREATE TABLE IF NOT EXISTS roomserver_events ( + event_nid BIGINT PRIMARY KEY DEFAULT nextval('roomserver_event_nid_seq'), + room_nid BIGINT NOT NULL +); + +INSERT INTO roomserver_events (event_nid, room_nid) VALUES (1, 1) +`) + assert.Nil(t, err) + + // insert duplicate prev events with different event_nids + stmt, err := db.PrepareContext(ctx.Context(), `INSERT INTO roomserver_previous_events (previous_event_id, event_nids, previous_reference_sha256) VALUES ($1, $2, $3)`) + assert.Nil(t, err) + assert.NotNil(t, stmt) + _, err = stmt.ExecContext(ctx.Context(), "1", pq.Array([]int64{1, 2}), "a") + assert.Nil(t, err) + _, err = stmt.ExecContext(ctx.Context(), "1", pq.Array([]int64{1, 2, 3}), "b") + assert.Nil(t, err) + // execute the migration + txn, err := db.Begin() + assert.Nil(t, err) + assert.NotNil(t, txn) + defer txn.Rollback() + err = UpDropEventReferenceSHAPrevEvents(ctx.Context(), txn) + assert.NoError(t, err) +} diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 19cde54105..c5c206cfb6 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -37,7 +37,7 @@ type Database struct { } // Open a postgres database. -func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { +func Open(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { var d Database var err error db, writer, err := conMan.Connection(dbProperties) @@ -131,6 +131,9 @@ func (d *Database) create(db *sql.DB) error { if err := CreateRedactionsTable(db); err != nil { return err } + if err := CreateUserRoomKeysTable(db); err != nil { + return err + } return nil } @@ -192,6 +195,11 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } + userRoomKeys, err := PrepareUserRoomKeysTable(db) + if err != nil { + return err + } + d.Database = shared.Database{ DB: db, EventDatabase: shared.EventDatabase{ @@ -215,6 +223,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room MembershipTable: membership, PublishedTable: published, Purge: purge, + UserRoomKeyTable: userRoomKeys, } return nil } diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go new file mode 100644 index 0000000000..217ee957f4 --- /dev/null +++ b/roomserver/storage/postgres/user_room_keys_table.go @@ -0,0 +1,177 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "crypto/ed25519" + "database/sql" + "errors" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib/spec" +) + +const userRoomKeysSchema = ` +CREATE TABLE IF NOT EXISTS roomserver_user_room_keys ( + user_nid INTEGER NOT NULL, + room_nid INTEGER NOT NULL, + pseudo_id_key BYTEA NULL, -- may be null for users not local to the server + pseudo_id_pub_key BYTEA NOT NULL, + CONSTRAINT roomserver_user_room_keys_pk PRIMARY KEY (user_nid, room_nid) +); +` + +const insertUserRoomPrivateKeySQL = ` + INSERT INTO roomserver_user_room_keys (user_nid, room_nid, pseudo_id_key, pseudo_id_pub_key) VALUES ($1, $2, $3, $4) + ON CONFLICT ON CONSTRAINT roomserver_user_room_keys_pk DO UPDATE SET pseudo_id_key = roomserver_user_room_keys.pseudo_id_key + RETURNING (pseudo_id_key) +` + +const insertUserRoomPublicKeySQL = ` + INSERT INTO roomserver_user_room_keys (user_nid, room_nid, pseudo_id_pub_key) VALUES ($1, $2, $3) + ON CONFLICT ON CONSTRAINT roomserver_user_room_keys_pk DO UPDATE SET pseudo_id_pub_key = $3 + RETURNING (pseudo_id_pub_key) +` + +const selectUserRoomKeySQL = `SELECT pseudo_id_key FROM roomserver_user_room_keys WHERE user_nid = $1 AND room_nid = $2` + +const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1 AND room_nid = $2` + +const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid = ANY($1) AND pseudo_id_pub_key = ANY($2)` + +const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1` + +type userRoomKeysStatements struct { + insertUserRoomPrivateKeyStmt *sql.Stmt + insertUserRoomPublicKeyStmt *sql.Stmt + selectUserRoomKeyStmt *sql.Stmt + selectUserRoomPublicKeyStmt *sql.Stmt + selectUserNIDsStmt *sql.Stmt + selectAllUserRoomPublicKeysForUser *sql.Stmt +} + +func CreateUserRoomKeysTable(db *sql.DB) error { + _, err := db.Exec(userRoomKeysSchema) + return err +} + +func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) { + s := &userRoomKeysStatements{} + return s, sqlutil.StatementList{ + {&s.insertUserRoomPrivateKeyStmt, insertUserRoomPrivateKeySQL}, + {&s.insertUserRoomPublicKeyStmt, insertUserRoomPublicKeySQL}, + {&s.selectUserRoomKeyStmt, selectUserRoomKeySQL}, + {&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL}, + {&s.selectUserNIDsStmt, selectUserNIDsSQL}, + {&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL}, + }.Prepare(db) +} + +func (s *userRoomKeysStatements) InsertUserRoomPrivatePublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PrivateKey) (result ed25519.PrivateKey, err error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.insertUserRoomPrivateKeyStmt) + err = stmt.QueryRowContext(ctx, userNID, roomNID, key, key.Public()).Scan(&result) + return result, err +} + +func (s *userRoomKeysStatements) InsertUserRoomPublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PublicKey) (result ed25519.PublicKey, err error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.insertUserRoomPublicKeyStmt) + err = stmt.QueryRowContext(ctx, userNID, roomNID, key).Scan(&result) + return result, err +} + +func (s *userRoomKeysStatements) SelectUserRoomPrivateKey( + ctx context.Context, + txn *sql.Tx, + userNID types.EventStateKeyNID, + roomNID types.RoomNID, +) (ed25519.PrivateKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectUserRoomKeyStmt) + var result ed25519.PrivateKey + err := stmt.QueryRowContext(ctx, userNID, roomNID).Scan(&result) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return result, err +} + +func (s *userRoomKeysStatements) SelectUserRoomPublicKey( + ctx context.Context, + txn *sql.Tx, + userNID types.EventStateKeyNID, + roomNID types.RoomNID, +) (ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectUserRoomPublicKeyStmt) + var result ed25519.PublicKey + err := stmt.QueryRowContext(ctx, userNID, roomNID).Scan(&result) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return result, err +} + +func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sql.Tx, senderKeys map[types.RoomNID][]ed25519.PublicKey) (map[string]types.UserRoomKeyPair, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectUserNIDsStmt) + + roomNIDs := make([]types.RoomNID, 0, len(senderKeys)) + var senders [][]byte + for roomNID := range senderKeys { + roomNIDs = append(roomNIDs, roomNID) + for _, key := range senderKeys[roomNID] { + senders = append(senders, key) + } + } + rows, err := stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(senders)) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "failed to close rows") + + result := make(map[string]types.UserRoomKeyPair, len(senders)+len(roomNIDs)) + var publicKey []byte + userRoomKeyPair := types.UserRoomKeyPair{} + for rows.Next() { + if err = rows.Scan(&userRoomKeyPair.EventStateKeyNID, &userRoomKeyPair.RoomNID, &publicKey); err != nil { + return nil, err + } + result[spec.Base64Bytes(publicKey).Encode()] = userRoomKeyPair + } + return result, rows.Err() +} + +func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser) + + rows, err := stmt.QueryContext(ctx, userNID) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + resultMap := make(map[types.RoomNID]ed25519.PublicKey) + + var roomNID types.RoomNID + var pubkey ed25519.PublicKey + for rows.Next() { + if err = rows.Scan(&roomNID, &pubkey); err != nil { + return nil, err + } + resultMap[roomNID] = pubkey + } + return resultMap, err +} diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go index f9c889cb15..a96e870721 100644 --- a/roomserver/storage/shared/membership_updater.go +++ b/roomserver/storage/shared/membership_updater.go @@ -101,7 +101,7 @@ func (u *MembershipUpdater) Update(newMembership tables.MembershipState, event * var inserted bool // Did the query result in a membership change? var retired []string // Did we retire any updates in the process? return inserted, retired, u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender()) + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, string(event.SenderID())) if err != nil { return fmt.Errorf("u.d.AssignStateKeyNID: %w", err) } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index cefa58a3d0..3331c60296 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -2,14 +2,19 @@ package shared import ( "context" + "crypto/ed25519" "database/sql" "encoding/json" + "errors" "fmt" "sort" + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/caching" @@ -41,6 +46,7 @@ type Database struct { MembershipTable tables.Membership PublishedTable tables.Published Purge tables.Purge + UserRoomKeyTable tables.UserRoomKeys GetRoomUpdaterFn func(ctx context.Context, roomInfo *types.RoomInfo) (*RoomUpdater, error) } @@ -276,17 +282,17 @@ func (d *Database) addState( var found bool for i := len(state) - 1; i >= 0; i-- { found = false + blocksLoop: for _, events := range blocks { for _, event := range events { if state[i].EventNID == event { found = true - break + break blocksLoop } } } if found { state = append(state[:i], state[i+1:]...) - i-- } } } @@ -485,10 +491,10 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { }) } -func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) { +func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderID spec.SenderID) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) { var requestSenderUserNID types.EventStateKeyNID err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID) + requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, string(requestSenderID)) return err }) if err != nil { @@ -657,6 +663,26 @@ func (d *Database) IsEventRejected(ctx context.Context, roomNID types.RoomNID, e return d.EventsTable.SelectEventRejected(ctx, nil, roomNID, eventID) } +func (d *Database) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) { + // This should already be checked, let's check it anyway. + _, err = gomatrixserverlib.GetRoomVersion(roomVersion) + if err != nil { + return 0, err + } + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + roomNID, err = d.assignRoomNID(ctx, txn, roomID.String(), roomVersion) + if err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + // Not setting caches, as assignRoomNID already does this + return roomNID, err +} + // GetOrCreateRoomInfo gets or creates a new RoomInfo, which is only safe to use with functions only needing a roomVersion or roomNID. func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserverlib.PDU) (roomInfo *types.RoomInfo, err error) { // Get the default room version. If the client doesn't supply a room_version @@ -670,8 +696,8 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver return nil, fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err) } - roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID()) - cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID()) + roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID().String()) + cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID().String()) // if we found both, the roomNID and version in our cache, no need to query the database if nidOK && versionOK { return &types.RoomInfo{ @@ -681,14 +707,14 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver } err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion) + roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID().String(), roomVersion) if err != nil { return err } return nil }) if roomVersion != "" { - d.Cache.StoreRoomVersion(event.RoomID(), roomVersion) + d.Cache.StoreRoomVersion(event.RoomID().String(), roomVersion) } return &types.RoomInfo{ RoomVersion: roomVersion, @@ -696,6 +722,22 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver }, err } +func (d *Database) GetRoomVersion(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(roomID) + if versionOK { + return cachedRoomVersion, nil + } + + roomInfo, err := d.RoomInfo(ctx, roomID) + if err != nil { + return "", err + } + if roomInfo == nil { + return "", nil + } + return roomInfo.RoomVersion, nil +} + func (d *Database) GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { if eventTypeNID, err = d.assignEventTypeNID(ctx, txn, eventType); err != nil { @@ -931,6 +973,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) ( return roomVersion, err } +// nolint:gocyclo // MaybeRedactEvent manages the redacted status of events. There's two cases to consider in order to comply with the spec: // "servers should not apply or send redactions to clients until both the redaction event and original event have been seen, and are valid." // https://matrix.org/docs/spec/rooms/v3#authorization-rules-for-events @@ -949,6 +992,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) ( // Returns the redaction event and the redacted event if this call resulted in a redaction. func (d *EventDatabase) MaybeRedactEvent( ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, + querier api.QuerySenderIDAPI, ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) { var ( redactionEvent, redactedEvent *types.Event @@ -982,14 +1026,22 @@ func (d *EventDatabase) MaybeRedactEvent( case validated || redactedEvent == nil || redactionEvent == nil: // we've seen this redaction before or there is nothing to redact return nil - case redactedEvent.RoomID() != redactionEvent.RoomID(): + case redactedEvent.RoomID().String() != redactionEvent.RoomID().String(): // redactions across rooms aren't allowed ignoreRedaction = true return nil } - _, sender1, _ := gomatrixserverlib.SplitID('@', redactedEvent.Sender()) - _, sender2, _ := gomatrixserverlib.SplitID('@', redactionEvent.Sender()) + sender1Domain := "" + sender1, err1 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactedEvent.SenderID()) + if err1 == nil { + sender1Domain = string(sender1.Domain()) + } + sender2Domain := "" + sender2, err2 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactionEvent.SenderID()) + if err2 == nil { + sender2Domain = string(sender2.Domain()) + } var powerlevels *gomatrixserverlib.PowerLevelContent powerlevels, err = plResolver.Resolve(ctx, redactionEvent.EventID()) if err != nil { @@ -997,9 +1049,9 @@ func (d *EventDatabase) MaybeRedactEvent( } switch { - case powerlevels.UserLevel(redactionEvent.Sender()) >= powerlevels.Redact: + case powerlevels.UserLevel(redactionEvent.SenderID()) >= powerlevels.Redact: // 1. The power level of the redaction event’s sender is greater than or equal to the redact level. - case sender1 == sender2: + case sender1Domain != "" && sender2Domain != "" && sender1Domain == sender2Domain: // 2. The domain of the redaction event’s sender matches that of the original event’s sender. default: ignoreRedaction = true @@ -1290,7 +1342,7 @@ func (d *Database) GetStateEventsWithEventType(ctx context.Context, roomID, evTy } // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key). -func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) { +func (d *Database) GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error) { var membershipState tables.MembershipState switch membership { case "join": @@ -1304,17 +1356,73 @@ func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership default: return nil, fmt.Errorf("GetRoomsByMembership: invalid membership %s", membership) } - stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID) + + // Convert provided user ID to NID + userNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID.String()) if err != nil { if err == sql.ErrNoRows { return nil, nil + } else { + return nil, fmt.Errorf("SelectEventStateKeyNID: cannot map user ID to state key NIDs: %w", err) } - return nil, fmt.Errorf("GetRoomsByMembership: cannot map user ID to state key NID: %w", err) } - roomNIDs, err := d.MembershipTable.SelectRoomsWithMembership(ctx, nil, stateKeyNID, membershipState) + + // Use this NID to fetch all associated room keys (for pseudo ID rooms) + roomKeyMap, err := d.UserRoomKeyTable.SelectAllPublicKeysForUser(ctx, nil, userNID) if err != nil { - return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err) + if err == sql.ErrNoRows { + roomKeyMap = map[types.RoomNID]ed25519.PublicKey{} + } else { + return nil, fmt.Errorf("SelectAllPublicKeysForUser: could not select user room public keys for user: %w", err) + } + } + + var eventStateKeyNIDs []types.EventStateKeyNID + + // If there are room keys (i.e. this user is in pseudo ID rooms), then gather the appropriate NIDs + if len(roomKeyMap) != 0 { + // Convert keys to string representation + userRoomKeys := make([]string, len(roomKeyMap)) + i := 0 + for _, key := range roomKeyMap { + userRoomKeys[i] = spec.Base64Bytes(key).Encode() + i += 1 + } + + // Convert the string representation to its NID + pseudoIDStateKeys, sqlErr := d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, nil, userRoomKeys) + if sqlErr != nil { + if sqlErr == sql.ErrNoRows { + pseudoIDStateKeys = map[string]types.EventStateKeyNID{} + } else { + return nil, fmt.Errorf("BulkSelectEventStateKeyNID: could not select state keys for public room keys: %w", err) + } + } + + // Collect all NIDs together + eventStateKeyNIDs = make([]types.EventStateKeyNID, len(pseudoIDStateKeys)+1) + eventStateKeyNIDs[0] = userNID + i = 1 + for _, nid := range pseudoIDStateKeys { + eventStateKeyNIDs[i] = nid + i += 1 + } + } else { + // If there are no room keys (so no pseudo ID rooms), we only need to care about the user ID NID. + eventStateKeyNIDs = []types.EventStateKeyNID{userNID} } + + // Fetch rooms that match membership for each NID + roomNIDs := []types.RoomNID{} + for _, nid := range eventStateKeyNIDs { + var roomNIDsChunk []types.RoomNID + roomNIDsChunk, err = d.MembershipTable.SelectRoomsWithMembership(ctx, nil, nid, membershipState) + if err != nil { + return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err) + } + roomNIDs = append(roomNIDs, roomNIDsChunk...) + } + roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, roomNIDs) if err != nil { return nil, fmt.Errorf("GetRoomsByMembership: failed to lookup room nids: %w", err) @@ -1409,7 +1517,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } result[i] = tables.StrippedEvent{ EventType: ev.Type(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StateKey: *ev.StateKey(), ContentValue: tables.ExtractContentValue(&types.HeaderedEvent{PDU: ev}), } @@ -1589,6 +1697,173 @@ func (d *Database) UpgradeRoom(ctx context.Context, oldRoomID, newRoomID, eventS }) } +// InsertUserRoomPrivatePublicKey inserts a new user room key for the given user and room. +// Returns the newly inserted private key or an existing private key. If there is +// an error talking to the database, returns that error. +func (d *Database) InsertUserRoomPrivatePublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID, key ed25519.PrivateKey) (result ed25519.PrivateKey, err error) { + uID := userID.String() + stateKeyNIDMap, sErr := d.eventStateKeyNIDs(ctx, nil, []string{uID}) + if sErr != nil { + return nil, sErr + } + stateKeyNID := stateKeyNIDMap[uID] + + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + roomInfo, rErr := d.roomInfo(ctx, txn, roomID.String()) + if rErr != nil { + return rErr + } + if roomInfo == nil { + return eventutil.ErrRoomNoExists{} + } + + var iErr error + result, iErr = d.UserRoomKeyTable.InsertUserRoomPrivatePublicKey(ctx, txn, stateKeyNID, roomInfo.RoomNID, key) + return iErr + }) + return result, err +} + +// InsertUserRoomPublicKey inserts a new user room key for the given user and room. +// Returns the newly inserted public key or an existing public key. If there is +// an error talking to the database, returns that error. +func (d *Database) InsertUserRoomPublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID, key ed25519.PublicKey) (result ed25519.PublicKey, err error) { + uID := userID.String() + stateKeyNIDMap, sErr := d.eventStateKeyNIDs(ctx, nil, []string{uID}) + if sErr != nil { + return nil, sErr + } + stateKeyNID := stateKeyNIDMap[uID] + + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + roomInfo, rErr := d.roomInfo(ctx, txn, roomID.String()) + if rErr != nil { + return rErr + } + if roomInfo == nil { + return eventutil.ErrRoomNoExists{} + } + + var iErr error + result, iErr = d.UserRoomKeyTable.InsertUserRoomPublicKey(ctx, txn, stateKeyNID, roomInfo.RoomNID, key) + return iErr + }) + return result, err +} + +// SelectUserRoomPrivateKey queries the users room private key. +// If no key exists, returns no key and no error. Otherwise returns +// the key and a database error, if any. +// TODO: Cache this? +func (d *Database) SelectUserRoomPrivateKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (key ed25519.PrivateKey, err error) { + uID := userID.String() + stateKeyNIDMap, sErr := d.eventStateKeyNIDs(ctx, nil, []string{uID}) + if sErr != nil { + return nil, sErr + } + stateKeyNID := stateKeyNIDMap[uID] + + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + roomInfo, rErr := d.roomInfo(ctx, txn, roomID.String()) + if rErr != nil { + return rErr + } + if roomInfo == nil { + return eventutil.ErrRoomNoExists{} + } + + key, sErr = d.UserRoomKeyTable.SelectUserRoomPrivateKey(ctx, txn, stateKeyNID, roomInfo.RoomNID) + if !errors.Is(sErr, sql.ErrNoRows) { + return sErr + } + return nil + }) + return +} + +// SelectUserRoomPublicKey queries the users room public key. +// If no key exists, returns no key and no error. Otherwise returns +// the key and a database error, if any. +func (d *Database) SelectUserRoomPublicKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (key ed25519.PublicKey, err error) { + uID := userID.String() + stateKeyNIDMap, sErr := d.eventStateKeyNIDs(ctx, nil, []string{uID}) + if sErr != nil { + return nil, sErr + } + stateKeyNID := stateKeyNIDMap[uID] + + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + roomInfo, rErr := d.roomInfo(ctx, txn, roomID.String()) + if rErr != nil { + return rErr + } + if roomInfo == nil { + return nil + } + + key, sErr = d.UserRoomKeyTable.SelectUserRoomPublicKey(ctx, txn, stateKeyNID, roomInfo.RoomNID) + if !errors.Is(sErr, sql.ErrNoRows) { + return sErr + } + return nil + }) + return +} + +// SelectUserIDsForPublicKeys returns a map from roomID -> map from senderKey -> userID +func (d *Database) SelectUserIDsForPublicKeys(ctx context.Context, publicKeys map[spec.RoomID][]ed25519.PublicKey) (result map[spec.RoomID]map[string]string, err error) { + result = make(map[spec.RoomID]map[string]string, len(publicKeys)) + + // map all roomIDs to roomNIDs + query := make(map[types.RoomNID][]ed25519.PublicKey) + rooms := make(map[types.RoomNID]spec.RoomID) + for roomID, keys := range publicKeys { + roomNID, ok := d.Cache.GetRoomServerRoomNID(roomID.String()) + if !ok { + roomInfo, rErr := d.roomInfo(ctx, nil, roomID.String()) + if rErr != nil { + return nil, rErr + } + if roomInfo == nil { + logrus.Warnf("missing room info for %s, there will be missing users in the response", roomID.String()) + continue + } + roomNID = roomInfo.RoomNID + } + + query[roomNID] = keys + rooms[roomNID] = roomID + } + + // get the user room key pars + userRoomKeyPairMap, sErr := d.UserRoomKeyTable.BulkSelectUserNIDs(ctx, nil, query) + if sErr != nil { + return nil, sErr + } + nids := make([]types.EventStateKeyNID, 0, len(userRoomKeyPairMap)) + for _, nid := range userRoomKeyPairMap { + nids = append(nids, nid.EventStateKeyNID) + } + // get the userIDs + nidMap, seErr := d.EventStateKeys(ctx, nids) + if seErr != nil { + return nil, seErr + } + + // build the result map (roomID -> map publicKey -> userID) + for publicKey, userRoomKeyPair := range userRoomKeyPairMap { + userID := nidMap[userRoomKeyPair.EventStateKeyNID] + roomID := rooms[userRoomKeyPair.RoomNID] + resMap, exists := result[roomID] + if !exists { + resMap = map[string]string{} + } + resMap[publicKey] = userID + result[roomID] = resMap + } + return result, err +} + // FIXME TODO: Remove all this - horrible dupe with roomserver/state. Can't use the original impl because of circular loops // it should live in this package! diff --git a/roomserver/storage/shared/storage_test.go b/roomserver/storage/shared/storage_test.go index 941e848021..612e4ef069 100644 --- a/roomserver/storage/shared/storage_test.go +++ b/roomserver/storage/shared/storage_test.go @@ -2,11 +2,16 @@ package shared_test import ( "context" + "crypto/ed25519" "testing" "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" + ed255192 "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" @@ -23,41 +28,62 @@ func mustCreateRoomserverDatabase(t *testing.T, dbType test.DBType) (*shared.Dat connStr, clearDB := test.PrepareDBConnectionString(t, dbType) dbOpts := &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)} - db, err := sqlutil.Open(dbOpts, sqlutil.NewExclusiveWriter()) + writer := sqlutil.NewExclusiveWriter() + db, err := sqlutil.Open(dbOpts, writer) assert.NoError(t, err) var membershipTable tables.Membership var stateKeyTable tables.EventStateKeys + var userRoomKeys tables.UserRoomKeys + var roomsTable tables.Rooms switch dbType { case test.DBTypePostgres: + err = postgres.CreateRoomsTable(db) + assert.NoError(t, err) err = postgres.CreateEventStateKeysTable(db) assert.NoError(t, err) err = postgres.CreateMembershipTable(db) assert.NoError(t, err) + err = postgres.CreateUserRoomKeysTable(db) + assert.NoError(t, err) + roomsTable, err = postgres.PrepareRoomsTable(db) + assert.NoError(t, err) membershipTable, err = postgres.PrepareMembershipTable(db) assert.NoError(t, err) stateKeyTable, err = postgres.PrepareEventStateKeysTable(db) + assert.NoError(t, err) + userRoomKeys, err = postgres.PrepareUserRoomKeysTable(db) case test.DBTypeSQLite: + err = sqlite3.CreateRoomsTable(db) + assert.NoError(t, err) err = sqlite3.CreateEventStateKeysTable(db) assert.NoError(t, err) err = sqlite3.CreateMembershipTable(db) assert.NoError(t, err) + err = sqlite3.CreateUserRoomKeysTable(db) + assert.NoError(t, err) + roomsTable, err = sqlite3.PrepareRoomsTable(db) + assert.NoError(t, err) membershipTable, err = sqlite3.PrepareMembershipTable(db) assert.NoError(t, err) stateKeyTable, err = sqlite3.PrepareEventStateKeysTable(db) + assert.NoError(t, err) + userRoomKeys, err = sqlite3.PrepareUserRoomKeysTable(db) } assert.NoError(t, err) cache := caching.NewRistrettoCache(8*1024*1024, time.Hour, false) - evDb := shared.EventDatabase{EventStateKeysTable: stateKeyTable, Cache: cache} + evDb := shared.EventDatabase{EventStateKeysTable: stateKeyTable, Cache: cache, Writer: writer} return &shared.Database{ - DB: db, - EventDatabase: evDb, - MembershipTable: membershipTable, - Writer: sqlutil.NewExclusiveWriter(), - Cache: cache, + DB: db, + EventDatabase: evDb, + MembershipTable: membershipTable, + UserRoomKeyTable: userRoomKeys, + RoomsTable: roomsTable, + Writer: writer, + Cache: cache, }, func() { clearDB() err = db.Close() @@ -97,3 +123,106 @@ func Test_GetLeftUsers(t *testing.T) { assert.ElementsMatch(t, expectedUserIDs, leftUsers) }) } + +func TestUserRoomKeys(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + userID, err := spec.NewUserID(alice.ID, true) + assert.NoError(t, err) + roomID, err := spec.NewRoomID(room.ID) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateRoomserverDatabase(t, dbType) + defer close() + + // create a room NID so we can query the room + _, err = db.RoomsTable.InsertRoomNID(ctx, nil, roomID.String(), gomatrixserverlib.RoomVersionV10) + assert.NoError(t, err) + doesNotExist, err := spec.NewRoomID("!doesnotexist:localhost") + assert.NoError(t, err) + _, err = db.RoomsTable.InsertRoomNID(ctx, nil, doesNotExist.String(), gomatrixserverlib.RoomVersionV10) + assert.NoError(t, err) + + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) + + gotKey, err := db.InsertUserRoomPrivatePublicKey(ctx, *userID, *roomID, key) + assert.NoError(t, err) + assert.Equal(t, gotKey, key) + + // again, this shouldn't result in an error, but return the existing key + _, key2, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) + gotKey, err = db.InsertUserRoomPrivatePublicKey(ctx, *userID, *roomID, key2) + assert.NoError(t, err) + assert.Equal(t, gotKey, key) + + gotKey, err = db.SelectUserRoomPrivateKey(context.Background(), *userID, *roomID) + assert.NoError(t, err) + assert.Equal(t, key, gotKey) + pubKey, err := db.SelectUserRoomPublicKey(context.Background(), *userID, *roomID) + assert.NoError(t, err) + assert.Equal(t, key.Public(), pubKey) + + // Key doesn't exist, we shouldn't get anything back + gotKey, err = db.SelectUserRoomPrivateKey(context.Background(), *userID, *doesNotExist) + assert.NoError(t, err) + assert.Nil(t, gotKey) + pubKey, err = db.SelectUserRoomPublicKey(context.Background(), *userID, *doesNotExist) + assert.NoError(t, err) + assert.Nil(t, pubKey) + + queryUserIDs := map[spec.RoomID][]ed25519.PublicKey{ + *roomID: {key.Public().(ed25519.PublicKey)}, + } + + userIDs, err := db.SelectUserIDsForPublicKeys(ctx, queryUserIDs) + assert.NoError(t, err) + wantKeys := map[spec.RoomID]map[string]string{ + *roomID: { + spec.Base64Bytes(key.Public().(ed25519.PublicKey)).Encode(): userID.String(), + }, + } + assert.Equal(t, wantKeys, userIDs) + + // insert key that came in over federation + var gotPublicKey, key4 ed255192.PublicKey + key4, _, err = ed25519.GenerateKey(nil) + assert.NoError(t, err) + gotPublicKey, err = db.InsertUserRoomPublicKey(context.Background(), *userID, *doesNotExist, key4) + assert.NoError(t, err) + assert.Equal(t, key4, gotPublicKey) + + // test invalid room + reallyDoesNotExist, err := spec.NewRoomID("!reallydoesnotexist:localhost") + assert.NoError(t, err) + _, err = db.InsertUserRoomPublicKey(context.Background(), *userID, *reallyDoesNotExist, key4) + assert.Error(t, err) + _, err = db.InsertUserRoomPrivatePublicKey(context.Background(), *userID, *reallyDoesNotExist, key) + assert.Error(t, err) + }) +} + +func TestAssignRoomNID(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + roomID, err := spec.NewRoomID(room.ID) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateRoomserverDatabase(t, dbType) + defer close() + + nid, err := db.AssignRoomNID(ctx, *roomID, room.Version) + assert.NoError(t, err) + assert.Greater(t, nid, types.EventNID(0)) + + _, err = db.AssignRoomNID(ctx, spec.RoomID{}, "notaroomversion") + assert.Error(t, err) + }) +} diff --git a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go index 452d72ace1..515bccc374 100644 --- a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go +++ b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go @@ -18,6 +18,10 @@ import ( "context" "database/sql" "fmt" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" ) func UpDropEventReferenceSHA(ctx context.Context, tx *sql.Tx) error { @@ -52,8 +56,72 @@ func UpDropEventReferenceSHAPrevEvents(ctx context.Context, tx *sql.Tx) error { return fmt.Errorf("tx.ExecContext: %w", err) } + // figure out if there are duplicates + dupeRows, err := tx.QueryContext(ctx, `SELECT previous_event_id FROM _roomserver_previous_events GROUP BY previous_event_id HAVING count(previous_event_id) > 1`) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeRows, "failed to close rows") + + var prevEvents []string + var prevEventID string + for dupeRows.Next() { + if err = dupeRows.Scan(&prevEventID); err != nil { + return err + } + prevEvents = append(prevEvents, prevEventID) + } + if dupeRows.Err() != nil { + return dupeRows.Err() + } + + // if we found duplicates, check if we can combine them, e.g. they are in the same room + for _, dupeID := range prevEvents { + var dupeNIDsRows *sql.Rows + dupeNIDsRows, err = tx.QueryContext(ctx, `SELECT event_nids FROM _roomserver_previous_events WHERE previous_event_id = $1`, dupeID) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeNIDsRows, "failed to close rows") + var dupeNIDs []int64 + for dupeNIDsRows.Next() { + var nids pq.Int64Array + if err = dupeNIDsRows.Scan(&nids); err != nil { + return err + } + dupeNIDs = append(dupeNIDs, nids...) + } + + if dupeNIDsRows.Err() != nil { + return dupeNIDsRows.Err() + } + // dedupe NIDs + dupeNIDs = dupeNIDs[:util.SortAndUnique(nids(dupeNIDs))] + // now that we have all NIDs, check which room they belong to + var roomCount int + err = tx.QueryRowContext(ctx, `SELECT count(distinct room_nid) FROM roomserver_events WHERE event_nid IN ($1)`, pq.Array(dupeNIDs)).Scan(&roomCount) + if err != nil { + return err + } + // if the events are from different rooms, that's bad and we can't continue + if roomCount > 1 { + return fmt.Errorf("detected events (%v) referenced for different rooms (%v)", dupeNIDs, roomCount) + } + // otherwise delete the dupes + _, err = tx.ExecContext(ctx, "DELETE FROM _roomserver_previous_events WHERE previous_event_id = $1", dupeID) + if err != nil { + return fmt.Errorf("unable to delete duplicates: %w", err) + } + + // insert combined values + _, err = tx.ExecContext(ctx, "INSERT INTO _roomserver_previous_events (previous_event_id, event_nids) VALUES ($1, $2)", dupeID, pq.Array(dupeNIDs)) + if err != nil { + return fmt.Errorf("unable to insert new event NIDs: %w", err) + } + } + // move data - if _, err := tx.ExecContext(ctx, ` + if _, err = tx.ExecContext(ctx, ` INSERT INTO roomserver_previous_events ( previous_event_id, event_nids @@ -64,9 +132,15 @@ INSERT return fmt.Errorf("tx.ExecContext: %w", err) } // drop old table - _, err := tx.ExecContext(ctx, `DROP TABLE _roomserver_previous_events;`) + _, err = tx.ExecContext(ctx, `DROP TABLE _roomserver_previous_events;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } return nil } + +type nids []int64 + +func (s nids) Len() int { return len(s) } +func (s nids) Less(i, j int) bool { return s[i] < s[j] } +func (s nids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go new file mode 100644 index 0000000000..547d9703be --- /dev/null +++ b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go @@ -0,0 +1,59 @@ +package deltas + +import ( + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/stretchr/testify/assert" +) + +func TestUpDropEventReferenceSHAPrevEvents(t *testing.T) { + + cfg, ctx, close := testrig.CreateConfig(t, test.DBTypeSQLite) + defer close() + + db, err := sqlutil.Open(&cfg.RoomServer.Database, sqlutil.NewExclusiveWriter()) + assert.Nil(t, err) + assert.NotNil(t, db) + defer db.Close() + + // create the table in the old layout + _, err = db.ExecContext(ctx.Context(), ` + CREATE TABLE IF NOT EXISTS roomserver_previous_events ( + previous_event_id TEXT NOT NULL, + previous_reference_sha256 BLOB, + event_nids TEXT NOT NULL, + UNIQUE (previous_event_id, previous_reference_sha256) + );`) + assert.Nil(t, err) + + // create the events table as well, slimmed down with one eventNID + _, err = db.ExecContext(ctx.Context(), ` + CREATE TABLE IF NOT EXISTS roomserver_events ( + event_nid INTEGER PRIMARY KEY AUTOINCREMENT, + room_nid INTEGER NOT NULL +); + +INSERT INTO roomserver_events (event_nid, room_nid) VALUES (1, 1) +`) + assert.Nil(t, err) + + // insert duplicate prev events with different event_nids + stmt, err := db.PrepareContext(ctx.Context(), `INSERT INTO roomserver_previous_events (previous_event_id, event_nids, previous_reference_sha256) VALUES ($1, $2, $3)`) + assert.Nil(t, err) + assert.NotNil(t, stmt) + _, err = stmt.ExecContext(ctx.Context(), "1", "{1,2}", "a") + assert.Nil(t, err) + _, err = stmt.ExecContext(ctx.Context(), "1", "{1,2,3}", "b") + assert.Nil(t, err) + + // execute the migration + txn, err := db.Begin() + assert.Nil(t, err) + assert.NotNil(t, txn) + err = UpDropEventReferenceSHAPrevEvents(ctx.Context(), txn) + defer txn.Rollback() + assert.NoError(t, err) +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 6ab427a84b..98d88f923f 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -36,7 +36,7 @@ type Database struct { } // Open a sqlite database. -func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { +func Open(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { var d Database var err error db, writer, err := conMan.Connection(dbProperties) @@ -138,6 +138,9 @@ func (d *Database) create(db *sql.DB) error { if err := CreateRedactionsTable(db); err != nil { return err } + if err := CreateUserRoomKeysTable(db); err != nil { + return err + } return nil } @@ -199,6 +202,10 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } + userRoomKeys, err := PrepareUserRoomKeysTable(db) + if err != nil { + return err + } d.Database = shared.Database{ DB: db, @@ -224,6 +231,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room PublishedTable: published, GetRoomUpdaterFn: d.GetRoomUpdater, Purge: purge, + UserRoomKeyTable: userRoomKeys, } return nil } diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go new file mode 100644 index 0000000000..434bad2959 --- /dev/null +++ b/roomserver/storage/sqlite3/user_room_keys_table.go @@ -0,0 +1,192 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "crypto/ed25519" + "database/sql" + "errors" + "strings" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib/spec" +) + +const userRoomKeysSchema = ` +CREATE TABLE IF NOT EXISTS roomserver_user_room_keys ( + user_nid INTEGER NOT NULL, + room_nid INTEGER NOT NULL, + pseudo_id_key TEXT NULL, -- may be null for users not local to the server + pseudo_id_pub_key TEXT NOT NULL, + CONSTRAINT roomserver_user_room_keys_pk PRIMARY KEY (user_nid, room_nid) +); +` + +const insertUserRoomKeySQL = ` + INSERT INTO roomserver_user_room_keys (user_nid, room_nid, pseudo_id_key, pseudo_id_pub_key) VALUES ($1, $2, $3, $4) + ON CONFLICT DO UPDATE SET pseudo_id_key = roomserver_user_room_keys.pseudo_id_key + RETURNING (pseudo_id_key) +` + +const insertUserRoomPublicKeySQL = ` + INSERT INTO roomserver_user_room_keys (user_nid, room_nid, pseudo_id_pub_key) VALUES ($1, $2, $3) + ON CONFLICT DO UPDATE SET pseudo_id_pub_key = $3 + RETURNING (pseudo_id_pub_key) +` + +const selectUserRoomKeySQL = `SELECT pseudo_id_key FROM roomserver_user_room_keys WHERE user_nid = $1 AND room_nid = $2` + +const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1 AND room_nid = $2` + +const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid IN ($1) AND pseudo_id_pub_key IN ($2)` + +const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1` + +type userRoomKeysStatements struct { + db *sql.DB + insertUserRoomPrivateKeyStmt *sql.Stmt + insertUserRoomPublicKeyStmt *sql.Stmt + selectUserRoomKeyStmt *sql.Stmt + selectUserRoomPublicKeyStmt *sql.Stmt + selectAllUserRoomPublicKeysForUser *sql.Stmt + //selectUserNIDsStmt *sql.Stmt //prepared at runtime +} + +func CreateUserRoomKeysTable(db *sql.DB) error { + _, err := db.Exec(userRoomKeysSchema) + return err +} + +func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) { + s := &userRoomKeysStatements{db: db} + return s, sqlutil.StatementList{ + {&s.insertUserRoomPrivateKeyStmt, insertUserRoomKeySQL}, + {&s.insertUserRoomPublicKeyStmt, insertUserRoomPublicKeySQL}, + {&s.selectUserRoomKeyStmt, selectUserRoomKeySQL}, + {&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL}, + {&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL}, + //{&s.selectUserNIDsStmt, selectUserNIDsSQL}, //prepared at runtime + }.Prepare(db) +} + +func (s *userRoomKeysStatements) InsertUserRoomPrivatePublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PrivateKey) (result ed25519.PrivateKey, err error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.insertUserRoomPrivateKeyStmt) + err = stmt.QueryRowContext(ctx, userNID, roomNID, key, key.Public()).Scan(&result) + return result, err +} + +func (s *userRoomKeysStatements) InsertUserRoomPublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PublicKey) (result ed25519.PublicKey, err error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.insertUserRoomPublicKeyStmt) + err = stmt.QueryRowContext(ctx, userNID, roomNID, key).Scan(&result) + return result, err +} + +func (s *userRoomKeysStatements) SelectUserRoomPrivateKey( + ctx context.Context, + txn *sql.Tx, + userNID types.EventStateKeyNID, + roomNID types.RoomNID, +) (ed25519.PrivateKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectUserRoomKeyStmt) + var result ed25519.PrivateKey + err := stmt.QueryRowContext(ctx, userNID, roomNID).Scan(&result) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return result, err +} + +func (s *userRoomKeysStatements) SelectUserRoomPublicKey( + ctx context.Context, + txn *sql.Tx, + userNID types.EventStateKeyNID, + roomNID types.RoomNID, +) (ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectUserRoomPublicKeyStmt) + var result ed25519.PublicKey + err := stmt.QueryRowContext(ctx, userNID, roomNID).Scan(&result) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return result, err +} + +func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sql.Tx, senderKeys map[types.RoomNID][]ed25519.PublicKey) (map[string]types.UserRoomKeyPair, error) { + + roomNIDs := make([]any, 0, len(senderKeys)) + var senders []any + for roomNID := range senderKeys { + roomNIDs = append(roomNIDs, roomNID) + + for _, key := range senderKeys[roomNID] { + senders = append(senders, []byte(key)) + } + } + + selectSQL := strings.Replace(selectUserNIDsSQL, "($2)", sqlutil.QueryVariadicOffset(len(senders), len(senderKeys)), 1) + selectSQL = strings.Replace(selectSQL, "($1)", sqlutil.QueryVariadic(len(senderKeys)), 1) // replace $1 with the roomNIDs + + selectStmt, err := s.db.Prepare(selectSQL) + if err != nil { + return nil, err + } + + params := append(roomNIDs, senders...) + + stmt := sqlutil.TxStmt(txn, selectStmt) + defer internal.CloseAndLogIfError(ctx, stmt, "failed to close statement") + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "failed to close rows") + + result := make(map[string]types.UserRoomKeyPair, len(params)) + var publicKey []byte + userRoomKeyPair := types.UserRoomKeyPair{} + for rows.Next() { + if err = rows.Scan(&userRoomKeyPair.EventStateKeyNID, &userRoomKeyPair.RoomNID, &publicKey); err != nil { + return nil, err + } + result[spec.Base64Bytes(publicKey).Encode()] = userRoomKeyPair + } + return result, rows.Err() +} + +func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser) + + rows, err := stmt.QueryContext(ctx, userNID) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + resultMap := make(map[types.RoomNID]ed25519.PublicKey) + + var roomNID types.RoomNID + var pubkey ed25519.PublicKey + for rows.Next() { + if err = rows.Scan(&roomNID, &pubkey); err != nil { + return nil, err + } + resultMap[roomNID] = pubkey + } + return resultMap, err +} diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 2b3b3bd85b..c3689f513a 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -29,7 +29,7 @@ import ( ) // Open opens a database connection. -func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { +func Open(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.Open(ctx, conMan, dbProperties, cache) diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 333483b324..0ae064e6b8 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -2,6 +2,7 @@ package tables import ( "context" + "crypto/ed25519" "database/sql" "errors" @@ -184,6 +185,23 @@ type Purge interface { ) error } +type UserRoomKeys interface { + // InsertUserRoomPrivatePublicKey inserts the given private key as well as the public key for it. This should be used + // when creating keys locally. + InsertUserRoomPrivatePublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PrivateKey) (ed25519.PrivateKey, error) + // InsertUserRoomPublicKey inserts the given public key, this should be used for users NOT local to this server + InsertUserRoomPublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID, key ed25519.PublicKey) (ed25519.PublicKey, error) + // SelectUserRoomPrivateKey selects the private key for the given user and room combination + SelectUserRoomPrivateKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID) (ed25519.PrivateKey, error) + // SelectUserRoomPublicKey selects the public key for the given user and room combination + SelectUserRoomPublicKey(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomNID types.RoomNID) (ed25519.PublicKey, error) + // BulkSelectUserNIDs selects all userIDs for the requested senderKeys. Returns a map from publicKey -> types.UserRoomKeyPair. + // If a senderKey can't be found, it is omitted in the result. + BulkSelectUserNIDs(ctx context.Context, txn *sql.Tx, senderKeys map[types.RoomNID][]ed25519.PublicKey) (map[string]types.UserRoomKeyPair, error) + // SelectAllPublicKeysForUser returns all known public keys for a user. Returns a map from room NID -> public key + SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) +} + // StrippedEvent represents a stripped event for returning extracted content values. type StrippedEvent struct { RoomID string diff --git a/roomserver/storage/tables/user_room_keys_table_test.go b/roomserver/storage/tables/user_room_keys_table_test.go new file mode 100644 index 0000000000..2809771b4c --- /dev/null +++ b/roomserver/storage/tables/user_room_keys_table_test.go @@ -0,0 +1,123 @@ +package tables_test + +import ( + "context" + "crypto/ed25519" + "database/sql" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" + ed255192 "golang.org/x/crypto/ed25519" +) + +func mustCreateUserRoomKeysTable(t *testing.T, dbType test.DBType) (tab tables.UserRoomKeys, db *sql.DB, close func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + assert.NoError(t, err) + switch dbType { + case test.DBTypePostgres: + err = postgres.CreateUserRoomKeysTable(db) + assert.NoError(t, err) + tab, err = postgres.PrepareUserRoomKeysTable(db) + case test.DBTypeSQLite: + err = sqlite3.CreateUserRoomKeysTable(db) + assert.NoError(t, err) + tab, err = sqlite3.PrepareUserRoomKeysTable(db) + } + assert.NoError(t, err) + + return tab, db, close +} + +func TestUserRoomKeysTable(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, db, close := mustCreateUserRoomKeysTable(t, dbType) + defer close() + userNID := types.EventStateKeyNID(1) + roomNID := types.RoomNID(1) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) + + err = sqlutil.WithTransaction(db, func(txn *sql.Tx) error { + var gotKey, key2, key3 ed25519.PrivateKey + var pubKey ed25519.PublicKey + gotKey, err = tab.InsertUserRoomPrivatePublicKey(context.Background(), txn, userNID, roomNID, key) + assert.NoError(t, err) + assert.Equal(t, gotKey, key) + + // again, this shouldn't result in an error, but return the existing key + _, key2, err = ed25519.GenerateKey(nil) + assert.NoError(t, err) + gotKey, err = tab.InsertUserRoomPrivatePublicKey(context.Background(), txn, userNID, roomNID, key2) + assert.NoError(t, err) + assert.Equal(t, gotKey, key) + + // add another user + _, key3, err = ed25519.GenerateKey(nil) + assert.NoError(t, err) + userNID2 := types.EventStateKeyNID(2) + _, err = tab.InsertUserRoomPrivatePublicKey(context.Background(), txn, userNID2, roomNID, key3) + assert.NoError(t, err) + + gotKey, err = tab.SelectUserRoomPrivateKey(context.Background(), txn, userNID, roomNID) + assert.NoError(t, err) + assert.Equal(t, key, gotKey) + pubKey, err = tab.SelectUserRoomPublicKey(context.Background(), txn, userNID, roomNID) + assert.NoError(t, err) + assert.Equal(t, key.Public(), pubKey) + + // try to update an existing key, this should only be done for users NOT on this homeserver + var gotPubKey ed25519.PublicKey + gotPubKey, err = tab.InsertUserRoomPublicKey(context.Background(), txn, userNID, roomNID, key2.Public().(ed25519.PublicKey)) + assert.NoError(t, err) + assert.Equal(t, key2.Public(), gotPubKey) + + // Key doesn't exist + gotKey, err = tab.SelectUserRoomPrivateKey(context.Background(), txn, userNID, 2) + assert.NoError(t, err) + assert.Nil(t, gotKey) + pubKey, err = tab.SelectUserRoomPublicKey(context.Background(), txn, userNID, 2) + assert.NoError(t, err) + assert.Nil(t, pubKey) + + // query user NIDs for senderKeys + var gotKeys map[string]types.UserRoomKeyPair + query := map[types.RoomNID][]ed25519.PublicKey{ + roomNID: {key2.Public().(ed25519.PublicKey), key3.Public().(ed25519.PublicKey)}, + types.RoomNID(2): {key.Public().(ed25519.PublicKey), key3.Public().(ed25519.PublicKey)}, // doesn't exist + } + gotKeys, err = tab.BulkSelectUserNIDs(context.Background(), txn, query) + assert.NoError(t, err) + assert.NotNil(t, gotKeys) + + wantKeys := map[string]types.UserRoomKeyPair{ + string(spec.Base64Bytes(key2.Public().(ed25519.PublicKey)).Encode()): {RoomNID: roomNID, EventStateKeyNID: userNID}, + string(spec.Base64Bytes(key3.Public().(ed25519.PublicKey)).Encode()): {RoomNID: roomNID, EventStateKeyNID: userNID2}, + } + assert.Equal(t, wantKeys, gotKeys) + + // insert key that came in over federation + var gotPublicKey, key4 ed255192.PublicKey + key4, _, err = ed25519.GenerateKey(nil) + assert.NoError(t, err) + gotPublicKey, err = tab.InsertUserRoomPublicKey(context.Background(), txn, userNID, 2, key4) + assert.NoError(t, err) + assert.Equal(t, key4, gotPublicKey) + + return nil + }) + assert.NoError(t, err) + + }) +} diff --git a/roomserver/types/headered_event.go b/roomserver/types/headered_event.go index 52d006bd90..7839998222 100644 --- a/roomserver/types/headered_event.go +++ b/roomserver/types/headered_event.go @@ -18,6 +18,7 @@ import ( "unsafe" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) // HeaderedEvent is an Event which serialises to the headered form, which includes @@ -25,6 +26,10 @@ import ( type HeaderedEvent struct { gomatrixserverlib.PDU Visibility gomatrixserverlib.HistoryVisibility + // TODO: Remove this. This is a temporary workaround to store the userID in the syncAPI. + // It really should be the userKey instead. + UserID spec.UserID + StateKeyResolved *string } func (h *HeaderedEvent) CacheCost() int { diff --git a/roomserver/types/types.go b/roomserver/types/types.go index f57978ad5b..fbff2cdab1 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -22,7 +22,9 @@ import ( "strings" "sync" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "golang.org/x/crypto/blake2b" ) @@ -44,6 +46,11 @@ type EventMetadata struct { RoomNID RoomNID } +type UserRoomKeyPair struct { + RoomNID RoomNID + EventStateKeyNID EventStateKeyNID +} + // StateSnapshotNID is a numeric ID for the state at an event. type StateSnapshotNID int64 @@ -331,3 +338,36 @@ func (r *RoomInfo) CopyFrom(r2 *RoomInfo) { } var ErrorInvalidRoomInfo = fmt.Errorf("room info is invalid") + +// Struct to represent a device or a server name. +// +// May be used to designate a caller for functions that can be called +// by a client (device) or by a server (server name). +// +// Exactly 1 of Device() and ServerName() will return a non-nil result. +type DeviceOrServerName struct { + device *userapi.Device + serverName *spec.ServerName +} + +func NewDeviceNotServerName(device userapi.Device) DeviceOrServerName { + return DeviceOrServerName{ + device: &device, + serverName: nil, + } +} + +func NewServerNameNotDevice(serverName spec.ServerName) DeviceOrServerName { + return DeviceOrServerName{ + device: nil, + serverName: &serverName, + } +} + +func (s *DeviceOrServerName) Device() *userapi.Device { + return s.device +} + +func (s *DeviceOrServerName) ServerName() *spec.ServerName { + return s.serverName +} diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 270d428972..e8bd890a0b 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -20,12 +20,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// DefaultRoomVersion contains the room version that will, by -// default, be used to create new rooms on this server. -func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV10 -} - // RoomVersions returns a map of all known room versions to this // server. func RoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.IRoomVersion { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index b6c74a75f0..44136e2a08 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -13,6 +13,10 @@ type ClientAPI struct { // secrets) RegistrationDisabled bool `yaml:"registration_disabled"` + // If set, requires users to submit a token during registration. + // Tokens can be managed using admin API. + RegistrationRequiresToken bool `yaml:"registration_requires_token"` + // Enable registration without captcha verification or shared secret. // This option is populated by the -really-enable-open-registration // command line parameter as it is not recommended. @@ -56,6 +60,7 @@ type ClientAPI struct { func (c *ClientAPI) Defaults(opts DefaultOpts) { c.RegistrationSharedSecret = "" + c.RegistrationRequiresToken = false c.RecaptchaPublicKey = "" c.RecaptchaPrivateKey = "" c.RecaptchaEnabled = false diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 1622bf3576..5b4ccf4002 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -48,6 +48,10 @@ type Global struct { // The server name to delegate client-server communications to, with optional port WellKnownClientName string `yaml:"well_known_client_name"` + // The server name to delegate sliding sync communications to, with optional port. + // Requires `well_known_client_name` to also be configured. + WellKnownSlidingSyncProxy string `yaml:"well_known_sliding_sync_proxy"` + // Disables federation. Dendrite will not be able to make any outbound HTTP requests // to other servers and the federation API will not be exposed. DisableFederation bool `yaml:"disable_federation"` diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index 21d4b4da04..ce491cd724 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -7,7 +7,6 @@ type MSCs struct { // 'msc2444': Peeking over federation - https://github.com/matrix-org/matrix-doc/pull/2444 // 'msc2753': Peeking via /sync - https://github.com/matrix-org/matrix-doc/pull/2753 // 'msc2836': Threading - https://github.com/matrix-org/matrix-doc/pull/2836 - // 'msc2946': Spaces Summary - https://github.com/matrix-org/matrix-doc/pull/2946 MSCs []string `yaml:"mscs"` Database DatabaseOptions `yaml:"database,omitempty"` diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index 319c2419cb..06e7757fbc 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -1,12 +1,22 @@ package config +import ( + "fmt" + + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + type RoomServer struct { Matrix *Global `yaml:"-"` + DefaultRoomVersion gomatrixserverlib.RoomVersion `yaml:"default_room_version,omitempty"` + Database DatabaseOptions `yaml:"database,omitempty"` } func (c *RoomServer) Defaults(opts DefaultOpts) { + c.DefaultRoomVersion = gomatrixserverlib.RoomVersionV10 if opts.Generate { if !opts.SingleDatabase { c.Database.ConnectionString = "file:roomserver.db" @@ -18,4 +28,10 @@ func (c *RoomServer) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) } + + if !gomatrixserverlib.KnownRoomVersion(c.DefaultRoomVersion) { + configErrs.Add(fmt.Sprintf("invalid value for config key 'room_server.default_room_version': unsupported room version: %q", c.DefaultRoomVersion)) + } else if !gomatrixserverlib.StableRoomVersion(c.DefaultRoomVersion) { + log.Warnf("WARNING: Provided default room version %q is unstable", c.DefaultRoomVersion) + } } diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 06a58d5422..8820e86b2b 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -70,7 +70,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS process.ComponentFinished() }() } - if !s.ReadyForConnections(time.Second * 10) { + if !s.ReadyForConnections(time.Second * 60) { logrus.Fatalln("NATS did not start in time") } // reuse existing connections @@ -87,6 +87,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS return js, nc } +// nolint:gocyclo func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { if nc == nil { var err error @@ -126,16 +127,32 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc subjects = []string{name, name + ".>"} } if info != nil { + // If the stream config doesn't match what we expect, try to update + // it. If that doesn't work then try to blow it away and we'll then + // recreate it in the next section. + // Each specific option that we set must be checked by hand, as if + // you DeepEqual the whole config struct, it will always show that + // there's a difference because the NATS Server will return defaults + // in the stream info. switch { case !reflect.DeepEqual(info.Config.Subjects, subjects): fallthrough case info.Config.Retention != stream.Retention: fallthrough case info.Config.Storage != stream.Storage: - if err = s.DeleteStream(name); err != nil { - logrus.WithError(err).Fatal("Unable to delete stream") + fallthrough + case info.Config.MaxAge != stream.MaxAge: + // Try updating the stream first, as many things can be updated + // non-destructively. + if info, err = s.UpdateStream(stream); err != nil { + logrus.WithError(err).Warnf("Unable to update stream %q, recreating...", name) + // We failed to update the stream, this is a last attempt to get + // things working but may result in data loss. + if err = s.DeleteStream(name); err != nil { + logrus.WithError(err).Fatalf("Unable to delete stream %q", name) + } + info = nil } - info = nil } } if info == nil { diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 590f0cbd9d..741407926d 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -48,6 +48,7 @@ var streams = []*nats.StreamConfig{ Name: InputRoomEvent, Retention: nats.InterestPolicy, Storage: nats.FileStorage, + MaxAge: time.Hour * 24, }, { Name: InputDeviceListUpdate, diff --git a/setup/monolith.go b/setup/monolith.go index 848dfe9c7c..4856d6e835 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -61,7 +61,7 @@ func (m *Monolith) AddAllPublicRoutes( processCtx *process.ProcessContext, cfg *config.Dendrite, routers httputil.Routers, - cm sqlutil.Connections, + cm *sqlutil.Connections, natsInstance *jetstream.NATSInstance, caches *caching.Caches, enableMetrics bool, diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index f468b048a6..15811710d7 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -92,9 +92,11 @@ type MSC2836EventRelationshipsResponse struct { ParsedAuthChain []gomatrixserverlib.PDU } -func toClientResponse(res *MSC2836EventRelationshipsResponse) *EventRelationshipResponse { +func toClientResponse(ctx context.Context, res *MSC2836EventRelationshipsResponse, rsAPI roomserver.RoomserverInternalAPI) *EventRelationshipResponse { out := &EventRelationshipResponse{ - Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(res.ParsedEvents), synctypes.FormatAll), + Events: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(res.ParsedEvents), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }), Limited: res.Limited, NextBatch: res.NextBatch, } @@ -103,7 +105,7 @@ func toClientResponse(res *MSC2836EventRelationshipsResponse) *EventRelationship // Enable this MSC func Enable( - cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, + cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier, ) error { db, err := NewDatabase(cm, &cfg.MSCs.Database) @@ -152,7 +154,7 @@ type reqCtx struct { rsAPI roomserver.RoomserverInternalAPI db Database req *EventRelationshipRequest - userID string + userID spec.UserID roomVersion gomatrixserverlib.RoomVersion // federated request args @@ -171,10 +173,17 @@ func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAP JSON: spec.BadJSON(fmt.Sprintf("invalid json: %s", err)), } } + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: 400, + JSON: spec.BadJSON(fmt.Sprintf("invalid json: %s", err)), + } + } rc := reqCtx{ ctx: req.Context(), req: relation, - userID: device.UserID, + userID: *userID, rsAPI: rsAPI, fsAPI: fsAPI, isFederatedRequest: false, @@ -187,7 +196,7 @@ func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAP return util.JSONResponse{ Code: 200, - JSON: toClientResponse(res), + JSON: toClientResponse(req.Context(), res, rsAPI), } } } @@ -262,7 +271,7 @@ func (rc *reqCtx) process() (*MSC2836EventRelationshipsResponse, *util.JSONRespo event = rc.fetchUnknownEvent(rc.req.EventID, rc.req.RoomID) } if rc.req.RoomID == "" && event != nil { - rc.req.RoomID = event.RoomID() + rc.req.RoomID = event.RoomID().String() } if event == nil || !rc.authorisedToSeeEvent(event) { return nil, &util.JSONResponse{ @@ -517,7 +526,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool { // make sure the server is in this room var res fs.QueryJoinedHostServerNamesInRoomResponse err := rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), }, &res) if err != nil { util.GetLogger(rc.ctx).WithError(err).Error("authorisedToSeeEvent: failed to QueryJoinedHostServerNamesInRoom") @@ -536,7 +545,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool { // TODO: This does not honour m.room.create content var queryMembershipRes roomserver.QueryMembershipForUserResponse err := rc.rsAPI.QueryMembershipForUser(rc.ctx, &roomserver.QueryMembershipForUserRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), UserID: rc.userID, }, &queryMembershipRes) if err != nil { @@ -603,7 +612,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent { // inject all the events into the roomserver then return the event in question rc.injectResponseToRoomserver(queryRes) for _, ev := range queryRes.ParsedEvents { - if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID() { + if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID().String() { return &types.HeaderedEvent{PDU: ev} } } @@ -620,7 +629,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent { } } } - if rc.req.RoomID == event.RoomID() { + if rc.req.RoomID == event.RoomID().String() { return event } return nil diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 2c6f63d452..ecbab706f1 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -525,6 +525,15 @@ type testRoomserverAPI struct { events map[string]*types.HeaderedEvent } +func (r *testRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + +func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil +} + func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver.QueryEventsByIDRequest, res *roomserver.QueryEventsByIDResponse) error { for _, eventID := range req.EventIDs { ev := r.events[eventID] @@ -536,7 +545,7 @@ func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver } func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roomserver.QueryMembershipForUserRequest, res *roomserver.QueryMembershipForUserResponse) error { - rooms := r.userToJoinedRooms[req.UserID] + rooms := r.userToJoinedRooms[req.UserID.String()] for _, roomID := range rooms { if roomID == req.RoomID { res.IsInRoom = true @@ -586,7 +595,7 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve seed := make([]byte, ed25519.SeedSize) // zero seed key := ed25519.NewKeyFromSeed(seed) eb := gomatrixserverlib.MustGetRoomVersion(roomVer).NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ - Sender: ev.Sender, + SenderID: ev.Sender, Depth: 999, Type: ev.Type, StateKey: ev.StateKey, diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index 6a45f08a4e..ade2a1616c 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -59,14 +59,14 @@ type DB struct { } // NewDatabase loads the database for msc2836 -func NewDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { +func NewDatabase(conMan *sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { if dbOpts.ConnectionString.IsPostgres() { return newPostgresDatabase(conMan, dbOpts) } return newSQLiteDatabase(conMan, dbOpts) } -func newPostgresDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { +func newPostgresDatabase(conMan *sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { d := DB{} var err error if d.db, d.writer, err = conMan.Connection(dbOpts); err != nil { @@ -144,7 +144,7 @@ func newPostgresDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOpti return &d, err } -func newSQLiteDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { +func newSQLiteDatabase(conMan *sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { d := DB{} var err error if d.db, d.writer, err = conMan.Connection(dbOpts); err != nil { @@ -239,7 +239,7 @@ func (p *DB) StoreRelation(ctx context.Context, ev *types.HeaderedEvent) error { return err } util.GetLogger(ctx).Infof("StoreRelation child=%s parent=%s rel_type=%s", child, parent, relType) - _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID(), count, base64.RawStdEncoding.EncodeToString(hash), 0) + _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID().String(), count, base64.RawStdEncoding.EncodeToString(hash), 0) return err }) } diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go deleted file mode 100644 index 291e0f3b20..0000000000 --- a/setup/mscs/msc2946/msc2946.go +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package msc2946 'Spaces Summary' implements https://github.com/matrix-org/matrix-doc/pull/2946 -package msc2946 - -import ( - "context" - "encoding/json" - "net/http" - "net/url" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/google/uuid" - "github.com/gorilla/mux" - fs "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" - "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/util" - "github.com/tidwall/gjson" -) - -const ( - ConstCreateEventContentKey = "type" - ConstCreateEventContentValueSpace = "m.space" - ConstSpaceChildEventType = "m.space.child" - ConstSpaceParentEventType = "m.space.parent" -) - -type MSC2946ClientResponse struct { - Rooms []fclient.MSC2946Room `json:"rooms"` - NextBatch string `json:"next_batch,omitempty"` -} - -// Enable this MSC -func Enable( - cfg *config.Dendrite, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, - fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.SpaceSummaryRoomsCache, -) error { - clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, cfg.Global.ServerName), httputil.WithAllowGuests()) - routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - routers.Client.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - - fedAPI := httputil.MakeExternalAPI( - "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { - fedReq, errResp := fclient.VerifyHTTPRequest( - req, time.Now(), cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, - ) - if fedReq == nil { - return errResp - } - // Extract the room ID from the request. Sanity check request data. - params, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - roomID := params["roomID"] - return federatedSpacesHandler(req.Context(), fedReq, roomID, cache, rsAPI, fsAPI, cfg.Global.ServerName) - }, - ) - routers.Federation.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - routers.Federation.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - return nil -} - -func federatedSpacesHandler( - ctx context.Context, fedReq *fclient.FederationRequest, roomID string, - cache caching.SpaceSummaryRoomsCache, - rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, - thisServer spec.ServerName, -) util.JSONResponse { - u, err := url.Parse(fedReq.RequestURI()) - if err != nil { - return util.JSONResponse{ - Code: 400, - JSON: spec.InvalidParam("bad request uri"), - } - } - - w := walker{ - rootRoomID: roomID, - serverName: fedReq.Origin(), - thisServer: thisServer, - ctx: ctx, - cache: cache, - suggestedOnly: u.Query().Get("suggested_only") == "true", - limit: 1000, - // The main difference is that it does not recurse into spaces and does not support pagination. - // This is somewhat equivalent to a Client-Server request with a max_depth=1. - maxDepth: 1, - - rsAPI: rsAPI, - fsAPI: fsAPI, - // inline cache as we don't have pagination in federation mode - paginationCache: make(map[string]paginationInfo), - } - return w.walk() -} - -func spacesHandler( - rsAPI roomserver.RoomserverInternalAPI, - fsAPI fs.FederationInternalAPI, - cache caching.SpaceSummaryRoomsCache, - thisServer spec.ServerName, -) func(*http.Request, *userapi.Device) util.JSONResponse { - // declared outside the returned handler so it persists between calls - // TODO: clear based on... time? - paginationCache := make(map[string]paginationInfo) - - return func(req *http.Request, device *userapi.Device) util.JSONResponse { - // Extract the room ID from the request. Sanity check request data. - params, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - roomID := params["roomID"] - w := walker{ - suggestedOnly: req.URL.Query().Get("suggested_only") == "true", - limit: parseInt(req.URL.Query().Get("limit"), 1000), - maxDepth: parseInt(req.URL.Query().Get("max_depth"), -1), - paginationToken: req.URL.Query().Get("from"), - rootRoomID: roomID, - caller: device, - thisServer: thisServer, - ctx: req.Context(), - cache: cache, - - rsAPI: rsAPI, - fsAPI: fsAPI, - paginationCache: paginationCache, - } - return w.walk() - } -} - -type paginationInfo struct { - processed set - unvisited []roomVisit -} - -type walker struct { - rootRoomID string - caller *userapi.Device - serverName spec.ServerName - thisServer spec.ServerName - rsAPI roomserver.RoomserverInternalAPI - fsAPI fs.FederationInternalAPI - ctx context.Context - cache caching.SpaceSummaryRoomsCache - suggestedOnly bool - limit int - maxDepth int - paginationToken string - - paginationCache map[string]paginationInfo - mu sync.Mutex -} - -func (w *walker) newPaginationCache() (string, paginationInfo) { - p := paginationInfo{ - processed: make(set), - unvisited: nil, - } - tok := uuid.NewString() - return tok, p -} - -func (w *walker) loadPaginationCache(paginationToken string) *paginationInfo { - w.mu.Lock() - defer w.mu.Unlock() - p := w.paginationCache[paginationToken] - return &p -} - -func (w *walker) storePaginationCache(paginationToken string, cache paginationInfo) { - w.mu.Lock() - defer w.mu.Unlock() - w.paginationCache[paginationToken] = cache -} - -type roomVisit struct { - roomID string - parentRoomID string - depth int - vias []string // vias to query this room by -} - -func (w *walker) walk() util.JSONResponse { - if authorised, _ := w.authorised(w.rootRoomID, ""); !authorised { - if w.caller != nil { - // CS API format - return util.JSONResponse{ - Code: 403, - JSON: spec.Forbidden("room is unknown/forbidden"), - } - } else { - // SS API format - return util.JSONResponse{ - Code: 404, - JSON: spec.NotFound("room is unknown/forbidden"), - } - } - } - - var discoveredRooms []fclient.MSC2946Room - - var cache *paginationInfo - if w.paginationToken != "" { - cache = w.loadPaginationCache(w.paginationToken) - if cache == nil { - return util.JSONResponse{ - Code: 400, - JSON: spec.InvalidParam("invalid from"), - } - } - } else { - tok, c := w.newPaginationCache() - cache = &c - w.paginationToken = tok - // Begin walking the graph starting with the room ID in the request in a queue of unvisited rooms - c.unvisited = append(c.unvisited, roomVisit{ - roomID: w.rootRoomID, - parentRoomID: "", - depth: 0, - }) - } - - processed := cache.processed - unvisited := cache.unvisited - - // Depth first -> stack data structure - for len(unvisited) > 0 { - if len(discoveredRooms) >= w.limit { - break - } - - // pop the stack - rv := unvisited[len(unvisited)-1] - unvisited = unvisited[:len(unvisited)-1] - // If this room has already been processed, skip. - // If this room exceeds the specified depth, skip. - if processed.isSet(rv.roomID) || rv.roomID == "" || (w.maxDepth > 0 && rv.depth > w.maxDepth) { - continue - } - - // Mark this room as processed. - processed.set(rv.roomID) - - // if this room is not a space room, skip. - var roomType string - create := w.stateEvent(rv.roomID, spec.MRoomCreate, "") - if create != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - } - - // Collect rooms/events to send back (either locally or fetched via federation) - var discoveredChildEvents []fclient.MSC2946StrippedEvent - - // If we know about this room and the caller is authorised (joined/world_readable) then pull - // events locally - roomExists := w.roomExists(rv.roomID) - if !roomExists { - // attempt to query this room over federation, as either we've never heard of it before - // or we've left it and hence are not authorised (but info may be exposed regardless) - fedRes := w.federatedRoomInfo(rv.roomID, rv.vias) - if fedRes != nil { - discoveredChildEvents = fedRes.Room.ChildrenState - discoveredRooms = append(discoveredRooms, fedRes.Room) - if len(fedRes.Children) > 0 { - discoveredRooms = append(discoveredRooms, fedRes.Children...) - } - // mark this room as a space room as the federated server responded. - // we need to do this so we add the children of this room to the unvisited stack - // as these children may be rooms we do know about. - roomType = ConstCreateEventContentValueSpace - } - } else if authorised, isJoinedOrInvited := w.authorised(rv.roomID, rv.parentRoomID); authorised { - // Get all `m.space.child` state events for this room - events, err := w.childReferences(rv.roomID) - if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("room_id", rv.roomID).Error("failed to extract references for room") - continue - } - discoveredChildEvents = events - - pubRoom := w.publicRoomsChunk(rv.roomID) - - discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ - PublicRoom: *pubRoom, - RoomType: roomType, - ChildrenState: events, - }) - // don't walk children if the user is not joined/invited to the space - if !isJoinedOrInvited { - continue - } - } else { - // room exists but user is not authorised - continue - } - - // don't walk the children - // if the parent is not a space room - if roomType != ConstCreateEventContentValueSpace { - continue - } - - // For each referenced room ID in the child events being returned to the caller - // add the room ID to the queue of unvisited rooms. Loop from the beginning. - // We need to invert the order here because the child events are lo->hi on the timestamp, - // so we need to ensure we pop in the same lo->hi order, which won't be the case if we - // insert the highest timestamp last in a stack. - for i := len(discoveredChildEvents) - 1; i >= 0; i-- { - spaceContent := struct { - Via []string `json:"via"` - }{} - ev := discoveredChildEvents[i] - _ = json.Unmarshal(ev.Content, &spaceContent) - unvisited = append(unvisited, roomVisit{ - roomID: ev.StateKey, - parentRoomID: rv.roomID, - depth: rv.depth + 1, - vias: spaceContent.Via, - }) - } - } - - if len(unvisited) > 0 { - // we still have more rooms so we need to send back a pagination token, - // we probably hit a room limit - cache.processed = processed - cache.unvisited = unvisited - w.storePaginationCache(w.paginationToken, *cache) - } else { - // clear the pagination token so we don't send it back to the client - // Note we do NOT nuke the cache just in case this response is lost - // and the client retries it. - w.paginationToken = "" - } - - if w.caller != nil { - // return CS API format - return util.JSONResponse{ - Code: 200, - JSON: MSC2946ClientResponse{ - Rooms: discoveredRooms, - NextBatch: w.paginationToken, - }, - } - } - // return SS API format - // the first discovered room will be the room asked for, and subsequent ones the depth=1 children - if len(discoveredRooms) == 0 { - return util.JSONResponse{ - Code: 404, - JSON: spec.NotFound("room is unknown/forbidden"), - } - } - return util.JSONResponse{ - Code: 200, - JSON: fclient.MSC2946SpacesResponse{ - Room: discoveredRooms[0], - Children: discoveredRooms[1:], - }, - } -} - -func (w *walker) stateEvent(roomID, evType, stateKey string) *types.HeaderedEvent { - var queryRes roomserver.QueryCurrentStateResponse - tuple := gomatrixserverlib.StateKeyTuple{ - EventType: evType, - StateKey: stateKey, - } - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, - }, &queryRes) - if err != nil { - return nil - } - return queryRes.StateEvents[tuple] -} - -func (w *walker) publicRoomsChunk(roomID string) *fclient.PublicRoom { - pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") - return nil - } - if len(pubRooms) == 0 { - return nil - } - return &pubRooms[0] -} - -// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was -// unsuccessful. -func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC2946SpacesResponse { - // only do federated requests for client requests - if w.caller == nil { - return nil - } - resp, ok := w.cache.GetSpaceSummary(roomID) - if ok { - util.GetLogger(w.ctx).Debugf("Returning cached response for %s", roomID) - return &resp - } - util.GetLogger(w.ctx).Debugf("Querying %s via %+v", roomID, vias) - ctx := context.Background() - // query more of the spaces graph using these servers - for _, serverName := range vias { - if serverName == string(w.thisServer) { - continue - } - res, err := w.fsAPI.MSC2946Spaces(ctx, w.thisServer, spec.ServerName(serverName), roomID, w.suggestedOnly) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) - continue - } - // ensure nil slices are empty as we send this to the client sometimes - if res.Room.ChildrenState == nil { - res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} - } - for i := 0; i < len(res.Children); i++ { - child := res.Children[i] - if child.ChildrenState == nil { - child.ChildrenState = []fclient.MSC2946StrippedEvent{} - } - res.Children[i] = child - } - w.cache.StoreSpaceSummary(roomID, res) - - return &res - } - return nil -} - -func (w *walker) roomExists(roomID string) bool { - var queryRes roomserver.QueryServerJoinedToRoomResponse - err := w.rsAPI.QueryServerJoinedToRoom(w.ctx, &roomserver.QueryServerJoinedToRoomRequest{ - RoomID: roomID, - ServerName: w.thisServer, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") - return false - } - // if the room exists but we aren't in the room then we might have stale data so we want to fetch - // it fresh via federation - return queryRes.RoomExists && queryRes.IsInRoom -} - -// authorised returns true iff the user is joined this room or the room is world_readable -func (w *walker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvited bool) { - if w.caller != nil { - return w.authorisedUser(roomID, parentRoomID) - } - return w.authorisedServer(roomID), false -} - -// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable -func (w *walker) authorisedServer(roomID string) bool { - // Check history visibility / join rules first - hisVisTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomHistoryVisibility, - StateKey: "", - } - joinRuleTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomJoinRules, - StateKey: "", - } - var queryRoomRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - hisVisTuple, joinRuleTuple, - }, - }, &queryRoomRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") - return false - } - hisVisEv := queryRoomRes.StateEvents[hisVisTuple] - if hisVisEv != nil { - hisVis, _ := hisVisEv.HistoryVisibility() - if hisVis == "world_readable" { - return true - } - } - - // check if this room is a restricted room and if so, we need to check if the server is joined to an allowed room ID - // in addition to the actual room ID (but always do the actual one first as it's quicker in the common case) - allowJoinedToRoomIDs := []string{roomID} - joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple] - - if joinRuleEv != nil { - rule, ruleErr := joinRuleEv.JoinRule() - if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") - return false - } - - if rule == spec.Public || rule == spec.Knock { - return true - } - - if rule == spec.Restricted { - allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...) - } - } - - // check if server is joined to any allowed room - for _, allowedRoomID := range allowJoinedToRoomIDs { - var queryRes fs.QueryJoinedHostServerNamesInRoomResponse - err = w.fsAPI.QueryJoinedHostServerNamesInRoom(w.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: allowedRoomID, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") - continue - } - for _, srv := range queryRes.ServerNames { - if srv == w.serverName { - return true - } - } - } - - return false -} - -// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable -// or if the room has a public or knock join rule. -// Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. -func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoinedOrInvited bool) { - hisVisTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomHistoryVisibility, - StateKey: "", - } - joinRuleTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomJoinRules, - StateKey: "", - } - roomMemberTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomMember, - StateKey: w.caller.UserID, - } - var queryRes roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - hisVisTuple, joinRuleTuple, roomMemberTuple, - }, - }, &queryRes) - if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") - return false, false - } - memberEv := queryRes.StateEvents[roomMemberTuple] - if memberEv != nil { - membership, _ := memberEv.Membership() - if membership == spec.Join || membership == spec.Invite { - return true, true - } - } - hisVisEv := queryRes.StateEvents[hisVisTuple] - if hisVisEv != nil { - hisVis, _ := hisVisEv.HistoryVisibility() - if hisVis == "world_readable" { - return true, false - } - } - joinRuleEv := queryRes.StateEvents[joinRuleTuple] - if parentRoomID != "" && joinRuleEv != nil { - var allowed bool - rule, ruleErr := joinRuleEv.JoinRule() - if ruleErr != nil { - util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule") - } else if rule == spec.Public || rule == spec.Knock { - allowed = true - } else if rule == spec.Restricted { - allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership") - // check parent is in the allowed set - for _, a := range allowedRoomIDs { - if parentRoomID == a { - allowed = true - break - } - } - } - if allowed { - // ensure caller is joined to the parent room - var queryRes2 roomserver.QueryCurrentStateResponse - err = w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ - RoomID: parentRoomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - roomMemberTuple, - }, - }, &queryRes2) - if err != nil { - util.GetLogger(w.ctx).WithError(err).WithField("parent_room_id", parentRoomID).Warn("failed to check user is joined to parent room") - } else { - memberEv = queryRes2.StateEvents[roomMemberTuple] - if memberEv != nil { - membership, _ := memberEv.Membership() - if membership == spec.Join { - return true, false - } - } - } - } - } - return false, false -} - -func (w *walker) restrictedJoinRuleAllowedRooms(joinRuleEv *types.HeaderedEvent, allowType string) (allows []string) { - rule, _ := joinRuleEv.JoinRule() - if rule != spec.Restricted { - return nil - } - var jrContent gomatrixserverlib.JoinRuleContent - if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { - util.GetLogger(w.ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) - return nil - } - for _, allow := range jrContent.Allow { - if allow.Type == allowType { - allows = append(allows, allow.RoomID) - } - } - return -} - -// references returns all child references pointing to or from this room. -func (w *walker) childReferences(roomID string) ([]fclient.MSC2946StrippedEvent, error) { - createTuple := gomatrixserverlib.StateKeyTuple{ - EventType: spec.MRoomCreate, - StateKey: "", - } - var res roomserver.QueryCurrentStateResponse - err := w.rsAPI.QueryCurrentState(context.Background(), &roomserver.QueryCurrentStateRequest{ - RoomID: roomID, - AllowWildcards: true, - StateTuples: []gomatrixserverlib.StateKeyTuple{ - createTuple, { - EventType: ConstSpaceChildEventType, - StateKey: "*", - }, - }, - }, &res) - if err != nil { - return nil, err - } - - // don't return any child refs if the room is not a space room - if res.StateEvents[createTuple] != nil { - // escape the `.`s so gjson doesn't think it's nested - roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str - if roomType != ConstCreateEventContentValueSpace { - return []fclient.MSC2946StrippedEvent{}, nil - } - } - delete(res.StateEvents, createTuple) - - el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) - for _, ev := range res.StateEvents { - content := gjson.ParseBytes(ev.Content()) - // only return events that have a `via` key as per MSC1772 - // else we'll incorrectly walk redacted events (as the link - // is in the state_key) - if content.Get("via").Exists() { - strip := stripped(ev.PDU) - if strip == nil { - continue - } - // if suggested only and this child isn't suggested, skip it. - // if suggested only = false we include everything so don't need to check the content. - if w.suggestedOnly && !content.Get("suggested").Bool() { - continue - } - el = append(el, *strip) - } - } - // sort by origin_server_ts as per MSC2946 - sort.Slice(el, func(i, j int) bool { - return el[i].OriginServerTS < el[j].OriginServerTS - }) - - return el, nil -} - -type set map[string]struct{} - -func (s set) set(val string) { - s[val] = struct{}{} -} -func (s set) isSet(val string) bool { - _, ok := s[val] - return ok -} - -func stripped(ev gomatrixserverlib.PDU) *fclient.MSC2946StrippedEvent { - if ev.StateKey() == nil { - return nil - } - return &fclient.MSC2946StrippedEvent{ - Type: ev.Type(), - StateKey: *ev.StateKey(), - Content: ev.Content(), - Sender: ev.Sender(), - OriginServerTS: ev.OriginServerTS(), - } -} - -func parseInt(intstr string, defaultVal int) int { - i, err := strconv.ParseInt(intstr, 10, 32) - if err != nil { - return defaultVal - } - return int(i) -} diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index 9cd5eed1cd..7a942cf7c8 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -17,7 +17,6 @@ package mscs import ( "context" - "fmt" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -25,12 +24,12 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2836" - "github.com/matrix-org/dendrite/setup/mscs/msc2946" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // Enable MSCs - returns an error on unknown MSCs -func Enable(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, caches *caching.Caches) error { +func Enable(cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, caches *caching.Caches) error { for _, msc := range cfg.MSCs.MSCs { util.GetLogger(context.Background()).WithField("msc", msc).Info("Enabling MSC") if err := EnableMSC(cfg, cm, routers, monolith, msc, caches); err != nil { @@ -40,16 +39,14 @@ func Enable(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Route return nil } -func EnableMSC(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, msc string, caches *caching.Caches) error { +func EnableMSC(cfg *config.Dendrite, cm *sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, msc string, caches *caching.Caches) error { switch msc { case "msc2836": return msc2836.Enable(cfg, cm, routers, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) - case "msc2946": - return msc2946.Enable(cfg, routers, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, caches) case "msc2444": // enabled inside federationapi case "msc2753": // enabled inside clientapi default: - return fmt.Errorf("EnableMSC: unknown msc '%s'", msc) + logrus.Warnf("EnableMSC: unknown MSC '%s', this MSC is either not supported or is natively supported by Dendrite", msc) } return nil } diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 3ed455e9f4..76b447133b 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -113,7 +113,7 @@ func (s *OutputClientDataConsumer) Start() error { id = streamPos e := fulltext.IndexElement{ EventID: ev.EventID(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StreamPosition: streamPos, } e.SetContentType(ev.Type()) diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 56285dbf4a..666f900d7b 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -17,16 +17,12 @@ package consumers import ( "context" "database/sql" + "encoding/base64" "encoding/json" + "errors" "fmt" "github.com/getsentry/sentry-go" - "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" - "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -37,7 +33,13 @@ import ( "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" ) // OutputRoomEventConsumer consumes events that originated in the room server. @@ -141,7 +143,14 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms ) } if err != nil { - log.WithError(err).Error("roomserver output log: failed to process event") + if errors.As(err, new(base64.CorruptInputError)) { + // no matter how often we retry this event, we will always get this error, discard the event + return true + } + log.WithFields(log.Fields{ + "type": output.Type, + }).WithError(err).Error("roomserver output log: failed to process event") + sentry.CaptureException(err) return false } @@ -151,15 +160,15 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms func (s *OutputRoomEventConsumer) onRedactEvent( ctx context.Context, msg api.OutputRedactedEvent, ) error { - err := s.db.RedactEvent(ctx, msg.RedactedEventID, msg.RedactedBecause) + err := s.db.RedactEvent(ctx, msg.RedactedEventID, msg.RedactedBecause, s.rsAPI) if err != nil { log.WithError(err).Error("RedactEvent error'd") return err } - if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil { + if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID().String(), msg.RedactedEventID); err != nil { log.WithFields(log.Fields{ - "room_id": msg.RedactedBecause.RoomID(), + "room_id": msg.RedactedBecause.RoomID().String(), "event_id": msg.RedactedBecause.EventID(), "redacted_event_id": msg.RedactedEventID, }).WithError(err).Warn("Failed to redact relations") @@ -213,7 +222,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( // Finally, work out if there are any more events missing. if len(missingEventIDs) > 0 { eventsReq := &api.QueryEventsByIDRequest{ - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), EventIDs: missingEventIDs, } eventsRes := &api.QueryEventsByIDResponse{} @@ -237,35 +246,30 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( ev, err := s.updateStateEvent(ev) if err != nil { - sentry.CaptureException(err) return err } for i := range addsStateEvents { addsStateEvents[i], err = s.updateStateEvent(addsStateEvents[i]) if err != nil { - sentry.CaptureException(err) return err } } if msg.RewritesState { - if err = s.db.PurgeRoomState(ctx, ev.RoomID()); err != nil { - sentry.CaptureException(err) + if err = s.db.PurgeRoomState(ctx, ev.RoomID().String()); err != nil { return fmt.Errorf("s.db.PurgeRoom: %w", err) } } - pduPos, err := s.db.WriteEvent( - ctx, - ev, - addsStateEvents, - msg.AddsStateEventIDs, - msg.RemovesStateEventIDs, - msg.TransactionID, - false, - msg.HistoryVisibility, - ) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID()) + if err != nil { + return err + } + + ev.UserID = *userID + + pduPos, err := s.db.WriteEvent(ctx, ev, addsStateEvents, msg.AddsStateEventIDs, msg.RemovesStateEventIDs, msg.TransactionID, false, msg.HistoryVisibility) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -286,7 +290,6 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil { log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos) - sentry.CaptureException(err) return err } @@ -299,7 +302,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } s.pduStream.Advance(pduPos) - s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos}) return nil } @@ -315,16 +318,14 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( // hack but until we have some better strategy for dealing with // old events in the sync API, this should at least prevent us // from confusing clients into thinking they've joined/left rooms. - pduPos, err := s.db.WriteEvent( - ctx, - ev, - []*rstypes.HeaderedEvent{}, - []string{}, // adds no state - []string{}, // removes no state - nil, // no transaction - ev.StateKey() != nil, // exclude from sync?, - msg.HistoryVisibility, - ) + + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID()) + if err != nil { + return err + } + ev.UserID = *userID + + pduPos, err := s.db.WriteEvent(ctx, ev, []*rstypes.HeaderedEvent{}, []string{}, []string{}, nil, ev.StateKey() != nil, msg.HistoryVisibility) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -344,7 +345,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( if err = s.db.UpdateRelations(ctx, ev); err != nil { log.WithFields(log.Fields{ - "room_id": ev.RoomID(), + "room_id": ev.RoomID().String(), "event_id": ev.EventID(), "type": ev.Type(), }).WithError(err).Warn("Failed to update relations") @@ -357,7 +358,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( } s.pduStream.Advance(pduPos) - s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos}) return nil } @@ -373,12 +374,20 @@ func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *rst // TODO: check that it's a join and not a profile change (means unmarshalling prev_content) if membership == spec.Join { // check it's a local join - if _, _, err := s.cfg.Matrix.SplitLocalID('@', *ev.StateKey()); err != nil { + if ev.StateKey() == nil { + return sp, fmt.Errorf("unexpected nil state_key") + } + + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey())) + if err != nil || userID == nil { + return sp, fmt.Errorf("failed getting userID for sender: %w", err) + } + if !s.cfg.Matrix.IsLocalServerName(userID.Domain()) { return sp, nil } // cancel any peeks for it - peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID(), *ev.StateKey()) + peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID().String(), *ev.StateKey()) if peekErr != nil { return sp, fmt.Errorf("s.db.DeletePeeks: %w", peekErr) } @@ -395,12 +404,19 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( if msg.Event.StateKey() == nil { return } - if _, _, err := s.cfg.Matrix.SplitLocalID('@', *msg.Event.StateKey()); err != nil { + + userID, err := s.rsAPI.QueryUserIDForSender(ctx, msg.Event.RoomID(), spec.SenderID(*msg.Event.StateKey())) + if err != nil || userID == nil { + return + } + if !s.cfg.Matrix.IsLocalServerName(userID.Domain()) { return } + + msg.Event.UserID = *userID + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { - sentry.CaptureException(err) // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ "event_id": msg.Event.EventID(), @@ -422,7 +438,6 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( // It's possible we just haven't heard of this invite yet, so // we should not panic if we try to retire it. if err != nil && err != sql.ErrNoRows { - sentry.CaptureException(err) // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ "event_id": msg.EventID, @@ -440,7 +455,25 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( // Notify any active sync requests that the invite has been retired. s.inviteStream.Advance(pduPos) - s.notifier.OnNewInvite(types.StreamingToken{InvitePosition: pduPos}, msg.TargetUserID) + validRoomID, err := spec.NewRoomID(msg.RoomID) + if err != nil { + log.WithFields(log.Fields{ + "event_id": msg.EventID, + "room_id": msg.RoomID, + log.ErrorKey: err, + }).Errorf("roomID is invalid") + return + } + userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, msg.TargetSenderID) + if err != nil || userID == nil { + log.WithFields(log.Fields{ + "event_id": msg.EventID, + "sender_id": msg.TargetSenderID, + log.ErrorKey: err, + }).Errorf("failed to find userID for sender") + return + } + s.notifier.OnNewInvite(types.StreamingToken{InvitePosition: pduPos}, userID.String()) } func (s *OutputRoomEventConsumer) onNewPeek( @@ -448,7 +481,6 @@ func (s *OutputRoomEventConsumer) onNewPeek( ) { sp, err := s.db.AddPeek(ctx, msg.RoomID, msg.UserID, msg.DeviceID) if err != nil { - sentry.CaptureException(err) // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ log.ErrorKey: err, @@ -497,6 +529,7 @@ func (s *OutputRoomEventConsumer) onPurgeRoom( } func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent) (*rstypes.HeaderedEvent, error) { + event.StateKeyResolved = event.StateKey() if event.StateKey() == nil { return event, nil } @@ -509,21 +542,38 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent) var succeeded bool defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + sKeyUser := "" + if stateKey != "" { + var sku *spec.UserID + sku, err = s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), spec.SenderID(stateKey)) + if err == nil && sku != nil { + sKeyUser = sku.String() + event.StateKeyResolved = &sKeyUser + } + } + prevEvent, err := snapshot.GetStateEvent( - s.ctx, event.RoomID(), event.Type(), stateKey, + s.ctx, event.RoomID().String(), event.Type(), sKeyUser, ) if err != nil { return event, err } + userID, err := s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), event.SenderID()) + if err != nil { + return event, err + } + + event.UserID = *userID + if prevEvent == nil || prevEvent.EventID() == event.EventID() { return event, nil } - prev := types.PrevEventRef{ + prev := synctypes.PrevEventRef{ PrevContent: prevEvent.Content(), ReplacesState: prevEvent.EventID(), - PrevSender: prevEvent.Sender(), + PrevSenderID: string(prevEvent.SenderID()), } event.PDU, err = event.SetUnsigned(prev) @@ -537,7 +587,7 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio } e := fulltext.IndexElement{ EventID: ev.EventID(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StreamPosition: int64(pduPosition), } e.SetContentType(ev.Type()) diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index 7449b46472..48475327de 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -16,6 +16,7 @@ package internal import ( "context" + "fmt" "math" "time" @@ -101,13 +102,15 @@ func (ev eventVisibility) allowed() (allowed bool) { // ApplyHistoryVisibilityFilter applies the room history visibility filter on types.HeaderedEvents. // Returns the filtered events and an error, if any. +// +// This function assumes that all provided events are from the same room. func ApplyHistoryVisibilityFilter( ctx context.Context, syncDB storage.DatabaseTransaction, rsAPI api.SyncRoomserverAPI, events []*types.HeaderedEvent, alwaysIncludeEventIDs map[string]struct{}, - userID, endpoint string, + userID spec.UserID, endpoint string, ) ([]*types.HeaderedEvent, error) { if len(events) == 0 { return events, nil @@ -115,15 +118,26 @@ func ApplyHistoryVisibilityFilter( start := time.Now() // try to get the current membership of the user - membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID, math.MaxInt64) + membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID().String(), userID.String(), math.MaxInt64) if err != nil { return nil, err } // Get the mapping from eventID -> eventVisibility eventsFiltered := make([]*types.HeaderedEvent, 0, len(events)) - visibilities := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID()) + firstEvRoomID := events[0].RoomID() + senderID, err := rsAPI.QuerySenderIDForUser(ctx, firstEvRoomID, userID) + if err != nil { + return nil, err + } + visibilities := visibilityForEvents(ctx, rsAPI, events, senderID, firstEvRoomID) + for _, ev := range events { + // Validate same room assumption + if ev.RoomID().String() != firstEvRoomID.String() { + return nil, fmt.Errorf("events from different rooms supplied to ApplyHistoryVisibilityFilter") + } + evVis := visibilities[ev.EventID()] evVis.membershipCurrent = membershipCurrent // Always include specific state events for /sync responses @@ -133,26 +147,36 @@ func ApplyHistoryVisibilityFilter( continue } } + // NOTSPEC: Always allow user to see their own membership events (spec contains more "rules") - if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(userID) { - eventsFiltered = append(eventsFiltered, ev) - continue + if senderID != nil { + if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(*senderID)) { + eventsFiltered = append(eventsFiltered, ev) + continue + } } + // Always allow history evVis events on boundaries. This is done // by setting the effective evVis to the least restrictive // of the old vs new. // https://spec.matrix.org/v1.3/client-server-api/#server-behaviour-5 - if hisVis, err := ev.HistoryVisibility(); err == nil { - prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String() - oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)] - // if we can't get the previous history visibility, default to shared. - if !ok { - oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared] - } - // no OK check, since this should have been validated when setting the value - newPrio := historyVisibilityPriority[hisVis] - if oldPrio < newPrio { - evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis) + if ev.Type() == spec.MRoomHistoryVisibility { + hisVis, err := ev.HistoryVisibility() + + if err == nil && hisVis != "" { + prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String() + oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)] + // if we can't get the previous history visibility, default to shared. + if !ok { + oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared] + } + // no OK check, since this should have been validated when setting the value + newPrio := historyVisibilityPriority[hisVis] + if oldPrio < newPrio { + evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis) + } else { + evVis.visibility = hisVis + } } } // do the actual check @@ -166,13 +190,13 @@ func ApplyHistoryVisibilityFilter( } // visibilityForEvents returns a map from eventID to eventVisibility containing the visibility and the membership -// of `userID` at the given event. +// of `senderID` at the given event. If provided sender ID is nil, assume that membership is Leave // Returns an error if the roomserver can't calculate the memberships. func visibilityForEvents( ctx context.Context, rsAPI api.SyncRoomserverAPI, events []*types.HeaderedEvent, - userID, roomID string, + senderID *spec.SenderID, roomID spec.RoomID, ) map[string]eventVisibility { eventIDs := make([]string, len(events)) for i := range events { @@ -182,15 +206,13 @@ func visibilityForEvents( result := make(map[string]eventVisibility, len(eventIDs)) // get the membership events for all eventIDs - membershipResp := &api.QueryMembershipAtEventResponse{} - - err := rsAPI.QueryMembershipAtEvent(ctx, &api.QueryMembershipAtEventRequest{ - RoomID: roomID, - EventIDs: eventIDs, - UserID: userID, - }, membershipResp) - if err != nil { - logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") + var err error + membershipEvents := make(map[string]*types.HeaderedEvent) + if senderID != nil { + membershipEvents, err = rsAPI.QueryMembershipAtEvent(ctx, roomID, eventIDs, *senderID) + if err != nil { + logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") + } } // Create a map from eventID -> eventVisibility @@ -200,7 +222,7 @@ func visibilityForEvents( membershipAtEvent: spec.Leave, // default to leave, to not expose events by accident visibility: event.Visibility, } - ev, ok := membershipResp.Membership[eventID] + ev, ok := membershipEvents[eventID] if !ok || ev == nil { result[eventID] = vis continue diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go new file mode 100644 index 0000000000..24515bbb2d --- /dev/null +++ b/syncapi/internal/history_visibility_test.go @@ -0,0 +1,214 @@ +package internal + +import ( + "context" + "fmt" + "math" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "gotest.tools/v3/assert" +) + +type mockHisVisRoomserverAPI struct { + rsapi.RoomserverInternalAPI + events []*types.HeaderedEvent + roomID string +} + +func (s *mockHisVisRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, roomID spec.RoomID, eventIDs []string, senderID spec.SenderID) (map[string]*types.HeaderedEvent, error) { + if roomID.String() == s.roomID { + membershipMap := map[string]*types.HeaderedEvent{} + + for _, queriedEventID := range eventIDs { + for _, event := range s.events { + if event.EventID() == queriedEventID { + membershipMap[queriedEventID] = event + } + } + } + + return membershipMap, nil + } else { + return nil, fmt.Errorf("room not found: \"%v\"", roomID) + } +} + +func (s *mockHisVisRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderIDFromUserID(userID) + return &senderID, nil +} + +func (s *mockHisVisRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + userID := senderID.ToUserID() + if userID == nil { + return nil, fmt.Errorf("sender ID not user ID") + } + return userID, nil +} + +type mockDB struct { + storage.DatabaseTransaction + // user ID -> membership (i.e. 'join', 'leave', etc.) + currentMembership map[string]string + roomID string +} + +func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int64, error) { + if roomID == s.roomID { + membership, ok := s.currentMembership[userID] + if !ok { + return spec.Leave, math.MaxInt64, nil + } + return membership, math.MaxInt64, nil + } + + return "", 0, fmt.Errorf("room not found: \"%v\"", roomID) +} + +// Tests logic around history visibility boundaries +// +// Specifically that if a room's history visibility before or after a particular history visibility event +// allows them to see events (a boundary), then the history visibility event itself should be shown +// ( spec: https://spec.matrix.org/v1.8/client-server-api/#server-behaviour-5 ) +// +// This also aims to emulate "Only see history_visibility changes on bounadries" in sytest/tests/30rooms/30history-visibility.pl +func Test_ApplyHistoryVisbility_Boundaries(t *testing.T) { + ctx := context.Background() + + roomID := "!roomid:domain" + + creatorUserID := spec.NewUserIDOrPanic("@creator:domain", false) + otherUserID := spec.NewUserIDOrPanic("@other:domain", false) + roomVersion := gomatrixserverlib.RoomVersionV10 + roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVersion) + + eventsJSON := []struct { + id string + json string + }{ + {id: "$create-event", json: fmt.Sprintf(`{ + "type": "m.room.create", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"creator": "%v", "room_version": "%v"} + }`, roomID, creatorUserID.String(), creatorUserID.String(), roomVersion)}, + {id: "$creator-joined", json: fmt.Sprintf(`{ + "type": "m.room.member", "state_key": "%v", + "room_id": "%v", "sender": "%v", + "content": {"membership": "join"} + }`, creatorUserID.String(), roomID, creatorUserID.String())}, + {id: "$hisvis-1", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "shared"} + }`, roomID, creatorUserID.String())}, + {id: "$msg-1", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "1"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-2", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "joined"}, + "unsigned": {"prev_content": {"history_visibility": "shared"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-2", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "1"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-3", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "invited"}, + "unsigned": {"prev_content": {"history_visibility": "joined"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-3", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "2"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-4", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "shared"}, + "unsigned": {"prev_content": {"history_visibility": "invited"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-4", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "3"} + }`, roomID, creatorUserID.String())}, + {id: "$other-joined", json: fmt.Sprintf(`{ + "type": "m.room.member", "state_key": "%v", + "room_id": "%v", "sender": "%v", + "content": {"membership": "join"} + }`, otherUserID.String(), roomID, otherUserID.String())}, + } + + events := make([]*types.HeaderedEvent, len(eventsJSON)) + + hisVis := gomatrixserverlib.HistoryVisibilityShared + + for i, eventJSON := range eventsJSON { + pdu, err := roomVerImpl.NewEventFromTrustedJSONWithEventID(eventJSON.id, []byte(eventJSON.json), false) + if err != nil { + t.Fatalf("failed to prepare event %s for test: %s", eventJSON.id, err.Error()) + } + events[i] = &types.HeaderedEvent{PDU: pdu} + + // 'Visibility' should be the visibility of the room just before this event was sent + // (according to processRoomEvent in roomserver/internal/input/input_events.go) + events[i].Visibility = hisVis + if pdu.Type() == spec.MRoomHistoryVisibility { + newHisVis, err := pdu.HistoryVisibility() + if err != nil { + t.Fatalf("failed to prepare history visibility event: %s", err.Error()) + } + hisVis = newHisVis + } + } + + rsAPI := &mockHisVisRoomserverAPI{ + events: events, + roomID: roomID, + } + syncDB := &mockDB{ + roomID: roomID, + currentMembership: map[string]string{ + creatorUserID.String(): spec.Join, + otherUserID.String(): spec.Join, + }, + } + + filteredEvents, err := ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, events, nil, otherUserID, "hisVisTest") + if err != nil { + t.Fatalf("ApplyHistoryVisibility returned non-nil error: %s", err.Error()) + } + + filteredEventIDs := make([]string, len(filteredEvents)) + for i, event := range filteredEvents { + filteredEventIDs[i] = event.EventID() + } + + assert.DeepEqual(t, + []string{ + "$create-event", // Always see m.room.create + "$creator-joined", // Always see membership + "$hisvis-1", // Sets room to shared (technically the room is already shared since shared is default) + "$msg-1", // Room currently 'shared' + "$hisvis-2", // Room changed from 'shared' to 'joined', so boundary event and should be shared + // Other events hidden, as other is not joined yet + // hisvis-3 is also hidden, as it changes from joined to invited, neither of which is visible to other + "$hisvis-4", // Changes from 'invited' to 'shared', so is a boundary event and visible + "$msg-4", // Room is 'shared', so visible + "$other-joined", // other's membership + }, + filteredEventIDs, + ) +} diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go index ad5935cdc4..24ffcc0414 100644 --- a/syncapi/internal/keychange.go +++ b/syncapi/internal/keychange.go @@ -169,12 +169,20 @@ func TrackChangedUsers( if err != nil { return nil, nil, err } - for _, state := range stateRes.Rooms { + for roomID, state := range stateRes.Rooms { + validRoomID, roomErr := spec.NewRoomID(roomID) + if roomErr != nil { + continue + } for tuple, membership := range state { if membership != spec.Join { continue } - queryRes.UserIDsToCount[tuple.StateKey]-- + user, queryErr := rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(tuple.StateKey)) + if queryErr != nil || user == nil { + continue + } + queryRes.UserIDsToCount[user.String()]-- } } @@ -211,14 +219,22 @@ func TrackChangedUsers( if err != nil { return nil, left, err } - for _, state := range stateRes.Rooms { + for roomID, state := range stateRes.Rooms { + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + continue + } for tuple, membership := range state { if membership != spec.Join { continue } // new user who we weren't previously sharing rooms with if _, ok := queryRes.UserIDsToCount[tuple.StateKey]; !ok { - changed = append(changed, tuple.StateKey) // changed is returned + user, err := rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(tuple.StateKey)) + if err != nil || user == nil { + continue + } + changed = append(changed, user.String()) // changed is returned } } } diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 23c2ecbaac..56954cfa03 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -59,18 +59,22 @@ func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *userapi.Query func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *userapi.QuerySignaturesRequest, res *userapi.QuerySignaturesResponse) { } -type mockRoomserverAPI struct { +type keyChangeMockRoomserverAPI struct { api.RoomserverInternalAPI roomIDToJoinedMembers map[string][]string } +func (s *keyChangeMockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + // QueryRoomsForUser retrieves a list of room IDs matching the given query. -func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - return nil +func (s *keyChangeMockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + return nil, nil } // QueryBulkStateContent does a bulk query for state event content in the given rooms. -func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { +func (s *keyChangeMockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string) if req.AllowWildcards && len(req.StateTuples) == 1 && req.StateTuples[0].EventType == spec.MRoomMember && req.StateTuples[0].StateKey == "*" { for _, roomID := range req.RoomIDs { @@ -87,7 +91,7 @@ func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api. } // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. -func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { +func (s *keyChangeMockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { roomsToQuery := req.IncludeRoomIDs for roomID, members := range s.roomIDToJoinedMembers { exclude := false @@ -119,7 +123,7 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query // This is actually a database function, but seeing as we track the state inside the // *mockRoomserverAPI, we'll just comply with the interface here instead. -func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { +func (s *keyChangeMockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { commonUsers := []string{} for _, members := range s.roomIDToJoinedMembers { for _, member := range members { @@ -207,7 +211,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser}, "!another:room": {syncingUser}, @@ -230,7 +234,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {removeUser}, "!another:room": {syncingUser}, @@ -253,7 +257,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, existingUser}, "!another:room": {syncingUser, existingUser}, @@ -275,7 +279,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {existingUser}, "!another:room": {syncingUser, existingUser}, @@ -339,7 +343,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {syncingUser, existingUser}, }, @@ -365,7 +369,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) { syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser, newShareUser2}, newlyLeftRoom: {newlyLeftUser, newlyLeftUser2}, @@ -455,7 +459,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {newShareUser, newShareUser2}, "!another:room": {syncingUser}, diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index f764568598..07b80b1650 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -20,6 +20,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" @@ -36,7 +37,8 @@ import ( // the event, but the token has already advanced by the time they fetch it, resulting // in missed events. type Notifier struct { - lock *sync.RWMutex + lock *sync.RWMutex + rsAPI api.SyncRoomserverAPI // A map of RoomID => Set : Must only be accessed by the OnNewEvent goroutine roomIDToJoinedUsers map[string]*userIDSet // A map of RoomID => Set : Must only be accessed by the OnNewEvent goroutine @@ -55,8 +57,9 @@ type Notifier struct { // NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier() *Notifier { +func NewNotifier(rsAPI api.SyncRoomserverAPI) *Notifier { return &Notifier{ + rsAPI: rsAPI, roomIDToJoinedUsers: make(map[string]*userIDSet), roomIDToPeekingDevices: make(map[string]peekingDeviceSet), userDeviceStreams: make(map[string]map[string]*UserDeviceStream), @@ -99,31 +102,37 @@ func (n *Notifier) OnNewEvent( if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. - usersToNotify := n._joinedUsers(ev.RoomID()) + usersToNotify := n._joinedUsers(ev.RoomID().String()) // Map this event's room_id to a list of peeking devices, and wake them up. - peekingDevicesToNotify := n._peekingDevices(ev.RoomID()) + peekingDevicesToNotify := n._peekingDevices(ev.RoomID().String()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { - targetUserID := *ev.StateKey() - membership, err := ev.Membership() - if err != nil { + targetUserID, err := n.rsAPI.QueryUserIDForSender(context.Background(), ev.RoomID(), spec.SenderID(*ev.StateKey())) + if err != nil || targetUserID == nil { log.WithError(err).WithField("event_id", ev.EventID()).Errorf( - "Notifier.OnNewEvent: Failed to unmarshal member event", + "Notifier.OnNewEvent: Failed to find the userID for this event", ) } else { - // Keep the joined user map up-to-date - switch membership { - case spec.Invite: - usersToNotify = append(usersToNotify, targetUserID) - case spec.Join: - // Manually append the new user's ID so they get notified - // along all members in the room - usersToNotify = append(usersToNotify, targetUserID) - n._addJoinedUser(ev.RoomID(), targetUserID) - case spec.Leave: - fallthrough - case spec.Ban: - n._removeJoinedUser(ev.RoomID(), targetUserID) + membership, err := ev.Membership() + if err != nil { + log.WithError(err).WithField("event_id", ev.EventID()).Errorf( + "Notifier.OnNewEvent: Failed to unmarshal member event", + ) + } else { + // Keep the joined user map up-to-date + switch membership { + case spec.Invite: + usersToNotify = append(usersToNotify, targetUserID.String()) + case spec.Join: + // Manually append the new user's ID so they get notified + // along all members in the room + usersToNotify = append(usersToNotify, targetUserID.String()) + n._addJoinedUser(ev.RoomID().String(), targetUserID.String()) + case spec.Leave: + fallthrough + case spec.Ban: + n._removeJoinedUser(ev.RoomID().String(), targetUserID.String()) + } } } } diff --git a/syncapi/notifier/notifier_test.go b/syncapi/notifier/notifier_test.go index 36577a0ee7..f86301a060 100644 --- a/syncapi/notifier/notifier_test.go +++ b/syncapi/notifier/notifier_test.go @@ -22,9 +22,11 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/roomserver/api" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -105,9 +107,15 @@ func mustEqualPositions(t *testing.T, got, want types.StreamingToken) { } } +type TestRoomServer struct{ api.SyncRoomserverAPI } + +func (t *TestRoomServer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) pos, err := waitForEvents(n, newTestSyncRequest(alice, aliceDev, syncPositionVeryOld)) if err != nil { @@ -118,7 +126,7 @@ func TestImmediateNotification(t *testing.T) { // Test that new events to a joined room unblocks the request. func TestNewEventAndJoinedToRoom(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, @@ -144,7 +152,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { } func TestCorrectStream(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) stream := lockedFetchUserStream(n, bob, bobDev) if stream.UserID != bob { @@ -156,7 +164,7 @@ func TestCorrectStream(t *testing.T) { } func TestCorrectStreamWakeup(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) awoken := make(chan string) @@ -184,7 +192,7 @@ func TestCorrectStreamWakeup(t *testing.T) { // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, @@ -241,7 +249,7 @@ func TestEDUWakeup(t *testing.T) { // Test that all blocked requests get woken up on a new event. func TestMultipleRequestWakeup(t *testing.T) { - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, @@ -278,7 +286,7 @@ func TestMultipleRequestWakeup(t *testing.T) { func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // listen as bob. Make bob leave room. Make alice send event to room. // Make sure alice gets woken up only and not bob as well. - n := NewNotifier() + n := NewNotifier(&TestRoomServer{}) n.SetCurrentPosition(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index ac17d39d2b..b0c91c40b7 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -85,9 +85,16 @@ func Context( *filter.Rooms = append(*filter.Rooms, roomID) } + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Device UserID is invalid"), + } + } ctx := req.Context() membershipRes := roomserver.QueryMembershipForUserResponse{} - membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID} + membershipReq := roomserver.QueryMembershipForUserRequest{UserID: *userID, RoomID: roomID} if err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil { logrus.WithError(err).Error("unable to query membership") return util.JSONResponse{ @@ -131,7 +138,7 @@ func Context( // verify the user is allowed to see the context for this room/event startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, *userID, "context") if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return util.JSONResponse{ @@ -169,7 +176,7 @@ func Context( } startTime = time.Now() - eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, device.UserID) + eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, *userID) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return util.JSONResponse{ @@ -193,14 +200,20 @@ func Context( } } - eventsBeforeClient := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsBeforeFiltered), synctypes.FormatAll) - eventsAfterClient := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsAfterFiltered), synctypes.FormatAll) + eventsBeforeClient := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsBeforeFiltered), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + eventsAfterClient := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsAfterFiltered), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) newState := state if filter.LazyLoadMembers { allEvents := append(eventsBeforeFiltered, eventsAfterFiltered...) allEvents = append(allEvents, &requestedEvent) - evs := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(allEvents), synctypes.FormatAll) + evs := synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(allEvents), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) newState, err = applyLazyLoadMembers(ctx, device, snapshot, roomID, evs, lazyLoadCache) if err != nil { logrus.WithError(err).Error("unable to load membership events") @@ -211,12 +224,16 @@ func Context( } } - ev := synctypes.ToClientEvent(&requestedEvent, synctypes.FormatAll) + ev := synctypes.ToClientEventDefault(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, requestedEvent) response := ContextRespsonse{ Event: &ev, EventsAfter: eventsAfterClient, EventsBefore: eventsBeforeClient, - State: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(newState), synctypes.FormatAll), + State: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(newState), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }), } if len(response.State) > filter.Limit { @@ -240,7 +257,7 @@ func Context( func applyHistoryVisibilityOnContextEvents( ctx context.Context, snapshot storage.DatabaseTransaction, rsAPI roomserver.SyncRoomserverAPI, eventsBefore, eventsAfter []*rstypes.HeaderedEvent, - userID string, + userID spec.UserID, ) (filteredBefore, filteredAfter []*rstypes.HeaderedEvent, err error) { eventIDsBefore := make(map[string]struct{}, len(eventsBefore)) eventIDsAfter := make(map[string]struct{}, len(eventsAfter)) diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index 0d3d412f64..c089539f05 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -37,7 +37,7 @@ import ( func GetEvent( req *http.Request, device *userapi.Device, - roomID string, + rawRoomID string, eventID string, cfg *config.SyncAPI, syncDB storage.Database, @@ -47,7 +47,7 @@ func GetEvent( db, err := syncDB.NewDatabaseTransaction(ctx) logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": eventID, - "room_id": roomID, + "room_id": rawRoomID, }) if err != nil { logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed") @@ -57,6 +57,14 @@ func GetEvent( } } + roomID, err := spec.NewRoomID(rawRoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + events, err := db.Events(ctx, []string{eventID}) if err != nil { logger.WithError(err).Error("GetEvent: syncDB.Events failed") @@ -76,13 +84,22 @@ func GetEvent( } // If the request is coming from an appservice, get the user from the request - userID := device.UserID + rawUserID := device.UserID if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" { - userID = asUserID + rawUserID = asUserID + } + + userID, err := spec.NewUserID(rawUserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("invalid device.UserID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } } // Apply history visibility to determine if the user is allowed to view the event - events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event") + events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, *userID, "event") if err != nil { logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed") return util.JSONResponse{ @@ -101,8 +118,19 @@ func GetEvent( } } + clientEvent, err := synctypes.ToClientEvent(events[0], synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll), + JSON: *clientEvent, } } diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index 7d2e137d3e..e849adf6de 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -59,14 +59,21 @@ func GetMemberships( syncDB storage.Database, rsAPI api.SyncRoomserverAPI, joinedOnly bool, membership, notMembership *string, at string, ) util.JSONResponse { + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Device UserID is invalid"), + } + } queryReq := api.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: device.UserID, + UserID: *userID, } var queryRes api.QueryMembershipForUserResponse - if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") + if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil { + util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, @@ -144,7 +151,22 @@ func GetMemberships( JSON: spec.InternalServerError{}, } } - res.Joined[ev.Sender()] = joinedMember(content) + + userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID()) + if err != nil || userID == nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"), + } + } + res.Joined[userID.String()] = joinedMember(content) } return util.JSONResponse{ Code: http.StatusOK, @@ -153,6 +175,8 @@ func GetMemberships( } return util.JSONResponse{ Code: http.StatusOK, - JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll)}, + JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + })}, } } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 58f663d0ba..3333cb54dc 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -50,9 +50,11 @@ type messagesReq struct { from *types.TopologyToken to *types.TopologyToken device *userapi.Device + deviceUserID spec.UserID wasToProvided bool backwardOrdering bool filter *synctypes.RoomEventFilter + didBackfill bool } type messagesResp struct { @@ -76,6 +78,15 @@ func OnIncomingMessagesRequest( ) util.JSONResponse { var err error + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + // NewDatabaseTransaction is used here instead of NewDatabaseSnapshot as we // expect to be able to write to the database in response to a /messages // request that requires backfilling from the roomserver or federation. @@ -239,9 +250,10 @@ func OnIncomingMessagesRequest( filter: filter, backwardOrdering: backwardOrdering, device: device, + deviceUserID: *deviceUserID, } - clientEvents, start, end, err := mReq.retrieveEvents() + clientEvents, start, end, err := mReq.retrieveEvents(req.Context(), rsAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("mreq.retrieveEvents failed") return util.JSONResponse{ @@ -250,13 +262,20 @@ func OnIncomingMessagesRequest( } } + // If start and end are equal, we either reached the beginning or something else + // is wrong. If we have nothing to return set end to 0. + if start == end || len(clientEvents) == 0 { + end = types.TopologyToken{} + } + util.GetLogger(req.Context()).WithFields(logrus.Fields{ - "from": from.String(), - "to": to.String(), - "limit": filter.Limit, - "backwards": backwardOrdering, - "return_start": start.String(), - "return_end": end.String(), + "request_from": from.String(), + "request_to": to.String(), + "limit": filter.Limit, + "backwards": backwardOrdering, + "response_start": start.String(), + "response_end": end.String(), + "backfilled": mReq.didBackfill, }).Info("Responding") res := messagesResp{ @@ -273,14 +292,11 @@ func OnIncomingMessagesRequest( JSON: spec.InternalServerError{}, } } - res.State = append(res.State, synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(membershipEvents), synctypes.FormatAll)...) + res.State = append(res.State, synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(membershipEvents), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + })...) } - // If we didn't return any events, set the end to an empty string, so it will be omitted - // in the response JSON. - if len(res.Chunk) == 0 { - res.End = "" - } if fromStream != nil { res.StartStream = fromStream.String() } @@ -294,9 +310,13 @@ func OnIncomingMessagesRequest( } func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (resp api.QueryMembershipForUserResponse, err error) { + fullUserID, err := spec.NewUserID(userID, true) + if err != nil { + return resp, err + } req := api.QueryMembershipForUserRequest{ RoomID: roomID, - UserID: userID, + UserID: *fullUserID, } if err := rsAPI.QueryMembershipForUser(ctx, &req, &resp); err != nil { return api.QueryMembershipForUserResponse{}, err @@ -310,16 +330,18 @@ func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api. // homeserver in the room for older events. // Returns an error if there was an issue talking to the database or with the // remote homeserver. -func (r *messagesReq) retrieveEvents() ( +func (r *messagesReq) retrieveEvents(ctx context.Context, rsAPI api.SyncRoomserverAPI) ( clientEvents []synctypes.ClientEvent, start, end types.TopologyToken, err error, ) { + emptyToken := types.TopologyToken{} // Retrieve the events from the local database. - streamEvents, err := r.snapshot.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering) + streamEvents, _, end, err := r.snapshot.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering) if err != nil { err = fmt.Errorf("GetEventsInRange: %w", err) - return + return []synctypes.ClientEvent{}, *r.from, emptyToken, err } + end.Decrement() var events []*rstypes.HeaderedEvent util.GetLogger(r.ctx).WithFields(logrus.Fields{ @@ -333,32 +355,54 @@ func (r *messagesReq) retrieveEvents() ( // on the ordering), or we've reached a backward extremity. if len(streamEvents) == 0 { if events, err = r.handleEmptyEventsSlice(); err != nil { - return + return []synctypes.ClientEvent{}, *r.from, emptyToken, err } } else { if events, err = r.handleNonEmptyEventsSlice(streamEvents); err != nil { - return + return []synctypes.ClientEvent{}, *r.from, emptyToken, err } } // If we didn't get any event, we don't need to proceed any further. if len(events) == 0 { - return []synctypes.ClientEvent{}, *r.from, *r.to, nil + return []synctypes.ClientEvent{}, *r.from, emptyToken, nil } - // Get the position of the first and the last event in the room's topology. - // This position is currently determined by the event's depth, so we could - // also use it instead of retrieving from the database. However, if we ever - // change the way topological positions are defined (as depth isn't the most - // reliable way to define it), it would be easier and less troublesome to - // only have to change it in one place, i.e. the database. - start, end, err = r.getStartEnd(events) + // Apply room history visibility filter + startTime := time.Now() + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.deviceUserID, "messages") if err != nil { - return []synctypes.ClientEvent{}, *r.from, *r.to, err + return []synctypes.ClientEvent{}, *r.from, *r.to, nil + } + logrus.WithFields(logrus.Fields{ + "duration": time.Since(startTime), + "room_id": r.roomID, + "events_before": len(events), + "events_after": len(filteredEvents), + }).Debug("applied history visibility (messages)") + + // No events left after applying history visibility + if len(filteredEvents) == 0 { + return []synctypes.ClientEvent{}, *r.from, emptyToken, nil + } + + // If we backfilled in the process of getting events, we need + // to re-fetch the start/end positions + if r.didBackfill { + _, end, err = r.getStartEnd(filteredEvents) + if err != nil { + return []synctypes.ClientEvent{}, *r.from, *r.to, err + } } // Sort the events to ensure we send them in the right order. if r.backwardOrdering { + if events[len(events)-1].Type() == spec.MRoomCreate { + // NOTSPEC: We've hit the beginning of the room so there's really nowhere + // else to go. This seems to fix Element iOS from looping on /messages endlessly. + end = types.TopologyToken{} + } + // This reverses the array from old->new to new->old reversed := func(in []*rstypes.HeaderedEvent) []*rstypes.HeaderedEvent { out := make([]*rstypes.HeaderedEvent, len(in)) @@ -367,22 +411,14 @@ func (r *messagesReq) retrieveEvents() ( } return out } - events = reversed(events) - } - if len(events) == 0 { - return []synctypes.ClientEvent{}, *r.from, *r.to, nil + filteredEvents = reversed(filteredEvents) } - // Apply room history visibility filter - startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.device.UserID, "messages") - logrus.WithFields(logrus.Fields{ - "duration": time.Since(startTime), - "room_id": r.roomID, - "events_before": len(events), - "events_after": len(filteredEvents), - }).Debug("applied history visibility (messages)") - return synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(filteredEvents), synctypes.FormatAll), start, end, err + start = *r.from + + return synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(filteredEvents), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }), start, end, nil } func (r *messagesReq) getStartEnd(events []*rstypes.HeaderedEvent) (start, end types.TopologyToken, err error) { @@ -435,6 +471,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( if err != nil { return } + r.didBackfill = true } else { // If not, it means the slice was empty because we reached the room's // creation, so return an empty slice. @@ -484,13 +521,13 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent if err != nil { return } - + r.didBackfill = true // Append the PDUs to the list to send back to the client. events = append(events, pdus...) } // Append the events ve previously retrieved locally. - events = append(events, r.snapshot.StreamEventsToEvents(nil, streamEvents)...) + events = append(events, r.snapshot.StreamEventsToEvents(r.ctx, nil, streamEvents, r.rsAPI)...) sort.Sort(eventsByDepth(events)) return @@ -546,15 +583,17 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] if res.HistoryVisibility == "" { res.HistoryVisibility = gomatrixserverlib.HistoryVisibilityShared } - for i := range res.Events { + events := res.Events + for i := range events { + events[i].Visibility = res.HistoryVisibility _, err = r.db.WriteEvent( context.Background(), - res.Events[i], + events[i], []*rstypes.HeaderedEvent{}, []string{}, []string{}, nil, true, - res.HistoryVisibility, + events[i].Visibility, ) if err != nil { return nil, err @@ -562,14 +601,10 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] } // we may have got more than the requested limit so resize now - events := res.Events if len(events) > limit { // last `limit` events events = events[len(events)-limit:] } - for _, ev := range events { - ev.Visibility = res.HistoryVisibility - } return events, nil } diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go index 8374bf5b04..935ba83b30 100644 --- a/syncapi/routing/relations.go +++ b/syncapi/routing/relations.go @@ -43,9 +43,25 @@ func Relations( req *http.Request, device *userapi.Device, syncDB storage.Database, rsAPI api.SyncRoomserverAPI, - roomID, eventID, relType, eventType string, + rawRoomID, eventID, relType, eventType string, ) util.JSONResponse { - var err error + roomID, err := spec.NewRoomID(rawRoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + var from, to types.StreamPosition var limit int dir := req.URL.Query().Get("dir") @@ -93,7 +109,7 @@ func Relations( } var events []types.StreamEvent events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor( - req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit, + req.Context(), roomID.String(), eventID, relType, eventType, from, to, dir == "b", limit, ) if err != nil { return util.ErrorResponse(err) @@ -105,7 +121,7 @@ func Relations( } // Apply history visibility to the result events. - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, *userID, "relations") if err != nil { return util.ErrorResponse(err) } @@ -114,9 +130,16 @@ func Relations( // type if it was specified. res.Chunk = make([]synctypes.ClientEvent, 0, len(filteredEvents)) for _, event := range filteredEvents { + clientEvent, err := synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + }) + if err != nil { + util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent") + continue + } res.Chunk = append( res.Chunk, - synctypes.ToClientEvent(event.PDU, synctypes.FormatAll), + *clientEvent, ) } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 9ad0c04766..a837e1696e 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -43,6 +43,7 @@ func Setup( cfg *config.SyncAPI, lazyLoadCache caching.LazyLoadCache, fts fulltext.Indexer, + rateLimits *httputil.RateLimits, ) { v1unstablemux := csMux.PathPrefix("/{apiversion:(?:v1|unstable)}/").Subrouter() v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() @@ -53,6 +54,10 @@ func Setup( }, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/messages", httputil.MakeAuthAPI("room_messages", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + // not specced, but ensure we're rate limiting requests to this endpoint + if r := rateLimits.Limit(req, device); r != nil { + return *r + } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -171,7 +176,7 @@ func Setup( nb := req.FormValue("next_batch") nextBatch = &nb } - return Search(req, device, syncDB, fts, nextBatch) + return Search(req, device, syncDB, fts, nextBatch, rsAPI) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index b7191873ec..4a8be9f495 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/synctypes" @@ -38,7 +39,7 @@ import ( ) // nolint:gocyclo -func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts fulltext.Indexer, from *string) util.JSONResponse { +func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts fulltext.Indexer, from *string, rsAPI roomserverAPI.SyncRoomserverAPI) util.JSONResponse { start := time.Now() var ( searchReq SearchRequest @@ -204,11 +205,17 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts profileInfos := make(map[string]ProfileInfoResponse) for _, ev := range append(eventsBefore, eventsAfter...) { - profile, ok := knownUsersProfiles[event.Sender()] + userID, queryErr := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID()) + if queryErr != nil { + logrus.WithError(queryErr).WithField("sender_id", ev.SenderID()).Warn("failed to query userprofile") + continue + } + + profile, ok := knownUsersProfiles[userID.String()] if !ok { - stateEvent, err := snapshot.GetStateEvent(ctx, ev.RoomID(), spec.MRoomMember, ev.Sender()) - if err != nil { - logrus.WithError(err).WithField("user_id", event.Sender()).Warn("failed to query userprofile") + stateEvent, stateErr := snapshot.GetStateEvent(ctx, ev.RoomID().String(), spec.MRoomMember, string(ev.SenderID())) + if stateErr != nil { + logrus.WithError(stateErr).WithField("sender_id", event.SenderID()).Warn("failed to query userprofile") continue } if stateEvent == nil { @@ -218,28 +225,40 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts AvatarURL: gjson.GetBytes(stateEvent.Content(), "avatar_url").Str, DisplayName: gjson.GetBytes(stateEvent.Content(), "displayname").Str, } - knownUsersProfiles[event.Sender()] = profile + knownUsersProfiles[userID.String()] = profile } - profileInfos[ev.Sender()] = profile + profileInfos[userID.String()] = profile + } + + clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + util.GetLogger(req.Context()).WithError(err).WithField("senderID", event.SenderID()).Error("Failed converting to ClientEvent") + continue } results = append(results, Result{ Context: SearchContextResponse{ - Start: startToken.String(), - End: endToken.String(), - EventsAfter: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsAfter), synctypes.FormatSync), - EventsBefore: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsBefore), synctypes.FormatSync), - ProfileInfo: profileInfos, + Start: startToken.String(), + End: endToken.String(), + EventsAfter: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsAfter), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + }), + EventsBefore: synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(eventsBefore), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + }), + ProfileInfo: profileInfos, }, Rank: eventScore[event.EventID()].Score, - Result: synctypes.ToClientEvent(event, synctypes.FormatAll), + Result: *clientEvent, }) - roomGroup := groups[event.RoomID()] + roomGroup := groups[event.RoomID().String()] roomGroup.Results = append(roomGroup.Results, event.EventID()) - groups[event.RoomID()] = roomGroup - if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { + groups[event.RoomID().String()] = roomGroup + if _, ok := stateForRooms[event.RoomID().String()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { stateFilter := synctypes.DefaultStateFilter() - state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil) + state, err := snapshot.CurrentState(ctx, event.RoomID().String(), &stateFilter, nil) if err != nil { logrus.WithError(err).Error("unable to get current state") return util.JSONResponse{ @@ -247,7 +266,9 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts JSON: spec.InternalServerError{}, } } - stateForRooms[event.RoomID()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync) + stateForRooms[event.RoomID().String()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + }) } } @@ -291,19 +312,19 @@ func contextEvents( roomFilter *synctypes.RoomEventFilter, searchReq SearchRequest, ) ([]*types.HeaderedEvent, []*types.HeaderedEvent, error) { - id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID()) + id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID().String(), event.EventID()) if err != nil { logrus.WithError(err).Error("failed to query context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit - eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter) + eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID().String(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query before context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit - _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter) + _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID().String(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query after context event") return nil, nil, err diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go index 1cc95a873a..a983bb7b5b 100644 --- a/syncapi/routing/search_test.go +++ b/syncapi/routing/search_test.go @@ -2,6 +2,7 @@ package routing import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -9,6 +10,7 @@ import ( "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" + rsapi "github.com/matrix-org/dendrite/roomserver/api" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/synctypes" @@ -21,6 +23,12 @@ import ( "github.com/stretchr/testify/assert" ) +type FakeSyncRoomserverAPI struct{ rsapi.SyncRoomserverAPI } + +func (f *FakeSyncRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func TestSearch(t *testing.T) { alice := test.NewUser(t) aliceDevice := userapi.Device{UserID: alice.ID} @@ -222,6 +230,7 @@ func TestSearch(t *testing.T) { stateEvents = append(stateEvents, x) stateEventIDs = append(stateEventIDs, x.EventID()) } + x.StateKeyResolved = x.StateKey() sp, err = db.WriteEvent(processCtx.Context(), x, stateEvents, stateEventIDs, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared) assert.NoError(t, err) if x.Type() != "m.room.message" { @@ -229,7 +238,7 @@ func TestSearch(t *testing.T) { } elements = append(elements, fulltext.IndexElement{ EventID: x.EventID(), - RoomID: x.RoomID(), + RoomID: x.RoomID().String(), Content: string(x.Content()), ContentType: x.Type(), StreamPosition: int64(sp), @@ -247,7 +256,7 @@ func TestSearch(t *testing.T) { assert.NoError(t, err) req := httptest.NewRequest(http.MethodPost, "/", reqBody) - res := Search(req, tc.device, db, fts, tc.from) + res := Search(req, tc.device, db, fts, tc.from, &FakeSyncRoomserverAPI{}) if !tc.wantOK && !res.Is2xx() { return } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 302b9bad88..97c781b9b1 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -44,8 +44,8 @@ type DatabaseTransaction interface { MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) CurrentState(ctx context.Context, roomID string, stateFilterPart *synctypes.StateFilter, excludeEventIDs []string) ([]*rstypes.HeaderedEvent, error) - GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter) ([]types.StateDelta, []string, error) - GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter) ([]types.StateDelta, []string, error) + GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter, rsAPI api.SyncRoomserverAPI) ([]types.StateDelta, []string, error) + GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter, rsAPI api.SyncRoomserverAPI) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) GetRoomSummary(ctx context.Context, roomID, userID string) (summary *types.Summary, err error) @@ -81,8 +81,11 @@ type DatabaseTransaction interface { // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *synctypes.EventFilter) (map[string][]string, types.StreamPosition, error) - // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last. - GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *synctypes.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) + // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. + // If backwardsOrdering is true, the most recent event must be first, else last. + // Returns the filtered StreamEvents on success. Returns **unfiltered** StreamEvents and ErrNoEventsForFilter if + // the provided filter removed all events, this can be used to still calculate the start/end position. (e.g for `/messages`) + GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *synctypes.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, start, end types.TopologyToken, err error) // EventPositionInTopology returns the depth and stream position of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events. @@ -90,7 +93,7 @@ type DatabaseTransaction interface { // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. - StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*rstypes.HeaderedEvent + StreamEventsToEvents(ctx context.Context, device *userapi.Device, in []types.StreamEvent, rsAPI api.SyncRoomserverAPI) []*rstypes.HeaderedEvent // SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns the // relevant events within the given ranges for the supplied user ID and device ID. SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, from, to types.StreamPosition) (pos types.StreamPosition, events []types.SendToDeviceEvent, err error) @@ -104,7 +107,7 @@ type DatabaseTransaction interface { // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. - SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) @@ -174,7 +177,7 @@ type Database interface { // goes wrong. PutFilter(ctx context.Context, localpart string, filter *synctypes.Filter) (string, error) // RedactEvent wipes an event in the database and sets the unsigned.redacted_because key to the redaction event - RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *rstypes.HeaderedEvent) error + RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *rstypes.HeaderedEvent, querier api.QuerySenderIDAPI) error // StoreReceipt stores new receipt events StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp spec.Timestamp) (pos types.StreamPosition, err error) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 0cc9637318..b0148bef54 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -340,12 +340,12 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), - event.Sender(), + event.UserID.String(), containsURL, - *event.StateKey(), + *event.StateKeyResolved, headeredJSON, membership, addedAt, diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 267209bba6..1f46cd09db 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -99,9 +99,9 @@ func (s *inviteEventsStatements) InsertInviteEvent( err = sqlutil.TxStmt(txn, s.insertInviteEventStmt).QueryRowContext( ctx, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), - *inviteEvent.StateKey(), + inviteEvent.UserID.String(), headeredJSON, ).Scan(&streamPos) return diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 3905f9abb2..4fe4260da8 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -108,8 +108,8 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), - *event.StateKey(), + event.RoomID().String(), + event.StateKeyResolved, membership, event.EventID(), streamPos, @@ -131,7 +131,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 3aadbccf85..b2d1911113 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -334,7 +334,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } - needSet := stateNeeded[ev.RoomID()] + needSet := stateNeeded[ev.RoomID().String()] if needSet == nil { // make set if required needSet = make(map[string]bool) } @@ -344,7 +344,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for _, id := range addIDs { needSet[id] = true } - stateNeeded[ev.RoomID()] = needSet + stateNeeded[ev.RoomID().String()] = needSet ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ @@ -403,11 +403,11 @@ func (s *outputRoomEventsStatements) InsertEvent( stmt := sqlutil.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), headeredJSON, event.Type(), - event.Sender(), + event.UserID.String(), containsURL, pq.StringArray(addState), pq.StringArray(removeState), diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 7140a92fc1..2158d99ecb 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -48,14 +48,14 @@ const insertEventInTopologySQL = "" + " RETURNING topological_position" const selectEventIDsInRangeASCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + + "SELECT event_id, topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position >= $5)" + ") ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + + "SELECT event_id, topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + @@ -107,18 +107,19 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition, ) (topoPos types.StreamPosition, err error) { err = sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).QueryRowContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), pos, + ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos, ).Scan(&topoPos) return } // SelectEventIDsInRange selects the IDs of events which positions are within a -// given range in a given room's topological order. +// given range in a given room's topological order. Returns the start/end topological tokens for +// the returned eventIDs. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool, -) (eventIDs []string, err error) { +) (eventIDs []string, start, end types.TopologyToken, err error) { // Decide on the selection's order according to whether chronological order // is requested or not. var stmt *sql.Stmt @@ -132,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( rows, err := stmt.QueryContext(ctx, roomID, minDepth, maxDepth, maxDepth, maxStreamPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. - return []string{}, nil + return []string{}, start, end, nil } else if err != nil { return } @@ -140,14 +141,23 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( // Return the IDs. var eventID string + var token types.TopologyToken + var tokens []types.TopologyToken for rows.Next() { - if err = rows.Scan(&eventID); err != nil { + if err = rows.Scan(&eventID, &token.Depth, &token.PDUPosition); err != nil { return } eventIDs = append(eventIDs, eventID) + tokens = append(tokens, token) } - return eventIDs, rows.Err() + // The values are already ordered by SQL, so we can use them as is. + if len(tokens) > 0 { + start = tokens[0] + end = tokens[len(tokens)-1] + } + + return eventIDs, start, end, rows.Err() } // SelectPositionInTopology returns the position of a given event in the diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 9f9de28d97..2105bcae25 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -36,7 +36,7 @@ type SyncServerDatasource struct { } // NewDatabase creates a new sync server database -func NewDatabase(ctx context.Context, cm sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { +func NewDatabase(ctx context.Context, cm *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, d.writer, err = cm.Connection(dbProperties); err != nil { diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index ecfd418fc7..923ead9bdd 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -99,7 +99,41 @@ func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*rstypes.He // We don't include a device here as we only include transaction IDs in // incremental syncs. - return d.StreamEventsToEvents(nil, streamEvents), nil + return d.StreamEventsToEvents(ctx, nil, streamEvents, nil), nil +} + +func (d *Database) StreamEventsToEvents(ctx context.Context, device *userapi.Device, in []types.StreamEvent, rsAPI api.SyncRoomserverAPI) []*rstypes.HeaderedEvent { + out := make([]*rstypes.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[i].HeaderedEvent + if device != nil && in[i].TransactionID != nil { + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + continue + } + deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, in[i].RoomID(), *userID) + if err != nil || deviceSenderID == nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + continue + } + if *deviceSenderID == in[i].SenderID() && device.SessionID == in[i].TransactionID.SessionID { + err := out[i].SetUnsignedField( + "transaction_id", in[i].TransactionID.TransactionID, + ) + if err != nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + } + } + } + } + return out } // AddInviteEvent stores a new invite event for a user. @@ -190,32 +224,12 @@ func (d *Database) UpsertAccountData( return } -func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*rstypes.HeaderedEvent { - out := make([]*rstypes.HeaderedEvent, len(in)) - for i := 0; i < len(in); i++ { - out[i] = in[i].HeaderedEvent - if device != nil && in[i].TransactionID != nil { - if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { - err := out[i].SetUnsignedField( - "transaction_id", in[i].TransactionID.TransactionID, - ) - if err != nil { - logrus.WithFields(logrus.Fields{ - "event_id": out[i].EventID(), - }).WithError(err).Warnf("Failed to add transaction ID to event") - } - } - } - } - return out -} - // handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. // This function should always be called within a sqlutil.Writer for safety in SQLite. func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *rstypes.HeaderedEvent) error { - if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID()); err != nil { return err } @@ -236,7 +250,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e // If the event is missing, consider it a backward extremity. if !found { - if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID(), eID); err != nil { return err } } @@ -343,7 +357,7 @@ func (d *Database) PutFilter( return filterID, err } -func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *rstypes.HeaderedEvent) error { +func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *rstypes.HeaderedEvent, querier api.QuerySenderIDAPI) error { redactedEvents, err := d.Events(ctx, []string{redactedEventID}) if err != nil { return err @@ -354,7 +368,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda } eventToRedact := redactedEvents[0].PDU redactionEvent := redactedBecause.PDU - if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil { + if err = eventutil.RedactEvent(ctx, redactionEvent, eventToRedact, querier); err != nil { return err } @@ -405,7 +419,7 @@ func (d *Database) fetchStateEvents( } // we know we got them all otherwise an error would've been returned, so just loop the events for _, ev := range evs { - roomID := ev.RoomID() + roomID := ev.RoomID().String() stateBetween[roomID] = append(stateBetween[roomID], ev) } } @@ -493,8 +507,20 @@ func (d *Database) CleanSendToDeviceUpdates( // getMembershipFromEvent returns the value of content.membership iff the event is a state event // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev gomatrixserverlib.PDU, userID string) (string, string) { - if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) { +func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userID string, rsAPI api.SyncRoomserverAPI) (string, string) { + if ev.StateKey() == nil || *ev.StateKey() == "" { + return "", "" + } + fullUser, err := spec.NewUserID(userID, true) + if err != nil { + return "", "" + } + senderID, err := rsAPI.QuerySenderIDForUser(ctx, ev.RoomID(), *fullUser) + if err != nil || senderID == nil { + return "", "" + } + + if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(*senderID)) { return "", "" } membership, err := ev.Membership() @@ -557,7 +583,7 @@ func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types return d.Presence.GetPresenceForUsers(ctx, nil, userIDs) } -func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { +func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) { return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } @@ -589,7 +615,7 @@ func (d *Database) UpdateRelations(ctx context.Context, event *rstypes.HeaderedE default: return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.Relations.InsertRelation( - ctx, txn, event.RoomID(), content.Relations.EventID, + ctx, txn, event.RoomID().String(), content.Relations.EventID, event.EventID(), event.Type(), content.Relations.RelationType, ) }) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index f2b1c58dc9..cd17fdc699 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -10,6 +10,7 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" @@ -186,7 +187,7 @@ func (d *DatabaseTransaction) Events(ctx context.Context, eventIDs []string) ([] // We don't include a device here as we only include transaction IDs in // incremental syncs. - return d.StreamEventsToEvents(nil, streamEvents), nil + return d.StreamEventsToEvents(ctx, nil, streamEvents, nil), nil } func (d *DatabaseTransaction) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { @@ -236,7 +237,7 @@ func (d *DatabaseTransaction) GetEventsInTopologicalRange( roomID string, filter *synctypes.RoomEventFilter, backwardOrdering bool, -) (events []types.StreamEvent, err error) { +) (events []types.StreamEvent, start, end types.TopologyToken, err error) { var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition if backwardOrdering { // Backward ordering means the 'from' token has a higher depth than the 'to' token @@ -254,7 +255,7 @@ func (d *DatabaseTransaction) GetEventsInTopologicalRange( // Select the event IDs from the defined range. var eIDs []string - eIDs, err = d.Topology.SelectEventIDsInRange( + eIDs, start, end, err = d.Topology.SelectEventIDsInRange( ctx, d.txn, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering, ) if err != nil { @@ -263,6 +264,10 @@ func (d *DatabaseTransaction) GetEventsInTopologicalRange( // Retrieve the events' contents using their IDs. events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eIDs, filter, true) + if err != nil { + return + } + return } @@ -325,7 +330,7 @@ func (d *DatabaseTransaction) GetBackwardTopologyPos( func (d *DatabaseTransaction) GetStateDeltas( ctx context.Context, device *userapi.Device, r types.Range, userID string, - stateFilter *synctypes.StateFilter, + stateFilter *synctypes.StateFilter, rsAPI api.SyncRoomserverAPI, ) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -417,7 +422,7 @@ func (d *DatabaseTransaction) GetStateDeltas( if !peek.Deleted { deltas = append(deltas, types.StateDelta{ Membership: spec.Peek, - StateEvents: d.StreamEventsToEvents(device, state[peek.RoomID]), + StateEvents: d.StreamEventsToEvents(ctx, device, state[peek.RoomID], rsAPI), RoomID: peek.RoomID, }) } @@ -429,7 +434,7 @@ func (d *DatabaseTransaction) GetStateDeltas( for _, ev := range stateStreamEvents { // Look for our membership in the state events and skip over any // membership events that are not related to us. - membership, prevMembership := getMembershipFromEvent(ev.PDU, userID) + membership, prevMembership := getMembershipFromEvent(ctx, ev.PDU, userID, rsAPI) if membership == "" { continue } @@ -462,7 +467,7 @@ func (d *DatabaseTransaction) GetStateDeltas( deltas = append(deltas, types.StateDelta{ Membership: membership, MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateFiltered[roomID]), + StateEvents: d.StreamEventsToEvents(ctx, device, stateFiltered[roomID], rsAPI), RoomID: roomID, }) break @@ -474,7 +479,7 @@ func (d *DatabaseTransaction) GetStateDeltas( for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, types.StateDelta{ Membership: spec.Join, - StateEvents: d.StreamEventsToEvents(device, stateFiltered[joinedRoomID]), + StateEvents: d.StreamEventsToEvents(ctx, device, stateFiltered[joinedRoomID], rsAPI), RoomID: joinedRoomID, NewlyJoined: newlyJoinedRooms[joinedRoomID], }) @@ -490,7 +495,7 @@ func (d *DatabaseTransaction) GetStateDeltas( func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( ctx context.Context, device *userapi.Device, r types.Range, userID string, - stateFilter *synctypes.StateFilter, + stateFilter *synctypes.StateFilter, rsAPI api.SyncRoomserverAPI, ) ([]types.StateDelta, []string, error) { // Look up all memberships for the user. We only care about rooms that a // user has ever interacted with — joined to, kicked/banned from, left. @@ -531,7 +536,7 @@ func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( } deltas[peek.RoomID] = types.StateDelta{ Membership: spec.Peek, - StateEvents: d.StreamEventsToEvents(device, s), + StateEvents: d.StreamEventsToEvents(ctx, device, s, rsAPI), RoomID: peek.RoomID, } } @@ -555,12 +560,12 @@ func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership, _ := getMembershipFromEvent(ev.PDU, userID); membership != "" { + if membership, _ := getMembershipFromEvent(ctx, ev.PDU, userID, rsAPI); membership != "" { if membership != spec.Join { // We've already added full state for all joined rooms above. deltas[roomID] = types.StateDelta{ Membership: membership, MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + StateEvents: d.StreamEventsToEvents(ctx, device, stateStreamEvents, rsAPI), RoomID: roomID, } } @@ -581,7 +586,7 @@ func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( } deltas[joinedRoomID] = types.StateDelta{ Membership: spec.Join, - StateEvents: d.StreamEventsToEvents(device, s), + StateEvents: d.StreamEventsToEvents(ctx, device, s, rsAPI), RoomID: joinedRoomID, } } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index b49c2f7015..132bd80c8f 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -130,7 +130,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( if pos == 0 { pos = r.High() } - return data, pos, nil + return data, pos, rows.Err() } func (s *accountDataStatements) SelectMaxAccountDataID( diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 1b8632eb67..78b2e397cf 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -339,12 +339,12 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), - event.Sender(), + event.UserID.String(), containsURL, - *event.StateKey(), + *event.StateKeyResolved, headeredJSON, membership, addedAt, diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 347523cf79..ebb469d24a 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -106,9 +106,9 @@ func (s *inviteEventsStatements) InsertInviteEvent( _, err = stmt.ExecContext( ctx, streamPos, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), - *inviteEvent.StateKey(), + inviteEvent.UserID.String(), headeredJSON, ) return diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index c09fa1510d..a1b16306c1 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -111,8 +111,8 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), - *event.StateKey(), + event.RoomID().String(), + event.StateKeyResolved, membership, event.EventID(), streamPos, @@ -134,7 +134,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index d63e760672..93caee8063 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -254,7 +254,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } - needSet := stateNeeded[ev.RoomID()] + needSet := stateNeeded[ev.RoomID().String()] if needSet == nil { // make set if required needSet = make(map[string]bool) } @@ -264,7 +264,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for _, id := range addIDs { needSet[id] = true } - stateNeeded[ev.RoomID()] = needSet + stateNeeded[ev.RoomID().String()] = needSet ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ @@ -344,11 +344,11 @@ func (s *outputRoomEventsStatements) InsertEvent( _, err = insertStmt.ExecContext( ctx, streamPos, - event.RoomID(), + event.RoomID().String(), event.EventID(), headeredJSON, event.Type(), - event.Sender(), + event.UserID.String(), containsURL, string(addStateJSON), string(removeStateJSON), diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 68b75f5b18..36967d1e73 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -44,14 +44,14 @@ const insertEventInTopologySQL = "" + " ON CONFLICT DO NOTHING" const selectEventIDsInRangeASCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + + "SELECT event_id, topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position >= $5)" + ") ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + + "SELECT event_id, topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + @@ -106,16 +106,20 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition, ) (types.StreamPosition, error) { _, err := sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), pos, + ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos, ) return types.StreamPosition(event.Depth()), err } +// SelectEventIDsInRange selects the IDs of events which positions are within a +// given range in a given room's topological order. Returns the start/end topological tokens for +// the returned eventIDs. +// Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool, -) (eventIDs []string, err error) { +) (eventIDs []string, start, end types.TopologyToken, err error) { // Decide on the selection's order according to whether chronological order // is requested or not. var stmt *sql.Stmt @@ -129,18 +133,27 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( rows, err := stmt.QueryContext(ctx, roomID, minDepth, maxDepth, maxDepth, maxStreamPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. - return []string{}, nil + return []string{}, start, end, nil } else if err != nil { return } // Return the IDs. var eventID string + var token types.TopologyToken + var tokens []types.TopologyToken for rows.Next() { - if err = rows.Scan(&eventID); err != nil { + if err = rows.Scan(&eventID, &token.Depth, &token.PDUPosition); err != nil { return } eventIDs = append(eventIDs, eventID) + tokens = append(tokens, token) + } + + // The values are already ordered by SQL, so we can use them as is. + if len(tokens) > 0 { + start = tokens[0] + end = tokens[len(tokens)-1] } return diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 3f1ca355e3..e1372f10bf 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -36,7 +36,7 @@ type SyncServerDatasource struct { // NewDatabase creates a new sync server database // nolint: gocyclo -func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { +func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 8714ec5e24..e05f9d9115 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -28,7 +28,7 @@ import ( ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { +func NewSyncServerDatasource(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(ctx, conMan, dbProperties) diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 08ca99a766..ce7ca3fc7d 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage" @@ -19,6 +20,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" ) var ctx = context.Background() @@ -41,6 +43,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*rstypes.Header var addStateEventIDs []string var removeStateEventIDs []string if ev.StateKey() != nil { + ev.StateKeyResolved = ev.StateKey() addStateEvents = append(addStateEvents, ev) addStateEventIDs = append(addStateEventIDs, ev.EventID()) } @@ -210,12 +213,48 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { // backpaginate 5 messages starting at the latest position. filter := &synctypes.RoomEventFilter{Limit: 5} - paginatedEvents, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) + paginatedEvents, start, end, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) if err != nil { t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) } - gots := snapshot.StreamEventsToEvents(nil, paginatedEvents) + gots := snapshot.StreamEventsToEvents(context.Background(), nil, paginatedEvents, nil) test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:])) + assert.Equal(t, types.TopologyToken{Depth: 15, PDUPosition: 15}, start) + assert.Equal(t, types.TopologyToken{Depth: 11, PDUPosition: 11}, end) + }) + }) +} + +// The purpose of this test is to ensure that backfilling returns no start/end if a given filter removes +// all events. +func TestGetEventsInRangeWithTopologyTokenNoEventsForFilter(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser(t) + r := test.NewRoom(t, alice) + for i := 0; i < 10; i++ { + r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)}) + } + events := r.Events() + _ = MustWriteEvents(t, db, events) + + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + from := types.TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64} + t.Logf("max topo pos = %+v", from) + // head towards the beginning of time + to := types.TopologyToken{} + + // backpaginate 20 messages starting at the latest position. + notTypes := []string{spec.MRoomRedaction} + senders := []string{alice.ID} + filter := &synctypes.RoomEventFilter{Limit: 20, NotTypes: ¬Types, Senders: &senders} + paginatedEvents, start, end, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) + assert.NoError(t, err) + assert.Equal(t, 0, len(paginatedEvents)) + // Even if we didn't get anything back due to the filter, we should still have start/end + assert.Equal(t, types.TopologyToken{Depth: 15, PDUPosition: 15}, start) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 1}, end) }) }) } @@ -978,3 +1017,52 @@ func TestRecentEvents(t *testing.T) { } }) } + +type FakeQuerier struct { + api.QuerySenderIDAPI +} + +func (f *FakeQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + +func TestRedaction(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + redactedEvent := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hi"}) + redactionEvent := room.CreateEvent(t, alice, spec.MRoomRedaction, map[string]string{"redacts": redactedEvent.EventID()}) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + t.Cleanup(close) + MustWriteEvents(t, db, room.Events()) + + err := db.RedactEvent(context.Background(), redactedEvent.EventID(), redactionEvent, &FakeQuerier{}) + if err != nil { + t.Fatal(err) + } + + evs, err := db.Events(context.Background(), []string{redactedEvent.EventID()}) + if err != nil { + t.Fatal(err) + } + + if len(evs) != 1 { + t.Fatalf("expected 1 event, got %d", len(evs)) + } + + // check a few fields which shouldn't be there in unsigned + authEvs := gjson.GetBytes(evs[0].Unsigned(), "redacted_because.auth_events") + if authEvs.Exists() { + t.Error("unexpected auth_events in redacted event") + } + prevEvs := gjson.GetBytes(evs[0].Unsigned(), "redacted_because.prev_events") + if prevEvs.Exists() { + t.Error("unexpected auth_events in redacted event") + } + depth := gjson.GetBytes(evs[0].Unsigned(), "redacted_because.depth") + if depth.Exists() { + t.Error("unexpected auth_events in redacted event") + } + }) +} diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go index 7d4ec812ce..2df111a267 100644 --- a/syncapi/storage/tables/current_room_state_test.go +++ b/syncapi/storage/tables/current_room_state_test.go @@ -54,7 +54,13 @@ func TestCurrentRoomStateTable(t *testing.T) { events := room.CurrentState() err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { for i, ev := range events { - err := tab.UpsertRoomState(ctx, txn, ev, nil, types.StreamPosition(i)) + ev.StateKeyResolved = ev.StateKey() + userID, err := spec.NewUserID(string(ev.SenderID()), true) + if err != nil { + return err + } + ev.UserID = *userID + err = tab.UpsertRoomState(ctx, txn, ev, nil, types.StreamPosition(i)) if err != nil { return fmt.Errorf("failed to UpsertRoomState: %w", err) } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 854292bd2c..45117d6d30 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -89,11 +89,11 @@ type Topology interface { // InsertEventInTopology inserts the given event in the room's topology, based on the event's depth. // `pos` is the stream position of this event in the events table, and is used to order events which have the same depth. InsertEventInTopology(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition) (topoPos types.StreamPosition, err error) - // SelectEventIDsInRange selects the IDs of events whose depths are within a given range in a given room's topological order. - // Events with `minDepth` are *exclusive*, as is the event which has exactly `minDepth`,`maxStreamPos`. + // SelectEventIDsInRange selects the IDs and the topological position of events whose depths are within a given range in a given room's topological order. + // Events with `minDepth` are *exclusive*, as is the event which has exactly `minDepth`,`maxStreamPos`. Returns the eventIDs and start/end topological tokens. // `maxStreamPos` is only used when events have the same depth as `maxDepth`, which results in events less than `maxStreamPos` being returned. // Returns an empty slice if no events match the given range. - SelectEventIDsInRange(ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool) (eventIDs []string, err error) + SelectEventIDsInRange(ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool) (eventIDs []string, start, end types.TopologyToken, err error) // SelectPositionInTopology returns the depth and stream position of a given event in the topology of the room it belongs to. SelectPositionInTopology(ctx context.Context, txn *sql.Tx, eventID string) (depth, spos types.StreamPosition, err error) // SelectStreamToTopologicalPosition converts a stream position to a topological position by finding the nearest topological position in the room. @@ -194,7 +194,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) - SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) PurgeMemberships(ctx context.Context, txn *sql.Tx, roomID string) error SelectMemberships( ctx context.Context, txn *sql.Tx, diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go index 4afa2ac5b2..0a36f5887b 100644 --- a/syncapi/storage/tables/memberships_test.go +++ b/syncapi/storage/tables/memberships_test.go @@ -80,6 +80,7 @@ func TestMembershipsTable(t *testing.T) { defer cancel() for _, ev := range userEvents { + ev.StateKeyResolved = ev.StateKey() if err := table.UpsertMembership(ctx, nil, ev, types.StreamPosition(ev.Depth()), 1); err != nil { t.Fatalf("failed to upsert membership: %s", err) } @@ -123,7 +124,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem if err != nil { t.Fatalf("failed to select membership: %s", err) } - expectedPos := 1 + var expectedPos int64 = 1 if pos != expectedPos { t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) } @@ -134,6 +135,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem ev := room.CreateAndInsert(t, user, spec.MRoomMember, map[string]interface{}{ "membership": spec.Join, }, test.WithStateKey(user.ID)) + ev.StateKeyResolved = ev.StateKey() // Insert the same event again, but with different positions, which should get updated if err = table.UpsertMembership(ctx, nil, ev, 2, 2); err != nil { t.Fatalf("failed to upsert membership: %s", err) diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go index f4f75bdf31..7691cc5f84 100644 --- a/syncapi/storage/tables/topology_test.go +++ b/syncapi/storage/tables/topology_test.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" ) func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.DB, func()) { @@ -60,28 +61,37 @@ func TestTopologyTable(t *testing.T) { highestPos = topoPos + 1 } // check ordering works without limit - eventIDs, err := tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, true) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } + eventIDs, start, end, err := tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, true) + assert.NoError(t, err, "failed to SelectEventIDsInRange") test.AssertEventIDsEqual(t, eventIDs, events[:]) - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, false) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, start) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, end) + + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[:])) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, start) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, end) + // check ordering works with limit - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, true) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, true) + assert.NoError(t, err, "failed to SelectEventIDsInRange") test.AssertEventIDsEqual(t, eventIDs, events[:3]) - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, false) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, start) + assert.Equal(t, types.TopologyToken{Depth: 3, PDUPosition: 2}, end) + + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[len(events)-3:])) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, start) + assert.Equal(t, types.TopologyToken{Depth: 3, PDUPosition: 2}, end) + // Check that we return no values for invalid rooms + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, "!doesnotexist:localhost", 0, highestPos, highestPos, 10, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") + assert.Equal(t, 0, len(eventIDs)) + assert.Equal(t, types.TopologyToken{}, start) + assert.Equal(t, types.TopologyToken{}, end) return nil }) if err != nil { diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index becd863a98..a3634c03fe 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" @@ -17,6 +18,7 @@ import ( type InviteStreamProvider struct { DefaultStreamProvider + rsAPI api.SyncRoomserverAPI } func (p *InviteStreamProvider) Setup( @@ -61,12 +63,27 @@ func (p *InviteStreamProvider) IncrementalSync( return from } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + for roomID, inviteEvent := range invites { + user := spec.UserID{} + sender, err := p.rsAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), inviteEvent.SenderID()) + if err == nil && sender != nil { + user = *sender + } + // skip ignored user events - if _, ok := req.IgnoredUsers.List[inviteEvent.Sender()]; ok { + if _, ok := req.IgnoredUsers.List[user.String()]; ok { + continue + } + ir, err := types.NewInviteResponse(ctx, p.rsAPI, inviteEvent, eventFormat) + if err != nil { + req.Log.WithError(err).Error("failed creating invite response") continue } - ir := types.NewInviteResponse(inviteEvent) req.Response.Rooms.Invite[roomID] = ir } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index b5fd5be8e8..3abb0b3c6d 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -85,6 +85,11 @@ func (p *PDUStreamProvider) CompleteSync( req.Log.WithError(err).Error("unable to update event filter with ignored users") } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + recentEvents, err := snapshot.RecentEvents(ctx, joinedRoomIDs, r, &eventFilter, true, true) if err != nil { return from @@ -102,7 +107,7 @@ func (p *PDUStreamProvider) CompleteSync( // get the join response for each room jr, jerr := p.getJoinResponseForCompleteSync( ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, false, - events.Events, events.Limited, + events.Events, events.Limited, eventFormat, ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") @@ -139,7 +144,7 @@ func (p *PDUStreamProvider) CompleteSync( events := recentEvents[roomID] jr, err = p.getJoinResponseForCompleteSync( ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, true, - events.Events, events.Limited, + events.Events, events.Limited, eventFormat, ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") @@ -175,12 +180,12 @@ func (p *PDUStreamProvider) IncrementalSync( eventFilter := req.Filter.Room.Timeline if req.WantFullState { - if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { + if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter, p.rsAPI); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") return from } } else { - if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { + if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter, p.rsAPI); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") return from } @@ -275,7 +280,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( limited := dbEvents[delta.RoomID].Limited recEvents := gomatrixserverlib.ReverseTopologicalOrdering( - gomatrixserverlib.ToPDUs(snapshot.StreamEventsToEvents(device, recentStreamEvents)), + gomatrixserverlib.ToPDUs(snapshot.StreamEventsToEvents(ctx, device, recentStreamEvents, p.rsAPI)), gomatrixserverlib.TopologicalOrderByPrevEvents, ) recentEvents := make([]*rstypes.HeaderedEvent, len(recEvents)) @@ -343,6 +348,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err) } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + // Now that we've filtered the timeline, work out which state events are still // left. Anything that appears in the filtered timeline will be removed from the // "state" section and kept in "timeline". @@ -351,9 +361,17 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( gomatrixserverlib.TopologicalOrderByAuthEvents, ) delta.StateEvents = make([]*rstypes.HeaderedEvent, len(sEvents)) + var skipped int for i := range sEvents { - delta.StateEvents[i] = sEvents[i].(*rstypes.HeaderedEvent) + ev := sEvents[i] + he, ok := ev.(*rstypes.HeaderedEvent) + if !ok { + skipped++ + continue + } + delta.StateEvents[i-skipped] = he } + delta.StateEvents = delta.StateEvents[:len(sEvents)-skipped] if len(delta.StateEvents) > 0 { if last := delta.StateEvents[len(delta.StateEvents)-1]; last != nil { @@ -376,20 +394,28 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } } jr.Timeline.PrevBatch = &prevBatch - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync) + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = (limited && len(events) == len(recentEvents)) || delta.NewlyJoined - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync) + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) req.Response.Rooms.Join[delta.RoomID] = jr case spec.Peek: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = &prevBatch // TODO: Apply history visibility on peeked rooms - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), synctypes.FormatSync) + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) jr.Timeline.Limited = limited - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync) + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) req.Response.Rooms.Peek[delta.RoomID] = jr case spec.Leave: @@ -398,11 +424,15 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( case spec.Ban: lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = &prevBatch - lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync) + lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) - lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync) + lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) req.Response.Rooms.Leave[delta.RoomID] = lr } @@ -425,7 +455,7 @@ func applyHistoryVisibilityFilter( for _, ev := range recentEvents { if ev.StateKey() != nil { stateTypes = append(stateTypes, ev.Type()) - senders = append(senders, ev.Sender()) + senders = append(senders, string(ev.SenderID())) } } @@ -444,8 +474,13 @@ func applyHistoryVisibilityFilter( } } + parsedUserID, err := spec.NewUserID(userID, true) + if err != nil { + return nil, err + } + startTime := time.Now() - events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync") + events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, *parsedUserID, "sync") if err != nil { return nil, err } @@ -458,6 +493,7 @@ func applyHistoryVisibilityFilter( return events, nil } +// nolint: gocyclo func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, snapshot storage.DatabaseTransaction, @@ -468,6 +504,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( isPeek bool, recentStreamEvents []types.StreamEvent, limited bool, + eventFormat synctypes.ClientEventFormat, ) (jr *types.JoinResponse, err error) { jr = types.NewJoinResponse() // TODO: When filters are added, we may need to call this multiple times to get enough events. @@ -489,7 +526,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( stateEvents, err := snapshot.CurrentState(ctx, roomID, stateFilter, excludingEventIDs) if err != nil { - return + return jr, err } jr.Summary, err = snapshot.GetRoomSummary(ctx, roomID, device.UserID) @@ -500,7 +537,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: // "Can sync a room with a message with a transaction id" - which does a complete sync to check. - recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents) + recentEvents := snapshot.StreamEventsToEvents(ctx, device, recentStreamEvents, p.rsAPI) events := recentEvents // Only apply history visibility checks if the response is for joined rooms @@ -542,7 +579,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } backwardTopologyPos, backwardStreamPos, err = snapshot.PositionInTopology(ctx, event.EventID()) if err != nil { - return + return jr, err } prevBatch = &types.TopologyToken{ Depth: backwardTopologyPos, @@ -552,11 +589,15 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } jr.Timeline.PrevBatch = prevBatch - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync) + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), synctypes.FormatSync) + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) return jr, nil } @@ -569,6 +610,10 @@ func (p *PDUStreamProvider) lazyLoadMembers( if len(timelineEvents) == 0 { return stateEvents, nil } + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return nil, err + } // Work out which memberships to include timelineUsers := make(map[string]struct{}) if !incremental { @@ -577,8 +622,8 @@ func (p *PDUStreamProvider) lazyLoadMembers( // Add all users the client doesn't know about yet to a list for _, event := range timelineEvents { // Membership is not yet cached, add it to the list - if _, ok := p.lazyLoadCache.IsLazyLoadedUserCached(device, roomID, event.Sender()); !ok { - timelineUsers[event.Sender()] = struct{}{} + if _, ok := p.lazyLoadCache.IsLazyLoadedUserCached(device, roomID, string(event.SenderID())); !ok { + timelineUsers[string(event.SenderID())] = struct{}{} } } // Preallocate with the same amount, even if it will end up with fewer values @@ -589,13 +634,17 @@ func (p *PDUStreamProvider) lazyLoadMembers( // If this is a gapped incremental sync, we still want this membership isGappedIncremental := limited && incremental // We want this users membership event, keep it in the list - stateKey := *event.StateKey() - if _, ok := timelineUsers[stateKey]; ok || isGappedIncremental || stateKey == device.UserID { + userID := "" + stateKeyUserID, queryErr := p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*event.StateKey())) + if queryErr == nil && stateKeyUserID != nil { + userID = stateKeyUserID.String() + } + if _, ok := timelineUsers[userID]; ok || isGappedIncremental || userID == device.UserID { newStateEvents = append(newStateEvents, event) if !stateFilter.IncludeRedundantMembers { - p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, stateKey, event.EventID()) + p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, userID, event.EventID()) } - delete(timelineUsers, stateKey) + delete(timelineUsers, userID) } } else { newStateEvents = append(newStateEvents, event) diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index a35491acf4..f25bc978fb 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -45,6 +45,7 @@ func NewSyncStreamProviders( }, InviteStreamProvider: &InviteStreamProvider{ DefaultStreamProvider: DefaultStreamProvider{DB: d}, + rsAPI: rsAPI, }, SendToDeviceStreamProvider: &SendToDeviceStreamProvider{ DefaultStreamProvider: DefaultStreamProvider{DB: d}, diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index ecbe05dd84..091e3db413 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -45,7 +45,7 @@ func AddPublicRoutes( processContext *process.ProcessContext, routers httputil.Routers, dendriteCfg *config.Dendrite, - cm sqlutil.Connections, + cm *sqlutil.Connections, natsInstance *jetstream.NATSInstance, userAPI userapi.SyncUserAPI, rsAPI api.SyncRoomserverAPI, @@ -60,7 +60,7 @@ func AddPublicRoutes( } eduCache := caching.NewTypingCache() - notifier := notifier.NewNotifier() + notifier := notifier.NewNotifier(rsAPI) streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, eduCache, caches, notifier) notifier.SetCurrentPosition(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { @@ -144,8 +144,11 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start receipts consumer") } + rateLimits := httputil.NewRateLimits(&dendriteCfg.ClientAPI.RateLimiting) + routing.Setup( routers.Client, requestPool, syncDB, userAPI, rsAPI, &dendriteCfg.SyncAPI, caches, fts, + rateLimits, ) } diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index bc766e6633..ac52685110 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -40,6 +40,15 @@ type syncRoomserverAPI struct { rooms []*test.Room } +func (s *syncRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + +func (s *syncRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil +} + func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { var room *test.Room for _, r := range s.rooms { @@ -70,8 +79,13 @@ func (s *syncRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *rsa return nil } -func (s *syncRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, req *rsapi.QueryMembershipAtEventRequest, res *rsapi.QueryMembershipAtEventResponse) error { - return nil +func (s *syncRoomserverAPI) QueryMembershipAtEvent( + ctx context.Context, + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, +) (map[string]*rstypes.HeaderedEvent, error) { + return map[string]*rstypes.HeaderedEvent{}, nil } type syncUserAPI struct { @@ -195,6 +209,156 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { } } +func TestSyncAPIEventFormatPowerLevels(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncEventFormatPowerLevels(t, dbType) + }) +} + +func testSyncEventFormatPowerLevels(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + setRoomVersion := func(t *testing.T, r *test.Room) { r.Version = gomatrixserverlib.RoomVersionPseudoIDs } + room := test.NewRoom(t, user, setRoomVersion) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + }, + }, test.WithStateKey("")) + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + msgs := toNATSMsgs(t, cfg, room.Events()...) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + testCases := []struct { + name string + wantCode int + wantJoinedRooms []string + eventFormat synctypes.ClientEventFormat + }{ + { + name: "Client format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSync, + }, + { + name: "Federation format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSyncFederation, + }, + } + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + for _, tc := range testCases { + format := "" + if tc.eventFormat == synctypes.FormatSyncFederation { + format = "federation" + } + + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + }))) + if w.Code != tc.wantCode { + t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) + } + if tc.wantJoinedRooms != nil { + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Fatalf("%s: failed to decode response body: %s", tc.name, err) + } + if len(res.Rooms.Join) != len(tc.wantJoinedRooms) { + t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res) + } + t.Logf("res: %+v", res.Rooms.Join[room.ID]) + + gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for i, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[i] = ev.EventID + } + test.AssertEventIDsEqual(t, gotEventIDs, room.Events()) + + event := room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + "@otheruser:localhost": 50, + }, + }, test.WithStateKey("")) + + msgs := toNATSMsgs(t, cfg, event) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + since := res.NextBatch.String() + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + "since": since, + }))) + if w.Code != 200 { + t.Errorf("since=%s got HTTP %d want 200", since, w.Code) + } + + res = *types.NewResponse() + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + if len(res.Rooms.Join) != 1 { + t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join)) + } + gotEventIDs = make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for j, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[j] = ev.EventID + if ev.Type == spec.MRoomPowerLevels { + content := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal(ev.Content, &content) + if err != nil { + t.Errorf("failed to unmarshal power level content: %s", err) + } + otherUserLevel := content.UserLevel("@otheruser:localhost") + if otherUserLevel != 50 { + t.Errorf("Expected user PL of %d but got %d", 50, otherUserLevel) + } + } + } + events := []*rstypes.HeaderedEvent{room.Events()[len(room.Events())-1]} + test.AssertEventIDsEqual(t, gotEventIDs, events) + } + } +} + // Tests what happens when we create a room and then /sync before all events from /createRoom have // been sent to the syncapi func TestSyncAPICreateRoomSyncEarly(t *testing.T) { @@ -429,6 +593,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { } cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RateLimiting = config.RateLimiting{Enabled: false} routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) @@ -1236,7 +1401,7 @@ func toNATSMsgs(t *testing.T, cfg *config.Dendrite, input ...*rstypes.HeaderedEv if ev.StateKey() != nil { addsStateIDs = append(addsStateIDs, ev.EventID()) } - result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID(), api.OutputEvent{ + result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID().String(), api.OutputEvent{ Type: rsapi.OutputTypeNewRoomEvent, NewRoomEvent: &rsapi.OutputNewRoomEvent{ Event: ev, diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go index c722fe60a2..fe4f6c07f0 100644 --- a/syncapi/synctypes/clientevent.go +++ b/syncapi/synctypes/clientevent.go @@ -16,10 +16,23 @@ package synctypes import ( + "encoding/json" + "fmt" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) +// PrevEventRef represents a reference to a previous event in a state event upgrade +type PrevEventRef struct { + PrevContent json.RawMessage `json:"prev_content"` + ReplacesState string `json:"replaces_state"` + PrevSenderID string `json:"prev_sender"` +} + type ClientEventFormat int const ( @@ -28,8 +41,21 @@ const ( // FormatSync will include only the event keys required by the /sync API. Notably, this // means the 'room_id' will be missing from the events. FormatSync + // FormatSyncFederation will include all event keys normally included in federated events. + // This allows clients to request federated formatted events via the /sync API. + FormatSyncFederation ) +// ClientFederationFields extends a ClientEvent to contain the additional fields present in a +// federation event. Used when the client requests `event_format` of type `federation`. +type ClientFederationFields struct { + Depth int64 `json:"depth,omitempty"` + PrevEvents []string `json:"prev_events,omitempty"` + AuthEvents []string `json:"auth_events,omitempty"` + Signatures spec.RawJSON `json:"signatures,omitempty"` + Hashes spec.RawJSON `json:"hashes,omitempty"` +} + // ClientEvent is an event which is fit for consumption by clients, in accordance with the specification. type ClientEvent struct { Content spec.RawJSON `json:"content"` @@ -37,38 +63,368 @@ type ClientEvent struct { OriginServerTS spec.Timestamp `json:"origin_server_ts,omitempty"` // OriginServerTS is omitted on receipt events RoomID string `json:"room_id,omitempty"` // RoomID is omitted on /sync responses Sender string `json:"sender,omitempty"` // Sender is omitted on receipt events + SenderKey spec.SenderID `json:"sender_key,omitempty"` // The SenderKey for events in pseudo ID rooms StateKey *string `json:"state_key,omitempty"` Type string `json:"type"` Unsigned spec.RawJSON `json:"unsigned,omitempty"` Redacts string `json:"redacts,omitempty"` + + // Only sent to clients when `event_format` == `federation`. + ClientFederationFields } // ToClientEvents converts server events to client events. -func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat) []ClientEvent { +func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat, userIDForSender spec.UserIDForSender) []ClientEvent { evs := make([]ClientEvent, 0, len(serverEvs)) for _, se := range serverEvs { if se == nil { continue // TODO: shouldn't happen? } - evs = append(evs, ToClientEvent(se, format)) + ev, err := ToClientEvent(se, format, userIDForSender) + if err != nil { + logrus.WithError(err).Warn("Failed converting event to ClientEvent") + continue + } + evs = append(evs, *ev) } return evs } +// ToClientEventDefault converts a single server event to a client event. +// It provides default logic for event.SenderID & event.StateKey -> userID conversions. +func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent { + ev, err := ToClientEvent(event, FormatAll, userIDQuery) + if err != nil { + return ClientEvent{} + } + return *ev +} + +// If provided state key is a user ID (state keys beginning with @ are reserved for this purpose) +// fetch it's associated sender ID and use that instead. Otherwise returns the same state key back. +// +// # This function either returns the state key that should be used, or an error +// +// TODO: handle failure cases better (e.g. no sender ID) +func FromClientStateKey(roomID spec.RoomID, stateKey string, senderIDQuery spec.SenderIDForUser) (*string, error) { + if len(stateKey) >= 1 && stateKey[0] == '@' { + parsedStateKey, err := spec.NewUserID(stateKey, true) + if err != nil { + // If invalid user ID, then there is no associated state event. + return nil, fmt.Errorf("Provided state key begins with @ but is not a valid user ID: %w", err) + } + senderID, err := senderIDQuery(roomID, *parsedStateKey) + if err != nil { + return nil, fmt.Errorf("Failed to query sender ID: %w", err) + } + if senderID == nil { + // If no sender ID, then there is no associated state event. + return nil, fmt.Errorf("No associated sender ID found.") + } + newStateKey := string(*senderID) + return &newStateKey, nil + } else { + return &stateKey, nil + } +} + // ToClientEvent converts a single server event to a client event. -func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat) ClientEvent { +func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, userIDForSender spec.UserIDForSender) (*ClientEvent, error) { ce := ClientEvent{ - Content: spec.RawJSON(se.Content()), - Sender: se.Sender(), + Content: se.Content(), + Sender: string(se.SenderID()), Type: se.Type(), StateKey: se.StateKey(), - Unsigned: spec.RawJSON(se.Unsigned()), + Unsigned: se.Unsigned(), OriginServerTS: se.OriginServerTS(), EventID: se.EventID(), Redacts: se.Redacts(), } - if format == FormatAll { - ce.RoomID = se.RoomID() + + switch format { + case FormatAll: + ce.RoomID = se.RoomID().String() + case FormatSync: + case FormatSyncFederation: + ce.RoomID = se.RoomID().String() + ce.AuthEvents = se.AuthEventIDs() + ce.PrevEvents = se.PrevEventIDs() + ce.Depth = se.Depth() + // TODO: Set Signatures & Hashes fields + } + + if format != FormatSyncFederation && se.Version() == gomatrixserverlib.RoomVersionPseudoIDs { + err := updatePseudoIDs(&ce, se, userIDForSender, format) + if err != nil { + return nil, err + } + } + + return &ce, nil +} + +func updatePseudoIDs(ce *ClientEvent, se gomatrixserverlib.PDU, userIDForSender spec.UserIDForSender, format ClientEventFormat) error { + ce.SenderKey = se.SenderID() + + userID, err := userIDForSender(se.RoomID(), se.SenderID()) + if err == nil && userID != nil { + ce.Sender = userID.String() } - return ce + + sk := se.StateKey() + if sk != nil && *sk != "" { + skUserID, err := userIDForSender(se.RoomID(), spec.SenderID(*sk)) + if err == nil && skUserID != nil { + skString := skUserID.String() + ce.StateKey = &skString + } + } + + var prev PrevEventRef + if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" { + prevUserID, err := userIDForSender(se.RoomID(), spec.SenderID(prev.PrevSenderID)) + if err == nil && userID != nil { + prev.PrevSenderID = prevUserID.String() + } else { + errString := "userID unknown" + if err != nil { + errString = err.Error() + } + logrus.Warnf("Failed to find userID for prev_sender in ClientEvent: %s", errString) + // NOTE: Not much can be done here, so leave the previous value in place. + } + ce.Unsigned, err = json.Marshal(prev) + if err != nil { + err = fmt.Errorf("Failed to marshal unsigned content for ClientEvent: %w", err) + return err + } + } + + switch se.Type() { + case spec.MRoomCreate: + updatedContent, err := updateCreateEvent(se.Content(), userIDForSender, se.RoomID()) + if err != nil { + err = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", err) + return err + } + ce.Content = updatedContent + case spec.MRoomMember: + updatedEvent, err := updateInviteEvent(userIDForSender, se, format) + if err != nil { + err = fmt.Errorf("Failed to update m.room.member event for ClientEvent: %w", err) + return err + } + if updatedEvent != nil { + ce.Unsigned = updatedEvent.Unsigned() + } + case spec.MRoomPowerLevels: + updatedEvent, err := updatePowerLevelEvent(userIDForSender, se, format) + if err != nil { + err = fmt.Errorf("Failed update m.room.power_levels event for ClientEvent: %w", err) + return err + } + if updatedEvent != nil { + ce.Content = updatedEvent.Content() + ce.Unsigned = updatedEvent.Unsigned() + } + } + + return nil +} + +func updateCreateEvent(content spec.RawJSON, userIDForSender spec.UserIDForSender, roomID spec.RoomID) (spec.RawJSON, error) { + if creator := gjson.GetBytes(content, "creator"); creator.Exists() { + oldCreator := creator.Str + userID, err := userIDForSender(roomID, spec.SenderID(oldCreator)) + if err != nil { + err = fmt.Errorf("Failed to find userID for creator in ClientEvent: %w", err) + return nil, err + } + + if userID != nil { + var newCreatorBytes, newContent []byte + newCreatorBytes, err = json.Marshal(userID.String()) + if err != nil { + err = fmt.Errorf("Failed to marshal new creator for ClientEvent: %w", err) + return nil, err + } + + newContent, err = sjson.SetRawBytes([]byte(content), "creator", newCreatorBytes) + if err != nil { + err = fmt.Errorf("Failed to set new creator for ClientEvent: %w", err) + return nil, err + } + + return newContent, nil + } + } + + return content, nil +} + +func updateInviteEvent(userIDForSender spec.UserIDForSender, ev gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) { + if inviteRoomState := gjson.GetBytes(ev.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + userID, err := userIDForSender(ev.RoomID(), ev.SenderID()) + if err != nil || userID == nil { + if err != nil { + err = fmt.Errorf("invalid userID found when updating invite_room_state: %w", err) + } + return nil, err + } + + newState, err := GetUpdatedInviteRoomState(userIDForSender, inviteRoomState, ev, ev.RoomID(), eventFormat) + if err != nil { + return nil, err + } + + var newEv []byte + newEv, err = sjson.SetRawBytes(ev.JSON(), "unsigned.invite_room_state", newState) + if err != nil { + return nil, err + } + + return gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSON(newEv, false) + } + + return ev, nil +} + +type InviteRoomStateEvent struct { + Content spec.RawJSON `json:"content"` + SenderID string `json:"sender"` + StateKey *string `json:"state_key"` + Type string `json:"type"` +} + +func GetUpdatedInviteRoomState(userIDForSender spec.UserIDForSender, inviteRoomState gjson.Result, event gomatrixserverlib.PDU, roomID spec.RoomID, eventFormat ClientEventFormat) (spec.RawJSON, error) { + var res spec.RawJSON + inviteStateEvents := []InviteRoomStateEvent{} + err := json.Unmarshal([]byte(inviteRoomState.Raw), &inviteStateEvents) + if err != nil { + return nil, err + } + + if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != FormatSyncFederation { + for i, ev := range inviteStateEvents { + userID, userIDErr := userIDForSender(roomID, spec.SenderID(ev.SenderID)) + if userIDErr != nil { + return nil, userIDErr + } + if userID != nil { + inviteStateEvents[i].SenderID = userID.String() + } + + if ev.StateKey != nil && *ev.StateKey != "" { + userID, senderErr := userIDForSender(roomID, spec.SenderID(*ev.StateKey)) + if senderErr != nil { + return nil, senderErr + } + if userID != nil { + user := userID.String() + inviteStateEvents[i].StateKey = &user + } + } + + updatedContent, updateErr := updateCreateEvent(ev.Content, userIDForSender, roomID) + if updateErr != nil { + updateErr = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", userIDErr) + return nil, updateErr + } + inviteStateEvents[i].Content = updatedContent + } + } + + res, err = json.Marshal(inviteStateEvents) + if err != nil { + return nil, err + } + + return res, nil +} + +func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) { + if !se.StateKeyEquals("") { + return se, nil + } + + newEv := se.JSON() + + usersField := gjson.GetBytes(se.JSON(), "content.users") + if usersField.Exists() { + pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + var userID *spec.UserID + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + if err != nil { + return nil, err + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) + if err != nil { + return nil, err + } + } + + // do the same for prev content + prevUsersField := gjson.GetBytes(se.JSON(), "unsigned.prev_content.users") + if prevUsersField.Exists() { + prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") + if !prevContent.Exists() { + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) + if err != nil { + return nil, err + } + + return evNew, err + } + pls := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal([]byte(prevContent.Raw), &pls) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, userErr := userIDForSender(se.RoomID(), spec.SenderID(user)) + if userErr != nil { + return nil, userErr + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) + if err != nil { + return nil, err + } + } + + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) + if err != nil { + return nil, err + } + + return evNew, err } diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go index b914e64f1d..662f9ea43b 100644 --- a/syncapi/synctypes/clientevent_test.go +++ b/syncapi/synctypes/clientevent_test.go @@ -18,11 +18,77 @@ package synctypes import ( "bytes" "encoding/json" + "fmt" + "reflect" "testing" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) +func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + +const testSenderID = "testSenderID" +const testUserID = "@test:localhost" + +type EventFieldsToVerify struct { + EventID string + Type string + OriginServerTS spec.Timestamp + StateKey *string + Content spec.RawJSON + Unsigned spec.RawJSON + Sender string + Depth int64 + PrevEvents []string + AuthEvents []string +} + +func verifyEventFields(t *testing.T, got EventFieldsToVerify, want EventFieldsToVerify) { + if got.EventID != want.EventID { + t.Errorf("ClientEvent.EventID: wanted %s, got %s", want.EventID, got.EventID) + } + if got.OriginServerTS != want.OriginServerTS { + t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", want.OriginServerTS, got.OriginServerTS) + } + if got.StateKey == nil && want.StateKey != nil { + t.Errorf("ClientEvent.StateKey: no state key present when one was wanted: %s", *want.StateKey) + } + if got.StateKey != nil && want.StateKey == nil { + t.Errorf("ClientEvent.StateKey: state key present when one was not wanted: %s", *got.StateKey) + } + if got.StateKey != nil && want.StateKey != nil && *got.StateKey != *want.StateKey { + t.Errorf("ClientEvent.StateKey: wanted %s, got %s", *want.StateKey, *got.StateKey) + } + if got.Type != want.Type { + t.Errorf("ClientEvent.Type: wanted %s, got %s", want.Type, got.Type) + } + if !bytes.Equal(got.Content, want.Content) { + t.Errorf("ClientEvent.Content: wanted %s, got %s", string(want.Content), string(got.Content)) + } + if !bytes.Equal(got.Unsigned, want.Unsigned) { + t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(want.Unsigned), string(got.Unsigned)) + } + if got.Sender != want.Sender { + t.Errorf("ClientEvent.Sender: wanted %s, got %s", want.Sender, got.Sender) + } + if got.Depth != want.Depth { + t.Errorf("ClientEvent.Depth: wanted %d, got %d", want.Depth, got.Depth) + } + if !reflect.DeepEqual(got.PrevEvents, want.PrevEvents) { + t.Errorf("ClientEvent.PrevEvents: wanted %v, got %v", want.PrevEvents, got.PrevEvents) + } + if !reflect.DeepEqual(got.AuthEvents, want.AuthEvents) { + t.Errorf("ClientEvent.AuthEvents: wanted %v, got %v", want.AuthEvents, got.AuthEvents) + } +} + func TestToClientEvent(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1).NewEventFromTrustedJSON([]byte(`{ "type": "m.room.name", @@ -43,28 +109,38 @@ func TestToClientEvent(t *testing.T) { // nolint: gocyclo if err != nil { t.Fatalf("failed to create Event: %s", err) } - ce := ToClientEvent(ev, FormatAll) - if ce.EventID != ev.EventID() { - t.Errorf("ClientEvent.EventID: wanted %s, got %s", ev.EventID(), ce.EventID) - } - if ce.OriginServerTS != ev.OriginServerTS() { - t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", ev.OriginServerTS(), ce.OriginServerTS) - } - if ce.StateKey == nil || *ce.StateKey != "" { - t.Errorf("ClientEvent.StateKey: wanted '', got %v", ce.StateKey) - } - if ce.Type != ev.Type() { - t.Errorf("ClientEvent.Type: wanted %s, got %s", ev.Type(), ce.Type) - } - if !bytes.Equal(ce.Content, ev.Content()) { - t.Errorf("ClientEvent.Content: wanted %s, got %s", string(ev.Content()), string(ce.Content)) - } - if !bytes.Equal(ce.Unsigned, ev.Unsigned()) { - t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(ev.Unsigned()), string(ce.Unsigned)) + userID, err := spec.NewUserID("@test:localhost", true) + if err != nil { + t.Fatalf("failed to create userID: %s", err) } - if ce.Sender != ev.Sender() { - t.Errorf("ClientEvent.Sender: wanted %s, got %s", ev.Sender(), ce.Sender) + sk := "" + ce, err := ToClientEvent(ev, FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) } + + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: userID.String(), + }) + j, err := json.Marshal(ce) if err != nil { t.Fatalf("failed to Marshal ClientEvent: %s", err) @@ -98,8 +174,388 @@ func TestToClientFormatSync(t *testing.T) { if err != nil { t.Fatalf("failed to create Event: %s", err) } - ce := ToClientEvent(ev, FormatSync) + ce, err := ToClientEvent(ev, FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) + } if ce.RoomID != "" { t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID) } } + +func TestToClientEventFormatSyncFederation(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + userID, err := spec.NewUserID("@test:localhost", true) + if err != nil { + t.Fatalf("failed to create userID: %s", err) + } + sk := "" + ce, err := ToClientEvent(ev, FormatSyncFederation, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) + } + + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + Depth: ce.Depth, + PrevEvents: ce.PrevEvents, + AuthEvents: ce.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: userID.String(), + Depth: ev.Depth(), + PrevEvents: ev.PrevEventIDs(), + AuthEvents: ev.AuthEventIDs(), + }) +} + +func userIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "unknownSenderID" { + return nil, fmt.Errorf("Cannot find userID") + } + return spec.NewUserID(testUserID, true) +} + +func TestToClientEventsFormatSyncFederation(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSyncFederation, userIDForSender) + ce := clientEvents[0] + sk := testSenderID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + Depth: ce.Depth, + PrevEvents: ce.PrevEvents, + AuthEvents: ce.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testSenderID, + Depth: ev.Depth(), + PrevEvents: ev.PrevEventIDs(), + AuthEvents: ev.AuthEventIDs(), + }) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + Depth: ce2.Depth, + PrevEvents: ce2.PrevEvents, + AuthEvents: ce2.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: ev2.Unsigned(), + Sender: testSenderID, + Depth: ev2.Depth(), + PrevEvents: ev2.PrevEventIDs(), + AuthEvents: ev2.AuthEventIDs(), + }) +} + +func TestToClientEventsFormatSync(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9 + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender) + ce := clientEvents[0] + sk := testUserID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testUserID, + }) + + var prev PrevEventRef + prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`) + prev.PrevSenderID = testUserID + expectedUnsigned, _ := json.Marshal(prev) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: expectedUnsigned, + Sender: testUserID, + }) +} + +func TestToClientEventsFormatSyncUnknownPrevSender(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "unknownSenderID" + }, + "depth": 9 + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender) + ce := clientEvents[0] + sk := testUserID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testUserID, + }) + + var prev PrevEventRef + prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`) + prev.PrevSenderID = "unknownSenderID" + expectedUnsigned, _ := json.Marshal(prev) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: expectedUnsigned, + Sender: testUserID, + }) +} diff --git a/syncapi/synctypes/filter.go b/syncapi/synctypes/filter.go index c994ddb96f..8998d44330 100644 --- a/syncapi/synctypes/filter.go +++ b/syncapi/synctypes/filter.go @@ -78,9 +78,14 @@ type RoomEventFilter struct { ContainsURL *bool `json:"contains_url,omitempty"` } +const ( + EventFormatClient = "client" + EventFormatFederation = "federation" +) + // Validate checks if the filter contains valid property values func (filter *Filter) Validate() error { - if filter.EventFormat != "" && filter.EventFormat != "client" && filter.EventFormat != "federation" { + if filter.EventFormat != "" && filter.EventFormat != EventFormatClient && filter.EventFormat != EventFormatFederation { return errors.New("Bad event_format value. Must be one of [\"client\", \"federation\"]") } return nil diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 22c27fea5d..bca11855c9 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -15,6 +15,7 @@ package types import ( + "context" "encoding/json" "errors" "fmt" @@ -339,13 +340,6 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { return token, nil } -// PrevEventRef represents a reference to a previous event in a state event upgrade -type PrevEventRef struct { - PrevContent json.RawMessage `json:"prev_content"` - ReplacesState string `json:"replaces_state"` - PrevSender string `json:"prev_sender"` -} - type DeviceLists struct { Changed []string `json:"changed,omitempty"` Left []string `json:"left,omitempty"` @@ -539,7 +533,7 @@ type InviteResponse struct { } // NewInviteResponse creates an empty response with initialised arrays. -func NewInviteResponse(event *types.HeaderedEvent) *InviteResponse { +func NewInviteResponse(ctx context.Context, rsAPI api.QuerySenderIDAPI, event *types.HeaderedEvent, eventFormat synctypes.ClientEventFormat) (*InviteResponse, error) { res := InviteResponse{} res.InviteState.Events = []json.RawMessage{} @@ -547,18 +541,42 @@ func NewInviteResponse(event *types.HeaderedEvent) *InviteResponse { // If there is then unmarshal it into the response. This will contain the // partial room state such as join rules, room name etc. if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { - _ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events) + if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != synctypes.FormatSyncFederation { + updatedInvite, err := synctypes.GetUpdatedInviteRoomState(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, inviteRoomState, event.PDU, event.RoomID(), eventFormat) + if err != nil { + return nil, err + } + _ = json.Unmarshal(updatedInvite, &res.InviteState.Events) + } else { + _ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events) + } + } + + // Clear unsigned so it doesn't have pseudoIDs converted during ToClientEvent + eventNoUnsigned, err := event.SetUnsigned(nil) + if err != nil { + return nil, err } // Then we'll see if we can create a partial of the invite event itself. // This is needed for clients to work out *who* sent the invite. - inviteEvent := synctypes.ToClientEvent(event.PDU, synctypes.FormatSync) + inviteEvent, err := synctypes.ToClientEvent(eventNoUnsigned, eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + return nil, err + } + + // Ensure unsigned field is empty so it isn't marshalled into the final JSON inviteEvent.Unsigned = nil - if ev, err := json.Marshal(inviteEvent); err == nil { + + if ev, err := json.Marshal(*inviteEvent); err == nil { res.InviteState.Events = append(res.InviteState.Events, ev) } - return &res + return &res, nil } // LeaveResponse represents a /sync response for a room which is under the 'leave' key. diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 8e0448fe76..35e1882cba 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -1,6 +1,7 @@ package types import ( + "context" "encoding/json" "reflect" "testing" @@ -8,8 +9,24 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) +type FakeRoomserverAPI struct{} + +func (f *FakeRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + +func (f *FakeRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + sender := spec.SenderID(userID.String()) + return &sender, nil +} + func TestSyncTokens(t *testing.T) { shouldPass := map[string]string{ "s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(), @@ -56,14 +73,18 @@ func TestNewInviteResponse(t *testing.T) { t.Fatal(err) } - res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}) + rsAPI := FakeRoomserverAPI{} + res, err := NewInviteResponse(context.Background(), &rsAPI, &types.HeaderedEvent{PDU: ev}, synctypes.FormatSync) + if err != nil { + t.Fatal(err) + } j, err := json.Marshal(res) if err != nil { t.Fatal(err) } if string(j) != expected { - t.Fatalf("Invite response didn't contain correct info") + t.Fatalf("Invite response didn't contain correct info, \nexpected: %s \ngot: %s", expected, string(j)) } } diff --git a/sytest-blacklist b/sytest-blacklist index 49a3cc8700..d6fadc7e1b 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -8,6 +8,7 @@ Events in rooms with AS-hosted room aliases are sent to AS server Inviting an AS-hosted user asks the AS server Accesing an AS-hosted room alias asks the AS server If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes +New federated private chats get full presence information (SYN-115) # This will fail in HTTP API mode, so blacklisted for now If a device list update goes missing, the server resyncs on the next one diff --git a/sytest-whitelist b/sytest-whitelist index c61e0bc3c6..492c756baf 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -783,4 +783,16 @@ Invited user can reject invite for empty room Invited user can reject local invite after originator leaves Guest users can join guest_access rooms Forgotten room messages cannot be paginated -Local device key changes get to remote servers with correct prev_id \ No newline at end of file +Local device key changes get to remote servers with correct prev_id +HS provides query metadata +HS can provide query metadata on a single protocol +Invites over federation are correctly pushed +Invites over federation are correctly pushed with name +User can create and send/receive messages in a room with version 11 +local user can join room with version 11 +User can invite local user to room with version 11 +remote user can join room with version 11 +User can invite remote user to room with version 11 +Remote user can backfill in a room with version 11 +Can reject invites over federation for rooms with version 11 +Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/test/room.go b/test/room.go index 852e31533b..da09de7c2a 100644 --- a/test/room.go +++ b/test/room.go @@ -39,6 +39,10 @@ var ( roomIDCounter = int64(0) ) +func UserIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + type Room struct { ID string Version gomatrixserverlib.RoomVersion @@ -164,7 +168,7 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten } builder := gomatrixserverlib.MustGetRoomVersion(r.Version).NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ - Sender: creator.ID, + SenderID: creator.ID, RoomID: r.ID, Type: eventType, StateKey: mod.stateKey, @@ -195,7 +199,7 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten if err != nil { t.Fatalf("CreateEvent[%s]: failed to build event: %s", eventType, err) } - if err = gomatrixserverlib.Allowed(ev, &r.authEvents); err != nil { + if err = gomatrixserverlib.Allowed(ev, &r.authEvents, UserIDForSender); err != nil { t.Fatalf("CreateEvent[%s]: failed to verify event was allowed: %s", eventType, err) } headeredEvent := &rstypes.HeaderedEvent{PDU: ev} diff --git a/userapi/api/api.go b/userapi/api/api.go index 0504026451..a0dce97589 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" ) @@ -94,6 +95,11 @@ type ClientUserAPI interface { QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error QueryPushRules(ctx context.Context, userID string) (*pushrules.AccountRuleSets, error) QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error + PerformAdminCreateRegistrationToken(ctx context.Context, registrationToken *clientapi.RegistrationToken) (bool, error) + PerformAdminListRegistrationTokens(ctx context.Context, returnAll bool, valid bool) ([]clientapi.RegistrationToken, error) + PerformAdminGetRegistrationToken(ctx context.Context, tokenString string) (*clientapi.RegistrationToken, error) + PerformAdminDeleteRegistrationToken(ctx context.Context, tokenString string) error + PerformAdminUpdateRegistrationToken(ctx context.Context, tokenString string, newAttributes map[string]interface{}) (*clientapi.RegistrationToken, error) PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 3cfdc0ce92..6da41f8a10 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -92,30 +92,43 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called // Only handle events we care about - if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { - return true - } - var output rsapi.OutputEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return true - } - event := output.NewRoomEvent.Event - if event == nil { - log.Errorf("userapi consumer: expected event") + + var event *rstypes.HeaderedEvent + var isNewRoomEvent bool + switch rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) { + case rsapi.OutputTypeNewRoomEvent: + isNewRoomEvent = true + fallthrough + case rsapi.OutputTypeNewInviteEvent: + var output rsapi.OutputEvent + if err := json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("roomserver output log: message parse failure") + return true + } + if isNewRoomEvent { + event = output.NewRoomEvent.Event + } else { + event = output.NewInviteEvent.Event + } + + if event == nil { + log.Errorf("userapi consumer: expected event") + return true + } + + log.WithFields(log.Fields{ + "event_id": event.EventID(), + "event_type": event.Type(), + }).Tracef("Received message from roomserver: %#v", output) + default: return true } if s.cfg.Matrix.ReportStats.Enabled { - go s.storeMessageStats(ctx, event.Type(), event.Sender(), event.RoomID()) + go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String()) } - log.WithFields(log.Fields{ - "event_id": event.EventID(), - "event_type": event.Type(), - }).Tracef("Received message from roomserver: %#v", output) - metadata, err := msg.Metadata() if err != nil { return true @@ -253,8 +266,8 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, directChats := gjson.ParseBytes(directChatsRaw) newDirectChats := make(map[string][]string) // iterate over all userID -> roomIDs + var found bool directChats.ForEach(func(userID, roomIDs gjson.Result) bool { - var found bool for _, roomID := range roomIDs.Array() { newDirectChats[userID.Str] = append(newDirectChats[userID.Str], roomID.Str) // add the new roomID to m.direct @@ -263,22 +276,21 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, newDirectChats[userID.Str] = append(newDirectChats[userID.Str], newRoomID) } } - // Only hit the database if we found the old room as a DM for this user - if found { - var data []byte - data, err = json.Marshal(newDirectChats) - if err != nil { - return true - } - if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { - return true - } - } return true }) - if err != nil { - return fmt.Errorf("failed to update m.direct state") + + // Only hit the database if we found the old room as a DM for this user + if found { + var data []byte + data, err = json.Marshal(newDirectChats) + if err != nil { + return err + } + if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { + return fmt.Errorf("failed to update m.direct state: %w", err) + } } + return nil } @@ -294,16 +306,21 @@ func (s *OutputRoomEventConsumer) copyTags(ctx context.Context, oldRoomID, newRo } func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rstypes.HeaderedEvent, streamPos uint64) error { - members, roomSize, err := s.localRoomMembers(ctx, event.RoomID()) + members, roomSize, err := s.localRoomMembers(ctx, event.RoomID().String()) if err != nil { return fmt.Errorf("s.localRoomMembers: %w", err) } switch { case event.Type() == spec.MRoomMember: - cevent := synctypes.ToClientEvent(event, synctypes.FormatAll) + cevent, clientEvErr := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if clientEvErr != nil { + return clientEvErr + } var member *localMembership - member, err = newLocalMembership(&cevent) + member, err = newLocalMembership(cevent) if err != nil { return fmt.Errorf("newLocalMembership: %w", err) } @@ -314,7 +331,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst } case event.Type() == "m.room.tombstone" && event.StateKeyEquals(""): // Handle room upgrades - oldRoomID := event.RoomID() + oldRoomID := event.RoomID().String() newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str if err = s.handleRoomUpgrade(ctx, oldRoomID, newRoomID, members, roomSize); err != nil { // while inconvenient, this shouldn't stop us from sending push notifications @@ -331,7 +348,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "num_members": len(members), "room_size": roomSize, }).Tracef("Notifying members") @@ -385,18 +402,25 @@ func newLocalMembership(event *synctypes.ClientEvent) (*localMembership, error) // localRoomMembers fetches the current local members of a room, and // the total number of members. func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID string) ([]*localMembership, int, error) { + // Get only locally joined users to avoid unmarshalling and caching + // membership events we only use to calculate the room size. req := &rsapi.QueryMembershipsForRoomRequest{ RoomID: roomID, JoinedOnly: true, + LocalOnly: true, } var res rsapi.QueryMembershipsForRoomResponse - - // XXX: This could potentially race if the state for the event is not known yet - // e.g. the event came over federation but we do not have the full state persisted. if err := s.rsAPI.QueryMembershipsForRoom(ctx, req, &res); err != nil { return nil, 0, err } + // Since we only queried locally joined users above, + // we also need to ask the roomserver about the joined user count. + totalCount, err := s.rsAPI.JoinedUserCount(ctx, roomID) + if err != nil { + return nil, 0, err + } + var members []*localMembership for _, event := range res.JoinEvents { // Filter out invalid join events @@ -406,31 +430,18 @@ func (s *OutputRoomEventConsumer) localRoomMembers(ctx context.Context, roomID s if *event.StateKey == "" { continue } - _, serverName, err := gomatrixserverlib.SplitID('@', *event.StateKey) - if err != nil { - log.WithError(err).Error("failed to get servername from statekey") - continue - } - // Only get memberships for our server - if serverName != s.serverName { - continue - } + // We're going to trust the Query from above to really just return + // local users member, err := newLocalMembership(&event) if err != nil { log.WithError(err).Errorf("Parsing MemberContent") continue } - if member.Membership != spec.Join { - continue - } - if member.Domain != s.cfg.Matrix.ServerName { - continue - } members = append(members, member) } - return members, len(res.JoinEvents), nil + return members, totalCount, nil } // roomName returns the name in the event (if type==m.room.name), or @@ -449,8 +460,21 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H } } + // Special case for invites, as we don't store any "current state" for these events, + // we need to make sure that, if present, the m.room.name is sent as well. + if event.Type() == spec.MRoomMember && + gjson.GetBytes(event.Content(), "membership").Str == "invite" { + invState := gjson.GetBytes(event.JSON(), "unsigned.invite_room_state") + for _, ev := range invState.Array() { + if ev.Get("type").Str == spec.MRoomName { + name := ev.Get("content.name").Str + return name, nil + } + } + } + req := &rsapi.QueryCurrentStateRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple}, } var res rsapi.QueryCurrentStateResponse @@ -518,7 +542,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if a != pushrules.NotifyAction && a != pushrules.CoalesceAction { log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, }).Tracef("Push rule evaluation rejected the event") return nil @@ -528,26 +552,32 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if err != nil { return fmt.Errorf("s.localPushDevices: %w", err) } + clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + return err + } n := &api.Notification{ Actions: actions, // UNSPEC: the spec doesn't say this is a ClientEvent, but the // fields seem to match. room_id should be missing, which // matches the behaviour of FormatSync. - Event: synctypes.ToClientEvent(event, synctypes.FormatSync), + Event: *clientEvent, // TODO: this is per-device, but it's not part of the primary // key. So inserting one notification per profile tag doesn't // make sense. What is this supposed to be? Sytests require it // to "work", but they only use a single device. ProfileTag: profileTag, - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), TS: spec.AsTimestamp(time.Now()), } if err = s.db.InsertNotification(ctx, mem.Localpart, mem.Domain, event.EventID(), streamPos, tweaks, n); err != nil { return fmt.Errorf("s.db.InsertNotification: %w", err) } - if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID()); err != nil { + if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID().String()); err != nil { return fmt.Errorf("s.syncProducer.GetAndSendNotificationData: %w", err) } @@ -559,7 +589,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, "num_urls": len(devicesByURLAndFormat), "num_unread": userNumUnreadNotifs, @@ -615,7 +645,12 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype // evaluatePushRules fetches and evaluates the push rules of a local // user. Returns actions (including dont_notify). func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *rstypes.HeaderedEvent, mem *localMembership, roomSize int) ([]*pushrules.Action, error) { - if event.Sender() == mem.UserID { + user := "" + sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) + if err == nil { + user = sender.String() + } + if user == mem.UserID { // SPEC: Homeservers MUST NOT notify the Push Gateway for // events that the user has sent themselves. return nil, nil @@ -632,9 +667,8 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event * if err != nil { return nil, err } - sender := event.Sender() - if _, ok := ignored.List[sender]; ok { - return nil, fmt.Errorf("user %s is ignored", sender) + if _, ok := ignored.List[sender.String()]; ok { + return nil, fmt.Errorf("user %s is ignored", sender.String()) } } ruleSets, err := s.db.QueryPushRules(ctx, mem.Localpart, mem.Domain) @@ -646,11 +680,13 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event * ctx: ctx, rsAPI: s.rsAPI, mem: mem, - roomID: event.RoomID(), + roomID: event.RoomID().String(), roomSize: roomSize, } eval := pushrules.NewRuleSetEvaluator(ec, &ruleSets.Global) - rule, err := eval.MatchEvent(event.PDU) + rule, err := eval.MatchEvent(event.PDU, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { return nil, err } @@ -662,7 +698,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event * log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, "rule_id": rule.RuleID, }).Trace("Matched a push rule") @@ -682,7 +718,7 @@ func (rse *ruleSetEvalContext) UserDisplayName() string { return rse.mem.Display func (rse *ruleSetEvalContext) RoomMemberCount() (int, error) { return rse.roomSize, nil } -func (rse *ruleSetEvalContext) HasPowerLevel(userID, levelKey string) (bool, error) { +func (rse *ruleSetEvalContext) HasPowerLevel(senderID spec.SenderID, levelKey string) (bool, error) { req := &rsapi.QueryLatestEventsAndStateRequest{ RoomID: rse.roomID, StateToFetch: []gomatrixserverlib.StateKeyTuple{ @@ -702,7 +738,7 @@ func (rse *ruleSetEvalContext) HasPowerLevel(userID, levelKey string) (bool, err if err != nil { return false, err } - return plc.UserLevel(userID) >= plc.NotificationLevel(levelKey), nil + return plc.UserLevel(senderID) >= plc.NotificationLevel(levelKey), nil } return true, nil } @@ -751,11 +787,16 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes }, Devices: devices, EventID: event.EventID(), - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), }, } default: + sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) + if err != nil { + logger.WithError(err).Errorf("Failed to get userID for sender %s", event.SenderID()) + return nil, err + } req = pushgateway.NotifyRequest{ Notification: pushgateway.Notification{ Content: event.Content(), @@ -765,16 +806,29 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes Devices: devices, EventID: event.EventID(), ID: event.EventID(), - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), RoomName: roomName, - Sender: event.Sender(), + Sender: sender.String(), Type: event.Type(), }, } - if mem, err := event.Membership(); err == nil { + if mem, memberErr := event.Membership(); memberErr == nil { req.Notification.Membership = mem } - if event.StateKey() != nil && *event.StateKey() == fmt.Sprintf("@%s:%s", localpart, s.cfg.Matrix.ServerName) { + userID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", localpart, s.cfg.Matrix.ServerName), true) + if err != nil { + logger.WithError(err).Errorf("Failed to convert local user to userID %s", localpart) + return nil, err + } + localSender, err := s.rsAPI.QuerySenderIDForUser(ctx, event.RoomID(), *userID) + if err != nil { + logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String()) + return nil, err + } else if localSender == nil { + logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String()) + return nil, fmt.Errorf("no sender ID for user %s in %s", userID.String(), event.RoomID().String()) + } + if event.StateKey() != nil && *event.StateKey() == string(*localSender) { req.Notification.UserIsTarget = true } } diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 53977206f7..49dd5b2381 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -2,18 +2,25 @@ package consumers import ( "context" + "crypto/ed25519" "reflect" "sync" "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/internal/pushrules" + rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi/storage" @@ -44,13 +51,19 @@ func mustCreateEvent(t *testing.T, content string) *types.HeaderedEvent { return &types.HeaderedEvent{PDU: ev} } +type FakeUserRoomserverAPI struct{ rsapi.UserRoomserverAPI } + +func (f *FakeUserRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return spec.NewUserID(string(senderID), true) +} + func Test_evaluatePushRules(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := mustCreateDatabase(t, dbType) defer close() - consumer := OutputRoomEventConsumer{db: db} + consumer := OutputRoomEventConsumer{db: db, rsAPI: &FakeUserRoomserverAPI{}} testCases := []struct { name string @@ -61,13 +74,13 @@ func Test_evaluatePushRules(t *testing.T) { }{ { name: "m.receipt doesn't notify", - eventContent: `{"type":"m.receipt"}`, + eventContent: `{"type":"m.receipt","room_id":"!room:example.com"}`, wantAction: pushrules.UnknownAction, wantActions: nil, }, { name: "m.reaction doesn't notify", - eventContent: `{"type":"m.reaction"}`, + eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`, wantAction: pushrules.DontNotifyAction, wantActions: []*pushrules.Action{ { @@ -77,7 +90,7 @@ func Test_evaluatePushRules(t *testing.T) { }, { name: "m.room.message notifies", - eventContent: `{"type":"m.room.message"}`, + eventContent: `{"type":"m.room.message","room_id":"!room:example.com"}`, wantNotify: true, wantAction: pushrules.NotifyAction, wantActions: []*pushrules.Action{ @@ -86,7 +99,7 @@ func Test_evaluatePushRules(t *testing.T) { }, { name: "m.room.message highlights", - eventContent: `{"type":"m.room.message", "content": {"body": "test"} }`, + eventContent: `{"type":"m.room.message", "content": {"body": "test"},"room_id":"!room:example.com"}`, wantNotify: true, wantAction: pushrules.NotifyAction, wantActions: []*pushrules.Action{ @@ -132,6 +145,42 @@ func Test_evaluatePushRules(t *testing.T) { }) } +func TestLocalRoomMembers(t *testing.T) { + alice := test.NewUser(t) + _, sk, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) + bob := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + charlie := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + + room := test.NewRoom(t, alice) + room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(charlie.ID)) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, cfg.Global.ServerName, bcrypt.MinCost, 1000, 1000, "") + assert.NoError(t, err) + + err = rsapi.SendEvents(processCtx.Context(), rsAPI, rsapi.KindNew, room.Events(), "", "test", "test", nil, false) + assert.NoError(t, err) + + consumer := OutputRoomEventConsumer{db: db, rsAPI: rsAPI, serverName: "test", cfg: &cfg.UserAPI} + members, count, err := consumer.localRoomMembers(processCtx.Context(), room.ID) + assert.NoError(t, err) + assert.Equal(t, 3, count) + expectedLocalMember := &localMembership{UserID: alice.ID, Localpart: alice.Localpart, Domain: "test", MemberContent: gomatrixserverlib.MemberContent{Membership: spec.Join}} + assert.Equal(t, expectedLocalMember, members[0]) + }) + +} + func TestMessageStats(t *testing.T) { type args struct { eventType string @@ -250,3 +299,42 @@ func TestMessageStats(t *testing.T) { } }) } + +func BenchmarkLocalRoomMembers(b *testing.B) { + t := &testing.T{} + + cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypePostgres) + defer close() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, cfg.Global.ServerName, bcrypt.MinCost, 1000, 1000, "") + assert.NoError(b, err) + + consumer := OutputRoomEventConsumer{db: db, rsAPI: rsAPI, serverName: "test", cfg: &cfg.UserAPI} + _, sk, err := ed25519.GenerateKey(nil) + assert.NoError(b, err) + + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + for i := 0; i < 100; i++ { + user := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + room.CreateAndInsert(t, user, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(user.ID)) + } + + err = rsapi.SendEvents(processCtx.Context(), rsAPI, rsapi.KindNew, room.Events(), "", "test", "test", nil, false) + assert.NoError(b, err) + + expectedLocalMember := &localMembership{UserID: alice.ID, Localpart: alice.Localpart, Domain: "test", MemberContent: gomatrixserverlib.MemberContent{Membership: spec.Join}} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + members, count, err := consumer.localRoomMembers(processCtx.Context(), room.ID) + assert.NoError(b, err) + assert.Equal(b, 101, count) + assert.Equal(b, expectedLocalMember, members[0]) + } +} diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 32f3d84b5a..4e3c2671a1 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -33,6 +33,7 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/crypto/bcrypt" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/pushgateway" @@ -63,6 +64,37 @@ type UserInternalAPI struct { Updater *DeviceListUpdater } +func (a *UserInternalAPI) PerformAdminCreateRegistrationToken(ctx context.Context, registrationToken *clientapi.RegistrationToken) (bool, error) { + exists, err := a.DB.RegistrationTokenExists(ctx, *registrationToken.Token) + if err != nil { + return false, err + } + if exists { + return false, fmt.Errorf("token: %s already exists", *registrationToken.Token) + } + _, err = a.DB.InsertRegistrationToken(ctx, registrationToken) + if err != nil { + return false, fmt.Errorf("Error creating token: %s"+err.Error(), *registrationToken.Token) + } + return true, nil +} + +func (a *UserInternalAPI) PerformAdminListRegistrationTokens(ctx context.Context, returnAll bool, valid bool) ([]clientapi.RegistrationToken, error) { + return a.DB.ListRegistrationTokens(ctx, returnAll, valid) +} + +func (a *UserInternalAPI) PerformAdminGetRegistrationToken(ctx context.Context, tokenString string) (*clientapi.RegistrationToken, error) { + return a.DB.GetRegistrationToken(ctx, tokenString) +} + +func (a *UserInternalAPI) PerformAdminDeleteRegistrationToken(ctx context.Context, tokenString string) error { + return a.DB.DeleteRegistrationToken(ctx, tokenString) +} + +func (a *UserInternalAPI) PerformAdminUpdateRegistrationToken(ctx context.Context, tokenString string, newAttributes map[string]interface{}) (*clientapi.RegistrationToken, error) { + return a.DB.UpdateRegistrationToken(ctx, tokenString, newAttributes) +} + func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { @@ -531,12 +563,15 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) - if err != nil { - res.Err = err.Error() - } - res.Device = appServiceDevice + if err != nil || appServiceDevice != nil { + if err != nil { + res.Err = err.Error() + } + res.Device = appServiceDevice - return nil + return nil + } + // If the provided token wasn't an as_token (both err and appServiceDevice are nil), continue with normal auth. } device, err := a.DB.GetDeviceByAccessToken(ctx, req.AccessToken) if err != nil { diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index 4f5e99a8a1..125b315853 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/userapi/api" @@ -30,6 +31,15 @@ import ( "github.com/matrix-org/dendrite/userapi/types" ) +type RegistrationTokens interface { + RegistrationTokenExists(ctx context.Context, token string) (bool, error) + InsertRegistrationToken(ctx context.Context, registrationToken *clientapi.RegistrationToken) (bool, error) + ListRegistrationTokens(ctx context.Context, returnAll bool, valid bool) ([]clientapi.RegistrationToken, error) + GetRegistrationToken(ctx context.Context, tokenString string) (*clientapi.RegistrationToken, error) + DeleteRegistrationToken(ctx context.Context, tokenString string) error + UpdateRegistrationToken(ctx context.Context, tokenString string, newAttributes map[string]interface{}) (*clientapi.RegistrationToken, error) +} + type Profile interface { GetProfileByLocalpart(ctx context.Context, localpart string, serverName spec.ServerName) (*authtypes.Profile, error) SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) @@ -144,6 +154,7 @@ type UserDatabase interface { Pusher Statistics ThreePID + RegistrationTokens } type KeyChangeDatabase interface { diff --git a/userapi/storage/postgres/registration_tokens_table.go b/userapi/storage/postgres/registration_tokens_table.go new file mode 100644 index 0000000000..3c3e3fdd93 --- /dev/null +++ b/userapi/storage/postgres/registration_tokens_table.go @@ -0,0 +1,222 @@ +package postgres + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/clientapi/api" + internal "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "golang.org/x/exp/constraints" +) + +const registrationTokensSchema = ` +CREATE TABLE IF NOT EXISTS userapi_registration_tokens ( + token TEXT PRIMARY KEY, + pending BIGINT, + completed BIGINT, + uses_allowed BIGINT, + expiry_time BIGINT +); +` + +const selectTokenSQL = "" + + "SELECT token FROM userapi_registration_tokens WHERE token = $1" + +const insertTokenSQL = "" + + "INSERT INTO userapi_registration_tokens (token, uses_allowed, expiry_time, pending, completed) VALUES ($1, $2, $3, $4, $5)" + +const listAllTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens" + +const listValidTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens WHERE" + + "(uses_allowed > pending + completed OR uses_allowed IS NULL) AND" + + "(expiry_time > $1 OR expiry_time IS NULL)" + +const listInvalidTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens WHERE" + + "(uses_allowed <= pending + completed OR expiry_time <= $1)" + +const getTokenSQL = "" + + "SELECT pending, completed, uses_allowed, expiry_time FROM userapi_registration_tokens WHERE token = $1" + +const deleteTokenSQL = "" + + "DELETE FROM userapi_registration_tokens WHERE token = $1" + +const updateTokenUsesAllowedAndExpiryTimeSQL = "" + + "UPDATE userapi_registration_tokens SET uses_allowed = $2, expiry_time = $3 WHERE token = $1" + +const updateTokenUsesAllowedSQL = "" + + "UPDATE userapi_registration_tokens SET uses_allowed = $2 WHERE token = $1" + +const updateTokenExpiryTimeSQL = "" + + "UPDATE userapi_registration_tokens SET expiry_time = $2 WHERE token = $1" + +type registrationTokenStatements struct { + selectTokenStatement *sql.Stmt + insertTokenStatement *sql.Stmt + listAllTokensStatement *sql.Stmt + listValidTokensStatement *sql.Stmt + listInvalidTokenStatement *sql.Stmt + getTokenStatement *sql.Stmt + deleteTokenStatement *sql.Stmt + updateTokenUsesAllowedAndExpiryTimeStatement *sql.Stmt + updateTokenUsesAllowedStatement *sql.Stmt + updateTokenExpiryTimeStatement *sql.Stmt +} + +func NewPostgresRegistrationTokensTable(db *sql.DB) (tables.RegistrationTokensTable, error) { + s := ®istrationTokenStatements{} + _, err := db.Exec(registrationTokensSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectTokenStatement, selectTokenSQL}, + {&s.insertTokenStatement, insertTokenSQL}, + {&s.listAllTokensStatement, listAllTokensSQL}, + {&s.listValidTokensStatement, listValidTokensSQL}, + {&s.listInvalidTokenStatement, listInvalidTokensSQL}, + {&s.getTokenStatement, getTokenSQL}, + {&s.deleteTokenStatement, deleteTokenSQL}, + {&s.updateTokenUsesAllowedAndExpiryTimeStatement, updateTokenUsesAllowedAndExpiryTimeSQL}, + {&s.updateTokenUsesAllowedStatement, updateTokenUsesAllowedSQL}, + {&s.updateTokenExpiryTimeStatement, updateTokenExpiryTimeSQL}, + }.Prepare(db) +} + +func (s *registrationTokenStatements) RegistrationTokenExists(ctx context.Context, tx *sql.Tx, token string) (bool, error) { + var existingToken string + stmt := sqlutil.TxStmt(tx, s.selectTokenStatement) + err := stmt.QueryRowContext(ctx, token).Scan(&existingToken) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *registrationTokenStatements) InsertRegistrationToken(ctx context.Context, tx *sql.Tx, registrationToken *api.RegistrationToken) (bool, error) { + stmt := sqlutil.TxStmt(tx, s.insertTokenStatement) + _, err := stmt.ExecContext( + ctx, + *registrationToken.Token, + getInsertValue(registrationToken.UsesAllowed), + getInsertValue(registrationToken.ExpiryTime), + *registrationToken.Pending, + *registrationToken.Completed) + if err != nil { + return false, err + } + return true, nil +} + +func getInsertValue[t constraints.Integer](in *t) any { + if in == nil { + return nil + } + return *in +} + +func (s *registrationTokenStatements) ListRegistrationTokens(ctx context.Context, tx *sql.Tx, returnAll bool, valid bool) ([]api.RegistrationToken, error) { + var stmt *sql.Stmt + var tokens []api.RegistrationToken + var tokenString string + var pending, completed, usesAllowed *int32 + var expiryTime *int64 + var rows *sql.Rows + var err error + if returnAll { + stmt = sqlutil.TxStmt(tx, s.listAllTokensStatement) + rows, err = stmt.QueryContext(ctx) + } else if valid { + stmt = sqlutil.TxStmt(tx, s.listValidTokensStatement) + rows, err = stmt.QueryContext(ctx, time.Now().UnixNano()/int64(time.Millisecond)) + } else { + stmt = sqlutil.TxStmt(tx, s.listInvalidTokenStatement) + rows, err = stmt.QueryContext(ctx, time.Now().UnixNano()/int64(time.Millisecond)) + } + if err != nil { + return tokens, err + } + defer internal.CloseAndLogIfError(ctx, rows, "ListRegistrationTokens: rows.close() failed") + for rows.Next() { + err = rows.Scan(&tokenString, &pending, &completed, &usesAllowed, &expiryTime) + if err != nil { + return tokens, err + } + tokenString := tokenString + pending := pending + completed := completed + usesAllowed := usesAllowed + expiryTime := expiryTime + + tokenMap := api.RegistrationToken{ + Token: &tokenString, + Pending: pending, + Completed: completed, + UsesAllowed: usesAllowed, + ExpiryTime: expiryTime, + } + tokens = append(tokens, tokenMap) + } + return tokens, rows.Err() +} + +func (s *registrationTokenStatements) GetRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string) (*api.RegistrationToken, error) { + stmt := sqlutil.TxStmt(tx, s.getTokenStatement) + var pending, completed, usesAllowed *int32 + var expiryTime *int64 + err := stmt.QueryRowContext(ctx, tokenString).Scan(&pending, &completed, &usesAllowed, &expiryTime) + if err != nil { + return nil, err + } + token := api.RegistrationToken{ + Token: &tokenString, + Pending: pending, + Completed: completed, + UsesAllowed: usesAllowed, + ExpiryTime: expiryTime, + } + return &token, nil +} + +func (s *registrationTokenStatements) DeleteRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string) error { + stmt := sqlutil.TxStmt(tx, s.deleteTokenStatement) + _, err := stmt.ExecContext(ctx, tokenString) + if err != nil { + return err + } + return nil +} + +func (s *registrationTokenStatements) UpdateRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string, newAttributes map[string]interface{}) (*api.RegistrationToken, error) { + var stmt *sql.Stmt + usesAllowed, usesAllowedPresent := newAttributes["usesAllowed"] + expiryTime, expiryTimePresent := newAttributes["expiryTime"] + if usesAllowedPresent && expiryTimePresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenUsesAllowedAndExpiryTimeStatement) + _, err := stmt.ExecContext(ctx, tokenString, usesAllowed, expiryTime) + if err != nil { + return nil, err + } + } else if usesAllowedPresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenUsesAllowedStatement) + _, err := stmt.ExecContext(ctx, tokenString, usesAllowed) + if err != nil { + return nil, err + } + } else if expiryTimePresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenExpiryTimeStatement) + _, err := stmt.ExecContext(ctx, tokenString, expiryTime) + if err != nil { + return nil, err + } + } + return s.GetRegistrationToken(ctx, tx, tokenString) +} diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index 72e7c9cd90..b4edc80a95 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -31,7 +31,7 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName spec.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { +func NewDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName spec.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err @@ -53,6 +53,10 @@ func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties * return nil, err } + registationTokensTable, err := NewPostgresRegistrationTokensTable(db) + if err != nil { + return nil, fmt.Errorf("NewPostgresRegistrationsTokenTable: %w", err) + } accountsTable, err := NewPostgresAccountsTable(db, serverName) if err != nil { return nil, fmt.Errorf("NewPostgresAccountsTable: %w", err) @@ -125,6 +129,7 @@ func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties * ThreePIDs: threePIDTable, Pushers: pusherTable, Notifications: notificationsTable, + RegistrationTokens: registationTokensTable, Stats: statsTable, ServerName: serverName, DB: db, @@ -135,7 +140,7 @@ func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties * }, nil } -func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { +func NewKeyDatabase(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 705707571e..b7acb2035e 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/gomatrixserverlib/spec" "golang.org/x/crypto/bcrypt" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -43,6 +44,7 @@ import ( type Database struct { DB *sql.DB Writer sqlutil.Writer + RegistrationTokens tables.RegistrationTokensTable Accounts tables.AccountsTable Profiles tables.ProfileTable AccountDatas tables.AccountDataTable @@ -78,6 +80,42 @@ const ( loginTokenByteLength = 32 ) +func (d *Database) RegistrationTokenExists(ctx context.Context, token string) (bool, error) { + return d.RegistrationTokens.RegistrationTokenExists(ctx, nil, token) +} + +func (d *Database) InsertRegistrationToken(ctx context.Context, registrationToken *clientapi.RegistrationToken) (created bool, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + created, err = d.RegistrationTokens.InsertRegistrationToken(ctx, txn, registrationToken) + return err + }) + return +} + +func (d *Database) ListRegistrationTokens(ctx context.Context, returnAll bool, valid bool) ([]clientapi.RegistrationToken, error) { + return d.RegistrationTokens.ListRegistrationTokens(ctx, nil, returnAll, valid) +} + +func (d *Database) GetRegistrationToken(ctx context.Context, tokenString string) (*clientapi.RegistrationToken, error) { + return d.RegistrationTokens.GetRegistrationToken(ctx, nil, tokenString) +} + +func (d *Database) DeleteRegistrationToken(ctx context.Context, tokenString string) (err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + err = d.RegistrationTokens.DeleteRegistrationToken(ctx, nil, tokenString) + return err + }) + return +} + +func (d *Database) UpdateRegistrationToken(ctx context.Context, tokenString string, newAttributes map[string]interface{}) (updatedToken *clientapi.RegistrationToken, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + updatedToken, err = d.RegistrationTokens.UpdateRegistrationToken(ctx, txn, tokenString, newAttributes) + return err + }) + return +} + // GetAccountByPassword returns the account associated with the given localpart and password. // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByPassword( @@ -644,7 +682,7 @@ func (d *Database) CreateDevice( for i := 1; i <= 5; i++ { newDeviceID, returnErr = generateDeviceID() if returnErr != nil { - return + return nil, returnErr } returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { @@ -653,7 +691,7 @@ func (d *Database) CreateDevice( return err }) if returnErr == nil { - return + return dev, nil } } } diff --git a/userapi/storage/sqlite3/registration_tokens_table.go b/userapi/storage/sqlite3/registration_tokens_table.go new file mode 100644 index 0000000000..8979547317 --- /dev/null +++ b/userapi/storage/sqlite3/registration_tokens_table.go @@ -0,0 +1,222 @@ +package sqlite3 + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/clientapi/api" + internal "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "golang.org/x/exp/constraints" +) + +const registrationTokensSchema = ` +CREATE TABLE IF NOT EXISTS userapi_registration_tokens ( + token TEXT PRIMARY KEY, + pending BIGINT, + completed BIGINT, + uses_allowed BIGINT, + expiry_time BIGINT +); +` + +const selectTokenSQL = "" + + "SELECT token FROM userapi_registration_tokens WHERE token = $1" + +const insertTokenSQL = "" + + "INSERT INTO userapi_registration_tokens (token, uses_allowed, expiry_time, pending, completed) VALUES ($1, $2, $3, $4, $5)" + +const listAllTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens" + +const listValidTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens WHERE" + + "(uses_allowed > pending + completed OR uses_allowed IS NULL) AND" + + "(expiry_time > $1 OR expiry_time IS NULL)" + +const listInvalidTokensSQL = "" + + "SELECT * FROM userapi_registration_tokens WHERE" + + "(uses_allowed <= pending + completed OR expiry_time <= $1)" + +const getTokenSQL = "" + + "SELECT pending, completed, uses_allowed, expiry_time FROM userapi_registration_tokens WHERE token = $1" + +const deleteTokenSQL = "" + + "DELETE FROM userapi_registration_tokens WHERE token = $1" + +const updateTokenUsesAllowedAndExpiryTimeSQL = "" + + "UPDATE userapi_registration_tokens SET uses_allowed = $2, expiry_time = $3 WHERE token = $1" + +const updateTokenUsesAllowedSQL = "" + + "UPDATE userapi_registration_tokens SET uses_allowed = $2 WHERE token = $1" + +const updateTokenExpiryTimeSQL = "" + + "UPDATE userapi_registration_tokens SET expiry_time = $2 WHERE token = $1" + +type registrationTokenStatements struct { + selectTokenStatement *sql.Stmt + insertTokenStatement *sql.Stmt + listAllTokensStatement *sql.Stmt + listValidTokensStatement *sql.Stmt + listInvalidTokenStatement *sql.Stmt + getTokenStatement *sql.Stmt + deleteTokenStatement *sql.Stmt + updateTokenUsesAllowedAndExpiryTimeStatement *sql.Stmt + updateTokenUsesAllowedStatement *sql.Stmt + updateTokenExpiryTimeStatement *sql.Stmt +} + +func NewSQLiteRegistrationTokensTable(db *sql.DB) (tables.RegistrationTokensTable, error) { + s := ®istrationTokenStatements{} + _, err := db.Exec(registrationTokensSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectTokenStatement, selectTokenSQL}, + {&s.insertTokenStatement, insertTokenSQL}, + {&s.listAllTokensStatement, listAllTokensSQL}, + {&s.listValidTokensStatement, listValidTokensSQL}, + {&s.listInvalidTokenStatement, listInvalidTokensSQL}, + {&s.getTokenStatement, getTokenSQL}, + {&s.deleteTokenStatement, deleteTokenSQL}, + {&s.updateTokenUsesAllowedAndExpiryTimeStatement, updateTokenUsesAllowedAndExpiryTimeSQL}, + {&s.updateTokenUsesAllowedStatement, updateTokenUsesAllowedSQL}, + {&s.updateTokenExpiryTimeStatement, updateTokenExpiryTimeSQL}, + }.Prepare(db) +} + +func (s *registrationTokenStatements) RegistrationTokenExists(ctx context.Context, tx *sql.Tx, token string) (bool, error) { + var existingToken string + stmt := sqlutil.TxStmt(tx, s.selectTokenStatement) + err := stmt.QueryRowContext(ctx, token).Scan(&existingToken) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *registrationTokenStatements) InsertRegistrationToken(ctx context.Context, tx *sql.Tx, registrationToken *api.RegistrationToken) (bool, error) { + stmt := sqlutil.TxStmt(tx, s.insertTokenStatement) + _, err := stmt.ExecContext( + ctx, + *registrationToken.Token, + getInsertValue(registrationToken.UsesAllowed), + getInsertValue(registrationToken.ExpiryTime), + *registrationToken.Pending, + *registrationToken.Completed) + if err != nil { + return false, err + } + return true, nil +} + +func getInsertValue[t constraints.Integer](in *t) any { + if in == nil { + return nil + } + return *in +} + +func (s *registrationTokenStatements) ListRegistrationTokens(ctx context.Context, tx *sql.Tx, returnAll bool, valid bool) ([]api.RegistrationToken, error) { + var stmt *sql.Stmt + var tokens []api.RegistrationToken + var tokenString string + var pending, completed, usesAllowed *int32 + var expiryTime *int64 + var rows *sql.Rows + var err error + if returnAll { + stmt = sqlutil.TxStmt(tx, s.listAllTokensStatement) + rows, err = stmt.QueryContext(ctx) + } else if valid { + stmt = sqlutil.TxStmt(tx, s.listValidTokensStatement) + rows, err = stmt.QueryContext(ctx, time.Now().UnixNano()/int64(time.Millisecond)) + } else { + stmt = sqlutil.TxStmt(tx, s.listInvalidTokenStatement) + rows, err = stmt.QueryContext(ctx, time.Now().UnixNano()/int64(time.Millisecond)) + } + if err != nil { + return tokens, err + } + defer internal.CloseAndLogIfError(ctx, rows, "ListRegistrationTokens: rows.close() failed") + for rows.Next() { + err = rows.Scan(&tokenString, &pending, &completed, &usesAllowed, &expiryTime) + if err != nil { + return tokens, err + } + tokenString := tokenString + pending := pending + completed := completed + usesAllowed := usesAllowed + expiryTime := expiryTime + + tokenMap := api.RegistrationToken{ + Token: &tokenString, + Pending: pending, + Completed: completed, + UsesAllowed: usesAllowed, + ExpiryTime: expiryTime, + } + tokens = append(tokens, tokenMap) + } + return tokens, rows.Err() +} + +func (s *registrationTokenStatements) GetRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string) (*api.RegistrationToken, error) { + stmt := sqlutil.TxStmt(tx, s.getTokenStatement) + var pending, completed, usesAllowed *int32 + var expiryTime *int64 + err := stmt.QueryRowContext(ctx, tokenString).Scan(&pending, &completed, &usesAllowed, &expiryTime) + if err != nil { + return nil, err + } + token := api.RegistrationToken{ + Token: &tokenString, + Pending: pending, + Completed: completed, + UsesAllowed: usesAllowed, + ExpiryTime: expiryTime, + } + return &token, nil +} + +func (s *registrationTokenStatements) DeleteRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string) error { + stmt := sqlutil.TxStmt(tx, s.deleteTokenStatement) + _, err := stmt.ExecContext(ctx, tokenString) + if err != nil { + return err + } + return nil +} + +func (s *registrationTokenStatements) UpdateRegistrationToken(ctx context.Context, tx *sql.Tx, tokenString string, newAttributes map[string]interface{}) (*api.RegistrationToken, error) { + var stmt *sql.Stmt + usesAllowed, usesAllowedPresent := newAttributes["usesAllowed"] + expiryTime, expiryTimePresent := newAttributes["expiryTime"] + if usesAllowedPresent && expiryTimePresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenUsesAllowedAndExpiryTimeStatement) + _, err := stmt.ExecContext(ctx, tokenString, usesAllowed, expiryTime) + if err != nil { + return nil, err + } + } else if usesAllowedPresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenUsesAllowedStatement) + _, err := stmt.ExecContext(ctx, tokenString, usesAllowed) + if err != nil { + return nil, err + } + } else if expiryTimePresent { + stmt = sqlutil.TxStmt(tx, s.updateTokenExpiryTimeStatement) + _, err := stmt.ExecContext(ctx, tokenString, expiryTime) + if err != nil { + return nil, err + } + } + return s.GetRegistrationToken(ctx, tx, tokenString) +} diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index acd9678f21..fc13dde576 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -29,7 +29,7 @@ import ( ) // NewUserDatabase creates a new accounts and profiles database -func NewUserDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName spec.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { +func NewUserDatabase(ctx context.Context, conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName spec.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err @@ -50,7 +50,10 @@ func NewUserDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperti if err = m.Up(ctx); err != nil { return nil, err } - + registationTokensTable, err := NewSQLiteRegistrationTokensTable(db) + if err != nil { + return nil, fmt.Errorf("NewSQLiteRegistrationsTokenTable: %w", err) + } accountsTable, err := NewSQLiteAccountsTable(db, serverName) if err != nil { return nil, fmt.Errorf("NewSQLiteAccountsTable: %w", err) @@ -130,10 +133,11 @@ func NewUserDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperti LoginTokenLifetime: loginTokenLifetime, BcryptCost: bcryptCost, OpenIDTokenLifetimeMS: openIDTokenLifetimeMS, + RegistrationTokens: registationTokensTable, }, nil } -func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { +func NewKeyDatabase(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index 39231b2240..701383fcbf 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -34,7 +34,7 @@ import ( // and sets postgres connection parameters func NewUserDatabase( ctx context.Context, - conMan sqlutil.Connections, + conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName spec.ServerName, bcryptCost int, @@ -54,7 +54,7 @@ func NewUserDatabase( // NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme) // and sets postgres connection parameters. -func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { +func NewKeyDatabase(conMan *sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewKeyDatabase(conMan, dbProperties) diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 3c6214e7c6..3a0be73e4a 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -25,10 +25,20 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + clientapi "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/userapi/types" ) +type RegistrationTokensTable interface { + RegistrationTokenExists(ctx context.Context, txn *sql.Tx, token string) (bool, error) + InsertRegistrationToken(ctx context.Context, txn *sql.Tx, registrationToken *clientapi.RegistrationToken) (bool, error) + ListRegistrationTokens(ctx context.Context, txn *sql.Tx, returnAll bool, valid bool) ([]clientapi.RegistrationToken, error) + GetRegistrationToken(ctx context.Context, txn *sql.Tx, tokenString string) (*clientapi.RegistrationToken, error) + DeleteRegistrationToken(ctx context.Context, txn *sql.Tx, tokenString string) error + UpdateRegistrationToken(ctx context.Context, txn *sql.Tx, tokenString string, newAttributes map[string]interface{}) (*clientapi.RegistrationToken, error) +} + type AccountDataTable interface { InsertAccountData(ctx context.Context, txn *sql.Tx, localpart string, serverName spec.ServerName, roomID, dataType string, content json.RawMessage) error SelectAccountData(ctx context.Context, localpart string, serverName spec.ServerName) (map[string]json.RawMessage, map[string]map[string]json.RawMessage, error) diff --git a/userapi/userapi.go b/userapi/userapi.go index 6dcbc121f5..6b6dac884d 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -36,10 +36,13 @@ import ( // NewInternalAPI returns a concrete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +// +// Creating a new instance of the user API requires a roomserver API with a federation API set +// using its `SetFederationAPI` method, other you may get nil-dereference errors. func NewInternalAPI( processContext *process.ProcessContext, dendriteCfg *config.Dendrite, - cm sqlutil.Connections, + cm *sqlutil.Connections, natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index e1c88d47f9..2ea978d692 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "golang.org/x/crypto/bcrypt" @@ -22,6 +23,14 @@ import ( userUtil "github.com/matrix-org/dendrite/userapi/util" ) +func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + func TestNotifyUserCountsAsync(t *testing.T) { alice := test.NewUser(t) aliceLocalpart, serverName, err := gomatrixserverlib.SplitID('@', alice.ID) @@ -87,7 +96,7 @@ func TestNotifyUserCountsAsync(t *testing.T) { } // Prepare pusher with our test server URL - if err := db.UpsertPusher(ctx, api.Pusher{ + if err = db.UpsertPusher(ctx, api.Pusher{ Kind: api.HTTPKind, AppID: appID, PushKey: pushKey, @@ -99,8 +108,14 @@ func TestNotifyUserCountsAsync(t *testing.T) { } // Insert a dummy event + ev, err := synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Error(err) + } if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{ - Event: synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll), + Event: *ev, }); err != nil { t.Error(err) }