Skip to content

Commit cc08f3e

Browse files
scylla-operator-bot[bot]scylla-operator-bot[bot]
authored
Merge pull request scylladb#1759 from tnozicka/validate-ports
Validate open ports
2 parents 342d352 + 6ad1abf commit cc08f3e

File tree

3 files changed

+572
-0
lines changed

3 files changed

+572
-0
lines changed

‎test/e2e/set/scyllacluster/scyllacluster_listen.go‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
// Copyright (c) 2024 ScyllaDB.
2+
3+
package scyllacluster
4+
5+
import (
6+
"context"
7+
"net"
8+
"strings"
9+
10+
g "github.com/onsi/ginkgo/v2"
11+
o "github.com/onsi/gomega"
12+
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
13+
"github.com/scylladb/scylla-operator/pkg/controllerhelpers"
14+
"github.com/scylladb/scylla-operator/pkg/naming"
15+
"github.com/scylladb/scylla-operator/test/e2e/framework"
16+
"github.com/scylladb/scylla-operator/test/e2e/tools"
17+
"github.com/scylladb/scylla-operator/test/e2e/utils"
18+
linuxnetutils "github.com/scylladb/scylla-operator/test/e2e/utils/linux/net"
19+
scyllaclusterverification "github.com/scylladb/scylla-operator/test/e2e/utils/verification/scyllacluster"
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
)
22+
23+
var _ = g.Describe("ScyllaCluster", func() {
24+
f := framework.NewFramework("scyllacluster")
25+
26+
g.It("listens only on secure ports", func() {
27+
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
28+
defer cancel()
29+
30+
sc := f.GetDefaultScyllaCluster()
31+
sc.Spec.Datacenter.Racks[0].Members = 1
32+
sc.Spec.ExposeOptions = &scyllav1.ExposeOptions{
33+
BroadcastOptions: &scyllav1.NodeBroadcastOptions{
34+
Nodes: scyllav1.BroadcastOptions{
35+
Type: scyllav1.BroadcastAddressTypePodIP,
36+
},
37+
Clients: scyllav1.BroadcastOptions{
38+
Type: scyllav1.BroadcastAddressTypeServiceClusterIP,
39+
},
40+
},
41+
}
42+
sc.Spec.Alternator = &scyllav1.AlternatorSpec{}
43+
44+
framework.By("Creating a ScyllaCluster with 1 member")
45+
sc, err := f.ScyllaClient().ScyllaV1().ScyllaClusters(f.Namespace()).Create(ctx, sc, metav1.CreateOptions{})
46+
o.Expect(err).NotTo(o.HaveOccurred())
47+
48+
framework.By("Waiting for the ScyllaCluster to rollout (RV=%s)", sc.ResourceVersion)
49+
waitCtx1, waitCtx1Cancel := utils.ContextForRollout(ctx, sc)
50+
defer waitCtx1Cancel()
51+
sc, err = controllerhelpers.WaitForScyllaClusterState(waitCtx1, f.ScyllaClient().ScyllaV1().ScyllaClusters(sc.Namespace), sc.Name, controllerhelpers.WaitForStateOptions{}, utils.IsScyllaClusterRolledOut)
52+
o.Expect(err).NotTo(o.HaveOccurred())
53+
54+
scyllaclusterverification.Verify(ctx, f.KubeClient(), f.ScyllaClient(), sc)
55+
56+
serviceName := naming.MemberServiceNameForScyllaCluster(sc.Spec.Datacenter.Racks[0], sc, 0)
57+
nodeService, err := f.KubeClient().CoreV1().Services(sc.Namespace).Get(ctx, serviceName, metav1.GetOptions{})
58+
o.Expect(err).NotTo(o.HaveOccurred())
59+
60+
nodeServiceIP := nodeService.Spec.ClusterIP
61+
o.Expect(nodeServiceIP).NotTo(o.BeEmpty())
62+
63+
nodePod, err := f.KubeClient().CoreV1().Pods(sc.Namespace).Get(ctx, naming.PodNameFromService(nodeService), metav1.GetOptions{})
64+
o.Expect(err).NotTo(o.HaveOccurred())
65+
66+
nodePodIP := nodePod.Status.PodIP
67+
o.Expect(nodePodIP).NotTo(o.BeEmpty())
68+
69+
framework.By("Fetching raw network information from /proc/net/tcp{,6}")
70+
// Because the network stack is shared between containers in the same Pod, it doesn't matter which container we use.
71+
// We could also create an ephemeral Container in this Pod with an image we need, but the dependency here
72+
// is minimal (bash + tail) so, as long as one of the images in this Pod has those, using exec is fine and simplifies the test.
73+
stdout, stderr, err := tools.PodExec(
74+
f.KubeClient().CoreV1().RESTClient(),
75+
f.ClientConfig(),
76+
nodePod.Namespace,
77+
nodePod.Name,
78+
"scylla",
79+
[]string{
80+
"/usr/bin/bash",
81+
"-euEo",
82+
"pipefail",
83+
"-O",
84+
"inherit_errexit",
85+
"-c",
86+
strings.TrimPrefix(`
87+
tail -n +2 /proc/net/tcp
88+
tail -n +2 /proc/net/tcp6
89+
`, "\n"),
90+
},
91+
nil,
92+
)
93+
o.Expect(err).NotTo(o.HaveOccurred())
94+
o.Expect(stderr).To(o.BeEmpty())
95+
o.Expect(stdout).NotTo(o.BeEmpty())
96+
97+
procNetEntries, err := linuxnetutils.ParseProcNetEntries(stdout)
98+
o.Expect(err).NotTo(o.HaveOccurred())
99+
o.Expect(procNetEntries).NotTo(o.BeEmpty())
100+
101+
listenProcNetEntries := procNetEntries.FilterListen()
102+
/*
103+
* This is a list of allowed ports to be exposed using default configuration.
104+
* All of these ports should use encryption and force AuthN/AuthZ, unless it is not appropriate
105+
* or required for the type of data, e.g. /readyz probes.
106+
* We have some techdebt in this area so this contains some historical ports that don't uphold
107+
* to this standard. All such entries should have an issue link attached.
108+
*
109+
* Note: Some applications bind dedicated IPv4 and IPv6 socket while some reuse the IPv6 socket for IPv4 as well.
110+
*/
111+
o.Expect(listenProcNetEntries).To(o.ConsistOf([]linuxnetutils.AddressPort{
112+
{
113+
Address: net.ParseIP("::"),
114+
Port: 5090, // ScyllaDB Manager agent - metrics (insecure)
115+
},
116+
{
117+
Address: net.ParseIP("127.0.0.1"),
118+
Port: 5112, // ScyllaDB Manager agent - debug (insecure)
119+
},
120+
{
121+
Address: net.ParseIP("0.0.0.0"),
122+
Port: 7000, // ScyllaDB inter-node (insecure)
123+
// FIXME: Remove
124+
// https://github.com/scylladb/scylla-operator/issues/1217
125+
},
126+
{
127+
Address: net.ParseIP("0.0.0.0"),
128+
Port: 8043, // Alternator TLS
129+
},
130+
{
131+
Address: net.ParseIP("::"),
132+
Port: 8080, // Scylla Operator probes (insecure, non-sensitive data)
133+
},
134+
{
135+
Address: net.ParseIP("::"),
136+
Port: 42081, // ScyllaDB ignition probe
137+
},
138+
{
139+
Address: net.ParseIP("127.0.0.1"),
140+
Port: 9001, // supervisord (planned for removal with cont)
141+
// FIXME: Remove
142+
// https://github.com/scylladb/scylla-operator/issues/1769
143+
},
144+
{
145+
Address: net.ParseIP("::"),
146+
Port: 9100, // Node exporter - metrics (insecure)
147+
},
148+
{
149+
Address: net.ParseIP("0.0.0.0"),
150+
Port: 9042, // CQL (insecure)
151+
// FIXME: Remove
152+
// https://github.com/scylladb/scylla-operator/issues/1764
153+
},
154+
{
155+
Address: net.ParseIP("0.0.0.0"),
156+
Port: 9142, // CQL TLS (not AuthZ by default)
157+
// FIXME: Enforce AuthN+AuthZ by default
158+
// https://github.com/scylladb/scylla-operator/issues/1770
159+
},
160+
{
161+
Address: net.ParseIP("0.0.0.0"),
162+
Port: 9180, // ScyllaDB - metrics (insecure)
163+
},
164+
{
165+
Address: net.ParseIP("127.0.0.1"),
166+
Port: 10000, // ScyllaDB API (insecure and unprotected)
167+
},
168+
{
169+
Address: net.ParseIP("::"),
170+
Port: 10001, // ScyllaDB Manager API (insecure but authorized)
171+
// FIXME: This needs to be replaced with TLS
172+
// https://github.com/scylladb/scylla-operator/issues/1772
173+
},
174+
{
175+
Address: net.ParseIP("0.0.0.0"),
176+
Port: 19042, // Shard-aware CQL (insecure)
177+
// FIXME: Remove in favour of port 19142
178+
// https://github.com/scylladb/scylla-operator/issues/1764
179+
},
180+
{
181+
Address: net.ParseIP("0.0.0.0"),
182+
Port: 19142, // Shard-aware CQL TLS
183+
// FIXME: Enforce AuthN+AuthZ by default
184+
// https://github.com/scylladb/scylla-operator/issues/1770
185+
},
186+
}))
187+
})
188+
})

0 commit comments

Comments
 (0)