Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for userns #3941

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/nerdctl/container/container_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,13 @@ func processContainerCreateOptions(cmd *cobra.Command) (types.ContainerCreateOpt
}
// #endregion

// #region for userns
opt.Userns, err = cmd.Flags().GetString("userns")
if err != nil {
return opt, err
}
// #endregion

return opt, nil
}

Expand Down
161 changes: 161 additions & 0 deletions cmd/nerdctl/container/container_create_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@ import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"

"github.com/opencontainers/go-digest"
"gotest.tools/v3/assert"
"gotest.tools/v3/icmd"

"github.com/containerd/containerd/v2/defaults"

Expand Down Expand Up @@ -324,3 +328,160 @@ func TestCreateFromOCIArchive(t *testing.T) {
base.Cmd("create", "--rm", "--name", containerName, fmt.Sprintf("oci-archive://%s", tarPath)).AssertOK()
base.Cmd("start", "--attach", containerName).AssertOutContains("test-nerdctl-create-from-oci-archive")
}

func TestUsernsMappingCreateCmd(t *testing.T) {
nerdtest.Setup()
testCase := &test.Case{
Require: test.Not(nerdtest.ContainerdV1),
SubTests: []*test.Case{
{
Description: "Test container start with valid userns",
NoParallel: true, // Changes system config so running in non parallel mode
Setup: func(data test.Data, helpers test.Helpers) {
data.Set("validUserns", "nerdctltestuser")
data.Set("expectedHostUID", "123456789")
// need to be compiled with containerd version >2.0.2 to support multi uidmap and gidmap.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

change comment to 2.1.x

if err := appendUsernsConfig(data.Get("validUserns"), data.Get("expectedHostUID")); err != nil {
t.Fatalf("Failed to append userns config: %v", err)
}
},
Cleanup: func(data test.Data, helpers test.Helpers) {
removeUsernsConfig(t, data.Get("validUserns"), data.Get("expectedHostUID"))
helpers.Anyhow("rm", "-f", data.Identifier())
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
helpers.Ensure("create", "--tty", "--userns", data.Get("validUserns"), "--name", data.Identifier(), testutil.NginxAlpineImage)
return helpers.Command("start", data.Identifier())
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 0,
Output: func(stdout string, info string, t *testing.T) {
actualHostUID, err := getContainerHostUID(helpers, data.Identifier())
if err != nil {
t.Fatalf("Failed to get container host UID: %v", err)
}
assert.Assert(t, actualHostUID == data.Get("expectedHostUID"), info)
},
}
},
},
{
Description: "Test container start with invalid userns",
Setup: func(data test.Data, helpers test.Helpers) {
data.Set("invalidUserns", "invaliduser")
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Command("create", "--tty", "--userns", data.Get("invalidUserns"), "--name", data.Identifier(), testutil.NginxAlpineImage)
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 1,
}
},
},
},
}
testCase.Run(t)
}

func runUsernsContainer(t *testing.T, name, userns, image, cmd string) *icmd.Result {
base := testutil.NewBase(t)
removeContainerArgs := []string{
"rm", "-f", name,
}
base.Cmd(removeContainerArgs...).Run()

args := []string{
"run", "-d", "--userns", userns, "--name", name, image, "sh", "-c", cmd,
}
return base.Cmd(args...).Run()
}

func getContainerHostUID(helpers test.Helpers, containerName string) (string, error) {
result := helpers.Capture("inspect", "--format", "{{.State.Pid}}", containerName)
pidStr := strings.TrimSpace(result)
pid, err := strconv.Atoi(pidStr)
if err != nil {
return "", fmt.Errorf("invalid PID: %v", err)
}

stat, err := os.Stat(fmt.Sprintf("/proc/%d", pid))
if err != nil {
return "", fmt.Errorf("failed to stat process: %v", err)
}

uid := int(stat.Sys().(*syscall.Stat_t).Uid)
return strconv.Itoa(uid), nil
}

func appendUsernsConfig(userns string, hostUid string) error {
if err := addUser(userns, hostUid); err != nil {
return fmt.Errorf("failed to add user %s: %w", userns, err)
}

entry := fmt.Sprintf("%s:%s:65536\n", userns, hostUid)

files := []string{"/etc/subuid", "/etc/subgid"}
for _, file := range files {
f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("failed to open %s: %w", file, err)
}
defer f.Close()

if _, err := f.WriteString(entry); err != nil {
return fmt.Errorf("failed to write to %s: %w", file, err)
}
}
return nil
}

func addUser(username string, hostId string) error {
Copy link
Member

@AkihiroSuda AkihiroSuda Feb 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is scary to run by default. Needs to have an opt-in flag like -test.allow-modify-user

Similar:

flag.BoolVar(&flagTestKillDaemon, "test.allow-kill-daemon", false, "enable tests that kill the daemon")

cmd := exec.Command("groupadd", "-g", hostId, username)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("groupadd failed: %s, %w", string(output), err)
}
cmd = exec.Command("useradd", "-u", hostId, "-g", hostId, "-s", "/bin/false", username)
output, err = cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("useradd failed: %s, %w", string(output), err)
}
return nil
}

func removeUsernsConfig(t *testing.T, userns string, hostUid string) {

if err := delUser(userns); err != nil {
t.Logf("failed to del user %s", userns)
}

entry := fmt.Sprintf("%s:%s:65536\n", userns, hostUid)

files := []string{"/etc/subuid", "/etc/subgid"}
for _, file := range files {
content, err := os.ReadFile(file)
if err != nil {
t.Logf("Failed to read %s: %v", file, err)
continue
}

newContent := strings.ReplaceAll(string(content), entry, "")
if err := os.WriteFile(file, []byte(newContent), 0644); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is scary too. Needs to be opt-in.

Also, the content of /etc/subuid and /etc/subgid has to be backed up and has to be restored with defer()

t.Logf("Failed to write to %s: %v", file, err)
}
}
}

func delUser(username string) error {
cmd := exec.Command("sudo", "userdel", username)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sudo shouldn't be needed when the test is running as the root

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ack

output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("userdel failed: %s, %w", string(output), err)
}
return nil
}
6 changes: 6 additions & 0 deletions cmd/nerdctl/container/container_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ func setCreateFlags(cmd *cobra.Command) {
cmd.Flags().String("ipfs-address", "", "multiaddr of IPFS API (default uses $IPFS_PATH env variable if defined or local directory ~/.ipfs)")

cmd.Flags().String("isolation", "default", "Specify isolation technology for container. On Linux the only valid value is default. Windows options are host, process and hyperv with process isolation as the default")
cmd.Flags().String("userns", "", "Support idmapping of containers. This options is only supported on linux. If `host` is passed, no idmapping is done. if a user name is passed, it does idmapping based on the uidmap and gidmap ranges specified in /etc/subuid and /etc/subgid respectively")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add docs/command-reference.md.

Also, the command line seems incompatible with Docker?
Docker doesn't accept a username here, and the name is hardcoded to "dockremap".
Maybe we should have its equivalent as "nerdremap"?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Podman accepts --subuidname string --subgidname string to specify a custom user name

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, the command line seems incompatible with Docker?
if they add host it will behave as docker as we check for that string and create the default snapshot.

For other names it behaves as docker daemon but at a container level rather than at daemon level. Will you suggest we configure it in nerdctl config instead?

cmd.RegisterFlagCompletionFunc("isolation", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if runtime.GOOS == "windows" {
return []string{"default", "host", "process", "hyperv"}, cobra.ShellCompDirectiveNoFileComp
Expand Down Expand Up @@ -317,6 +318,11 @@ func processCreateCommandFlagsInRun(cmd *cobra.Command) (types.ContainerCreateOp
return opt, err
}

opt.Userns, err = cmd.Flags().GetString("userns")
if err != nil {
return opt, err
}

validAttachFlag := true
for i, str := range opt.Attach {
opt.Attach[i] = strings.ToUpper(str)
Expand Down
89 changes: 89 additions & 0 deletions cmd/nerdctl/container/container_run_user_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@
package container

import (
"fmt"
"testing"

"github.com/containerd/nerdctl/v2/pkg/testutil"
"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
"github.com/containerd/nerdctl/v2/pkg/testutil/test"
"gotest.tools/v3/assert"
)

func TestRunUserGID(t *testing.T) {
Expand Down Expand Up @@ -181,3 +185,88 @@ func TestRunAddGroup_CVE_2023_25173(t *testing.T) {
base.Cmd(cmd...).AssertOutContains(testCase.expected + "\n")
}
}

func TestUsernsMappingRunCmd(t *testing.T) {
nerdtest.Setup()
testCase := &test.Case{
Require: test.Require(test.Not(nerdtest.ContainerdV1),
test.Not(nerdtest.Docker)),
SubTests: []*test.Case{
{
Description: "Test container start with valid userns",
NoParallel: true, // Changes system config so running in non parallel mode
Setup: func(data test.Data, helpers test.Helpers) {
data.Set("validUserns", "nerdctltestuser")
data.Set("expectedHostUID", "123456789")
if err := appendUsernsConfig(data.Get("validUserns"), data.Get("expectedHostUID")); err != nil {
t.Fatalf("Failed to append userns config: %v", err)
}
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
removeUsernsConfig(t, data.Get("validUserns"), data.Get("expectedHostUID"))
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Command("run", "--tty", "-d", "--userns", data.Get("validUserns"), "--name", data.Identifier(), testutil.NginxAlpineImage)
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 0,
Output: func(stdout string, info string, t *testing.T) {
actualHostUID, err := getContainerHostUID(helpers, data.Identifier())
if err != nil {
t.Fatalf("Failed to get container host UID: %v", err)
}
assert.Assert(t, actualHostUID == data.Get("expectedHostUID"), info)
},
}
},
},
{
Description: "Test container network share with valid userns",
NoParallel: true, // Changes system config so running in non parallel mode
Setup: func(data test.Data, helpers test.Helpers) {
data.Set("validUserns", "nerdctltestuser")
data.Set("expectedHostUID", "123456789")
data.Set("net-container", "net-container")
if err := appendUsernsConfig(data.Get("validUserns"), data.Get("expectedHostUID")); err != nil {
t.Fatalf("Failed to append userns config: %v", err)
}

helpers.Ensure("run", "--tty", "-d", "--userns", data.Get("validUserns"), "--name", data.Get("net-container"), testutil.NginxAlpineImage)
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
helpers.Anyhow("rm", "-f", data.Get("net-container"))
removeUsernsConfig(t, data.Get("validUserns"), data.Get("expectedHostUID"))
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Command("run", "--tty", "-d", "--userns", data.Get("validUserns"), "--net", fmt.Sprintf("container:%s", data.Get("net-container")), "--name", data.Identifier(), testutil.NginxAlpineImage)
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 0,
}
},
},
{
Description: "Test container start with invalid userns",
Setup: func(data test.Data, helpers test.Helpers) {
data.Set("invalidUserns", "invaliduser")
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Command("run", "--tty", "-d", "--userns", data.Get("invalidUserns"), "--name", data.Identifier(), testutil.NginxAlpineImage)
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 1,
}
},
},
},
}
testCase.Run(t)
}
31 changes: 12 additions & 19 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ go 1.22.7
require (
github.com/Masterminds/semver/v3 v3.3.1
github.com/Microsoft/go-winio v0.6.2
github.com/Microsoft/hcsshim v0.12.9
github.com/compose-spec/compose-go/v2 v2.4.8
github.com/containerd/accelerated-container-image v1.3.0
github.com/Microsoft/hcsshim v0.13.0-rc.3
github.com/compose-spec/compose-go/v2 v2.4.7
github.com/containerd/accelerated-container-image v1.2.3
github.com/containerd/cgroups/v3 v3.0.5
github.com/containerd/console v1.0.4
github.com/containerd/containerd/api v1.8.0
github.com/containerd/containerd/v2 v2.0.2
github.com/containerd/containerd/v2 v2.0.1-0.20250211161307-525332b29211
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be possible to cherry-pick the relevant commit to v2.0?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

may not be the commit entirely but only the relevant functions as it is client side.

github.com/containerd/continuity v0.4.5
github.com/containerd/errdefs v1.0.0
github.com/containerd/fifo v1.1.0
Expand Down Expand Up @@ -50,6 +50,7 @@ require (
github.com/moby/sys/mount v0.3.4
github.com/moby/sys/mountinfo v0.7.2
github.com/moby/sys/signal v0.7.1
github.com/moby/sys/user v0.3.0
github.com/moby/sys/userns v0.1.0
github.com/moby/term v0.5.2
github.com/muesli/cancelreader v0.2.2
Expand All @@ -76,8 +77,6 @@ require (
)

require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cilium/ebpf v0.16.0 // indirect
Expand Down Expand Up @@ -108,15 +107,13 @@ require (
github.com/moby/locker v1.0.1 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/symlink v0.3.0 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.13.0 // indirect
github.com/multiformats/go-multibase v0.2.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
github.com/opencontainers/selinux v1.11.1 // indirect
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
Expand All @@ -126,24 +123,20 @@ require (
github.com/smallstep/pkcs7 v0.1.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/tinylib/msgp v1.2.0 // indirect
github.com/vbatts/tar-split v0.11.6 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
go.opentelemetry.io/otel v1.31.0 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/otel/trace v1.31.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect
go.opentelemetry.io/otel v1.34.0 // indirect
go.opentelemetry.io/otel/metric v1.34.0 // indirect
go.opentelemetry.io/otel/trace v1.34.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
golang.org/x/mod v0.22.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
google.golang.org/grpc v1.69.4 // indirect
google.golang.org/protobuf v1.36.2 // indirect
google.golang.org/protobuf v1.36.3 // indirect
lukechampine.com/blake3 v1.3.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
tags.cncf.io/container-device-interface v0.8.0 // indirect
tags.cncf.io/container-device-interface/specs-go v0.8.0 // indirect
)
Loading
Loading