From 37218170affa6fd2c47f639c08513fb546eb5051 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 09:13:08 +0000 Subject: [PATCH 1/6] fix: merge --- internal/core/runner/python/python.go | 2 +- internal/core/runner/python/setup.go | 42 ++++++++++++++++++--------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/internal/core/runner/python/python.go b/internal/core/runner/python/python.go index ea9ca32..4cef062 100644 --- a/internal/core/runner/python/python.go +++ b/internal/core/runner/python/python.go @@ -99,7 +99,7 @@ func (p *PythonRunner) Run( func (p *PythonRunner) InitializeEnvironment(code string, preload string, options *types.RunnerOptions) (string, string, error) { if !checkLibAvaliable() { // ensure environment is reversed - releaseLibBinary() + releaseLibBinary(false) } // create a tmp dir and copy the python script diff --git a/internal/core/runner/python/setup.go b/internal/core/runner/python/setup.go index 65a2ed0..46eb2f4 100644 --- a/internal/core/runner/python/setup.go +++ b/internal/core/runner/python/setup.go @@ -26,28 +26,42 @@ const ( ) func init() { - releaseLibBinary() + releaseLibBinary(true) } -func releaseLibBinary() { +func releaseLibBinary(force_remove_old_lib bool) { log.Info("initializing python runner environment...") // remove the old lib if _, err := os.Stat(path.Join(LIB_PATH, LIB_NAME)); err == nil { - err := os.Remove(path.Join(LIB_PATH, LIB_NAME)) + if force_remove_old_lib { + err := os.Remove(path.Join(LIB_PATH, LIB_NAME)) + if err != nil { + log.Panic(fmt.Sprintf("failed to remove %s", path.Join(LIB_PATH, LIB_NAME))) + } + + // write the new lib + err = os.MkdirAll(LIB_PATH, 0755) + if err != nil { + log.Panic(fmt.Sprintf("failed to create %s", LIB_PATH)) + } + err = os.WriteFile(path.Join(LIB_PATH, LIB_NAME), python_lib, 0755) + if err != nil { + log.Panic(fmt.Sprintf("failed to write %s", path.Join(LIB_PATH, LIB_NAME))) + } + } + } else if err == os.ErrNotExist { + err = os.MkdirAll(LIB_PATH, 0755) if err != nil { - log.Panic(fmt.Sprintf("failed to remove %s", path.Join(LIB_PATH, LIB_NAME))) + log.Panic(fmt.Sprintf("failed to create %s", LIB_PATH)) } + err = os.WriteFile(path.Join(LIB_PATH, LIB_NAME), python_lib, 0755) + if err != nil { + log.Panic(fmt.Sprintf("failed to write %s", path.Join(LIB_PATH, LIB_NAME))) + } + log.Info("python runner environment initialized") + } else { + log.Panic(fmt.Sprintf("failed to initialize python runner environment: %v", err)) } - - err := os.MkdirAll(LIB_PATH, 0755) - if err != nil { - log.Panic(fmt.Sprintf("failed to create %s", LIB_PATH)) - } - err = os.WriteFile(path.Join(LIB_PATH, LIB_NAME), python_lib, 0755) - if err != nil { - log.Panic(fmt.Sprintf("failed to write %s", path.Join(LIB_PATH, LIB_NAME))) - } - log.Info("python runner environment initialized") } func checkLibAvaliable() bool { From 3f393fc182acd6ccf27b0a9180bf18535e0f4dbf Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 09:21:40 +0000 Subject: [PATCH 2/6] fix --- internal/core/runner/python/setup.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/core/runner/python/setup.go b/internal/core/runner/python/setup.go index 46eb2f4..a89772c 100644 --- a/internal/core/runner/python/setup.go +++ b/internal/core/runner/python/setup.go @@ -49,7 +49,7 @@ func releaseLibBinary(force_remove_old_lib bool) { log.Panic(fmt.Sprintf("failed to write %s", path.Join(LIB_PATH, LIB_NAME))) } } - } else if err == os.ErrNotExist { + } else { err = os.MkdirAll(LIB_PATH, 0755) if err != nil { log.Panic(fmt.Sprintf("failed to create %s", LIB_PATH)) @@ -59,8 +59,6 @@ func releaseLibBinary(force_remove_old_lib bool) { log.Panic(fmt.Sprintf("failed to write %s", path.Join(LIB_PATH, LIB_NAME))) } log.Info("python runner environment initialized") - } else { - log.Panic(fmt.Sprintf("failed to initialize python runner environment: %v", err)) } } From b7c6a8733c5c31b9a41317a38204f09ec23e3fef Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 09:28:56 +0000 Subject: [PATCH 3/6] fix: cleanup --- internal/core/runner/python/python.go | 67 +++++++++++---------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/internal/core/runner/python/python.go b/internal/core/runner/python/python.go index 4cef062..abb807d 100644 --- a/internal/core/runner/python/python.go +++ b/internal/core/runner/python/python.go @@ -25,12 +25,6 @@ type PythonRunner struct { //go:embed prescript.py var sandbox_fs []byte -var ( - REQUIRED_FS = []string{ - path.Join(LIB_PATH, LIB_NAME), - } -) - func (p *PythonRunner) Run( code string, timeout time.Duration, @@ -50,45 +44,36 @@ func (p *PythonRunner) Run( output_handler := runner.NewOutputCaptureRunner() output_handler.SetTimeout(timeout) - err = p.WithTempDir(LIB_PATH, REQUIRED_FS, func(root_path string) error { - // cleanup - output_handler.SetAfterExitHook(func() { - os.RemoveAll(root_path) - os.Remove(root_path) - }) - - // create a new process - cmd := exec.Command( - configuration.PythonPath, - untrusted_code_path, - LIB_PATH, - key, - ) - cmd.Env = []string{} - - if configuration.Proxy.Socks5 != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", configuration.Proxy.Socks5)) - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", configuration.Proxy.Socks5)) - } else if configuration.Proxy.Https != "" || configuration.Proxy.Http != "" { - if configuration.Proxy.Https != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", configuration.Proxy.Https)) - } - if configuration.Proxy.Http != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", configuration.Proxy.Http)) - } - } - if len(configuration.AllowedSyscalls) > 0 { - cmd.Env = append(cmd.Env, fmt.Sprintf("ALLOWED_SYSCALLS=%s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(configuration.AllowedSyscalls)), ","), "[]"))) + // create a new process + cmd := exec.Command( + configuration.PythonPath, + untrusted_code_path, + LIB_PATH, + key, + ) + cmd.Env = []string{} + + if configuration.Proxy.Socks5 != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", configuration.Proxy.Socks5)) + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", configuration.Proxy.Socks5)) + } else if configuration.Proxy.Https != "" || configuration.Proxy.Http != "" { + if configuration.Proxy.Https != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", configuration.Proxy.Https)) } - - err = output_handler.CaptureOutput(cmd) - if err != nil { - return err + if configuration.Proxy.Http != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", configuration.Proxy.Http)) } + } - return nil - }) + if len(configuration.AllowedSyscalls) > 0 { + cmd.Env = append(cmd.Env, + fmt.Sprintf("ALLOWED_SYSCALLS=%s", + strings.Trim(strings.Join(strings.Fields(fmt.Sprint(configuration.AllowedSyscalls)), ","), "[]"), + ), + ) + } + err = output_handler.CaptureOutput(cmd) if err != nil { return nil, nil, nil, err } From 60f47383016c60b4ab8a165604c8f4ccc2b1e0f6 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 09:50:23 +0000 Subject: [PATCH 4/6] fix: library relative path --- internal/core/runner/python/prescript.py | 2 +- internal/core/runner/python/python.go | 1 + internal/core/runner/python/setup.go | 1 - .../python_malicious_test.go | 27 ++++++++++++++++--- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/internal/core/runner/python/prescript.py b/internal/core/runner/python/prescript.py index b4035d4..f465e10 100644 --- a/internal/core/runner/python/prescript.py +++ b/internal/core/runner/python/prescript.py @@ -10,7 +10,7 @@ def excepthook(type, value, tb): sys.excepthook = excepthook -lib = ctypes.CDLL("./var/sandbox/sandbox-python/python.so") +lib = ctypes.CDLL("./python.so") lib.DifySeccomp.argtypes = [ctypes.c_uint32, ctypes.c_uint32, ctypes.c_bool] lib.DifySeccomp.restype = None diff --git a/internal/core/runner/python/python.go b/internal/core/runner/python/python.go index abb807d..6ad85c3 100644 --- a/internal/core/runner/python/python.go +++ b/internal/core/runner/python/python.go @@ -52,6 +52,7 @@ func (p *PythonRunner) Run( key, ) cmd.Env = []string{} + cmd.Dir = LIB_PATH if configuration.Proxy.Socks5 != "" { cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", configuration.Proxy.Socks5)) diff --git a/internal/core/runner/python/setup.go b/internal/core/runner/python/setup.go index a89772c..701c18e 100644 --- a/internal/core/runner/python/setup.go +++ b/internal/core/runner/python/setup.go @@ -100,7 +100,6 @@ func InstallDependencies(requirements string) error { runner := runner.TempDirRunner{} return runner.WithTempDir("/", []string{}, func(root_path string) error { - defer os.Remove(root_path) defer os.RemoveAll(root_path) // create a requirements file err := os.WriteFile(path.Join(root_path, "requirements.txt"), []byte(requirements), 0644) diff --git a/tests/integration_tests/python_malicious_test.go b/tests/integration_tests/python_malicious_test.go index f3b2d37..860b96a 100644 --- a/tests/integration_tests/python_malicious_test.go +++ b/tests/integration_tests/python_malicious_test.go @@ -14,7 +14,9 @@ func TestSysFork(t *testing.T) { import os print(os.fork()) print(123) - `, "", &types.RunnerOptions{}) + `, "", &types.RunnerOptions{ + EnableNetwork: true, + }) if resp.Code != 0 { t.Error(resp) @@ -30,7 +32,9 @@ func TestExec(t *testing.T) { resp := service.RunPython3Code(` import os os.execl("/bin/ls", "ls") - `, "", &types.RunnerOptions{}) + `, "", &types.RunnerOptions{ + EnableNetwork: true, + }) if resp.Code != 0 { t.Error(resp) } @@ -45,7 +49,9 @@ func TestRunCommand(t *testing.T) { resp := service.RunPython3Code(` import subprocess subprocess.run(["ls", "-l"]) - `, "", &types.RunnerOptions{}) + `, "", &types.RunnerOptions{ + EnableNetwork: true, + }) if resp.Code != 0 { t.Error(resp) } @@ -54,3 +60,18 @@ subprocess.run(["ls", "-l"]) t.Error(resp.Data.(*service.RunCodeResponse).Stderr) } } + +func TestReadEtcPasswd(t *testing.T) { + resp := service.RunPython3Code(` +print(open("/etc/passwd").read()) + `, "", &types.RunnerOptions{ + EnableNetwork: true, + }) + if resp.Code != 0 { + t.Error(resp) + } + + if !strings.Contains(resp.Data.(*service.RunCodeResponse).Stderr, "No such file or directory") { + t.Error(resp.Data.(*service.RunCodeResponse).Stderr) + } +} From 9c915f723c7401de0865f3e179a4a7daedd489a8 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 09:53:52 +0000 Subject: [PATCH 5/6] fix: nodejs --- internal/core/runner/nodejs/nodejs.go | 7 ++++++- tests/integration_tests/conf/config.yaml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/core/runner/nodejs/nodejs.go b/internal/core/runner/nodejs/nodejs.go index 34b21b0..33c23b6 100644 --- a/internal/core/runner/nodejs/nodejs.go +++ b/internal/core/runner/nodejs/nodejs.go @@ -70,7 +70,12 @@ func (p *NodeJsRunner) Run( cmd.Env = []string{} if len(configuration.AllowedSyscalls) > 0 { - cmd.Env = append(cmd.Env, fmt.Sprintf("ALLOWED_SYSCALLS=%s", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(configuration.AllowedSyscalls)), ","), "[]"))) + cmd.Env = append( + cmd.Env, + fmt.Sprintf("ALLOWED_SYSCALLS=%s", strings.Trim( + strings.Join(strings.Fields(fmt.Sprint(configuration.AllowedSyscalls)), ","), "[]", + )), + ) } // capture the output diff --git a/tests/integration_tests/conf/config.yaml b/tests/integration_tests/conf/config.yaml index 10a5034..55e27bf 100644 --- a/tests/integration_tests/conf/config.yaml +++ b/tests/integration_tests/conf/config.yaml @@ -4,7 +4,7 @@ app: key: dify-sandbox max_workers: 4 max_requests: 50 -worker_timeout: 5 +worker_timeout: 30 python_path: /usr/bin/python3 enable_network: True # please make sure there is no network risk in your environment allowed_syscalls: # please leave it empty if you have no idea how seccomp works From adb7d7ff5a8d1ee8ea46b8de88f91e095daa957b Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 6 Sep 2024 10:16:54 +0000 Subject: [PATCH 6/6] fix: upgrade expat --- docker/amd64/dockerfile | 2 +- docker/arm64/dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/amd64/dockerfile b/docker/amd64/dockerfile index e599b25..ed18b60 100644 --- a/docker/amd64/dockerfile +++ b/docker/amd64/dockerfile @@ -4,7 +4,7 @@ RUN apt-get clean \ && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \ && apt-get update \ && apt-get install -y pkg-config libseccomp-dev wget xz-utils \ - && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-1 perl=5.38.2-5 libsqlite3-0=3.46.0-1 + && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-2 perl=5.38.2-5 libsqlite3-0=3.46.0-1 # copy main binary to /main COPY main /main diff --git a/docker/arm64/dockerfile b/docker/arm64/dockerfile index 0ca1fd7..a0dd332 100644 --- a/docker/arm64/dockerfile +++ b/docker/arm64/dockerfile @@ -4,7 +4,7 @@ RUN apt-get clean \ && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \ && apt-get update \ && apt-get install -y pkg-config libseccomp-dev wget xz-utils \ - && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-1 perl=5.38.2-5 libsqlite3-0=3.46.0-1 + && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-2 perl=5.38.2-5 libsqlite3-0=3.46.0-1 # copy main binary to /main COPY main /main