wandb/local crashes on Apple Silicon: rebuild with Go 1.25.4+ to fix

[Buggy-McBugFace]

wandb/local:latest crashes immediately on Apple Silicon when running via QEMU (Podman/Docker).

runtime: taggedPointerPack invalid packing: ptr=0xffffa7a42a00 tag=0x1 packed=0xffffa7a42a000001 -> ptr=0xffffffffa7a42a00 tag=0x1
fatal error: taggedPointerPack

This is a known Go runtime bug affecting linux/amd64 binaries under QEMU on arm64: golang/go#69255

The image is built with Go 1.25.3. Rebuilding with Go 1.25.4+ fixes it: golang/go@bf95b76

Full stack trace

WARNING: image platform (linux/amd64) does not match the expected platform (linux/arm64)
*** Running /etc/my_init.d/01_enable-services.sh...
*** Enabling production mode
*** Running /etc/my_init.d/02_load-settings.sh...
*** Loading settings...
runtime: taggedPointerPack invalid packing: ptr=0xffffa7a42a00 tag=0x1 packed=0xffffa7a42a000001 -> ptr=0xffffffffa7a42a00 tag=0x1
fatal error: taggedPointerPack

goroutine 1 gp=0xc000002380 m=0 mp=0x52635c0 [running, locked to thread]:
runtime.throw({0x2f41b61?, 0x0?})
	/usr/local/go/src/runtime/panic.go:1094 +0x48 fp=0xc000196b68 sp=0xc000196b38 pc=0x47c048
runtime.taggedPointerPack(0xffffa7a42a00, 0x1)
	/usr/local/go/src/runtime/tagptr_64bit.go:60 +0x12a fp=0xc000196ba0 sp=0xc000196b68 pc=0x46642a
runtime.netpollopen(0x5288210?, 0x5286a30?)
	/usr/local/go/src/runtime/netpoll_epoll.go:52 +0x3e fp=0xc000196c08 sp=0xc000196ba0 pc=0x440c9e
internal/poll.runtime_pollOpen(0x4)
	/usr/local/go/src/runtime/netpoll.go:272 +0xee fp=0xc000196c48 sp=0xc000196c08 pc=0x47b10e
internal/poll.(*pollDesc).init(0xc0000d6080, 0xc0000d6060?)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:40 +0x4a fp=0xc000196c68 sp=0xc000196c48 pc=0x5145aa
internal/poll.(*FD).Init(0xc0000d6060, {0x2f29cc2?, 0xc00007e1c8?}, 0x80?)
	/usr/local/go/src/internal/poll/fd_unix.go:66 +0x45 fp=0xc000196c88 sp=0xc000196c68 pc=0x515485
os.newFile(0x4, {0xc00007e1c8, 0x14}, 0x1, 0x0)
	/usr/local/go/src/os/file_unix.go:218 +0x15d fp=0xc000196cc8 sp=0xc000196c88 pc=0x52417d
os.openFileNolog({0xc00007e1c8, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file_unix.go:278 +0x17d fp=0xc000196d20 sp=0xc000196cc8 pc=0x52439d
os.OpenFile({0xc00007e1c8, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file.go:412 +0x3e fp=0xc000196d50 sp=0xc000196d20 pc=0x52297e
os.Open(...)
	/usr/local/go/src/os/file.go:390
google.golang.org/protobuf/internal/detrand.binaryHash()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:46 +0x53 fp=0xc000196e18 sp=0xc000196d50 pc=0xac70d3
google.golang.org/protobuf/internal/detrand.init()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:38 +0xf fp=0xc000196e28 sp=0xc000196e18 pc=0xac706f
runtime.doInit1(0x506ecd0)
	/usr/local/go/src/runtime/proc.go:7670 +0xd5 fp=0xc000196f50 sp=0xc000196e28 pc=0x456f15
runtime.doInit(...)
	/usr/local/go/src/runtime/proc.go:7637
runtime.main()
	/usr/local/go/src/runtime/proc.go:256 +0x350 fp=0xc000196fe0 sp=0xc000196f50 pc=0x447d90
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000196fe8 sp=0xc000196fe0 pc=0x484001

[exalate-issue-sync]

Anshuman Singh commented:
This request was closed and merged into request #107108 "[SDK] wandb/local crashes on App...".

[exalate-issue-sync]

Anshuman Singh commented:
Request #107107 "wandb/local crashes on Apple Sil..." was closed and merged into this request. Last comment in request #107107:

wandb/local:latest crashes immediately on Apple Silicon when running via QEMU (Podman/Docker).

runtime: taggedPointerPack invalid packing: ptr=0xffffa7a42a00 tag=0x1 packed=0xffffa7a42a000001 -> ptr=0xffffffffa7a42a00 tag=0x1
fatal error: taggedPointerPack

This is a known Go runtime bug affecting linux/amd64 binaries under QEMU on arm64: golang/go#69255

The image is built with Go 1.25.3. Rebuilding with Go 1.25.4+ fixes it: golang/go@bf95b76

Full stack trace

WARNING: image platform (linux/amd64) does not match the expected platform (linux/arm64)
*** Running /etc/my_init.d/01_enable-services.sh...
*** Enabling production mode
*** Running /etc/my_init.d/02_load-settings.sh...
*** Loading settings...
runtime: taggedPointerPack invalid packing: ptr=0xffffa7a42a00 tag=0x1 packed=0xffffa7a42a000001 -> ptr=0xffffffffa7a42a00 tag=0x1
fatal error: taggedPointerPack

goroutine 1 gp=0xc000002380 m=0 mp=0x52635c0 [running, locked to thread]:
runtime.throw({0x2f41b61?, 0x0?})
	/usr/local/go/src/runtime/panic.go:1094 +0x48 fp=0xc000196b68 sp=0xc000196b38 pc=0x47c048
runtime.taggedPointerPack(0xffffa7a42a00, 0x1)
	/usr/local/go/src/runtime/tagptr_64bit.go:60 +0x12a fp=0xc000196ba0 sp=0xc000196b68 pc=0x46642a
runtime.netpollopen(0x5288210?, 0x5286a30?)
	/usr/local/go/src/runtime/netpoll_epoll.go:52 +0x3e fp=0xc000196c08 sp=0xc000196ba0 pc=0x440c9e
internal/poll.runtime_pollOpen(0x4)
	/usr/local/go/src/runtime/netpoll.go:272 +0xee fp=0xc000196c48 sp=0xc000196c08 pc=0x47b10e
internal/poll.(*pollDesc).init(0xc0000d6080, 0xc0000d6060?)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:40 +0x4a fp=0xc000196c68 sp=0xc000196c48 pc=0x5145aa
internal/poll.(*FD).Init(0xc0000d6060, {0x2f29cc2?, 0xc00007e1c8?}, 0x80?)
	/usr/local/go/src/internal/poll/fd_unix.go:66 +0x45 fp=0xc000196c88 sp=0xc000196c68 pc=0x515485
os.newFile(0x4, {0xc00007e1c8, 0x14}, 0x1, 0x0)
	/usr/local/go/src/os/file_unix.go:218 +0x15d fp=0xc000196cc8 sp=0xc000196c88 pc=0x52417d
os.openFileNolog({0xc00007e1c8, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file_unix.go:278 +0x17d fp=0xc000196d20 sp=0xc000196cc8 pc=0x52439d
os.OpenFile({0xc00007e1c8, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file.go:412 +0x3e fp=0xc000196d50 sp=0xc000196d20 pc=0x52297e
os.Open(...)
	/usr/local/go/src/os/file.go:390
google.golang.org/protobuf/internal/detrand.binaryHash()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:46 +0x53 fp=0xc000196e18 sp=0xc000196d50 pc=0xac70d3
google.golang.org/protobuf/internal/detrand.init()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:38 +0xf fp=0xc000196e28 sp=0xc000196e18 pc=0xac706f
runtime.doInit1(0x506ecd0)
	/usr/local/go/src/runtime/proc.go:7670 +0xd5 fp=0xc000196f50 sp=0xc000196e28 pc=0x456f15
runtime.doInit(...)
	/usr/local/go/src/runtime/proc.go:7637
runtime.main()
	/usr/local/go/src/runtime/proc.go:256 +0x350 fp=0xc000196fe0 sp=0xc000196f50 pc=0x447d90
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000196fe8 sp=0xc000196fe0 pc=0x484001

[Anshuman-WandB]

Hi @Buggy-McBugFace,

Thanks for the detailed report and for digging into the root cause, could you please share the specific command you are running to launch the container?

[Anshuman-WandB]

Hi @Buggy-McBugFace,

Following up on this, I attempted to reproduce this crash on my local Apple Silicon machine but haven't been able to trigger the error just yet. To ensure I am mirroring your setup exactly, could you please share the full command you are using to launch the container? Thanks!

[Buggy-McBugFace]

Hi @Anshuman-WandB,
Apologies for the delay - I worked around issue by deleting my podman machine and recreating it with downgraded podman machine OS version 5.4 (podman machine init --image docker://quay.io/podman/machine-os:5.4 --disk-size 100 --memory 4096 --now) which has Rosetta compatibility (Rosetta works with 5.4 or earlier's older Linux kernel). So to reproduce for you, I made a reproduction script which creates new test machine for reproduction, here is all the output (including reproduction of error):

Script:

set -x
podman machine stop
podman machine init podman-machine-test-reproduce --now
podman system connection default podman-machine-test-reproduce
podman version
# confirm wandb container has old go version that has the bug:
podman run --rm -v "$PWD:/tmp/extract" wandb/local:latest bash -c 'cp /usr/local/bin/megabinary /tmp/extract/ 2>&1' && go version ./megabinary
podman run -it wandb/local:latest
set +x

Output (reproduction):

:cc % zsh reproduce_wandb_error.zsh 
+reproduce_wandb_error.zsh:2> podman machine stop
Machine "podman-machine-default" stopped successfully
+reproduce_wandb_error.zsh:3> podman machine init podman-machine-test-reproduce --now
Error: podman-machine-test-reproduce: VM already exists
+reproduce_wandb_error.zsh:4> podman system connection default podman-machine-test-reproduce
+reproduce_wandb_error.zsh:5> podman version
Client:        Podman Engine
Version:       5.6.2
API Version:   5.6.2
Go Version:    go1.25.1
Built:         Wed Oct  1 00:20:46 2025
Build Origin:  brew
OS/Arch:       darwin/arm64

Server:       Podman Engine
Version:      5.6.2
API Version:  5.6.2
Go Version:   go1.24.7
Git Commit:   9dd5e1ed33830612bc200d7a13db00af6ab865a4
Built:        Tue Sep 30 09:30:00 2025
OS/Arch:      linux/arm64
+reproduce_wandb_error.zsh:7> podman run --rm -v /Users/c/cc:/tmp/extract wandb/local:latest bash -c 'cp /usr/local/bin/megabinary /tmp/extract/ 2>&1'
WARNING: image platform (linux/amd64) does not match the expected platform (linux/arm64)
+reproduce_wandb_error.zsh:7> go version ./megabinary
./megabinary: go1.25.3
+reproduce_wandb_error.zsh:8> podman run -it wandb/local:latest
WARNING: image platform (linux/amd64) does not match the expected platform (linux/arm64)
*** Running /etc/my_init.d/01_enable-services.sh...
*** Enabling production mode
*** Running /etc/my_init.d/02_load-settings.sh...
*** Loading settings...
runtime: taggedPointerPack invalid packing: ptr=0xffff80523e00 tag=0x1 packed=0xffff80523e000001 -> ptr=0xffffffff80523e00 tag=0x1
fatal error: taggedPointerPack

goroutine 1 gp=0xc000002380 m=0 mp=0x52635c0 [running, locked to thread]:
runtime.throw({0x2f41b61?, 0x0?})
	/usr/local/go/src/runtime/panic.go:1094 +0x48 fp=0xc0000deb68 sp=0xc0000deb38 pc=0x47c048
runtime.taggedPointerPack(0xffff80523e00, 0x1)
	/usr/local/go/src/runtime/tagptr_64bit.go:60 +0x12a fp=0xc0000deba0 sp=0xc0000deb68 pc=0x46642a
runtime.netpollopen(0x5288210?, 0x5286a30?)
	/usr/local/go/src/runtime/netpoll_epoll.go:52 +0x3e fp=0xc0000dec08 sp=0xc0000deba0 pc=0x440c9e
internal/poll.runtime_pollOpen(0x4)
	/usr/local/go/src/runtime/netpoll.go:272 +0xee fp=0xc0000dec48 sp=0xc0000dec08 pc=0x47b10e
internal/poll.(*pollDesc).init(0xc000180080, 0xc000180060?)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:40 +0x4a fp=0xc0000dec68 sp=0xc0000dec48 pc=0x5145aa
internal/poll.(*FD).Init(0xc000180060, {0x2f29cc2?, 0xc00018e000?}, 0x80?)
	/usr/local/go/src/internal/poll/fd_unix.go:66 +0x45 fp=0xc0000dec88 sp=0xc0000dec68 pc=0x515485
os.newFile(0x4, {0xc00018e000, 0x14}, 0x1, 0x0)
	/usr/local/go/src/os/file_unix.go:218 +0x15d fp=0xc0000decc8 sp=0xc0000dec88 pc=0x52417d
os.openFileNolog({0xc00018e000, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file_unix.go:278 +0x17d fp=0xc0000ded20 sp=0xc0000decc8 pc=0x52439d
os.OpenFile({0xc00018e000, 0x14}, 0x0, 0x0)
	/usr/local/go/src/os/file.go:412 +0x3e fp=0xc0000ded50 sp=0xc0000ded20 pc=0x52297e
os.Open(...)
	/usr/local/go/src/os/file.go:390
google.golang.org/protobuf/internal/detrand.binaryHash()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:46 +0x53 fp=0xc0000dee18 sp=0xc0000ded50 pc=0xac70d3
google.golang.org/protobuf/internal/detrand.init()
	/go/pkg/mod/google.golang.org/protobuf@v1.36.10/internal/detrand/rand.go:38 +0xf fp=0xc0000dee28 sp=0xc0000dee18 pc=0xac706f
runtime.doInit1(0x506ecd0)
	/usr/local/go/src/runtime/proc.go:7670 +0xd5 fp=0xc0000def50 sp=0xc0000dee28 pc=0x456f15
runtime.doInit(...)
	/usr/local/go/src/runtime/proc.go:7637
runtime.main()
	/usr/local/go/src/runtime/proc.go:256 +0x350 fp=0xc0000defe0 sp=0xc0000def50 pc=0x447d90
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000defe8 sp=0xc0000defe0 pc=0x484001

goroutine 2 gp=0xc0000028c0 m=nil [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000acfa8 sp=0xc0000acf88 pc=0x47c16e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:466
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:373 +0xb3 fp=0xc0000acfe0 sp=0xc0000acfa8 pc=0x448013
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000acfe8 sp=0xc0000acfe0 pc=0x484001
created by runtime.init.7 in goroutine 1
	/usr/local/go/src/runtime/proc.go:361 +0x1a

goroutine 17 gp=0xc000102380 m=nil [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000a8780 sp=0xc0000a8760 pc=0x47c16e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:466
runtime.bgsweep(0xc000116000)
	/usr/local/go/src/runtime/mgcsweep.go:279 +0x94 fp=0xc0000a87c8 sp=0xc0000a8780 pc=0x431e74
runtime.gcenable.gowrap1()
	/usr/local/go/src/runtime/mgc.go:212 +0x25 fp=0xc0000a87e0 sp=0xc0000a87c8 pc=0x425ec5
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000a87e8 sp=0xc0000a87e0 pc=0x484001
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:212 +0x66

goroutine 18 gp=0xc000102540 m=nil [GC scavenge wait]:
runtime.gopark(0xc000116000?, 0x345ce58?, 0x1?, 0x0?, 0xc000102540?)
	/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000a8f78 sp=0xc0000a8f58 pc=0x47c16e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:466
runtime.(*scavengerState).park(0x5260d40)
	/usr/local/go/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc0000a8fa8 sp=0xc0000a8f78 pc=0x42f949
runtime.bgscavenge(0xc000116000)
	/usr/local/go/src/runtime/mgcscavenge.go:653 +0x3c fp=0xc0000a8fc8 sp=0xc0000a8fa8 pc=0x42fedc
runtime.gcenable.gowrap2()
	/usr/local/go/src/runtime/mgc.go:213 +0x25 fp=0xc0000a8fe0 sp=0xc0000a8fc8 pc=0x425e65
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000a8fe8 sp=0xc0000a8fe0 pc=0x484001
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:213 +0xa5

goroutine 3 gp=0xc000003180 m=nil [GOMAXPROCS updater (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000ad788 sp=0xc0000ad768 pc=0x47c16e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:466
runtime.updateMaxProcsGoroutine()
	/usr/local/go/src/runtime/proc.go:6720 +0xe7 fp=0xc0000ad7e0 sp=0xc0000ad788 pc=0x455f47
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000ad7e8 sp=0xc0000ad7e0 pc=0x484001
created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
	/usr/local/go/src/runtime/proc.go:6708 +0x37

goroutine 4 gp=0xc000003340 m=nil [finalizer wait]:
runtime.gopark(0x456f15?, 0x42ee3c?, 0xc0?, 0xa4?, 0x490013?)
	/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000ac620 sp=0xc0000ac600 pc=0x47c16e
runtime.runFinalizers()
	/usr/local/go/src/runtime/mfinal.go:210 +0x107 fp=0xc0000ac7e0 sp=0xc0000ac620 pc=0x424dc7
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000ac7e8 sp=0xc0000ac7e0 pc=0x484001
created by runtime.createfing in goroutine 1
	/usr/local/go/src/runtime/mfinal.go:172 +0x3d
!!! Failed to load settings. If this issue persists, please contact support@wandb.com.
*** Booting runit daemon...
*** Runit started as PID 91
^C*** Shutting down runit daemon (PID 91)...
*** Running /etc/my_init.post_shutdown.d/10_syslog-ng.shutdown...
*** Init system aborted.
*** Killing all processes...
+reproduce_wandb_error.zsh:9> set +x
:cc % 

Edit, this is with 15.7.2 macOS fyi, there is a tiny chance Apple improved Rosetta kernel compatibility in macOS 26 (if you are running that yet, I usually wait at least 3-4 months before major OS upgrade).

% sw_vers
ProductName:		macOS
ProductVersion:		15.7.2
BuildVersion:		24G325

[Anshuman-WandB]

Hi @Buggy-McBugFace,

Thank you very much for the deep dive and the reproduction script. I am opening a bug ticket on our end and will keep you posted here with updates as soon as the fix is deployed. Thanks again for flagging this.