blob: 8058bba8c0bf3ea97fbbf3cb7f29aeb21c0112b2 [file] [log] [blame]
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
import (
"internal/abi"
"runtime/internal/atomic"
"unsafe"
)
const nsecInf = int64(0x7FFFFFFFFFFFFFFF)
func osyield()
//go:nosplit
func osyield_no_g() {
osyield()
}
//go:linkname cputicks syscall/zx.Sys_ticks_get
func cputicks() int64
//go:linkname sys_cprng_draw syscall/zx.Sys_cprng_draw
//go:noescape
func sys_cprng_draw(buffer unsafe.Pointer, size uint)
//go:linkname sys_system_get_num_cpus syscall/zx.Sys_system_get_num_cpus
func sys_system_get_num_cpus() uint32
//go:linkname sys_deadline_after syscall/zx.Sys_deadline_after
func sys_deadline_after(ns int64) int64
//go:linkname sys_futex_wait syscall/zx.Sys_futex_wait
//go:noescape
func sys_futex_wait(v *int32, cv int32, newOwner uint32, deadline int64) int32
//go:linkname sys_futex_wake syscall/zx.Sys_futex_wake
//go:noescape
func sys_futex_wake(v *int32, count uint32) int32
//go:linkname sys_thread_create syscall/zx.Sys_thread_create
//go:noescape
func sys_thread_create(p uint32, name *byte, nlen uint, opts int, out *uint32) int32
//go:linkname sys_thread_start syscall/zx.Sys_thread_start
//go:noescape
func sys_thread_start(h uint32, pc unsafe.Pointer, stk unsafe.Pointer, mp uintptr, arg2 uintptr) int32
//go:linkname sys_clock_get_monotonic syscall/zx.Sys_clock_get_monotonic
func sys_clock_get_monotonic() int64
//go:linkname sys_nanosleep syscall/zx.Sys_nanosleep
func sys_nanosleep(deadline int64) int32
//go:linkname sys_process_exit syscall/zx.Sys_process_exit
func sys_process_exit(retcode int64)
//go:linkname zx_set_stdio_handle syscall/zx.set_stdio_handle
func zx_set_stdio_handle(n int, h uint32, t uint32)
//go:linkname zx_set_proc_handle syscall/zx.set_proc_handle
func zx_set_proc_handle(h uint32)
//go:linkname zx_get_proc_handle syscall/zx.get_proc_handle
func zx_get_proc_handle() uint32
//go:linkname zx_set_vmar_root syscall/zx.set_vmar_root
func zx_set_vmar_root(h uint32)
//go:linkname zx_set_namespace syscall/zx.set_namespace
func zx_set_namespace(m map[string]uint32)
//go:linkname zx_notify_new_thread syscall/zx.notify_new_thread
func zx_notify_new_thread(h uint32, w byte)
//go:linkname zx_init_threads_channel syscall/zx.init_threads_channel
func zx_init_threads_channel()
type mOS struct {
// threadLock protects the fields below.
//
// This lock must be held to write the fields or read them from another
// thread. Other threads should hold threadLock to prevent the M from
// getting unminit'd from underneath them.
threadLock mutex
thread uint32
threadKoid uint64
}
//go:nosplit
func getThreadHandle() uint32 {
return getg().m.thread
}
//go:nosplit
func getThreadKoid() uint64 {
return getg().m.threadKoid
}
type sigset struct{}
//go:nosplit
func getRandomData(r []byte) {
if len(r) == 0 {
return
}
sys_cprng_draw(unsafe.Pointer(&r[0]), uint(len(r)))
}
//go:nosplit
func futexsleep(addr *uint32, val uint32, ns int64) {
deadline := sys_deadline_after(ns)
if ns < 0 {
deadline = nsecInf
}
sys_futex_wait((*int32)(unsafe.Pointer(addr)), int32(val), 0, deadline)
}
//go:nosplit
func futexwakeup(addr *uint32, cnt uint32) {
sys_futex_wake((*int32)(unsafe.Pointer(addr)), cnt)
}
// cgocallm0 calls a C function on the current stack.
//
// It is intended for use inside functions like osinit that run
// before mstart directly on an OS thread stack.
//
// If in doubt, do not use.
func cgocallm0(fn, arg unsafe.Pointer)
// exceptionInfo is defined in zircon/syscalls/exception.h.
type exceptionInfo struct {
pid uint64
tid uint64
exceptionType uint32
padding [4]uint8
}
// watchexceptions is called when pprof CPU tracing is enabled. It runs without
// a p, hence nowritebarrierrec.
//
//go:nosplit
//go:nowritebarrierrec
func watchexceptions() {
procHandle := zx_get_proc_handle()
var exceptionChannel uint32
if status := vdsoCall_zx_task_create_exception_channel(procHandle, 0, unsafe.Pointer(&exceptionChannel)); status != ZX_OK {
println("failed to create exception channel", status)
exit(2)
}
for {
var observed uint32
if status := fake_m_object_wait_one(exceptionChannel, ZX_CHANNEL_READABLE, ZX_TIME_INFINITE, &observed); status != ZX_OK {
println("failed to wait on exception channel", status)
exit(2)
}
var info exceptionInfo
var exceptionHandle uint32
var actualBytes, actualHandles uint32
if status := vdsoCall_zx_channel_read(
exceptionChannel,
0,
unsafe.Pointer(&info),
unsafe.Pointer(&exceptionHandle),
uint32(unsafe.Sizeof(info)),
1,
unsafe.Pointer(&actualBytes),
unsafe.Pointer(&actualHandles),
); status != ZX_OK {
println("failed to read from exception channel", status)
exit(2)
}
switch info.exceptionType {
// A fatal exception occurred, attempt to push a panic into the
// offending thread.
case ZX_EXCP_FATAL_PAGE_FAULT, ZX_EXCP_UNALIGNED_ACCESS, ZX_EXCP_UNDEFINED_INSTRUCTION:
var thread uint32
if status := vdsoCall_zx_exception_get_thread(exceptionHandle, unsafe.Pointer(&thread)); status != ZX_OK {
println("failed to get exception thread", status)
exit(2)
}
var ctx sigctxt
if status := vdsoCall_zx_thread_read_state(
thread,
ZX_THREAD_STATE_GENERAL_REGS,
unsafe.Pointer(&ctx.regs),
uint(unsafe.Sizeof(ctx.regs)),
); status != ZX_OK {
println("failed to read thread state", status)
exit(2)
}
// Prepare a panic, we'll call sigpanic on the dead thread.
ctx.prepareExceptionPanic()
if status := vdsoCall_zx_thread_write_state(
thread,
ZX_THREAD_STATE_GENERAL_REGS,
unsafe.Pointer(&ctx.regs),
uint(unsafe.Sizeof(ctx.regs)),
); status != ZX_OK {
println("failed to recover thread state on OS exception")
ctx.traceback()
exit(2)
}
handled := ZX_EXCEPTION_STATE_HANDLED
if status := vdsoCall_zx_object_set_property(
exceptionHandle,
ZX_PROP_EXCEPTION_STATE,
unsafe.Pointer(&handled),
uint(unsafe.Sizeof(handled)),
); status != ZX_OK {
println("failed to mark OS exception as handled")
ctx.traceback()
exit(2)
}
// Close the thread handle, we don't need it anymore, and we don't really
// care if it succeeds.
_ = vdsoCall_zx_handle_close(thread)
if status := vdsoCall_zx_handle_close(exceptionHandle); status != ZX_OK {
println("failed to close exception handle")
ctx.traceback()
exit(2)
}
default:
// We're not handling this exception, just close the handle.
vdsoCall_zx_handle_close(exceptionHandle)
}
}
}
var critlock mutex
func osinit() {
if _cgo_get_initial_handles != nil {
cgocallm0(_cgo_get_initial_handles, unsafe.Pointer(&fdioHandles))
for i := 0; i < 3; i = i + 1 {
zx_set_stdio_handle(i, fdioHandles.stdioClones[i], fdioHandles.stdioCloneTypes[i])
}
zx_set_proc_handle(fdioHandles.processSelf)
zx_set_vmar_root(fdioHandles.vmarRootSelf)
zx_init_threads_channel()
} else {
println("runtime: fuchsia requires cgo")
exit(2)
}
ncpu = int32(sys_system_get_num_cpus())
physPageSize = 4096
}
func parseRootNS() {
if fdioHandles.rootNSNumHandles > 0 {
const maxHandleCount = 1 << 20 // arbitrary
paths := (*(*[maxHandleCount]*byte)(unsafe.Pointer(fdioHandles.rootNSPaths)))[:fdioHandles.rootNSNumHandles]
handles := (*(*[maxHandleCount]uint32)(unsafe.Pointer(fdioHandles.rootNSHandles)))[:fdioHandles.rootNSNumHandles]
m := make(map[string]uint32)
for i, p := range paths {
m[gostring(p)] = handles[i]
}
zx_set_namespace(m)
}
}
// Filled in by runtime/cgo when linked into binary.
var (
_cgo_get_initial_handles unsafe.Pointer
_cgo_get_thread_self_handle unsafe.Pointer
_cgo_timespec_get unsafe.Pointer
_cgo_write1 unsafe.Pointer
)
type ThreadType byte
const (
_ ThreadType = iota
WorkerThread
UnknownRuntimeThread
SysmonThread
WatchExceptionsThread
ProfileLoopThread
TemplateThread
)
//go:nosplit
func minit() {
_g_ := getg()
mp := _g_.m
lock(&mp.threadLock)
var h uint32
asmcgocall(_cgo_get_thread_self_handle, unsafe.Pointer(&h))
mp.thread = h
// NB: Write barrier restrictions in this function disallow a nicer pattern
// to assign these variables.
isWorker := mp.p != 0 || mp.nextp != 0
threadName := []byte("go-worker")
threadType := WorkerThread
if !isWorker {
// There are some known functions that we can compare against,
// unfortunately we can't use FuncForPC here because of the write
// barriers imposed by the callers.
switch abi.FuncPCABIInternal(mp.mstartfn) {
case abi.FuncPCABIInternal(sysmon):
threadName = []byte("sysmon")
threadType = SysmonThread
case abi.FuncPCABIInternal(watchexceptions):
threadName = []byte("watchexceptions")
threadType = WatchExceptionsThread
case abi.FuncPCABIInternal(profileLoop):
threadName = []byte("profileLoop")
threadType = ProfileLoopThread
case abi.FuncPCABIInternal(templateThread):
threadName = []byte("templateThread")
threadType = TemplateThread
default:
threadName = []byte("unknown-runtime-thread")
threadType = UnknownRuntimeThread
}
}
if status := vdsoCall_zx_object_set_property(h, propName, unsafe.Pointer(&threadName[0]), uint(len(threadName))); status != 0 {
println("runtime: vdsoCall_zx_object_set_property failed: ", status)
exit(2)
}
var info zxInfoHandleBasic
var actualSize uint64
var availSize uint64
if status := vdsoCall_zx_object_get_info(h, ZX_INFO_HANDLE_BASIC, unsafe.Pointer(&info), uint(unsafe.Sizeof(info)), unsafe.Pointer(&actualSize), unsafe.Pointer(&availSize)); status != ZX_OK {
println("failed to get the thread koid for tracing", status)
} else {
mp.threadKoid = info.koid
}
zx_notify_new_thread(h, byte(threadType))
unlock(&mp.threadLock)
}
//go:nosplit
func unminit() {
mp := getg().m
lock(&mp.threadLock)
mp.thread = ZX_HANDLE_INVALID
mp.threadKoid = 0
unlock(&mp.threadLock)
}
func mdestroy(mp *m) {
// TODO
}
var exceptionError = error(errorString("OS exception"))
func (c *sigctxt) traceback() {
traceback(c.regs.rPC(), c.regs.rSP(), c.regs.rLR(), getg())
}
func sigpanic() {
if !canpanic() {
throw("encountered OS exception")
}
panicCheck2(exceptionError.Error())
panic(exceptionError)
}
func mpreinit(mp *m) {
// TODO
}
//go:nosplit
func sigsave(p *sigset) {
// TODO
}
//go:nosplit
func msigrestore(sigmask sigset) {
// TODO
}
func initsig(preinit bool) {
// We can't call newm if this is preinit.
if !preinit {
newm(watchexceptions, nil, -1)
}
}
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
initsig(true)
}
//go:nosplit
func sigblock(exiting bool) {
// TODO
}
func sigenable(sig uint32) {
// TODO
}
func sigdisable(sig uint32) {
// TODO
}
func sigignore(sig uint32) {
// TODO
}
func crash() {
*(*int32)(nil) = 0
}
func threadinit(mp *m)
var gothreadname = []byte("gothread")
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
p := uint32(0) // TODO: only works due to temporary hack in zircon
var h uint32
status := sys_thread_create(p, &gothreadname[0], uint(len(gothreadname)), 0, &h)
if status < 0 {
println("runtime: newosproc sys_thread_create failed:", h)
exit(2)
}
status = sys_thread_start(h, unsafe.Pointer(abi.FuncPCABI0(threadinit)), stk, uintptr(unsafe.Pointer(mp)), 0)
if status < 0 {
println("runtime: newosproc sys_thread_start failed:", h)
exit(2)
}
}
//go:nosplit
func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
stack := sysAlloc(stacksize, &memstats.stacks_sys)
if stack == nil {
writeErrStr(failallocatestack)
exit(1)
}
p := uint32(0) // TODO: only works due to temporary hack in zircon
var h uint32
status := sys_thread_create(p, &gothreadname[0], uint(len(gothreadname)), 0, &h)
if status < 0 {
writeErrStr(failthreadcreate)
exit(1)
}
status = sys_thread_start(h, fn, unsafe.Pointer(uintptr(stack)+stacksize), 0, 0)
if status < 0 {
writeErrStr(failthreadstart)
exit(1)
}
}
const failthreadstart = "runtime: failed to start new OS thread\n"
type zx_proc_args struct {
protocol uint32
version uint32
handle_info_off uint32
args_off uint32
args_num uint32
}
type zx_proc_info struct {
magic uint32
version uint32
next_tls_slot uint32
proc_args *zx_proc_args
handle *uint32
handle_info *uint32
handle_count int32
argv **byte
argc int32
}
type zx_tls_root struct {
self *zx_tls_root
proc *zx_proc_info
magic uint32
flags uint16
maxslots uint16
slots [8]uintptr // has length maxslots
}
func resetcpuprofiler(hz int32) {
// TODO
}
// fdio_init matches a Go struct of the same name in gcc_fuchsia.c.
type fdio_init struct {
stdioClones [3]uint32
stdioCloneNumHandles [3]uint32
stdioCloneTypes [3]uint32
processSelf uint32
vmarRootSelf uint32
envlen int32
environ **byte
_ [4]byte
rootNSNumHandles int32
rootNSHandles *uint32
rootNSPaths **byte
}
var fdioHandles fdio_init
func goenvs() {
if _cgo_get_initial_handles != nil {
const maxEnvSize = 1 << 20 // arbitrary
envp := (*(*[maxEnvSize]*byte)(unsafe.Pointer(fdioHandles.environ)))[:fdioHandles.envlen]
envs = make([]string, len(envp))
for i, e := range envp {
envs[i] = gostring(e)
}
// TODO: find a better place to call this
parseRootNS()
} else {
// TODO: implement cgo-less init
println("runtime: no fuchsia process handle without cgo yet")
exit(2)
}
}
const _NSIG = 65 // TODO
const (
ZX_CLOCK_MONOTONIC = 0
ZX_HANDLE_INVALID = 0
ZX_INFO_HANDLE_BASIC = uint32(2)
ZX_INFO_THREAD = uint32(10)
ZX_OK = int32(0)
ZX_THREAD_STATE_GENERAL_REGS = uint32(0)
ZX_THREAD_STATE_NEW = uint32(0)
ZX_THREAD_SUSPENDED = uint32(1 << 5)
ZX_THREAD_TERMINATED = uint32(1 << 3)
ZX_TIME_INFINITE = int64(^uint64(1 << 63))
ZX_TIMER_SIGNALED = uint32(1 << 3)
ZX_TIMER_SLACK_EARLY = 1
ZX_CHANNEL_READABLE = uint32(1 << 0)
ZX_EXCEPTION_STATE_HANDLED = uint32(1)
ZX_PROP_EXCEPTION_STATE = uint32(16)
// Exception types that are handled by the runtime. Defined in
// zircon/syscall/exception.h
ZX_EXCP_FATAL_PAGE_FAULT = uint32(0x108)
ZX_EXCP_UNDEFINED_INSTRUCTION = uint32(0x208)
ZX_EXCP_UNALIGNED_ACCESS = uint32(0x508)
)
//go:nosplit
func nanotime1() int64 {
return sys_clock_get_monotonic()
}
// Should match timespec struct definition from libc time.h
type timespec struct {
tv_sec int64
tv_nsec int64
}
//go:linkname time_now time.now
//go:nosplit
func time_now() (sec int64, nsec int32, mono int64) {
var ts timespec
if asmcgocall(_cgo_timespec_get, unsafe.Pointer(&ts)) != 0 {
panic("failed to get UTC time")
}
return ts.tv_sec, int32(ts.tv_nsec), nanotime()
}
func unixnanotime() int64 {
return nanotime()
}
type writeargs struct {
fd uint32
buf uintptr
count uint32
}
//go:nosplit
func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 {
w := writeargs{
fd: uint32(fd),
buf: uintptr(buf),
count: uint32(n),
}
return asmcgocall(_cgo_write1, unsafe.Pointer(&w))
}
//go:nosplit
func exit(code int32) {
sys_process_exit(int64(code))
}
//go:nosplit
func usleep(usec uint32) {
sys_nanosleep(sys_deadline_after(int64(usec * 1000)))
}
//go:nosplit
func usleep_no_g(usec uint32) {
usleep(usec)
}
func signame(sig uint32) string {
return "unknown_sig" // TODO
}
// zircon_mktls
//
//go:nosplit
func zircon_mktls() {
// TODO
}
//go:nosplit
func zircon_settls(val uintptr) {
// TODO: should call zx_set_object_property with property ZX_PROP_REGISTER_FS in amd64 or
// ZX_PROP_REGISTER_CP15 on arm.
}
//go:linkname os_sigpipe os.sigpipe
func os_sigpipe() {
// TODO
}
// gsignalStack is unused on fuchsia
type gsignalStack struct{}
const preemptMSupported = false
func preemptM(mp *m) {
// No threads, so nothing to do.
}
var (
profTimer uint32 // timer handle
profHz = ZX_TIME_INFINITE
)
type zxCpuSet struct {
mask [16]uint32
}
type zxInfoHandleBasic struct {
koid uint64
rights uint32
typ uint32
related_koid uint64
reserved uint32
padding [4]uint8
}
type zxInfoThread struct {
state uint32
waitExceptionPortType uint32
affinityMask zxCpuSet
}
//go:noescape
//go:nosplit
func fake_m_object_wait_one(handle uint32, signals uint32, deadline int64, observed *uint32) int32
func profilem(mp *m, thread uint32) {
// Ignore nascent threads.
var observed uint32
var info zxInfoThread
var actualSize uint64
var availSize uint64
if status := vdsoCall_zx_object_get_info(thread, ZX_INFO_THREAD, unsafe.Pointer(&info), uint(unsafe.Sizeof(info)), unsafe.Pointer(&actualSize), unsafe.Pointer(&availSize)); status != ZX_OK {
return
}
if info.state == ZX_THREAD_STATE_NEW {
return
}
// Wait for the thread to be suspended.
if status := fake_m_object_wait_one(thread, ZX_THREAD_SUSPENDED|ZX_THREAD_TERMINATED, ZX_TIME_INFINITE, &observed); status != ZX_OK {
return
}
// Ignore threads that were terminated as a result of suspension.
if observed&ZX_THREAD_TERMINATED == ZX_THREAD_TERMINATED {
return
}
var r generalRegs
if status := vdsoCall_zx_thread_read_state(thread, ZX_THREAD_STATE_GENERAL_REGS, unsafe.Pointer(&r), uint(unsafe.Sizeof(r))); status != ZX_OK {
return
}
gp := gFromSP(mp, r.rSP())
sigprof(r.rPC(), r.rSP(), r.rLR(), gp, mp)
}
func gFromSP(mp *m, sp uintptr) *g {
if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
return gp
}
if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
return gp
}
if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
return gp
}
return nil
}
// profileLoop is called when pprof CPU tracing is enabled. It runs without a p, hence nowritebarrierrec.
//
//go:nosplit
//go:nowritebarrierrec
func profileLoop() {
for {
var observed uint32
if status := fake_m_object_wait_one(profTimer, ZX_TIMER_SIGNALED, ZX_TIME_INFINITE, &observed); status != ZX_OK {
continue
}
// Scan over all threads and suspend them, then trace. Ideally, we'd scan over
// all of them and give them a chance to suspend before this, but I don't want
// to deal with allocation here.
first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
for mp := first; mp != nil; mp = mp.alllink {
if mp == getg().m {
// Don't profile ourselves.
continue
}
// We must synchronize with unminit to avoid profiling
// threads after unminit, at which point they may be in
// the calling C code which we can't extract a G from
// (as none exists).
lock(&mp.threadLock)
thread := mp.thread
// Do not profile threads blocked on Notes (this
// includes idle worker threads, idle timer thread,
// idle heap scavenger, etc.).
if thread == ZX_HANDLE_INVALID || mp.profilehz == 0 || mp.blocked {
unlock(&mp.threadLock)
continue
}
var token uint32
if status := vdsoCall_zx_task_suspend_token(thread, unsafe.Pointer(&token)); status == ZX_OK {
profilem(mp, thread)
}
vdsoCall_zx_handle_close(token)
unlock(&mp.threadLock)
}
due := vdsoCall_zx_deadline_after(profHz)
if status := vdsoCall_zx_timer_set(profTimer, due, ZX_TIMER_SLACK_EARLY); status != ZX_OK {
// TODO: what do?
return
}
}
}
// profileloop is defined in sys_fuchsia_{amd,arm}64.s
func profileloop()
func setProcessCPUProfiler(hz int32) {
// This function is entered in the critical section guarded by
// prof.signalLock in proc.go and is thus safe to assume the profiler's
// timer is created atomically.
if profTimer == ZX_HANDLE_INVALID {
if status := vdsoCall_zx_timer_create(ZX_TIMER_SLACK_EARLY, ZX_CLOCK_MONOTONIC, unsafe.Pointer(&profTimer)); status != ZX_OK {
// TODO: what do?
return
}
newm(profileLoop, nil, -1)
}
}
func setThreadCPUProfiler(hz int32) {
// Don't do anything if we don't have a timer yet.
if profTimer == ZX_HANDLE_INVALID {
return
}
atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
if hz > 0 {
ns := int64(int64(1000000000) / int64(hz))
// Bound tracing frequency to microseconds. This is likely
// already well above the granularity traces can actually
// provide.
if ns < 1000 {
ns = 1000
}
profHz = ns
due := vdsoCall_zx_deadline_after(ns)
if status := vdsoCall_zx_timer_set(profTimer, due, ZX_TIMER_SLACK_EARLY); status != ZX_OK {
// TODO: what do?
return
}
} else {
if status := vdsoCall_zx_timer_cancel(profTimer); status != ZX_OK {
return
}
}
}
func exitThread(wait *atomic.Uint32) {}
//go:nosplit
//go:nowritebarrierrec
func clearSignalHandlers() {}