version: bump snapshot

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

.gitignore

@@ -1,4 +1 @@
 wireguard-go
-vendor
-.gopath
-ireallywantobuildon_linux.go

Makefile

@@ -1,33 +1,19 @@
 PREFIX ?= /usr
 DESTDIR ?=
 BINDIR ?= $(PREFIX)/bin
-export GOPATH ?= $(CURDIR)/.gopath
 export GO111MODULE := on
 
 all: generate-version-and-build
 
-ifeq ($(shell go env GOOS)|$(wildcard .git),linux|)
-$(error Do not build this for Linux. Instead use the Linux kernel module. See wireguard.com/install/ for more info.)
-else ifeq ($(shell go env GOOS),linux)
-ireallywantobuildon_linux.go:
-	@printf "WARNING: This software is meant for use on non-Linux\nsystems. For Linux, please use the kernel module\ninstead. See wireguard.com/install/ for more info.\n\n" >&2
-	@printf 'package main\nconst UseTheKernelModuleInstead = 0xdeadbabe\n' > "$@"
-clean-ireallywantobuildon_linux.go:
-	@rm -f ireallywantobuildon_linux.go
-.PHONY: clean-ireallywantobuildon_linux.go
-clean: clean-ireallywantobuildon_linux.go
-wireguard-go: ireallywantobuildon_linux.go
-endif
-
 MAKEFLAGS += --no-print-directory
 
 generate-version-and-build:
 	@export GIT_CEILING_DIRECTORIES="$(realpath $(CURDIR)/..)" && \
 	tag="$$(git describe --dirty 2>/dev/null)" && \
-	ver="$$(printf 'package device\nconst WireGuardGoVersion = "%s"\n' "$$tag")" && \
-	[ "$$(cat device/version.go 2>/dev/null)" != "$$ver" ] && \
-	echo "$$ver" > device/version.go && \
-	git update-index --assume-unchanged device/version.go || true
+	ver="$$(printf 'package main\n\nconst Version = "%s"\n' "$$tag")" && \
+	[ "$$(cat version.go 2>/dev/null)" != "$$ver" ] && \
+	echo "$$ver" > version.go && \
+	git update-index --assume-unchanged version.go || true
 	@$(MAKE) wireguard-go
 
 wireguard-go: $(wildcard *.go) $(wildcard */*.go)
@@ -36,7 +22,10 @@ wireguard-go: $(wildcard *.go) $(wildcard */*.go)
 install: wireguard-go
 	@install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/wireguard-go"
 
+test:
+	go test -v ./...
+
 clean:
 	rm -f wireguard-go
 
-.PHONY: all clean install generate-version-and-build
+.PHONY: all clean test install generate-version-and-build

README.md

@@ -18,7 +18,7 @@ To run wireguard-go without forking to the background, pass `-f` or `--foregroun
 $ wireguard-go -f wg0
 ```
 
-When an interface is running, you may use [`wg(8)`](https://git.zx2c4.com/WireGuard/about/src/tools/man/wg.8) to configure it, as well as the usual `ip(8)` and `ifconfig(8)` commands.
+When an interface is running, you may use [`wg(8)`](https://git.zx2c4.com/wireguard-tools/about/src/man/wg.8) to configure it, as well as the usual `ip(8)` and `ifconfig(8)` commands.
 
 To run with more logging you may set the environment variable `LOG_LEVEL=debug`.
 
@@ -26,7 +26,7 @@ To run with more logging you may set the environment variable `LOG_LEVEL=debug`.
 
 ### Linux
 
-This will run on Linux; however **YOU SHOULD NOT RUN THIS ON LINUX**. Instead use the kernel module; see the [installation page](https://www.wireguard.com/install/) for instructions.
+This will run on Linux; however you should instead use the kernel module, which is faster and better integrated into the OS. See the [installation page](https://www.wireguard.com/install/) for instructions.
 
 ### macOS
 
@@ -46,7 +46,7 @@ This will run on OpenBSD. It does not yet support sticky sockets. Fwmark is mapp
 
 ## Building
 
-This requires an installation of [go](https://golang.org) ≥ 1.12.
+This requires an installation of [go](https://golang.org) ≥ 1.16.
 
 ```
 $ git clone https://git.zx2c4.com/wireguard-go
@@ -56,7 +56,7 @@ $ make
 
 ## License
 
-    Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+    Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
     
     Permission is hereby granted, free of charge, to any person obtaining a copy of
     this software and associated documentation files (the "Software"), to deal in

conn/bind_linux.go

@@ -0,0 +1,585 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+import (
+	"errors"
+	"net"
+	"strconv"
+	"sync"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+type ipv4Source struct {
+	Src     [4]byte
+	Ifindex int32
+}
+
+type ipv6Source struct {
+	src [16]byte
+	// ifindex belongs in dst.ZoneId
+}
+
+type LinuxSocketEndpoint struct {
+	mu   sync.Mutex
+	dst  [unsafe.Sizeof(unix.SockaddrInet6{})]byte
+	src  [unsafe.Sizeof(ipv6Source{})]byte
+	isV6 bool
+}
+
+func (endpoint *LinuxSocketEndpoint) Src4() *ipv4Source         { return endpoint.src4() }
+func (endpoint *LinuxSocketEndpoint) Dst4() *unix.SockaddrInet4 { return endpoint.dst4() }
+func (endpoint *LinuxSocketEndpoint) IsV6() bool                { return endpoint.isV6 }
+
+func (endpoint *LinuxSocketEndpoint) src4() *ipv4Source {
+	return (*ipv4Source)(unsafe.Pointer(&endpoint.src[0]))
+}
+
+func (endpoint *LinuxSocketEndpoint) src6() *ipv6Source {
+	return (*ipv6Source)(unsafe.Pointer(&endpoint.src[0]))
+}
+
+func (endpoint *LinuxSocketEndpoint) dst4() *unix.SockaddrInet4 {
+	return (*unix.SockaddrInet4)(unsafe.Pointer(&endpoint.dst[0]))
+}
+
+func (endpoint *LinuxSocketEndpoint) dst6() *unix.SockaddrInet6 {
+	return (*unix.SockaddrInet6)(unsafe.Pointer(&endpoint.dst[0]))
+}
+
+// LinuxSocketBind uses sendmsg and recvmsg to implement a full bind with sticky sockets on Linux.
+type LinuxSocketBind struct {
+	// mu guards sock4 and sock6 and the associated fds.
+	// As long as someone holds mu (read or write), the associated fds are valid.
+	mu    sync.RWMutex
+	sock4 int
+	sock6 int
+}
+
+func NewLinuxSocketBind() Bind { return &LinuxSocketBind{sock4: -1, sock6: -1} }
+func NewDefaultBind() Bind     { return NewLinuxSocketBind() }
+
+var _ Endpoint = (*LinuxSocketEndpoint)(nil)
+var _ Bind = (*LinuxSocketBind)(nil)
+
+func (*LinuxSocketBind) ParseEndpoint(s string) (Endpoint, error) {
+	var end LinuxSocketEndpoint
+	addr, err := parseEndpoint(s)
+	if err != nil {
+		return nil, err
+	}
+
+	ipv4 := addr.IP.To4()
+	if ipv4 != nil {
+		dst := end.dst4()
+		end.isV6 = false
+		dst.Port = addr.Port
+		copy(dst.Addr[:], ipv4)
+		end.ClearSrc()
+		return &end, nil
+	}
+
+	ipv6 := addr.IP.To16()
+	if ipv6 != nil {
+		zone, err := zoneToUint32(addr.Zone)
+		if err != nil {
+			return nil, err
+		}
+		dst := end.dst6()
+		end.isV6 = true
+		dst.Port = addr.Port
+		dst.ZoneId = zone
+		copy(dst.Addr[:], ipv6[:])
+		end.ClearSrc()
+		return &end, nil
+	}
+
+	return nil, errors.New("invalid IP address")
+}
+
+func (bind *LinuxSocketBind) Open(port uint16) ([]ReceiveFunc, uint16, error) {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+
+	var err error
+	var newPort uint16
+	var tries int
+
+	if bind.sock4 != -1 || bind.sock6 != -1 {
+		return nil, 0, ErrBindAlreadyOpen
+	}
+
+	originalPort := port
+
+again:
+	port = originalPort
+	var sock4, sock6 int
+	// Attempt ipv6 bind, update port if successful.
+	sock6, newPort, err = create6(port)
+	if err != nil {
+		if !errors.Is(err, syscall.EAFNOSUPPORT) {
+			return nil, 0, err
+		}
+	} else {
+		port = newPort
+	}
+
+	// Attempt ipv4 bind, update port if successful.
+	sock4, newPort, err = create4(port)
+	if err != nil {
+		if originalPort == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 {
+			unix.Close(sock6)
+			tries++
+			goto again
+		}
+		if !errors.Is(err, syscall.EAFNOSUPPORT) {
+			unix.Close(sock6)
+			return nil, 0, err
+		}
+	} else {
+		port = newPort
+	}
+
+	var fns []ReceiveFunc
+	if sock4 != -1 {
+		bind.sock4 = sock4
+		fns = append(fns, bind.receiveIPv4)
+	}
+	if sock6 != -1 {
+		bind.sock6 = sock6
+		fns = append(fns, bind.receiveIPv6)
+	}
+	if len(fns) == 0 {
+		return nil, 0, syscall.EAFNOSUPPORT
+	}
+	return fns, port, nil
+}
+
+func (bind *LinuxSocketBind) SetMark(value uint32) error {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+
+	if bind.sock6 != -1 {
+		err := unix.SetsockoptInt(
+			bind.sock6,
+			unix.SOL_SOCKET,
+			unix.SO_MARK,
+			int(value),
+		)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	if bind.sock4 != -1 {
+		err := unix.SetsockoptInt(
+			bind.sock4,
+			unix.SOL_SOCKET,
+			unix.SO_MARK,
+			int(value),
+		)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bind *LinuxSocketBind) Close() error {
+	// Take a readlock to shut down the sockets...
+	bind.mu.RLock()
+	if bind.sock6 != -1 {
+		unix.Shutdown(bind.sock6, unix.SHUT_RDWR)
+	}
+	if bind.sock4 != -1 {
+		unix.Shutdown(bind.sock4, unix.SHUT_RDWR)
+	}
+	bind.mu.RUnlock()
+	// ...and a write lock to close the fd.
+	// This ensures that no one else is using the fd.
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	var err1, err2 error
+	if bind.sock6 != -1 {
+		err1 = unix.Close(bind.sock6)
+		bind.sock6 = -1
+	}
+	if bind.sock4 != -1 {
+		err2 = unix.Close(bind.sock4)
+		bind.sock4 = -1
+	}
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (bind *LinuxSocketBind) receiveIPv4(buf []byte) (int, Endpoint, error) {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	if bind.sock4 == -1 {
+		return 0, nil, net.ErrClosed
+	}
+	var end LinuxSocketEndpoint
+	n, err := receive4(bind.sock4, buf, &end)
+	return n, &end, err
+}
+
+func (bind *LinuxSocketBind) receiveIPv6(buf []byte) (int, Endpoint, error) {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	if bind.sock6 == -1 {
+		return 0, nil, net.ErrClosed
+	}
+	var end LinuxSocketEndpoint
+	n, err := receive6(bind.sock6, buf, &end)
+	return n, &end, err
+}
+
+func (bind *LinuxSocketBind) Send(buff []byte, end Endpoint) error {
+	nend, ok := end.(*LinuxSocketEndpoint)
+	if !ok {
+		return ErrWrongEndpointType
+	}
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	if !nend.isV6 {
+		if bind.sock4 == -1 {
+			return net.ErrClosed
+		}
+		return send4(bind.sock4, nend, buff)
+	} else {
+		if bind.sock6 == -1 {
+			return net.ErrClosed
+		}
+		return send6(bind.sock6, nend, buff)
+	}
+}
+
+func (end *LinuxSocketEndpoint) SrcIP() net.IP {
+	if !end.isV6 {
+		return net.IPv4(
+			end.src4().Src[0],
+			end.src4().Src[1],
+			end.src4().Src[2],
+			end.src4().Src[3],
+		)
+	} else {
+		return end.src6().src[:]
+	}
+}
+
+func (end *LinuxSocketEndpoint) DstIP() net.IP {
+	if !end.isV6 {
+		return net.IPv4(
+			end.dst4().Addr[0],
+			end.dst4().Addr[1],
+			end.dst4().Addr[2],
+			end.dst4().Addr[3],
+		)
+	} else {
+		return end.dst6().Addr[:]
+	}
+}
+
+func (end *LinuxSocketEndpoint) DstToBytes() []byte {
+	if !end.isV6 {
+		return (*[unsafe.Offsetof(end.dst4().Addr) + unsafe.Sizeof(end.dst4().Addr)]byte)(unsafe.Pointer(end.dst4()))[:]
+	} else {
+		return (*[unsafe.Offsetof(end.dst6().Addr) + unsafe.Sizeof(end.dst6().Addr)]byte)(unsafe.Pointer(end.dst6()))[:]
+	}
+}
+
+func (end *LinuxSocketEndpoint) SrcToString() string {
+	return end.SrcIP().String()
+}
+
+func (end *LinuxSocketEndpoint) DstToString() string {
+	var udpAddr net.UDPAddr
+	udpAddr.IP = end.DstIP()
+	if !end.isV6 {
+		udpAddr.Port = end.dst4().Port
+	} else {
+		udpAddr.Port = end.dst6().Port
+	}
+	return udpAddr.String()
+}
+
+func (end *LinuxSocketEndpoint) ClearDst() {
+	for i := range end.dst {
+		end.dst[i] = 0
+	}
+}
+
+func (end *LinuxSocketEndpoint) ClearSrc() {
+	for i := range end.src {
+		end.src[i] = 0
+	}
+}
+
+func zoneToUint32(zone string) (uint32, error) {
+	if zone == "" {
+		return 0, nil
+	}
+	if intr, err := net.InterfaceByName(zone); err == nil {
+		return uint32(intr.Index), nil
+	}
+	n, err := strconv.ParseUint(zone, 10, 32)
+	return uint32(n), err
+}
+
+func create4(port uint16) (int, uint16, error) {
+
+	// create socket
+
+	fd, err := unix.Socket(
+		unix.AF_INET,
+		unix.SOCK_DGRAM,
+		0,
+	)
+
+	if err != nil {
+		return -1, 0, err
+	}
+
+	addr := unix.SockaddrInet4{
+		Port: int(port),
+	}
+
+	// set sockopts and bind
+
+	if err := func() error {
+		if err := unix.SetsockoptInt(
+			fd,
+			unix.IPPROTO_IP,
+			unix.IP_PKTINFO,
+			1,
+		); err != nil {
+			return err
+		}
+
+		return unix.Bind(fd, &addr)
+	}(); err != nil {
+		unix.Close(fd)
+		return -1, 0, err
+	}
+
+	sa, err := unix.Getsockname(fd)
+	if err == nil {
+		addr.Port = sa.(*unix.SockaddrInet4).Port
+	}
+
+	return fd, uint16(addr.Port), err
+}
+
+func create6(port uint16) (int, uint16, error) {
+
+	// create socket
+
+	fd, err := unix.Socket(
+		unix.AF_INET6,
+		unix.SOCK_DGRAM,
+		0,
+	)
+
+	if err != nil {
+		return -1, 0, err
+	}
+
+	// set sockopts and bind
+
+	addr := unix.SockaddrInet6{
+		Port: int(port),
+	}
+
+	if err := func() error {
+		if err := unix.SetsockoptInt(
+			fd,
+			unix.IPPROTO_IPV6,
+			unix.IPV6_RECVPKTINFO,
+			1,
+		); err != nil {
+			return err
+		}
+
+		if err := unix.SetsockoptInt(
+			fd,
+			unix.IPPROTO_IPV6,
+			unix.IPV6_V6ONLY,
+			1,
+		); err != nil {
+			return err
+		}
+
+		return unix.Bind(fd, &addr)
+
+	}(); err != nil {
+		unix.Close(fd)
+		return -1, 0, err
+	}
+
+	sa, err := unix.Getsockname(fd)
+	if err == nil {
+		addr.Port = sa.(*unix.SockaddrInet6).Port
+	}
+
+	return fd, uint16(addr.Port), err
+}
+
+func send4(sock int, end *LinuxSocketEndpoint, buff []byte) error {
+
+	// construct message header
+
+	cmsg := struct {
+		cmsghdr unix.Cmsghdr
+		pktinfo unix.Inet4Pktinfo
+	}{
+		unix.Cmsghdr{
+			Level: unix.IPPROTO_IP,
+			Type:  unix.IP_PKTINFO,
+			Len:   unix.SizeofInet4Pktinfo + unix.SizeofCmsghdr,
+		},
+		unix.Inet4Pktinfo{
+			Spec_dst: end.src4().Src,
+			Ifindex:  end.src4().Ifindex,
+		},
+	}
+
+	end.mu.Lock()
+	_, err := unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst4(), 0)
+	end.mu.Unlock()
+
+	if err == nil {
+		return nil
+	}
+
+	// clear src and retry
+
+	if err == unix.EINVAL {
+		end.ClearSrc()
+		cmsg.pktinfo = unix.Inet4Pktinfo{}
+		end.mu.Lock()
+		_, err = unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst4(), 0)
+		end.mu.Unlock()
+	}
+
+	return err
+}
+
+func send6(sock int, end *LinuxSocketEndpoint, buff []byte) error {
+
+	// construct message header
+
+	cmsg := struct {
+		cmsghdr unix.Cmsghdr
+		pktinfo unix.Inet6Pktinfo
+	}{
+		unix.Cmsghdr{
+			Level: unix.IPPROTO_IPV6,
+			Type:  unix.IPV6_PKTINFO,
+			Len:   unix.SizeofInet6Pktinfo + unix.SizeofCmsghdr,
+		},
+		unix.Inet6Pktinfo{
+			Addr:    end.src6().src,
+			Ifindex: end.dst6().ZoneId,
+		},
+	}
+
+	if cmsg.pktinfo.Addr == [16]byte{} {
+		cmsg.pktinfo.Ifindex = 0
+	}
+
+	end.mu.Lock()
+	_, err := unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst6(), 0)
+	end.mu.Unlock()
+
+	if err == nil {
+		return nil
+	}
+
+	// clear src and retry
+
+	if err == unix.EINVAL {
+		end.ClearSrc()
+		cmsg.pktinfo = unix.Inet6Pktinfo{}
+		end.mu.Lock()
+		_, err = unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst6(), 0)
+		end.mu.Unlock()
+	}
+
+	return err
+}
+
+func receive4(sock int, buff []byte, end *LinuxSocketEndpoint) (int, error) {
+
+	// construct message header
+
+	var cmsg struct {
+		cmsghdr unix.Cmsghdr
+		pktinfo unix.Inet4Pktinfo
+	}
+
+	size, _, _, newDst, err := unix.Recvmsg(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], 0)
+
+	if err != nil {
+		return 0, err
+	}
+	end.isV6 = false
+
+	if newDst4, ok := newDst.(*unix.SockaddrInet4); ok {
+		*end.dst4() = *newDst4
+	}
+
+	// update source cache
+
+	if cmsg.cmsghdr.Level == unix.IPPROTO_IP &&
+		cmsg.cmsghdr.Type == unix.IP_PKTINFO &&
+		cmsg.cmsghdr.Len >= unix.SizeofInet4Pktinfo {
+		end.src4().Src = cmsg.pktinfo.Spec_dst
+		end.src4().Ifindex = cmsg.pktinfo.Ifindex
+	}
+
+	return size, nil
+}
+
+func receive6(sock int, buff []byte, end *LinuxSocketEndpoint) (int, error) {
+
+	// construct message header
+
+	var cmsg struct {
+		cmsghdr unix.Cmsghdr
+		pktinfo unix.Inet6Pktinfo
+	}
+
+	size, _, _, newDst, err := unix.Recvmsg(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], 0)
+
+	if err != nil {
+		return 0, err
+	}
+	end.isV6 = true
+
+	if newDst6, ok := newDst.(*unix.SockaddrInet6); ok {
+		*end.dst6() = *newDst6
+	}
+
+	// update source cache
+
+	if cmsg.cmsghdr.Level == unix.IPPROTO_IPV6 &&
+		cmsg.cmsghdr.Type == unix.IPV6_PKTINFO &&
+		cmsg.cmsghdr.Len >= unix.SizeofInet6Pktinfo {
+		end.src6().src = cmsg.pktinfo.Addr
+		end.dst6().ZoneId = cmsg.pktinfo.Ifindex
+	}
+
+	return size, nil
+}

conn/bind_std.go

@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+import (
+	"errors"
+	"net"
+	"sync"
+	"syscall"
+)
+
+// StdNetBind is meant to be a temporary solution on platforms for which
+// the sticky socket / source caching behavior has not yet been implemented.
+// It uses the Go's net package to implement networking.
+// See LinuxSocketBind for a proper implementation on the Linux platform.
+type StdNetBind struct {
+	mu         sync.Mutex // protects following fields
+	ipv4       *net.UDPConn
+	ipv6       *net.UDPConn
+	blackhole4 bool
+	blackhole6 bool
+}
+
+func NewStdNetBind() Bind { return &StdNetBind{} }
+
+type StdNetEndpoint net.UDPAddr
+
+var _ Bind = (*StdNetBind)(nil)
+var _ Endpoint = (*StdNetEndpoint)(nil)
+
+func (*StdNetBind) ParseEndpoint(s string) (Endpoint, error) {
+	addr, err := parseEndpoint(s)
+	return (*StdNetEndpoint)(addr), err
+}
+
+func (*StdNetEndpoint) ClearSrc() {}
+
+func (e *StdNetEndpoint) DstIP() net.IP {
+	return (*net.UDPAddr)(e).IP
+}
+
+func (e *StdNetEndpoint) SrcIP() net.IP {
+	return nil // not supported
+}
+
+func (e *StdNetEndpoint) DstToBytes() []byte {
+	addr := (*net.UDPAddr)(e)
+	out := addr.IP.To4()
+	if out == nil {
+		out = addr.IP
+	}
+	out = append(out, byte(addr.Port&0xff))
+	out = append(out, byte((addr.Port>>8)&0xff))
+	return out
+}
+
+func (e *StdNetEndpoint) DstToString() string {
+	return (*net.UDPAddr)(e).String()
+}
+
+func (e *StdNetEndpoint) SrcToString() string {
+	return ""
+}
+
+func listenNet(network string, port int) (*net.UDPConn, int, error) {
+	conn, err := net.ListenUDP(network, &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// Retrieve port.
+	laddr := conn.LocalAddr()
+	uaddr, err := net.ResolveUDPAddr(
+		laddr.Network(),
+		laddr.String(),
+	)
+	if err != nil {
+		return nil, 0, err
+	}
+	return conn, uaddr.Port, nil
+}
+
+func (bind *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+
+	var err error
+	var tries int
+
+	if bind.ipv4 != nil || bind.ipv6 != nil {
+		return nil, 0, ErrBindAlreadyOpen
+	}
+
+	// Attempt to open ipv4 and ipv6 listeners on the same port.
+	// If uport is 0, we can retry on failure.
+again:
+	port := int(uport)
+	var ipv4, ipv6 *net.UDPConn
+
+	ipv4, port, err = listenNet("udp4", port)
+	if err != nil && !errors.Is(err, syscall.EAFNOSUPPORT) {
+		return nil, 0, err
+	}
+
+	// Listen on the same port as we're using for ipv4.
+	ipv6, port, err = listenNet("udp6", port)
+	if uport == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 {
+		ipv4.Close()
+		tries++
+		goto again
+	}
+	if err != nil && !errors.Is(err, syscall.EAFNOSUPPORT) {
+		ipv4.Close()
+		return nil, 0, err
+	}
+	var fns []ReceiveFunc
+	if ipv4 != nil {
+		fns = append(fns, bind.makeReceiveIPv4(ipv4))
+		bind.ipv4 = ipv4
+	}
+	if ipv6 != nil {
+		fns = append(fns, bind.makeReceiveIPv6(ipv6))
+		bind.ipv6 = ipv6
+	}
+	if len(fns) == 0 {
+		return nil, 0, syscall.EAFNOSUPPORT
+	}
+	return fns, uint16(port), nil
+}
+
+func (bind *StdNetBind) Close() error {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+
+	var err1, err2 error
+	if bind.ipv4 != nil {
+		err1 = bind.ipv4.Close()
+		bind.ipv4 = nil
+	}
+	if bind.ipv6 != nil {
+		err2 = bind.ipv6.Close()
+		bind.ipv6 = nil
+	}
+	bind.blackhole4 = false
+	bind.blackhole6 = false
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (*StdNetBind) makeReceiveIPv4(conn *net.UDPConn) ReceiveFunc {
+	return func(buff []byte) (int, Endpoint, error) {
+		n, endpoint, err := conn.ReadFromUDP(buff)
+		if endpoint != nil {
+			endpoint.IP = endpoint.IP.To4()
+		}
+		return n, (*StdNetEndpoint)(endpoint), err
+	}
+}
+
+func (*StdNetBind) makeReceiveIPv6(conn *net.UDPConn) ReceiveFunc {
+	return func(buff []byte) (int, Endpoint, error) {
+		n, endpoint, err := conn.ReadFromUDP(buff)
+		return n, (*StdNetEndpoint)(endpoint), err
+	}
+}
+
+func (bind *StdNetBind) Send(buff []byte, endpoint Endpoint) error {
+	var err error
+	nend, ok := endpoint.(*StdNetEndpoint)
+	if !ok {
+		return ErrWrongEndpointType
+	}
+
+	bind.mu.Lock()
+	blackhole := bind.blackhole4
+	conn := bind.ipv4
+	if nend.IP.To4() == nil {
+		blackhole = bind.blackhole6
+		conn = bind.ipv6
+	}
+	bind.mu.Unlock()
+
+	if blackhole {
+		return nil
+	}
+	if conn == nil {
+		return syscall.EAFNOSUPPORT
+	}
+	_, err = conn.WriteToUDP(buff, (*net.UDPAddr)(nend))
+	return err
+}

conn/bind_windows.go

@@ -0,0 +1,581 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+import (
+	"encoding/binary"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+
+	"golang.zx2c4.com/wireguard/conn/winrio"
+)
+
+const (
+	packetsPerRing = 1024
+	bytesPerPacket = 2048 - 32
+	receiveSpins   = 15
+)
+
+type ringPacket struct {
+	addr WinRingEndpoint
+	data [bytesPerPacket]byte
+}
+
+type ringBuffer struct {
+	packets    uintptr
+	head, tail uint32
+	id         winrio.BufferId
+	iocp       windows.Handle
+	isFull     bool
+	cq         winrio.Cq
+	mu         sync.Mutex
+	overlapped windows.Overlapped
+}
+
+func (rb *ringBuffer) Push() *ringPacket {
+	for rb.isFull {
+		panic("ring is full")
+	}
+	ret := (*ringPacket)(unsafe.Pointer(rb.packets + (uintptr(rb.tail%packetsPerRing) * unsafe.Sizeof(ringPacket{}))))
+	rb.tail += 1
+	if rb.tail%packetsPerRing == rb.head%packetsPerRing {
+		rb.isFull = true
+	}
+	return ret
+}
+
+func (rb *ringBuffer) Return(count uint32) {
+	if rb.head%packetsPerRing == rb.tail%packetsPerRing && !rb.isFull {
+		return
+	}
+	rb.head += count
+	rb.isFull = false
+}
+
+type afWinRingBind struct {
+	sock      windows.Handle
+	rx, tx    ringBuffer
+	rq        winrio.Rq
+	mu        sync.Mutex
+	blackhole bool
+}
+
+// WinRingBind uses Windows registered I/O for fast ring buffered networking.
+type WinRingBind struct {
+	v4, v6 afWinRingBind
+	mu     sync.RWMutex
+	isOpen uint32
+}
+
+func NewDefaultBind() Bind { return NewWinRingBind() }
+
+func NewWinRingBind() Bind {
+	if !winrio.Initialize() {
+		return NewStdNetBind()
+	}
+	return new(WinRingBind)
+}
+
+type WinRingEndpoint struct {
+	family uint16
+	data   [30]byte
+}
+
+var _ Bind = (*WinRingBind)(nil)
+var _ Endpoint = (*WinRingEndpoint)(nil)
+
+func (*WinRingBind) ParseEndpoint(s string) (Endpoint, error) {
+	host, port, err := net.SplitHostPort(s)
+	if err != nil {
+		return nil, err
+	}
+	host16, err := windows.UTF16PtrFromString(host)
+	if err != nil {
+		return nil, err
+	}
+	port16, err := windows.UTF16PtrFromString(port)
+	if err != nil {
+		return nil, err
+	}
+	hints := windows.AddrinfoW{
+		Flags:    windows.AI_NUMERICHOST,
+		Family:   windows.AF_UNSPEC,
+		Socktype: windows.SOCK_DGRAM,
+		Protocol: windows.IPPROTO_UDP,
+	}
+	var addrinfo *windows.AddrinfoW
+	err = windows.GetAddrInfoW(host16, port16, &hints, &addrinfo)
+	if err != nil {
+		return nil, err
+	}
+	defer windows.FreeAddrInfoW(addrinfo)
+	if (addrinfo.Family != windows.AF_INET && addrinfo.Family != windows.AF_INET6) || addrinfo.Addrlen > unsafe.Sizeof(WinRingEndpoint{}) {
+		return nil, windows.ERROR_INVALID_ADDRESS
+	}
+	var dst [unsafe.Sizeof(WinRingEndpoint{})]byte
+	copy(dst[:], unsafe.Slice((*byte)(unsafe.Pointer(addrinfo.Addr)), addrinfo.Addrlen))
+	return (*WinRingEndpoint)(unsafe.Pointer(&dst[0])), nil
+}
+
+func (*WinRingEndpoint) ClearSrc() {}
+
+func (e *WinRingEndpoint) DstIP() net.IP {
+	switch e.family {
+	case windows.AF_INET:
+		return append([]byte{}, e.data[2:6]...)
+	case windows.AF_INET6:
+		return append([]byte{}, e.data[6:22]...)
+	}
+	return nil
+}
+
+func (e *WinRingEndpoint) SrcIP() net.IP {
+	return nil // not supported
+}
+
+func (e *WinRingEndpoint) DstToBytes() []byte {
+	switch e.family {
+	case windows.AF_INET:
+		b := make([]byte, 0, 6)
+		b = append(b, e.data[2:6]...)
+		b = append(b, e.data[1], e.data[0])
+		return b
+	case windows.AF_INET6:
+		b := make([]byte, 0, 18)
+		b = append(b, e.data[6:22]...)
+		b = append(b, e.data[1], e.data[0])
+		return b
+	}
+	return nil
+}
+
+func (e *WinRingEndpoint) DstToString() string {
+	switch e.family {
+	case windows.AF_INET:
+		addr := net.UDPAddr{IP: e.data[2:6], Port: int(binary.BigEndian.Uint16(e.data[0:2]))}
+		return addr.String()
+	case windows.AF_INET6:
+		var zone string
+		if scope := *(*uint32)(unsafe.Pointer(&e.data[22])); scope > 0 {
+			zone = strconv.FormatUint(uint64(scope), 10)
+		}
+		addr := net.UDPAddr{IP: e.data[6:22], Zone: zone, Port: int(binary.BigEndian.Uint16(e.data[0:2]))}
+		return addr.String()
+	}
+	return ""
+}
+
+func (e *WinRingEndpoint) SrcToString() string {
+	return ""
+}
+
+func (ring *ringBuffer) CloseAndZero() {
+	if ring.cq != 0 {
+		winrio.CloseCompletionQueue(ring.cq)
+		ring.cq = 0
+	}
+	if ring.iocp != 0 {
+		windows.CloseHandle(ring.iocp)
+		ring.iocp = 0
+	}
+	if ring.id != 0 {
+		winrio.DeregisterBuffer(ring.id)
+		ring.id = 0
+	}
+	if ring.packets != 0 {
+		windows.VirtualFree(ring.packets, 0, windows.MEM_RELEASE)
+		ring.packets = 0
+	}
+	ring.head = 0
+	ring.tail = 0
+	ring.isFull = false
+}
+
+func (bind *afWinRingBind) CloseAndZero() {
+	bind.rx.CloseAndZero()
+	bind.tx.CloseAndZero()
+	if bind.sock != 0 {
+		windows.CloseHandle(bind.sock)
+		bind.sock = 0
+	}
+	bind.blackhole = false
+}
+
+func (bind *WinRingBind) closeAndZero() {
+	atomic.StoreUint32(&bind.isOpen, 0)
+	bind.v4.CloseAndZero()
+	bind.v6.CloseAndZero()
+}
+
+func (ring *ringBuffer) Open() error {
+	var err error
+	packetsLen := unsafe.Sizeof(ringPacket{}) * packetsPerRing
+	ring.packets, err = windows.VirtualAlloc(0, packetsLen, windows.MEM_COMMIT|windows.MEM_RESERVE, windows.PAGE_READWRITE)
+	if err != nil {
+		return err
+	}
+	ring.id, err = winrio.RegisterPointer(unsafe.Pointer(ring.packets), uint32(packetsLen))
+	if err != nil {
+		return err
+	}
+	ring.iocp, err = windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0)
+	if err != nil {
+		return err
+	}
+	ring.cq, err = winrio.CreateIOCPCompletionQueue(packetsPerRing, ring.iocp, 0, &ring.overlapped)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (bind *afWinRingBind) Open(family int32, sa windows.Sockaddr) (windows.Sockaddr, error) {
+	var err error
+	bind.sock, err = winrio.Socket(family, windows.SOCK_DGRAM, windows.IPPROTO_UDP)
+	if err != nil {
+		return nil, err
+	}
+	err = bind.rx.Open()
+	if err != nil {
+		return nil, err
+	}
+	err = bind.tx.Open()
+	if err != nil {
+		return nil, err
+	}
+	bind.rq, err = winrio.CreateRequestQueue(bind.sock, packetsPerRing, 1, packetsPerRing, 1, bind.rx.cq, bind.tx.cq, 0)
+	if err != nil {
+		return nil, err
+	}
+	err = windows.Bind(bind.sock, sa)
+	if err != nil {
+		return nil, err
+	}
+	sa, err = windows.Getsockname(bind.sock)
+	if err != nil {
+		return nil, err
+	}
+	return sa, nil
+}
+
+func (bind *WinRingBind) Open(port uint16) (recvFns []ReceiveFunc, selectedPort uint16, err error) {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	defer func() {
+		if err != nil {
+			bind.closeAndZero()
+		}
+	}()
+	if atomic.LoadUint32(&bind.isOpen) != 0 {
+		return nil, 0, ErrBindAlreadyOpen
+	}
+	var sa windows.Sockaddr
+	sa, err = bind.v4.Open(windows.AF_INET, &windows.SockaddrInet4{Port: int(port)})
+	if err != nil {
+		return nil, 0, err
+	}
+	sa, err = bind.v6.Open(windows.AF_INET6, &windows.SockaddrInet6{Port: sa.(*windows.SockaddrInet4).Port})
+	if err != nil {
+		return nil, 0, err
+	}
+	selectedPort = uint16(sa.(*windows.SockaddrInet6).Port)
+	for i := 0; i < packetsPerRing; i++ {
+		err = bind.v4.InsertReceiveRequest()
+		if err != nil {
+			return nil, 0, err
+		}
+		err = bind.v6.InsertReceiveRequest()
+		if err != nil {
+			return nil, 0, err
+		}
+	}
+	atomic.StoreUint32(&bind.isOpen, 1)
+	return []ReceiveFunc{bind.receiveIPv4, bind.receiveIPv6}, selectedPort, err
+}
+
+func (bind *WinRingBind) Close() error {
+	bind.mu.RLock()
+	if atomic.LoadUint32(&bind.isOpen) != 1 {
+		bind.mu.RUnlock()
+		return nil
+	}
+	atomic.StoreUint32(&bind.isOpen, 2)
+	windows.PostQueuedCompletionStatus(bind.v4.rx.iocp, 0, 0, nil)
+	windows.PostQueuedCompletionStatus(bind.v4.tx.iocp, 0, 0, nil)
+	windows.PostQueuedCompletionStatus(bind.v6.rx.iocp, 0, 0, nil)
+	windows.PostQueuedCompletionStatus(bind.v6.tx.iocp, 0, 0, nil)
+	bind.mu.RUnlock()
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	bind.closeAndZero()
+	return nil
+}
+
+func (bind *WinRingBind) SetMark(mark uint32) error {
+	return nil
+}
+
+func (bind *afWinRingBind) InsertReceiveRequest() error {
+	packet := bind.rx.Push()
+	dataBuffer := &winrio.Buffer{
+		Id:     bind.rx.id,
+		Offset: uint32(uintptr(unsafe.Pointer(&packet.data[0])) - bind.rx.packets),
+		Length: uint32(len(packet.data)),
+	}
+	addressBuffer := &winrio.Buffer{
+		Id:     bind.rx.id,
+		Offset: uint32(uintptr(unsafe.Pointer(&packet.addr)) - bind.rx.packets),
+		Length: uint32(unsafe.Sizeof(packet.addr)),
+	}
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	return winrio.ReceiveEx(bind.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, uintptr(unsafe.Pointer(packet)))
+}
+
+//go:linkname procyield runtime.procyield
+func procyield(cycles uint32)
+
+func (bind *afWinRingBind) Receive(buf []byte, isOpen *uint32) (int, Endpoint, error) {
+	if atomic.LoadUint32(isOpen) != 1 {
+		return 0, nil, net.ErrClosed
+	}
+	bind.rx.mu.Lock()
+	defer bind.rx.mu.Unlock()
+
+	var err error
+	var count uint32
+	var results [1]winrio.Result
+retry:
+	count = 0
+	for tries := 0; count == 0 && tries < receiveSpins; tries++ {
+		if tries > 0 {
+			if atomic.LoadUint32(isOpen) != 1 {
+				return 0, nil, net.ErrClosed
+			}
+			procyield(1)
+		}
+		count = winrio.DequeueCompletion(bind.rx.cq, results[:])
+	}
+	if count == 0 {
+		err = winrio.Notify(bind.rx.cq)
+		if err != nil {
+			return 0, nil, err
+		}
+		var bytes uint32
+		var key uintptr
+		var overlapped *windows.Overlapped
+		err = windows.GetQueuedCompletionStatus(bind.rx.iocp, &bytes, &key, &overlapped, windows.INFINITE)
+		if err != nil {
+			return 0, nil, err
+		}
+		if atomic.LoadUint32(isOpen) != 1 {
+			return 0, nil, net.ErrClosed
+		}
+		count = winrio.DequeueCompletion(bind.rx.cq, results[:])
+		if count == 0 {
+			return 0, nil, io.ErrNoProgress
+
+		}
+	}
+	bind.rx.Return(1)
+	err = bind.InsertReceiveRequest()
+	if err != nil {
+		return 0, nil, err
+	}
+	// We limit the MTU well below the 65k max for practicality, but this means a remote host can still send us
+	// huge packets. Just try again when this happens. The infinite loop this could cause is still limited to
+	// attacker bandwidth, just like the rest of the receive path.
+	if windows.Errno(results[0].Status) == windows.WSAEMSGSIZE {
+		if atomic.LoadUint32(isOpen) != 1 {
+			return 0, nil, net.ErrClosed
+		}
+		goto retry
+	}
+	if results[0].Status != 0 {
+		return 0, nil, windows.Errno(results[0].Status)
+	}
+	packet := (*ringPacket)(unsafe.Pointer(uintptr(results[0].RequestContext)))
+	ep := packet.addr
+	n := copy(buf, packet.data[:results[0].BytesTransferred])
+	return n, &ep, nil
+}
+
+func (bind *WinRingBind) receiveIPv4(buf []byte) (int, Endpoint, error) {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	return bind.v4.Receive(buf, &bind.isOpen)
+}
+
+func (bind *WinRingBind) receiveIPv6(buf []byte) (int, Endpoint, error) {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	return bind.v6.Receive(buf, &bind.isOpen)
+}
+
+func (bind *afWinRingBind) Send(buf []byte, nend *WinRingEndpoint, isOpen *uint32) error {
+	if atomic.LoadUint32(isOpen) != 1 {
+		return net.ErrClosed
+	}
+	if len(buf) > bytesPerPacket {
+		return io.ErrShortBuffer
+	}
+	bind.tx.mu.Lock()
+	defer bind.tx.mu.Unlock()
+	var results [packetsPerRing]winrio.Result
+	count := winrio.DequeueCompletion(bind.tx.cq, results[:])
+	if count == 0 && bind.tx.isFull {
+		err := winrio.Notify(bind.tx.cq)
+		if err != nil {
+			return err
+		}
+		var bytes uint32
+		var key uintptr
+		var overlapped *windows.Overlapped
+		err = windows.GetQueuedCompletionStatus(bind.tx.iocp, &bytes, &key, &overlapped, windows.INFINITE)
+		if err != nil {
+			return err
+		}
+		if atomic.LoadUint32(isOpen) != 1 {
+			return net.ErrClosed
+		}
+		count = winrio.DequeueCompletion(bind.tx.cq, results[:])
+		if count == 0 {
+			return io.ErrNoProgress
+		}
+	}
+	if count > 0 {
+		bind.tx.Return(count)
+	}
+	packet := bind.tx.Push()
+	packet.addr = *nend
+	copy(packet.data[:], buf)
+	dataBuffer := &winrio.Buffer{
+		Id:     bind.tx.id,
+		Offset: uint32(uintptr(unsafe.Pointer(&packet.data[0])) - bind.tx.packets),
+		Length: uint32(len(buf)),
+	}
+	addressBuffer := &winrio.Buffer{
+		Id:     bind.tx.id,
+		Offset: uint32(uintptr(unsafe.Pointer(&packet.addr)) - bind.tx.packets),
+		Length: uint32(unsafe.Sizeof(packet.addr)),
+	}
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	return winrio.SendEx(bind.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
+}
+
+func (bind *WinRingBind) Send(buf []byte, endpoint Endpoint) error {
+	nend, ok := endpoint.(*WinRingEndpoint)
+	if !ok {
+		return ErrWrongEndpointType
+	}
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	switch nend.family {
+	case windows.AF_INET:
+		if bind.v4.blackhole {
+			return nil
+		}
+		return bind.v4.Send(buf, nend, &bind.isOpen)
+	case windows.AF_INET6:
+		if bind.v6.blackhole {
+			return nil
+		}
+		return bind.v6.Send(buf, nend, &bind.isOpen)
+	}
+	return nil
+}
+
+func (bind *StdNetBind) BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	sysconn, err := bind.ipv4.SyscallConn()
+	if err != nil {
+		return err
+	}
+	err2 := sysconn.Control(func(fd uintptr) {
+		err = bindSocketToInterface4(windows.Handle(fd), interfaceIndex)
+	})
+	if err2 != nil {
+		return err2
+	}
+	if err != nil {
+		return err
+	}
+	bind.blackhole4 = blackhole
+	return nil
+}
+
+func (bind *StdNetBind) BindSocketToInterface6(interfaceIndex uint32, blackhole bool) error {
+	bind.mu.Lock()
+	defer bind.mu.Unlock()
+	sysconn, err := bind.ipv6.SyscallConn()
+	if err != nil {
+		return err
+	}
+	err2 := sysconn.Control(func(fd uintptr) {
+		err = bindSocketToInterface6(windows.Handle(fd), interfaceIndex)
+	})
+	if err2 != nil {
+		return err2
+	}
+	if err != nil {
+		return err
+	}
+	bind.blackhole6 = blackhole
+	return nil
+}
+func (bind *WinRingBind) BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	if atomic.LoadUint32(&bind.isOpen) != 1 {
+		return net.ErrClosed
+	}
+	err := bindSocketToInterface4(bind.v4.sock, interfaceIndex)
+	if err != nil {
+		return err
+	}
+	bind.v4.blackhole = blackhole
+	return nil
+}
+
+func (bind *WinRingBind) BindSocketToInterface6(interfaceIndex uint32, blackhole bool) error {
+	bind.mu.RLock()
+	defer bind.mu.RUnlock()
+	if atomic.LoadUint32(&bind.isOpen) != 1 {
+		return net.ErrClosed
+	}
+	err := bindSocketToInterface6(bind.v6.sock, interfaceIndex)
+	if err != nil {
+		return err
+	}
+	bind.v6.blackhole = blackhole
+	return nil
+}
+
+func bindSocketToInterface4(handle windows.Handle, interfaceIndex uint32) error {
+	const IP_UNICAST_IF = 31
+	/* MSDN says for IPv4 this needs to be in net byte order, so that it's like an IP address with leading zeros. */
+	var bytes [4]byte
+	binary.BigEndian.PutUint32(bytes[:], interfaceIndex)
+	interfaceIndex = *(*uint32)(unsafe.Pointer(&bytes[0]))
+	err := windows.SetsockoptInt(handle, windows.IPPROTO_IP, IP_UNICAST_IF, int(interfaceIndex))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func bindSocketToInterface6(handle windows.Handle, interfaceIndex uint32) error {
+	const IPV6_UNICAST_IF = 31
+	return windows.SetsockoptInt(handle, windows.IPPROTO_IPV6, IPV6_UNICAST_IF, int(interfaceIndex))
+}

conn/bindtest/bindtest.go

@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package bindtest
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+	"strconv"
+
+	"golang.zx2c4.com/wireguard/conn"
+)
+
+type ChannelBind struct {
+	rx4, tx4         *chan []byte
+	rx6, tx6         *chan []byte
+	closeSignal      chan bool
+	source4, source6 ChannelEndpoint
+	target4, target6 ChannelEndpoint
+}
+
+type ChannelEndpoint uint16
+
+var _ conn.Bind = (*ChannelBind)(nil)
+var _ conn.Endpoint = (*ChannelEndpoint)(nil)
+
+func NewChannelBinds() [2]conn.Bind {
+	arx4 := make(chan []byte, 8192)
+	brx4 := make(chan []byte, 8192)
+	arx6 := make(chan []byte, 8192)
+	brx6 := make(chan []byte, 8192)
+	var binds [2]ChannelBind
+	binds[0].rx4 = &arx4
+	binds[0].tx4 = &brx4
+	binds[1].rx4 = &brx4
+	binds[1].tx4 = &arx4
+	binds[0].rx6 = &arx6
+	binds[0].tx6 = &brx6
+	binds[1].rx6 = &brx6
+	binds[1].tx6 = &arx6
+	binds[0].target4 = ChannelEndpoint(1)
+	binds[1].target4 = ChannelEndpoint(2)
+	binds[0].target6 = ChannelEndpoint(3)
+	binds[1].target6 = ChannelEndpoint(4)
+	binds[0].source4 = binds[1].target4
+	binds[0].source6 = binds[1].target6
+	binds[1].source4 = binds[0].target4
+	binds[1].source6 = binds[0].target6
+	return [2]conn.Bind{&binds[0], &binds[1]}
+}
+
+func (c ChannelEndpoint) ClearSrc() {}
+
+func (c ChannelEndpoint) SrcToString() string { return "" }
+
+func (c ChannelEndpoint) DstToString() string { return fmt.Sprintf("127.0.0.1:%d", c) }
+
+func (c ChannelEndpoint) DstToBytes() []byte { return []byte{byte(c)} }
+
+func (c ChannelEndpoint) DstIP() net.IP { return net.IPv4(127, 0, 0, 1) }
+
+func (c ChannelEndpoint) SrcIP() net.IP { return nil }
+
+func (c *ChannelBind) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) {
+	c.closeSignal = make(chan bool)
+	fns = append(fns, c.makeReceiveFunc(*c.rx4))
+	fns = append(fns, c.makeReceiveFunc(*c.rx6))
+	if rand.Uint32()&1 == 0 {
+		return fns, uint16(c.source4), nil
+	} else {
+		return fns, uint16(c.source6), nil
+	}
+}
+
+func (c *ChannelBind) Close() error {
+	if c.closeSignal != nil {
+		select {
+		case <-c.closeSignal:
+		default:
+			close(c.closeSignal)
+		}
+	}
+	return nil
+}
+
+func (c *ChannelBind) SetMark(mark uint32) error { return nil }
+
+func (c *ChannelBind) makeReceiveFunc(ch chan []byte) conn.ReceiveFunc {
+	return func(b []byte) (n int, ep conn.Endpoint, err error) {
+		select {
+		case <-c.closeSignal:
+			return 0, nil, net.ErrClosed
+		case rx := <-ch:
+			return copy(b, rx), c.target6, nil
+		}
+	}
+}
+
+func (c *ChannelBind) Send(b []byte, ep conn.Endpoint) error {
+	select {
+	case <-c.closeSignal:
+		return net.ErrClosed
+	default:
+		bc := make([]byte, len(b))
+		copy(bc, b)
+		if ep.(ChannelEndpoint) == c.target4 {
+			*c.tx4 <- bc
+		} else if ep.(ChannelEndpoint) == c.target6 {
+			*c.tx6 <- bc
+		} else {
+			return os.ErrInvalid
+		}
+	}
+	return nil
+}
+
+func (c *ChannelBind) ParseEndpoint(s string) (conn.Endpoint, error) {
+	_, port, err := net.SplitHostPort(s)
+	if err != nil {
+		return nil, err
+	}
+	i, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		return nil, err
+	}
+	return ChannelEndpoint(i), nil
+}

conn/boundif_android.go

@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+func (bind *StdNetBind) PeekLookAtSocketFd4() (fd int, err error) {
+	sysconn, err := bind.ipv4.SyscallConn()
+	if err != nil {
+		return -1, err
+	}
+	err = sysconn.Control(func(f uintptr) {
+		fd = int(f)
+	})
+	if err != nil {
+		return -1, err
+	}
+	return
+}
+
+func (bind *StdNetBind) PeekLookAtSocketFd6() (fd int, err error) {
+	sysconn, err := bind.ipv6.SyscallConn()
+	if err != nil {
+		return -1, err
+	}
+	err = sysconn.Control(func(f uintptr) {
+		fd = int(f)
+	})
+	if err != nil {
+		return -1, err
+	}
+	return
+}

conn/conn.go

@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+// Package conn implements WireGuard's network connections.
+package conn
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"reflect"
+	"runtime"
+	"strings"
+)
+
+// A ReceiveFunc receives a single inbound packet from the network.
+// It writes the data into b. n is the length of the packet.
+// ep is the remote endpoint.
+type ReceiveFunc func(b []byte) (n int, ep Endpoint, err error)
+
+// A Bind listens on a port for both IPv6 and IPv4 UDP traffic.
+//
+// A Bind interface may also be a PeekLookAtSocketFd or BindSocketToInterface,
+// depending on the platform-specific implementation.
+type Bind interface {
+	// Open puts the Bind into a listening state on a given port and reports the actual
+	// port that it bound to. Passing zero results in a random selection.
+	// fns is the set of functions that will be called to receive packets.
+	Open(port uint16) (fns []ReceiveFunc, actualPort uint16, err error)
+
+	// Close closes the Bind listener.
+	// All fns returned by Open must return net.ErrClosed after a call to Close.
+	Close() error
+
+	// SetMark sets the mark for each packet sent through this Bind.
+	// This mark is passed to the kernel as the socket option SO_MARK.
+	SetMark(mark uint32) error
+
+	// Send writes a packet b to address ep.
+	Send(b []byte, ep Endpoint) error
+
+	// ParseEndpoint creates a new endpoint from a string.
+	ParseEndpoint(s string) (Endpoint, error)
+}
+
+// BindSocketToInterface is implemented by Bind objects that support being
+// tied to a single network interface. Used by wireguard-windows.
+type BindSocketToInterface interface {
+	BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error
+	BindSocketToInterface6(interfaceIndex uint32, blackhole bool) error
+}
+
+// PeekLookAtSocketFd is implemented by Bind objects that support having their
+// file descriptor peeked at. Used by wireguard-android.
+type PeekLookAtSocketFd interface {
+	PeekLookAtSocketFd4() (fd int, err error)
+	PeekLookAtSocketFd6() (fd int, err error)
+}
+
+// An Endpoint maintains the source/destination caching for a peer.
+//
+//	dst: the remote address of a peer ("endpoint" in uapi terminology)
+//	src: the local address from which datagrams originate going to the peer
+type Endpoint interface {
+	ClearSrc()           // clears the source address
+	SrcToString() string // returns the local source address (ip:port)
+	DstToString() string // returns the destination address (ip:port)
+	DstToBytes() []byte  // used for mac2 cookie calculations
+	DstIP() net.IP
+	SrcIP() net.IP
+}
+
+var (
+	ErrBindAlreadyOpen   = errors.New("bind is already open")
+	ErrWrongEndpointType = errors.New("endpoint type does not correspond with bind type")
+)
+
+func (fn ReceiveFunc) PrettyName() string {
+	name := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
+	// 0. cheese/taco.beansIPv6.func12.func21218-fm
+	name = strings.TrimSuffix(name, "-fm")
+	// 1. cheese/taco.beansIPv6.func12.func21218
+	if idx := strings.LastIndexByte(name, '/'); idx != -1 {
+		name = name[idx+1:]
+		// 2. taco.beansIPv6.func12.func21218
+	}
+	for {
+		var idx int
+		for idx = len(name) - 1; idx >= 0; idx-- {
+			if name[idx] < '0' || name[idx] > '9' {
+				break
+			}
+		}
+		if idx == len(name)-1 {
+			break
+		}
+		const dotFunc = ".func"
+		if !strings.HasSuffix(name[:idx+1], dotFunc) {
+			break
+		}
+		name = name[:idx+1-len(dotFunc)]
+		// 3. taco.beansIPv6.func12
+		// 4. taco.beansIPv6
+	}
+	if idx := strings.LastIndexByte(name, '.'); idx != -1 {
+		name = name[idx+1:]
+		// 5. beansIPv6
+	}
+	if name == "" {
+		return fmt.Sprintf("%p", fn)
+	}
+	if strings.HasSuffix(name, "IPv4") {
+		return "v4"
+	}
+	if strings.HasSuffix(name, "IPv6") {
+		return "v6"
+	}
+	return name
+}
+
+func parseEndpoint(s string) (*net.UDPAddr, error) {
+	// ensure that the host is an IP address
+
+	host, _, err := net.SplitHostPort(s)
+	if err != nil {
+		return nil, err
+	}
+	if i := strings.LastIndexByte(host, '%'); i > 0 && strings.IndexByte(host, ':') >= 0 {
+		// Remove the scope, if any. ResolveUDPAddr below will use it, but here we're just
+		// trying to make sure with a small sanity test that this is a real IP address and
+		// not something that's likely to incur DNS lookups.
+		host = host[:i]
+	}
+	if ip := net.ParseIP(host); ip == nil {
+		return nil, errors.New("Failed to parse IP address: " + host)
+	}
+
+	// parse address and port
+
+	addr, err := net.ResolveUDPAddr("udp", s)
+	if err != nil {
+		return nil, err
+	}
+	ip4 := addr.IP.To4()
+	if ip4 != nil {
+		addr.IP = ip4
+	}
+	return addr, err
+}

conn/default.go

@@ -0,0 +1,10 @@
+//go:build !linux && !windows
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+func NewDefaultBind() Bind { return NewStdNetBind() }

conn/mark_default.go

@@ -0,0 +1,12 @@
+//go:build !linux && !openbsd && !freebsd
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+func (bind *StdNetBind) SetMark(mark uint32) error {
+	return nil
+}

conn/mark_unix.go

@@ -1,11 +1,11 @@
-// +build android openbsd freebsd
+//go:build linux || openbsd || freebsd
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
-package device
+package conn
 
 import (
 	"runtime"
@@ -26,7 +26,7 @@ func init() {
 	}
 }
 
-func (bind *nativeBind) SetMark(mark uint32) error {
+func (bind *StdNetBind) SetMark(mark uint32) error {
 	var operr error
 	if fwmarkIoctl == 0 {
 		return nil

conn/winrio/rio_windows.go

@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package winrio
+
+import (
+	"log"
+	"sync"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+const (
+	MsgDontNotify = 1
+	MsgDefer      = 2
+	MsgWaitAll    = 4
+	MsgCommitOnly = 8
+
+	MaxCqSize = 0x8000000
+
+	invalidBufferId = 0xFFFFFFFF
+	invalidCq       = 0
+	invalidRq       = 0
+	corruptCq       = 0xFFFFFFFF
+)
+
+var extensionFunctionTable struct {
+	cbSize                   uint32
+	rioReceive               uintptr
+	rioReceiveEx             uintptr
+	rioSend                  uintptr
+	rioSendEx                uintptr
+	rioCloseCompletionQueue  uintptr
+	rioCreateCompletionQueue uintptr
+	rioCreateRequestQueue    uintptr
+	rioDequeueCompletion     uintptr
+	rioDeregisterBuffer      uintptr
+	rioNotify                uintptr
+	rioRegisterBuffer        uintptr
+	rioResizeCompletionQueue uintptr
+	rioResizeRequestQueue    uintptr
+}
+
+type Cq uintptr
+
+type Rq uintptr
+
+type BufferId uintptr
+
+type Buffer struct {
+	Id     BufferId
+	Offset uint32
+	Length uint32
+}
+
+type Result struct {
+	Status           int32
+	BytesTransferred uint32
+	SocketContext    uint64
+	RequestContext   uint64
+}
+
+type notificationCompletionType uint32
+
+const (
+	eventCompletion notificationCompletionType = 1
+	iocpCompletion  notificationCompletionType = 2
+)
+
+type eventNotificationCompletion struct {
+	completionType notificationCompletionType
+	event          windows.Handle
+	notifyReset    uint32
+}
+
+type iocpNotificationCompletion struct {
+	completionType notificationCompletionType
+	iocp           windows.Handle
+	key            uintptr
+	overlapped     *windows.Overlapped
+}
+
+var initialized sync.Once
+var available bool
+
+func Initialize() bool {
+	initialized.Do(func() {
+		var (
+			err    error
+			socket windows.Handle
+			cq     Cq
+		)
+		defer func() {
+			if err == nil {
+				return
+			}
+			if maj, _, _ := windows.RtlGetNtVersionNumbers(); maj <= 7 {
+				return
+			}
+			log.Printf("Registered I/O is unavailable: %v", err)
+		}()
+		socket, err = Socket(windows.AF_INET, windows.SOCK_DGRAM, windows.IPPROTO_UDP)
+		if err != nil {
+			return
+		}
+		defer windows.CloseHandle(socket)
+		var WSAID_MULTIPLE_RIO = &windows.GUID{0x8509e081, 0x96dd, 0x4005, [8]byte{0xb1, 0x65, 0x9e, 0x2e, 0xe8, 0xc7, 0x9e, 0x3f}}
+		const SIO_GET_MULTIPLE_EXTENSION_FUNCTION_POINTER = 0xc8000024
+		ob := uint32(0)
+		err = windows.WSAIoctl(socket, SIO_GET_MULTIPLE_EXTENSION_FUNCTION_POINTER,
+			(*byte)(unsafe.Pointer(WSAID_MULTIPLE_RIO)), uint32(unsafe.Sizeof(*WSAID_MULTIPLE_RIO)),
+			(*byte)(unsafe.Pointer(&extensionFunctionTable)), uint32(unsafe.Sizeof(extensionFunctionTable)),
+			&ob, nil, 0)
+		if err != nil {
+			return
+		}
+
+		// While we should be able to stop here, after getting the function pointers, some anti-virus actually causes
+		// failures in RIOCreateRequestQueue, so keep going to be certain this is supported.
+		var iocp windows.Handle
+		iocp, err = windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0)
+		if err != nil {
+			return
+		}
+		defer windows.CloseHandle(iocp)
+		var overlapped windows.Overlapped
+		cq, err = CreateIOCPCompletionQueue(2, iocp, 0, &overlapped)
+		if err != nil {
+			return
+		}
+		defer CloseCompletionQueue(cq)
+		_, err = CreateRequestQueue(socket, 1, 1, 1, 1, cq, cq, 0)
+		if err != nil {
+			return
+		}
+		available = true
+	})
+	return available
+}
+
+func Socket(af, typ, proto int32) (windows.Handle, error) {
+	return windows.WSASocket(af, typ, proto, nil, 0, windows.WSA_FLAG_REGISTERED_IO)
+}
+
+func CloseCompletionQueue(cq Cq) {
+	_, _, _ = syscall.Syscall(extensionFunctionTable.rioCloseCompletionQueue, 1, uintptr(cq), 0, 0)
+}
+
+func CreateEventCompletionQueue(queueSize uint32, event windows.Handle, notifyReset bool) (Cq, error) {
+	notificationCompletion := &eventNotificationCompletion{
+		completionType: eventCompletion,
+		event:          event,
+	}
+	if notifyReset {
+		notificationCompletion.notifyReset = 1
+	}
+	ret, _, err := syscall.Syscall(extensionFunctionTable.rioCreateCompletionQueue, 2, uintptr(queueSize), uintptr(unsafe.Pointer(notificationCompletion)), 0)
+	if ret == invalidCq {
+		return 0, err
+	}
+	return Cq(ret), nil
+}
+
+func CreateIOCPCompletionQueue(queueSize uint32, iocp windows.Handle, key uintptr, overlapped *windows.Overlapped) (Cq, error) {
+	notificationCompletion := &iocpNotificationCompletion{
+		completionType: iocpCompletion,
+		iocp:           iocp,
+		key:            key,
+		overlapped:     overlapped,
+	}
+	ret, _, err := syscall.Syscall(extensionFunctionTable.rioCreateCompletionQueue, 2, uintptr(queueSize), uintptr(unsafe.Pointer(notificationCompletion)), 0)
+	if ret == invalidCq {
+		return 0, err
+	}
+	return Cq(ret), nil
+}
+
+func CreatePolledCompletionQueue(queueSize uint32) (Cq, error) {
+	ret, _, err := syscall.Syscall(extensionFunctionTable.rioCreateCompletionQueue, 2, uintptr(queueSize), 0, 0)
+	if ret == invalidCq {
+		return 0, err
+	}
+	return Cq(ret), nil
+}
+
+func CreateRequestQueue(socket windows.Handle, maxOutstandingReceive, maxReceiveDataBuffers, maxOutstandingSend, maxSendDataBuffers uint32, receiveCq, sendCq Cq, socketContext uintptr) (Rq, error) {
+	ret, _, err := syscall.Syscall9(extensionFunctionTable.rioCreateRequestQueue, 8, uintptr(socket), uintptr(maxOutstandingReceive), uintptr(maxReceiveDataBuffers), uintptr(maxOutstandingSend), uintptr(maxSendDataBuffers), uintptr(receiveCq), uintptr(sendCq), socketContext, 0)
+	if ret == invalidRq {
+		return 0, err
+	}
+	return Rq(ret), nil
+}
+
+func DequeueCompletion(cq Cq, results []Result) uint32 {
+	var array uintptr
+	if len(results) > 0 {
+		array = uintptr(unsafe.Pointer(&results[0]))
+	}
+	ret, _, _ := syscall.Syscall(extensionFunctionTable.rioDequeueCompletion, 3, uintptr(cq), array, uintptr(len(results)))
+	if ret == corruptCq {
+		panic("cq is corrupt")
+	}
+	return uint32(ret)
+}
+
+func DeregisterBuffer(id BufferId) {
+	_, _, _ = syscall.Syscall(extensionFunctionTable.rioDeregisterBuffer, 1, uintptr(id), 0, 0)
+}
+
+func RegisterBuffer(buffer []byte) (BufferId, error) {
+	var buf unsafe.Pointer
+	if len(buffer) > 0 {
+		buf = unsafe.Pointer(&buffer[0])
+	}
+	return RegisterPointer(buf, uint32(len(buffer)))
+}
+
+func RegisterPointer(ptr unsafe.Pointer, size uint32) (BufferId, error) {
+	ret, _, err := syscall.Syscall(extensionFunctionTable.rioRegisterBuffer, 2, uintptr(ptr), uintptr(size), 0)
+	if ret == invalidBufferId {
+		return 0, err
+	}
+	return BufferId(ret), nil
+}
+
+func SendEx(rq Rq, buf *Buffer, dataBufferCount uint32, localAddress, remoteAddress, controlContext, flags *Buffer, sflags uint32, requestContext uintptr) error {
+	ret, _, err := syscall.Syscall9(extensionFunctionTable.rioSendEx, 9, uintptr(rq), uintptr(unsafe.Pointer(buf)), uintptr(dataBufferCount), uintptr(unsafe.Pointer(localAddress)), uintptr(unsafe.Pointer(remoteAddress)), uintptr(unsafe.Pointer(controlContext)), uintptr(unsafe.Pointer(flags)), uintptr(sflags), requestContext)
+	if ret == 0 {
+		return err
+	}
+	return nil
+}
+
+func ReceiveEx(rq Rq, buf *Buffer, dataBufferCount uint32, localAddress, remoteAddress, controlContext, flags *Buffer, sflags uint32, requestContext uintptr) error {
+	ret, _, err := syscall.Syscall9(extensionFunctionTable.rioReceiveEx, 9, uintptr(rq), uintptr(unsafe.Pointer(buf)), uintptr(dataBufferCount), uintptr(unsafe.Pointer(localAddress)), uintptr(unsafe.Pointer(remoteAddress)), uintptr(unsafe.Pointer(controlContext)), uintptr(unsafe.Pointer(flags)), uintptr(sflags), requestContext)
+	if ret == 0 {
+		return err
+	}
+	return nil
+}
+
+func Notify(cq Cq) error {
+	ret, _, _ := syscall.Syscall(extensionFunctionTable.rioNotify, 1, uintptr(cq), 0, 0)
+	if ret != 0 {
+		return windows.Errno(ret)
+	}
+	return nil
+}

device/alignment_test.go

@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"reflect"
+	"testing"
+	"unsafe"
+)
+
+func checkAlignment(t *testing.T, name string, offset uintptr) {
+	t.Helper()
+	if offset%8 != 0 {
+		t.Errorf("offset of %q within struct is %d bytes, which does not align to 64-bit word boundaries (missing %d bytes). Atomic operations will crash on 32-bit systems.", name, offset, 8-(offset%8))
+	}
+}
+
+// TestPeerAlignment checks that atomically-accessed fields are
+// aligned to 64-bit boundaries, as required by the atomic package.
+//
+// Unfortunately, violating this rule on 32-bit platforms results in a
+// hard segfault at runtime.
+func TestPeerAlignment(t *testing.T) {
+	var p Peer
+
+	typ := reflect.TypeOf(&p).Elem()
+	t.Logf("Peer type size: %d, with fields:", typ.Size())
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)",
+			field.Name,
+			field.Offset,
+			field.Type.Size(),
+			field.Type.Align(),
+		)
+	}
+
+	checkAlignment(t, "Peer.stats", unsafe.Offsetof(p.stats))
+	checkAlignment(t, "Peer.isRunning", unsafe.Offsetof(p.isRunning))
+}
+
+// TestDeviceAlignment checks that atomically-accessed fields are
+// aligned to 64-bit boundaries, as required by the atomic package.
+//
+// Unfortunately, violating this rule on 32-bit platforms results in a
+// hard segfault at runtime.
+func TestDeviceAlignment(t *testing.T) {
+	var d Device
+
+	typ := reflect.TypeOf(&d).Elem()
+	t.Logf("Device type size: %d, with fields:", typ.Size())
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)",
+			field.Name,
+			field.Offset,
+			field.Type.Size(),
+			field.Type.Align(),
+		)
+	}
+	checkAlignment(t, "Device.rate.underLoadUntil", unsafe.Offsetof(d.rate)+unsafe.Offsetof(d.rate.underLoadUntil))
+}

device/allowedips.go

@@ -1,11 +1,12 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
+	"container/list"
 	"errors"
 	"math/bits"
 	"net"
@@ -13,16 +14,20 @@ import (
 	"unsafe"
 )
 
+type parentIndirection struct {
+	parentBit     **trieEntry
+	parentBitType uint8
+}
+
 type trieEntry struct {
-	cidr  uint
-	child [2]*trieEntry
-	bits  net.IP
-	peer  *Peer
-
-	// index of "branching" bit
-
-	bit_at_byte  uint
-	bit_at_shift uint
+	peer        *Peer
+	child       [2]*trieEntry
+	parent      parentIndirection
+	cidr        uint8
+	bitAtByte   uint8
+	bitAtShift  uint8
+	bits        net.IP
+	perPeerElem *list.Element
 }
 
 func isLittleEndian() bool {
@@ -46,128 +51,170 @@ func swapU64(i uint64) uint64 {
 	return bits.ReverseBytes64(i)
 }
 
-func commonBits(ip1 net.IP, ip2 net.IP) uint {
+func commonBits(ip1 net.IP, ip2 net.IP) uint8 {
 	size := len(ip1)
 	if size == net.IPv4len {
 		a := (*uint32)(unsafe.Pointer(&ip1[0]))
 		b := (*uint32)(unsafe.Pointer(&ip2[0]))
 		x := *a ^ *b
-		return uint(bits.LeadingZeros32(swapU32(x)))
+		return uint8(bits.LeadingZeros32(swapU32(x)))
 	} else if size == net.IPv6len {
 		a := (*uint64)(unsafe.Pointer(&ip1[0]))
 		b := (*uint64)(unsafe.Pointer(&ip2[0]))
 		x := *a ^ *b
 		if x != 0 {
-			return uint(bits.LeadingZeros64(swapU64(x)))
+			return uint8(bits.LeadingZeros64(swapU64(x)))
 		}
 		a = (*uint64)(unsafe.Pointer(&ip1[8]))
 		b = (*uint64)(unsafe.Pointer(&ip2[8]))
 		x = *a ^ *b
-		return 64 + uint(bits.LeadingZeros64(swapU64(x)))
+		return 64 + uint8(bits.LeadingZeros64(swapU64(x)))
 	} else {
 		panic("Wrong size bit string")
 	}
 }
 
-func (node *trieEntry) removeByPeer(p *Peer) *trieEntry {
-	if node == nil {
-		return node
+func (node *trieEntry) addToPeerEntries() {
+	node.perPeerElem = node.peer.trieEntries.PushBack(node)
+}
+
+func (node *trieEntry) removeFromPeerEntries() {
+	if node.perPeerElem != nil {
+		node.peer.trieEntries.Remove(node.perPeerElem)
+		node.perPeerElem = nil
 	}
-
-	// walk recursively
-
-	node.child[0] = node.child[0].removeByPeer(p)
-	node.child[1] = node.child[1].removeByPeer(p)
-
-	if node.peer != p {
-		return node
-	}
-
-	// remove peer & merge
-
-	node.peer = nil
-	if node.child[0] == nil {
-		return node.child[1]
-	}
-	return node.child[0]
 }
 
 func (node *trieEntry) choose(ip net.IP) byte {
-	return (ip[node.bit_at_byte] >> node.bit_at_shift) & 1
+	return (ip[node.bitAtByte] >> node.bitAtShift) & 1
 }
 
-func (node *trieEntry) insert(ip net.IP, cidr uint, peer *Peer) *trieEntry {
-
-	// at leaf
-
-	if node == nil {
-		return &trieEntry{
-			bits:         ip,
-			peer:         peer,
-			cidr:         cidr,
-			bit_at_byte:  cidr / 8,
-			bit_at_shift: 7 - (cidr % 8),
-		}
+func (node *trieEntry) maskSelf() {
+	mask := net.CIDRMask(int(node.cidr), len(node.bits)*8)
+	for i := 0; i < len(mask); i++ {
+		node.bits[i] &= mask[i]
 	}
+}
 
-	// traverse deeper
+func (node *trieEntry) zeroizePointers() {
+	// Make the garbage collector's life slightly easier
+	node.peer = nil
+	node.child[0] = nil
+	node.child[1] = nil
+	node.parent.parentBit = nil
+}
 
-	common := commonBits(node.bits, ip)
-	if node.cidr <= cidr && common >= node.cidr {
-		if node.cidr == cidr {
-			node.peer = peer
-			return node
+func (node *trieEntry) nodePlacement(ip net.IP, cidr uint8) (parent *trieEntry, exact bool) {
+	for node != nil && node.cidr <= cidr && commonBits(node.bits, ip) >= node.cidr {
+		parent = node
+		if parent.cidr == cidr {
+			exact = true
+			return
 		}
 		bit := node.choose(ip)
-		node.child[bit] = node.child[bit].insert(ip, cidr, peer)
-		return node
+		node = node.child[bit]
 	}
+	return
+}
 
-	// split node
+func (trie parentIndirection) insert(ip net.IP, cidr uint8, peer *Peer) {
+	if *trie.parentBit == nil {
+		node := &trieEntry{
+			peer:       peer,
+			parent:     trie,
+			bits:       ip,
+			cidr:       cidr,
+			bitAtByte:  cidr / 8,
+			bitAtShift: 7 - (cidr % 8),
+		}
+		node.maskSelf()
+		node.addToPeerEntries()
+		*trie.parentBit = node
+		return
+	}
+	node, exact := (*trie.parentBit).nodePlacement(ip, cidr)
+	if exact {
+		node.removeFromPeerEntries()
+		node.peer = peer
+		node.addToPeerEntries()
+		return
+	}
 
 	newNode := &trieEntry{
-		bits:         ip,
-		peer:         peer,
-		cidr:         cidr,
-		bit_at_byte:  cidr / 8,
-		bit_at_shift: 7 - (cidr % 8),
+		peer:       peer,
+		bits:       ip,
+		cidr:       cidr,
+		bitAtByte:  cidr / 8,
+		bitAtShift: 7 - (cidr % 8),
 	}
+	newNode.maskSelf()
+	newNode.addToPeerEntries()
 
-	cidr = min(cidr, common)
-
-	// check for shorter prefix
+	var down *trieEntry
+	if node == nil {
+		down = *trie.parentBit
+	} else {
+		bit := node.choose(ip)
+		down = node.child[bit]
+		if down == nil {
+			newNode.parent = parentIndirection{&node.child[bit], bit}
+			node.child[bit] = newNode
+			return
+		}
+	}
+	common := commonBits(down.bits, ip)
+	if common < cidr {
+		cidr = common
+	}
+	parent := node
 
 	if newNode.cidr == cidr {
-		bit := newNode.choose(node.bits)
-		newNode.child[bit] = node
-		return newNode
+		bit := newNode.choose(down.bits)
+		down.parent = parentIndirection{&newNode.child[bit], bit}
+		newNode.child[bit] = down
+		if parent == nil {
+			newNode.parent = trie
+			*trie.parentBit = newNode
+		} else {
+			bit := parent.choose(newNode.bits)
+			newNode.parent = parentIndirection{&parent.child[bit], bit}
+			parent.child[bit] = newNode
+		}
+		return
 	}
 
-	// create new parent for node & newNode
-
-	parent := &trieEntry{
-		bits:         ip,
-		peer:         nil,
-		cidr:         cidr,
-		bit_at_byte:  cidr / 8,
-		bit_at_shift: 7 - (cidr % 8),
+	node = &trieEntry{
+		bits:       append([]byte{}, newNode.bits...),
+		cidr:       cidr,
+		bitAtByte:  cidr / 8,
+		bitAtShift: 7 - (cidr % 8),
 	}
+	node.maskSelf()
 
-	bit := parent.choose(ip)
-	parent.child[bit] = newNode
-	parent.child[bit^1] = node
-
-	return parent
+	bit := node.choose(down.bits)
+	down.parent = parentIndirection{&node.child[bit], bit}
+	node.child[bit] = down
+	bit = node.choose(newNode.bits)
+	newNode.parent = parentIndirection{&node.child[bit], bit}
+	node.child[bit] = newNode
+	if parent == nil {
+		node.parent = trie
+		*trie.parentBit = node
+	} else {
+		bit := parent.choose(node.bits)
+		node.parent = parentIndirection{&parent.child[bit], bit}
+		parent.child[bit] = node
+	}
 }
 
 func (node *trieEntry) lookup(ip net.IP) *Peer {
 	var found *Peer
-	size := uint(len(ip))
+	size := uint8(len(ip))
 	for node != nil && commonBits(node.bits, ip) >= node.cidr {
 		if node.peer != nil {
 			found = node.peer
 		}
-		if node.bit_at_byte == size {
+		if node.bitAtByte == size {
 			break
 		}
 		bit := node.choose(ip)
@@ -176,76 +223,89 @@ func (node *trieEntry) lookup(ip net.IP) *Peer {
 	return found
 }
 
-func (node *trieEntry) entriesForPeer(p *Peer, results []net.IPNet) []net.IPNet {
-	if node == nil {
-		return results
-	}
-	if node.peer == p {
-		mask := net.CIDRMask(int(node.cidr), len(node.bits)*8)
-		results = append(results, net.IPNet{
-			Mask: mask,
-			IP:   node.bits.Mask(mask),
-		})
-	}
-	results = node.child[0].entriesForPeer(p, results)
-	results = node.child[1].entriesForPeer(p, results)
-	return results
-}
-
 type AllowedIPs struct {
 	IPv4  *trieEntry
 	IPv6  *trieEntry
 	mutex sync.RWMutex
 }
 
-func (table *AllowedIPs) EntriesForPeer(peer *Peer) []net.IPNet {
+func (table *AllowedIPs) EntriesForPeer(peer *Peer, cb func(ip net.IP, cidr uint8) bool) {
 	table.mutex.RLock()
 	defer table.mutex.RUnlock()
 
-	allowed := make([]net.IPNet, 0, 10)
-	allowed = table.IPv4.entriesForPeer(peer, allowed)
-	allowed = table.IPv6.entriesForPeer(peer, allowed)
-	return allowed
-}
-
-func (table *AllowedIPs) Reset() {
-	table.mutex.Lock()
-	defer table.mutex.Unlock()
-
-	table.IPv4 = nil
-	table.IPv6 = nil
+	for elem := peer.trieEntries.Front(); elem != nil; elem = elem.Next() {
+		node := elem.Value.(*trieEntry)
+		if !cb(node.bits, node.cidr) {
+			return
+		}
+	}
 }
 
 func (table *AllowedIPs) RemoveByPeer(peer *Peer) {
 	table.mutex.Lock()
 	defer table.mutex.Unlock()
 
-	table.IPv4 = table.IPv4.removeByPeer(peer)
-	table.IPv6 = table.IPv6.removeByPeer(peer)
+	var next *list.Element
+	for elem := peer.trieEntries.Front(); elem != nil; elem = next {
+		next = elem.Next()
+		node := elem.Value.(*trieEntry)
+
+		node.removeFromPeerEntries()
+		node.peer = nil
+		if node.child[0] != nil && node.child[1] != nil {
+			continue
+		}
+		bit := 0
+		if node.child[0] == nil {
+			bit = 1
+		}
+		child := node.child[bit]
+		if child != nil {
+			child.parent = node.parent
+		}
+		*node.parent.parentBit = child
+		if node.child[0] != nil || node.child[1] != nil || node.parent.parentBitType > 1 {
+			node.zeroizePointers()
+			continue
+		}
+		parent := (*trieEntry)(unsafe.Pointer(uintptr(unsafe.Pointer(node.parent.parentBit)) - unsafe.Offsetof(node.child) - unsafe.Sizeof(node.child[0])*uintptr(node.parent.parentBitType)))
+		if parent.peer != nil {
+			node.zeroizePointers()
+			continue
+		}
+		child = parent.child[node.parent.parentBitType^1]
+		if child != nil {
+			child.parent = parent.parent
+		}
+		*parent.parent.parentBit = child
+		node.zeroizePointers()
+		parent.zeroizePointers()
+	}
 }
 
-func (table *AllowedIPs) Insert(ip net.IP, cidr uint, peer *Peer) {
+func (table *AllowedIPs) Insert(ip net.IP, cidr uint8, peer *Peer) {
 	table.mutex.Lock()
 	defer table.mutex.Unlock()
 
 	switch len(ip) {
 	case net.IPv6len:
-		table.IPv6 = table.IPv6.insert(ip, cidr, peer)
+		parentIndirection{&table.IPv6, 2}.insert(ip, cidr, peer)
 	case net.IPv4len:
-		table.IPv4 = table.IPv4.insert(ip, cidr, peer)
+		parentIndirection{&table.IPv4, 2}.insert(ip, cidr, peer)
 	default:
 		panic(errors.New("inserting unknown address type"))
 	}
 }
 
-func (table *AllowedIPs) LookupIPv4(address []byte) *Peer {
+func (table *AllowedIPs) Lookup(address []byte) *Peer {
 	table.mutex.RLock()
 	defer table.mutex.RUnlock()
-	return table.IPv4.lookup(address)
-}
-
-func (table *AllowedIPs) LookupIPv6(address []byte) *Peer {
-	table.mutex.RLock()
-	defer table.mutex.RUnlock()
-	return table.IPv6.lookup(address)
+	switch len(address) {
+	case net.IPv6len:
+		return table.IPv6.lookup(address)
+	case net.IPv4len:
+		return table.IPv4.lookup(address)
+	default:
+		panic(errors.New("looking up unknown address type"))
+	}
 }

device/allowedips_rand_test.go

@@ -1,25 +1,27 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
 	"math/rand"
+	"net"
 	"sort"
 	"testing"
 )
 
 const (
-	NumberOfPeers     = 100
-	NumberOfAddresses = 250
-	NumberOfTests     = 10000
+	NumberOfPeers        = 100
+	NumberOfPeerRemovals = 4
+	NumberOfAddresses    = 250
+	NumberOfTests        = 10000
 )
 
 type SlowNode struct {
 	peer *Peer
-	cidr uint
+	cidr uint8
 	bits []byte
 }
 
@@ -37,7 +39,7 @@ func (r SlowRouter) Swap(i, j int) {
 	r[i], r[j] = r[j], r[i]
 }
 
-func (r SlowRouter) Insert(addr []byte, cidr uint, peer *Peer) SlowRouter {
+func (r SlowRouter) Insert(addr []byte, cidr uint8, peer *Peer) SlowRouter {
 	for _, t := range r {
 		if t.cidr == cidr && commonBits(t.bits, addr) >= cidr {
 			t.peer = peer
@@ -64,68 +66,75 @@ func (r SlowRouter) Lookup(addr []byte) *Peer {
 	return nil
 }
 
-func TestTrieRandomIPv4(t *testing.T) {
-	var trie *trieEntry
-	var slow SlowRouter
+func (r SlowRouter) RemoveByPeer(peer *Peer) SlowRouter {
+	n := 0
+	for _, x := range r {
+		if x.peer != peer {
+			r[n] = x
+			n++
+		}
+	}
+	return r[:n]
+}
+
+func TestTrieRandom(t *testing.T) {
+	var slow4, slow6 SlowRouter
 	var peers []*Peer
+	var allowedIPs AllowedIPs
 
 	rand.Seed(1)
 
-	const AddressLength = 4
-
-	for n := 0; n < NumberOfPeers; n += 1 {
+	for n := 0; n < NumberOfPeers; n++ {
 		peers = append(peers, &Peer{})
 	}
 
-	for n := 0; n < NumberOfAddresses; n += 1 {
-		var addr [AddressLength]byte
-		rand.Read(addr[:])
-		cidr := uint(rand.Uint32() % (AddressLength * 8))
-		index := rand.Int() % NumberOfPeers
-		trie = trie.insert(addr[:], cidr, peers[index])
-		slow = slow.Insert(addr[:], cidr, peers[index])
+	for n := 0; n < NumberOfAddresses; n++ {
+		var addr4 [4]byte
+		rand.Read(addr4[:])
+		cidr := uint8(rand.Intn(32) + 1)
+		index := rand.Intn(NumberOfPeers)
+		allowedIPs.Insert(addr4[:], cidr, peers[index])
+		slow4 = slow4.Insert(addr4[:], cidr, peers[index])
+
+		var addr6 [16]byte
+		rand.Read(addr6[:])
+		cidr = uint8(rand.Intn(128) + 1)
+		index = rand.Intn(NumberOfPeers)
+		allowedIPs.Insert(addr6[:], cidr, peers[index])
+		slow6 = slow6.Insert(addr6[:], cidr, peers[index])
 	}
 
-	for n := 0; n < NumberOfTests; n += 1 {
-		var addr [AddressLength]byte
-		rand.Read(addr[:])
-		peer1 := slow.Lookup(addr[:])
-		peer2 := trie.lookup(addr[:])
-		if peer1 != peer2 {
-			t.Error("Trie did not match naive implementation, for:", addr)
-		}
-	}
-}
-
-func TestTrieRandomIPv6(t *testing.T) {
-	var trie *trieEntry
-	var slow SlowRouter
-	var peers []*Peer
-
-	rand.Seed(1)
-
-	const AddressLength = 16
-
-	for n := 0; n < NumberOfPeers; n += 1 {
-		peers = append(peers, &Peer{})
-	}
-
-	for n := 0; n < NumberOfAddresses; n += 1 {
-		var addr [AddressLength]byte
-		rand.Read(addr[:])
-		cidr := uint(rand.Uint32() % (AddressLength * 8))
-		index := rand.Int() % NumberOfPeers
-		trie = trie.insert(addr[:], cidr, peers[index])
-		slow = slow.Insert(addr[:], cidr, peers[index])
-	}
-
-	for n := 0; n < NumberOfTests; n += 1 {
-		var addr [AddressLength]byte
-		rand.Read(addr[:])
-		peer1 := slow.Lookup(addr[:])
-		peer2 := trie.lookup(addr[:])
-		if peer1 != peer2 {
-			t.Error("Trie did not match naive implementation, for:", addr)
+	var p int
+	for p = 0; ; p++ {
+		for n := 0; n < NumberOfTests; n++ {
+			var addr4 [4]byte
+			rand.Read(addr4[:])
+			peer1 := slow4.Lookup(addr4[:])
+			peer2 := allowedIPs.Lookup(addr4[:])
+			if peer1 != peer2 {
+				t.Errorf("Trie did not match naive implementation, for %v: want %p, got %p", net.IP(addr4[:]), peer1, peer2)
+			}
+
+			var addr6 [16]byte
+			rand.Read(addr6[:])
+			peer1 = slow6.Lookup(addr6[:])
+			peer2 = allowedIPs.Lookup(addr6[:])
+			if peer1 != peer2 {
+				t.Errorf("Trie did not match naive implementation, for %v: want %p, got %p", net.IP(addr6[:]), peer1, peer2)
+			}
 		}
+		if p >= len(peers) || p >= NumberOfPeerRemovals {
+			break
+		}
+		allowedIPs.RemoveByPeer(peers[p])
+		slow4 = slow4.RemoveByPeer(peers[p])
+		slow6 = slow6.RemoveByPeer(peers[p])
+	}
+	for ; p < len(peers); p++ {
+		allowedIPs.RemoveByPeer(peers[p])
+	}
+
+	if allowedIPs.IPv4 != nil || allowedIPs.IPv6 != nil {
+		t.Error("Failed to remove all nodes from trie by peer")
 	}
 }

device/allowedips_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -11,33 +11,10 @@ import (
 	"testing"
 )
 
-/* Todo: More comprehensive
- */
-
 type testPairCommonBits struct {
 	s1    []byte
 	s2    []byte
-	match uint
-}
-
-type testPairTrieInsert struct {
-	key  []byte
-	cidr uint
-	peer *Peer
-}
-
-type testPairTrieLookup struct {
-	key  []byte
-	peer *Peer
-}
-
-func printTrie(t *testing.T, p *trieEntry) {
-	if p == nil {
-		return
-	}
-	t.Log(p)
-	printTrie(t, p.child[0])
-	printTrie(t, p.child[1])
+	match uint8
 }
 
 func TestCommonBits(t *testing.T) {
@@ -65,24 +42,25 @@ func TestCommonBits(t *testing.T) {
 func benchmarkTrie(peerNumber int, addressNumber int, addressLength int, b *testing.B) {
 	var trie *trieEntry
 	var peers []*Peer
+	root := parentIndirection{&trie, 2}
 
 	rand.Seed(1)
 
 	const AddressLength = 4
 
-	for n := 0; n < peerNumber; n += 1 {
+	for n := 0; n < peerNumber; n++ {
 		peers = append(peers, &Peer{})
 	}
 
-	for n := 0; n < addressNumber; n += 1 {
+	for n := 0; n < addressNumber; n++ {
 		var addr [AddressLength]byte
 		rand.Read(addr[:])
-		cidr := uint(rand.Uint32() % (AddressLength * 8))
+		cidr := uint8(rand.Uint32() % (AddressLength * 8))
 		index := rand.Int() % peerNumber
-		trie = trie.insert(addr[:], cidr, peers[index])
+		root.insert(addr[:], cidr, peers[index])
 	}
 
-	for n := 0; n < b.N; n += 1 {
+	for n := 0; n < b.N; n++ {
 		var addr [AddressLength]byte
 		rand.Read(addr[:])
 		trie.lookup(addr[:])
@@ -117,21 +95,21 @@ func TestTrieIPv4(t *testing.T) {
 	g := &Peer{}
 	h := &Peer{}
 
-	var trie *trieEntry
+	var allowedIPs AllowedIPs
 
-	insert := func(peer *Peer, a, b, c, d byte, cidr uint) {
-		trie = trie.insert([]byte{a, b, c, d}, cidr, peer)
+	insert := func(peer *Peer, a, b, c, d byte, cidr uint8) {
+		allowedIPs.Insert([]byte{a, b, c, d}, cidr, peer)
 	}
 
 	assertEQ := func(peer *Peer, a, b, c, d byte) {
-		p := trie.lookup([]byte{a, b, c, d})
+		p := allowedIPs.Lookup([]byte{a, b, c, d})
 		if p != peer {
 			t.Error("Assert EQ failed")
 		}
 	}
 
 	assertNEQ := func(peer *Peer, a, b, c, d byte) {
-		p := trie.lookup([]byte{a, b, c, d})
+		p := allowedIPs.Lookup([]byte{a, b, c, d})
 		if p == peer {
 			t.Error("Assert NEQ failed")
 		}
@@ -173,7 +151,7 @@ func TestTrieIPv4(t *testing.T) {
 	assertEQ(a, 192, 0, 0, 0)
 	assertEQ(a, 255, 0, 0, 0)
 
-	trie = trie.removeByPeer(a)
+	allowedIPs.RemoveByPeer(a)
 
 	assertNEQ(a, 1, 0, 0, 0)
 	assertNEQ(a, 64, 0, 0, 0)
@@ -181,12 +159,21 @@ func TestTrieIPv4(t *testing.T) {
 	assertNEQ(a, 192, 0, 0, 0)
 	assertNEQ(a, 255, 0, 0, 0)
 
-	trie = nil
+	allowedIPs.RemoveByPeer(a)
+	allowedIPs.RemoveByPeer(b)
+	allowedIPs.RemoveByPeer(c)
+	allowedIPs.RemoveByPeer(d)
+	allowedIPs.RemoveByPeer(e)
+	allowedIPs.RemoveByPeer(g)
+	allowedIPs.RemoveByPeer(h)
+	if allowedIPs.IPv4 != nil || allowedIPs.IPv6 != nil {
+		t.Error("Expected removing all the peers to empty trie, but it did not")
+	}
 
 	insert(a, 192, 168, 0, 0, 16)
 	insert(a, 192, 168, 0, 0, 24)
 
-	trie = trie.removeByPeer(a)
+	allowedIPs.RemoveByPeer(a)
 
 	assertNEQ(a, 192, 168, 0, 1)
 }
@@ -204,7 +191,7 @@ func TestTrieIPv6(t *testing.T) {
 	g := &Peer{}
 	h := &Peer{}
 
-	var trie *trieEntry
+	var allowedIPs AllowedIPs
 
 	expand := func(a uint32) []byte {
 		var out [4]byte
@@ -215,13 +202,13 @@ func TestTrieIPv6(t *testing.T) {
 		return out[:]
 	}
 
-	insert := func(peer *Peer, a, b, c, d uint32, cidr uint) {
+	insert := func(peer *Peer, a, b, c, d uint32, cidr uint8) {
 		var addr []byte
 		addr = append(addr, expand(a)...)
 		addr = append(addr, expand(b)...)
 		addr = append(addr, expand(c)...)
 		addr = append(addr, expand(d)...)
-		trie = trie.insert(addr, cidr, peer)
+		allowedIPs.Insert(addr, cidr, peer)
 	}
 
 	assertEQ := func(peer *Peer, a, b, c, d uint32) {
@@ -230,7 +217,7 @@ func TestTrieIPv6(t *testing.T) {
 		addr = append(addr, expand(b)...)
 		addr = append(addr, expand(c)...)
 		addr = append(addr, expand(d)...)
-		p := trie.lookup(addr)
+		p := allowedIPs.Lookup(addr)
 		if p != peer {
 			t.Error("Assert EQ failed")
 		}

device/bind_test.go

@@ -1,23 +1,24 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
-import "errors"
+import (
+	"errors"
+
+	"golang.zx2c4.com/wireguard/conn"
+)
 
 type DummyDatagram struct {
 	msg      []byte
-	endpoint Endpoint
-	world    bool // better type
+	endpoint conn.Endpoint
 }
 
 type DummyBind struct {
 	in6    chan DummyDatagram
-	ou6    chan DummyDatagram
 	in4    chan DummyDatagram
-	ou4    chan DummyDatagram
 	closed bool
 }
 
@@ -25,7 +26,7 @@ func (b *DummyBind) SetMark(v uint32) error {
 	return nil
 }
 
-func (b *DummyBind) ReceiveIPv6(buff []byte) (int, Endpoint, error) {
+func (b *DummyBind) ReceiveIPv6(buff []byte) (int, conn.Endpoint, error) {
 	datagram, ok := <-b.in6
 	if !ok {
 		return 0, nil, errors.New("closed")
@@ -34,7 +35,7 @@ func (b *DummyBind) ReceiveIPv6(buff []byte) (int, Endpoint, error) {
 	return len(datagram.msg), datagram.endpoint, nil
 }
 
-func (b *DummyBind) ReceiveIPv4(buff []byte) (int, Endpoint, error) {
+func (b *DummyBind) ReceiveIPv4(buff []byte) (int, conn.Endpoint, error) {
 	datagram, ok := <-b.in4
 	if !ok {
 		return 0, nil, errors.New("closed")
@@ -50,6 +51,6 @@ func (b *DummyBind) Close() error {
 	return nil
 }
 
-func (b *DummyBind) Send(buff []byte, end Endpoint) error {
+func (b *DummyBind) Send(buff []byte, end conn.Endpoint) error {
 	return nil
 }

device/boundif_android.go

@@ -1,44 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-import "errors"
-
-func (device *Device) PeekLookAtSocketFd4() (fd int, err error) {
-	nb, ok := device.net.bind.(*nativeBind)
-	if !ok {
-		return 0, errors.New("no socket exists")
-	}
-	sysconn, err := nb.ipv4.SyscallConn()
-	if err != nil {
-		return
-	}
-	err = sysconn.Control(func(f uintptr) {
-		fd = int(f)
-	})
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (device *Device) PeekLookAtSocketFd6() (fd int, err error) {
-	nb, ok := device.net.bind.(*nativeBind)
-	if !ok {
-		return 0, errors.New("no socket exists")
-	}
-	sysconn, err := nb.ipv6.SyscallConn()
-	if err != nil {
-		return
-	}
-	err = sysconn.Control(func(f uintptr) {
-		fd = int(f)
-	})
-	if err != nil {
-		return
-	}
-	return
-}

device/boundif_windows.go

@@ -1,62 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-import (
-	"encoding/binary"
-	"errors"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-const (
-	sockoptIP_UNICAST_IF   = 31
-	sockoptIPV6_UNICAST_IF = 31
-)
-
-func (device *Device) BindSocketToInterface4(interfaceIndex uint32) error {
-	/* MSDN says for IPv4 this needs to be in net byte order, so that it's like an IP address with leading zeros. */
-	bytes := make([]byte, 4)
-	binary.BigEndian.PutUint32(bytes, interfaceIndex)
-	interfaceIndex = *(*uint32)(unsafe.Pointer(&bytes[0]))
-
-	if device.net.bind == nil {
-		return errors.New("Bind is not yet initialized")
-	}
-
-	sysconn, err := device.net.bind.(*nativeBind).ipv4.SyscallConn()
-	if err != nil {
-		return err
-	}
-	err2 := sysconn.Control(func(fd uintptr) {
-		err = windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IP, sockoptIP_UNICAST_IF, int(interfaceIndex))
-	})
-	if err2 != nil {
-		return err2
-	}
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (device *Device) BindSocketToInterface6(interfaceIndex uint32) error {
-	sysconn, err := device.net.bind.(*nativeBind).ipv6.SyscallConn()
-	if err != nil {
-		return err
-	}
-	err2 := sysconn.Control(func(fd uintptr) {
-		err = windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IPV6, sockoptIPV6_UNICAST_IF, int(interfaceIndex))
-	})
-	if err2 != nil {
-		return err2
-	}
-	if err != nil {
-		return err
-	}
-	return nil
-}

device/channels.go

@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"runtime"
+	"sync"
+)
+
+// An outboundQueue is a channel of QueueOutboundElements awaiting encryption.
+// An outboundQueue is ref-counted using its wg field.
+// An outboundQueue created with newOutboundQueue has one reference.
+// Every additional writer must call wg.Add(1).
+// Every completed writer must call wg.Done().
+// When no further writers will be added,
+// call wg.Done to remove the initial reference.
+// When the refcount hits 0, the queue's channel is closed.
+type outboundQueue struct {
+	c  chan *QueueOutboundElement
+	wg sync.WaitGroup
+}
+
+func newOutboundQueue() *outboundQueue {
+	q := &outboundQueue{
+		c: make(chan *QueueOutboundElement, QueueOutboundSize),
+	}
+	q.wg.Add(1)
+	go func() {
+		q.wg.Wait()
+		close(q.c)
+	}()
+	return q
+}
+
+// A inboundQueue is similar to an outboundQueue; see those docs.
+type inboundQueue struct {
+	c  chan *QueueInboundElement
+	wg sync.WaitGroup
+}
+
+func newInboundQueue() *inboundQueue {
+	q := &inboundQueue{
+		c: make(chan *QueueInboundElement, QueueInboundSize),
+	}
+	q.wg.Add(1)
+	go func() {
+		q.wg.Wait()
+		close(q.c)
+	}()
+	return q
+}
+
+// A handshakeQueue is similar to an outboundQueue; see those docs.
+type handshakeQueue struct {
+	c  chan QueueHandshakeElement
+	wg sync.WaitGroup
+}
+
+func newHandshakeQueue() *handshakeQueue {
+	q := &handshakeQueue{
+		c: make(chan QueueHandshakeElement, QueueHandshakeSize),
+	}
+	q.wg.Add(1)
+	go func() {
+		q.wg.Wait()
+		close(q.c)
+	}()
+	return q
+}
+
+type autodrainingInboundQueue struct {
+	c chan *QueueInboundElement
+}
+
+// newAutodrainingInboundQueue returns a channel that will be drained when it gets GC'd.
+// It is useful in cases in which is it hard to manage the lifetime of the channel.
+// The returned channel must not be closed. Senders should signal shutdown using
+// some other means, such as sending a sentinel nil values.
+func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue {
+	q := &autodrainingInboundQueue{
+		c: make(chan *QueueInboundElement, QueueInboundSize),
+	}
+	runtime.SetFinalizer(q, device.flushInboundQueue)
+	return q
+}
+
+func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) {
+	for {
+		select {
+		case elem := <-q.c:
+			elem.Lock()
+			device.PutMessageBuffer(elem.buffer)
+			device.PutInboundElement(elem)
+		default:
+			return
+		}
+	}
+}
+
+type autodrainingOutboundQueue struct {
+	c chan *QueueOutboundElement
+}
+
+// newAutodrainingOutboundQueue returns a channel that will be drained when it gets GC'd.
+// It is useful in cases in which is it hard to manage the lifetime of the channel.
+// The returned channel must not be closed. Senders should signal shutdown using
+// some other means, such as sending a sentinel nil values.
+// All sends to the channel must be best-effort, because there may be no receivers.
+func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue {
+	q := &autodrainingOutboundQueue{
+		c: make(chan *QueueOutboundElement, QueueOutboundSize),
+	}
+	runtime.SetFinalizer(q, device.flushOutboundQueue)
+	return q
+}
+
+func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) {
+	for {
+		select {
+		case elem := <-q.c:
+			elem.Lock()
+			device.PutMessageBuffer(elem.buffer)
+			device.PutOutboundElement(elem)
+		default:
+			return
+		}
+	}
+}

device/conn.go

@@ -1,187 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-import (
-	"errors"
-	"net"
-	"strings"
-
-	"golang.org/x/net/ipv4"
-	"golang.org/x/net/ipv6"
-)
-
-const (
-	ConnRoutineNumber = 2
-)
-
-/* A Bind handles listening on a port for both IPv6 and IPv4 UDP traffic
- */
-type Bind interface {
-	SetMark(value uint32) error
-	ReceiveIPv6(buff []byte) (int, Endpoint, error)
-	ReceiveIPv4(buff []byte) (int, Endpoint, error)
-	Send(buff []byte, end Endpoint) error
-	Close() error
-}
-
-/* An Endpoint maintains the source/destination caching for a peer
- *
- * dst : the remote address of a peer ("endpoint" in uapi terminology)
- * src : the local address from which datagrams originate going to the peer
- */
-type Endpoint interface {
-	ClearSrc()           // clears the source address
-	SrcToString() string // returns the local source address (ip:port)
-	DstToString() string // returns the destination address (ip:port)
-	DstToBytes() []byte  // used for mac2 cookie calculations
-	DstIP() net.IP
-	SrcIP() net.IP
-}
-
-func parseEndpoint(s string) (*net.UDPAddr, error) {
-	// ensure that the host is an IP address
-
-	host, _, err := net.SplitHostPort(s)
-	if err != nil {
-		return nil, err
-	}
-	if i := strings.LastIndexByte(host, '%'); i > 0 && strings.IndexByte(host, ':') >= 0 {
-		// Remove the scope, if any. ResolveUDPAddr below will use it, but here we're just
-		// trying to make sure with a small sanity test that this is a real IP address and
-		// not something that's likely to incur DNS lookups.
-		host = host[:i]
-	}
-	if ip := net.ParseIP(host); ip == nil {
-		return nil, errors.New("Failed to parse IP address: " + host)
-	}
-
-	// parse address and port
-
-	addr, err := net.ResolveUDPAddr("udp", s)
-	if err != nil {
-		return nil, err
-	}
-	ip4 := addr.IP.To4()
-	if ip4 != nil {
-		addr.IP = ip4
-	}
-	return addr, err
-}
-
-func unsafeCloseBind(device *Device) error {
-	var err error
-	netc := &device.net
-	if netc.bind != nil {
-		err = netc.bind.Close()
-		netc.bind = nil
-	}
-	netc.stopping.Wait()
-	return err
-}
-
-func (device *Device) BindSetMark(mark uint32) error {
-
-	device.net.Lock()
-	defer device.net.Unlock()
-
-	// check if modified
-
-	if device.net.fwmark == mark {
-		return nil
-	}
-
-	// update fwmark on existing bind
-
-	device.net.fwmark = mark
-	if device.isUp.Get() && device.net.bind != nil {
-		if err := device.net.bind.SetMark(mark); err != nil {
-			return err
-		}
-	}
-
-	// clear cached source addresses
-
-	device.peers.RLock()
-	for _, peer := range device.peers.keyMap {
-		peer.Lock()
-		defer peer.Unlock()
-		if peer.endpoint != nil {
-			peer.endpoint.ClearSrc()
-		}
-	}
-	device.peers.RUnlock()
-
-	return nil
-}
-
-func (device *Device) BindUpdate() error {
-
-	device.net.Lock()
-	defer device.net.Unlock()
-
-	// close existing sockets
-
-	if err := unsafeCloseBind(device); err != nil {
-		return err
-	}
-
-	// open new sockets
-
-	if device.isUp.Get() {
-
-		// bind to new port
-
-		var err error
-		netc := &device.net
-		netc.bind, netc.port, err = CreateBind(netc.port, device)
-		if err != nil {
-			netc.bind = nil
-			netc.port = 0
-			return err
-		}
-
-		// set fwmark
-
-		if netc.fwmark != 0 {
-			err = netc.bind.SetMark(netc.fwmark)
-			if err != nil {
-				return err
-			}
-		}
-
-		// clear cached source addresses
-
-		device.peers.RLock()
-		for _, peer := range device.peers.keyMap {
-			peer.Lock()
-			defer peer.Unlock()
-			if peer.endpoint != nil {
-				peer.endpoint.ClearSrc()
-			}
-		}
-		device.peers.RUnlock()
-
-		// start receiving routines
-
-		device.net.starting.Add(ConnRoutineNumber)
-		device.net.stopping.Add(ConnRoutineNumber)
-		go device.RoutineReceiveIncoming(ipv4.Version, netc.bind)
-		go device.RoutineReceiveIncoming(ipv6.Version, netc.bind)
-		device.net.starting.Wait()
-
-		device.log.Debug.Println("UDP bind has been updated")
-	}
-
-	return nil
-}
-
-func (device *Device) BindClose() error {
-	device.net.Lock()
-	err := unsafeCloseBind(device)
-	device.net.Unlock()
-	return err
-}

device/conn_default.go

@@ -1,170 +0,0 @@
-// +build !linux android
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-import (
-	"net"
-	"os"
-	"syscall"
-)
-
-/* This code is meant to be a temporary solution
- * on platforms for which the sticky socket / source caching behavior
- * has not yet been implemented.
- *
- * See conn_linux.go for an implementation on the linux platform.
- */
-
-type nativeBind struct {
-	ipv4 *net.UDPConn
-	ipv6 *net.UDPConn
-}
-
-type NativeEndpoint net.UDPAddr
-
-var _ Bind = (*nativeBind)(nil)
-var _ Endpoint = (*NativeEndpoint)(nil)
-
-func CreateEndpoint(s string) (Endpoint, error) {
-	addr, err := parseEndpoint(s)
-	return (*NativeEndpoint)(addr), err
-}
-
-func (_ *NativeEndpoint) ClearSrc() {}
-
-func (e *NativeEndpoint) DstIP() net.IP {
-	return (*net.UDPAddr)(e).IP
-}
-
-func (e *NativeEndpoint) SrcIP() net.IP {
-	return nil // not supported
-}
-
-func (e *NativeEndpoint) DstToBytes() []byte {
-	addr := (*net.UDPAddr)(e)
-	out := addr.IP.To4()
-	if out == nil {
-		out = addr.IP
-	}
-	out = append(out, byte(addr.Port&0xff))
-	out = append(out, byte((addr.Port>>8)&0xff))
-	return out
-}
-
-func (e *NativeEndpoint) DstToString() string {
-	return (*net.UDPAddr)(e).String()
-}
-
-func (e *NativeEndpoint) SrcToString() string {
-	return ""
-}
-
-func listenNet(network string, port int) (*net.UDPConn, int, error) {
-
-	// listen
-
-	conn, err := net.ListenUDP(network, &net.UDPAddr{Port: port})
-	if err != nil {
-		return nil, 0, err
-	}
-
-	// retrieve port
-
-	laddr := conn.LocalAddr()
-	uaddr, err := net.ResolveUDPAddr(
-		laddr.Network(),
-		laddr.String(),
-	)
-	if err != nil {
-		return nil, 0, err
-	}
-	return conn, uaddr.Port, nil
-}
-
-func extractErrno(err error) error {
-	opErr, ok := err.(*net.OpError)
-	if !ok {
-		return nil
-	}
-	syscallErr, ok := opErr.Err.(*os.SyscallError)
-	if !ok {
-		return nil
-	}
-	return syscallErr.Err
-}
-
-func CreateBind(uport uint16, device *Device) (Bind, uint16, error) {
-	var err error
-	var bind nativeBind
-
-	port := int(uport)
-
-	bind.ipv4, port, err = listenNet("udp4", port)
-	if err != nil && extractErrno(err) != syscall.EAFNOSUPPORT {
-		return nil, 0, err
-	}
-
-	bind.ipv6, port, err = listenNet("udp6", port)
-	if err != nil && extractErrno(err) != syscall.EAFNOSUPPORT {
-		bind.ipv4.Close()
-		bind.ipv4 = nil
-		return nil, 0, err
-	}
-
-	return &bind, uint16(port), nil
-}
-
-func (bind *nativeBind) Close() error {
-	var err1, err2 error
-	if bind.ipv4 != nil {
-		err1 = bind.ipv4.Close()
-	}
-	if bind.ipv6 != nil {
-		err2 = bind.ipv6.Close()
-	}
-	if err1 != nil {
-		return err1
-	}
-	return err2
-}
-
-func (bind *nativeBind) ReceiveIPv4(buff []byte) (int, Endpoint, error) {
-	if bind.ipv4 == nil {
-		return 0, nil, syscall.EAFNOSUPPORT
-	}
-	n, endpoint, err := bind.ipv4.ReadFromUDP(buff)
-	if endpoint != nil {
-		endpoint.IP = endpoint.IP.To4()
-	}
-	return n, (*NativeEndpoint)(endpoint), err
-}
-
-func (bind *nativeBind) ReceiveIPv6(buff []byte) (int, Endpoint, error) {
-	if bind.ipv6 == nil {
-		return 0, nil, syscall.EAFNOSUPPORT
-	}
-	n, endpoint, err := bind.ipv6.ReadFromUDP(buff)
-	return n, (*NativeEndpoint)(endpoint), err
-}
-
-func (bind *nativeBind) Send(buff []byte, endpoint Endpoint) error {
-	var err error
-	nend := endpoint.(*NativeEndpoint)
-	if nend.IP.To4() != nil {
-		if bind.ipv4 == nil {
-			return syscall.EAFNOSUPPORT
-		}
-		_, err = bind.ipv4.WriteToUDP(buff, (*net.UDPAddr)(nend))
-	} else {
-		if bind.ipv6 == nil {
-			return syscall.EAFNOSUPPORT
-		}
-		_, err = bind.ipv6.WriteToUDP(buff, (*net.UDPAddr)(nend))
-	}
-	return err
-}

device/conn_linux.go

@@ -1,747 +0,0 @@
-// +build !android
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- *
- * This implements userspace semantics of "sticky sockets", modeled after
- * WireGuard's kernelspace implementation. This is more or less a straight port
- * of the sticky-sockets.c example code:
- * https://git.zx2c4.com/WireGuard/tree/contrib/examples/sticky-sockets/sticky-sockets.c
- *
- * Currently there is no way to achieve this within the net package:
- * See e.g. https://github.com/golang/go/issues/17930
- * So this code is remains platform dependent.
- */
-
-package device
-
-import (
-	"errors"
-	"net"
-	"strconv"
-	"sync"
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-	"golang.zx2c4.com/wireguard/rwcancel"
-)
-
-const (
-	FD_ERR = -1
-)
-
-type IPv4Source struct {
-	src     [4]byte
-	ifindex int32
-}
-
-type IPv6Source struct {
-	src [16]byte
-	//ifindex belongs in dst.ZoneId
-}
-
-type NativeEndpoint struct {
-	dst  [unsafe.Sizeof(unix.SockaddrInet6{})]byte
-	src  [unsafe.Sizeof(IPv6Source{})]byte
-	isV6 bool
-}
-
-func (endpoint *NativeEndpoint) src4() *IPv4Source {
-	return (*IPv4Source)(unsafe.Pointer(&endpoint.src[0]))
-}
-
-func (endpoint *NativeEndpoint) src6() *IPv6Source {
-	return (*IPv6Source)(unsafe.Pointer(&endpoint.src[0]))
-}
-
-func (endpoint *NativeEndpoint) dst4() *unix.SockaddrInet4 {
-	return (*unix.SockaddrInet4)(unsafe.Pointer(&endpoint.dst[0]))
-}
-
-func (endpoint *NativeEndpoint) dst6() *unix.SockaddrInet6 {
-	return (*unix.SockaddrInet6)(unsafe.Pointer(&endpoint.dst[0]))
-}
-
-type nativeBind struct {
-	sock4         int
-	sock6         int
-	netlinkSock   int
-	netlinkCancel *rwcancel.RWCancel
-	lastMark      uint32
-}
-
-var _ Endpoint = (*NativeEndpoint)(nil)
-var _ Bind = (*nativeBind)(nil)
-
-func CreateEndpoint(s string) (Endpoint, error) {
-	var end NativeEndpoint
-	addr, err := parseEndpoint(s)
-	if err != nil {
-		return nil, err
-	}
-
-	ipv4 := addr.IP.To4()
-	if ipv4 != nil {
-		dst := end.dst4()
-		end.isV6 = false
-		dst.Port = addr.Port
-		copy(dst.Addr[:], ipv4)
-		end.ClearSrc()
-		return &end, nil
-	}
-
-	ipv6 := addr.IP.To16()
-	if ipv6 != nil {
-		zone, err := zoneToUint32(addr.Zone)
-		if err != nil {
-			return nil, err
-		}
-		dst := end.dst6()
-		end.isV6 = true
-		dst.Port = addr.Port
-		dst.ZoneId = zone
-		copy(dst.Addr[:], ipv6[:])
-		end.ClearSrc()
-		return &end, nil
-	}
-
-	return nil, errors.New("Invalid IP address")
-}
-
-func createNetlinkRouteSocket() (int, error) {
-	sock, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_ROUTE)
-	if err != nil {
-		return -1, err
-	}
-	saddr := &unix.SockaddrNetlink{
-		Family: unix.AF_NETLINK,
-		Groups: uint32(1 << (unix.RTNLGRP_IPV4_ROUTE - 1)),
-	}
-	err = unix.Bind(sock, saddr)
-	if err != nil {
-		unix.Close(sock)
-		return -1, err
-	}
-	return sock, nil
-
-}
-
-func CreateBind(port uint16, device *Device) (*nativeBind, uint16, error) {
-	var err error
-	var bind nativeBind
-	var newPort uint16
-
-	bind.netlinkSock, err = createNetlinkRouteSocket()
-	if err != nil {
-		return nil, 0, err
-	}
-	bind.netlinkCancel, err = rwcancel.NewRWCancel(bind.netlinkSock)
-	if err != nil {
-		unix.Close(bind.netlinkSock)
-		return nil, 0, err
-	}
-
-	go bind.routineRouteListener(device)
-
-	// attempt ipv6 bind, update port if succesful
-
-	bind.sock6, newPort, err = create6(port)
-	if err != nil {
-		if err != syscall.EAFNOSUPPORT {
-			bind.netlinkCancel.Cancel()
-			return nil, 0, err
-		}
-	} else {
-		port = newPort
-	}
-
-	// attempt ipv4 bind, update port if succesful
-
-	bind.sock4, newPort, err = create4(port)
-	if err != nil {
-		if err != syscall.EAFNOSUPPORT {
-			bind.netlinkCancel.Cancel()
-			unix.Close(bind.sock6)
-			return nil, 0, err
-		}
-	} else {
-		port = newPort
-	}
-
-	if bind.sock4 == FD_ERR && bind.sock6 == FD_ERR {
-		return nil, 0, errors.New("ipv4 and ipv6 not supported")
-	}
-
-	return &bind, port, nil
-}
-
-func (bind *nativeBind) SetMark(value uint32) error {
-	if bind.sock6 != -1 {
-		err := unix.SetsockoptInt(
-			bind.sock6,
-			unix.SOL_SOCKET,
-			unix.SO_MARK,
-			int(value),
-		)
-
-		if err != nil {
-			return err
-		}
-	}
-
-	if bind.sock4 != -1 {
-		err := unix.SetsockoptInt(
-			bind.sock4,
-			unix.SOL_SOCKET,
-			unix.SO_MARK,
-			int(value),
-		)
-
-		if err != nil {
-			return err
-		}
-	}
-
-	bind.lastMark = value
-	return nil
-}
-
-func closeUnblock(fd int) error {
-	// shutdown to unblock readers and writers
-	unix.Shutdown(fd, unix.SHUT_RDWR)
-	return unix.Close(fd)
-}
-
-func (bind *nativeBind) Close() error {
-	var err1, err2, err3 error
-	if bind.sock6 != -1 {
-		err1 = closeUnblock(bind.sock6)
-	}
-	if bind.sock4 != -1 {
-		err2 = closeUnblock(bind.sock4)
-	}
-	err3 = bind.netlinkCancel.Cancel()
-
-	if err1 != nil {
-		return err1
-	}
-	if err2 != nil {
-		return err2
-	}
-	return err3
-}
-
-func (bind *nativeBind) ReceiveIPv6(buff []byte) (int, Endpoint, error) {
-	var end NativeEndpoint
-	if bind.sock6 == -1 {
-		return 0, nil, syscall.EAFNOSUPPORT
-	}
-	n, err := receive6(
-		bind.sock6,
-		buff,
-		&end,
-	)
-	return n, &end, err
-}
-
-func (bind *nativeBind) ReceiveIPv4(buff []byte) (int, Endpoint, error) {
-	var end NativeEndpoint
-	if bind.sock4 == -1 {
-		return 0, nil, syscall.EAFNOSUPPORT
-	}
-	n, err := receive4(
-		bind.sock4,
-		buff,
-		&end,
-	)
-	return n, &end, err
-}
-
-func (bind *nativeBind) Send(buff []byte, end Endpoint) error {
-	nend := end.(*NativeEndpoint)
-	if !nend.isV6 {
-		if bind.sock4 == -1 {
-			return syscall.EAFNOSUPPORT
-		}
-		return send4(bind.sock4, nend, buff)
-	} else {
-		if bind.sock6 == -1 {
-			return syscall.EAFNOSUPPORT
-		}
-		return send6(bind.sock6, nend, buff)
-	}
-}
-
-func (end *NativeEndpoint) SrcIP() net.IP {
-	if !end.isV6 {
-		return net.IPv4(
-			end.src4().src[0],
-			end.src4().src[1],
-			end.src4().src[2],
-			end.src4().src[3],
-		)
-	} else {
-		return end.src6().src[:]
-	}
-}
-
-func (end *NativeEndpoint) DstIP() net.IP {
-	if !end.isV6 {
-		return net.IPv4(
-			end.dst4().Addr[0],
-			end.dst4().Addr[1],
-			end.dst4().Addr[2],
-			end.dst4().Addr[3],
-		)
-	} else {
-		return end.dst6().Addr[:]
-	}
-}
-
-func (end *NativeEndpoint) DstToBytes() []byte {
-	if !end.isV6 {
-		return (*[unsafe.Offsetof(end.dst4().Addr) + unsafe.Sizeof(end.dst4().Addr)]byte)(unsafe.Pointer(end.dst4()))[:]
-	} else {
-		return (*[unsafe.Offsetof(end.dst6().Addr) + unsafe.Sizeof(end.dst6().Addr)]byte)(unsafe.Pointer(end.dst6()))[:]
-	}
-}
-
-func (end *NativeEndpoint) SrcToString() string {
-	return end.SrcIP().String()
-}
-
-func (end *NativeEndpoint) DstToString() string {
-	var udpAddr net.UDPAddr
-	udpAddr.IP = end.DstIP()
-	if !end.isV6 {
-		udpAddr.Port = end.dst4().Port
-	} else {
-		udpAddr.Port = end.dst6().Port
-	}
-	return udpAddr.String()
-}
-
-func (end *NativeEndpoint) ClearDst() {
-	for i := range end.dst {
-		end.dst[i] = 0
-	}
-}
-
-func (end *NativeEndpoint) ClearSrc() {
-	for i := range end.src {
-		end.src[i] = 0
-	}
-}
-
-func zoneToUint32(zone string) (uint32, error) {
-	if zone == "" {
-		return 0, nil
-	}
-	if intr, err := net.InterfaceByName(zone); err == nil {
-		return uint32(intr.Index), nil
-	}
-	n, err := strconv.ParseUint(zone, 10, 32)
-	return uint32(n), err
-}
-
-func create4(port uint16) (int, uint16, error) {
-
-	// create socket
-
-	fd, err := unix.Socket(
-		unix.AF_INET,
-		unix.SOCK_DGRAM,
-		0,
-	)
-
-	if err != nil {
-		return FD_ERR, 0, err
-	}
-
-	addr := unix.SockaddrInet4{
-		Port: int(port),
-	}
-
-	// set sockopts and bind
-
-	if err := func() error {
-		if err := unix.SetsockoptInt(
-			fd,
-			unix.SOL_SOCKET,
-			unix.SO_REUSEADDR,
-			1,
-		); err != nil {
-			return err
-		}
-
-		if err := unix.SetsockoptInt(
-			fd,
-			unix.IPPROTO_IP,
-			unix.IP_PKTINFO,
-			1,
-		); err != nil {
-			return err
-		}
-
-		return unix.Bind(fd, &addr)
-	}(); err != nil {
-		unix.Close(fd)
-		return FD_ERR, 0, err
-	}
-
-	return fd, uint16(addr.Port), err
-}
-
-func create6(port uint16) (int, uint16, error) {
-
-	// create socket
-
-	fd, err := unix.Socket(
-		unix.AF_INET6,
-		unix.SOCK_DGRAM,
-		0,
-	)
-
-	if err != nil {
-		return FD_ERR, 0, err
-	}
-
-	// set sockopts and bind
-
-	addr := unix.SockaddrInet6{
-		Port: int(port),
-	}
-
-	if err := func() error {
-
-		if err := unix.SetsockoptInt(
-			fd,
-			unix.SOL_SOCKET,
-			unix.SO_REUSEADDR,
-			1,
-		); err != nil {
-			return err
-		}
-
-		if err := unix.SetsockoptInt(
-			fd,
-			unix.IPPROTO_IPV6,
-			unix.IPV6_RECVPKTINFO,
-			1,
-		); err != nil {
-			return err
-		}
-
-		if err := unix.SetsockoptInt(
-			fd,
-			unix.IPPROTO_IPV6,
-			unix.IPV6_V6ONLY,
-			1,
-		); err != nil {
-			return err
-		}
-
-		return unix.Bind(fd, &addr)
-
-	}(); err != nil {
-		unix.Close(fd)
-		return FD_ERR, 0, err
-	}
-
-	return fd, uint16(addr.Port), err
-}
-
-func send4(sock int, end *NativeEndpoint, buff []byte) error {
-
-	// construct message header
-
-	cmsg := struct {
-		cmsghdr unix.Cmsghdr
-		pktinfo unix.Inet4Pktinfo
-	}{
-		unix.Cmsghdr{
-			Level: unix.IPPROTO_IP,
-			Type:  unix.IP_PKTINFO,
-			Len:   unix.SizeofInet4Pktinfo + unix.SizeofCmsghdr,
-		},
-		unix.Inet4Pktinfo{
-			Spec_dst: end.src4().src,
-			Ifindex:  end.src4().ifindex,
-		},
-	}
-
-	_, err := unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst4(), 0)
-
-	if err == nil {
-		return nil
-	}
-
-	// clear src and retry
-
-	if err == unix.EINVAL {
-		end.ClearSrc()
-		cmsg.pktinfo = unix.Inet4Pktinfo{}
-		_, err = unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst4(), 0)
-	}
-
-	return err
-}
-
-func send6(sock int, end *NativeEndpoint, buff []byte) error {
-
-	// construct message header
-
-	cmsg := struct {
-		cmsghdr unix.Cmsghdr
-		pktinfo unix.Inet6Pktinfo
-	}{
-		unix.Cmsghdr{
-			Level: unix.IPPROTO_IPV6,
-			Type:  unix.IPV6_PKTINFO,
-			Len:   unix.SizeofInet6Pktinfo + unix.SizeofCmsghdr,
-		},
-		unix.Inet6Pktinfo{
-			Addr:    end.src6().src,
-			Ifindex: end.dst6().ZoneId,
-		},
-	}
-
-	if cmsg.pktinfo.Addr == [16]byte{} {
-		cmsg.pktinfo.Ifindex = 0
-	}
-
-	_, err := unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst6(), 0)
-
-	if err == nil {
-		return nil
-	}
-
-	// clear src and retry
-
-	if err == unix.EINVAL {
-		end.ClearSrc()
-		cmsg.pktinfo = unix.Inet6Pktinfo{}
-		_, err = unix.SendmsgN(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], end.dst6(), 0)
-	}
-
-	return err
-}
-
-func receive4(sock int, buff []byte, end *NativeEndpoint) (int, error) {
-
-	// contruct message header
-
-	var cmsg struct {
-		cmsghdr unix.Cmsghdr
-		pktinfo unix.Inet4Pktinfo
-	}
-
-	size, _, _, newDst, err := unix.Recvmsg(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], 0)
-
-	if err != nil {
-		return 0, err
-	}
-	end.isV6 = false
-
-	if newDst4, ok := newDst.(*unix.SockaddrInet4); ok {
-		*end.dst4() = *newDst4
-	}
-
-	// update source cache
-
-	if cmsg.cmsghdr.Level == unix.IPPROTO_IP &&
-		cmsg.cmsghdr.Type == unix.IP_PKTINFO &&
-		cmsg.cmsghdr.Len >= unix.SizeofInet4Pktinfo {
-		end.src4().src = cmsg.pktinfo.Spec_dst
-		end.src4().ifindex = cmsg.pktinfo.Ifindex
-	}
-
-	return size, nil
-}
-
-func receive6(sock int, buff []byte, end *NativeEndpoint) (int, error) {
-
-	// contruct message header
-
-	var cmsg struct {
-		cmsghdr unix.Cmsghdr
-		pktinfo unix.Inet6Pktinfo
-	}
-
-	size, _, _, newDst, err := unix.Recvmsg(sock, buff, (*[unsafe.Sizeof(cmsg)]byte)(unsafe.Pointer(&cmsg))[:], 0)
-
-	if err != nil {
-		return 0, err
-	}
-	end.isV6 = true
-
-	if newDst6, ok := newDst.(*unix.SockaddrInet6); ok {
-		*end.dst6() = *newDst6
-	}
-
-	// update source cache
-
-	if cmsg.cmsghdr.Level == unix.IPPROTO_IPV6 &&
-		cmsg.cmsghdr.Type == unix.IPV6_PKTINFO &&
-		cmsg.cmsghdr.Len >= unix.SizeofInet6Pktinfo {
-		end.src6().src = cmsg.pktinfo.Addr
-		end.dst6().ZoneId = cmsg.pktinfo.Ifindex
-	}
-
-	return size, nil
-}
-
-func (bind *nativeBind) routineRouteListener(device *Device) {
-	type peerEndpointPtr struct {
-		peer     *Peer
-		endpoint *Endpoint
-	}
-	var reqPeer map[uint32]peerEndpointPtr
-	var reqPeerLock sync.Mutex
-
-	defer unix.Close(bind.netlinkSock)
-
-	for msg := make([]byte, 1<<16); ; {
-		var err error
-		var msgn int
-		for {
-			msgn, _, _, _, err = unix.Recvmsg(bind.netlinkSock, msg[:], nil, 0)
-			if err == nil || !rwcancel.RetryAfterError(err) {
-				break
-			}
-			if !bind.netlinkCancel.ReadyRead() {
-				return
-			}
-		}
-		if err != nil {
-			return
-		}
-
-		for remain := msg[:msgn]; len(remain) >= unix.SizeofNlMsghdr; {
-
-			hdr := *(*unix.NlMsghdr)(unsafe.Pointer(&remain[0]))
-
-			if uint(hdr.Len) > uint(len(remain)) {
-				break
-			}
-
-			switch hdr.Type {
-			case unix.RTM_NEWROUTE, unix.RTM_DELROUTE:
-				if hdr.Seq <= MaxPeers && hdr.Seq > 0 {
-					if uint(len(remain)) < uint(hdr.Len) {
-						break
-					}
-					if hdr.Len > unix.SizeofNlMsghdr+unix.SizeofRtMsg {
-						attr := remain[unix.SizeofNlMsghdr+unix.SizeofRtMsg:]
-						for {
-							if uint(len(attr)) < uint(unix.SizeofRtAttr) {
-								break
-							}
-							attrhdr := *(*unix.RtAttr)(unsafe.Pointer(&attr[0]))
-							if attrhdr.Len < unix.SizeofRtAttr || uint(len(attr)) < uint(attrhdr.Len) {
-								break
-							}
-							if attrhdr.Type == unix.RTA_OIF && attrhdr.Len == unix.SizeofRtAttr+4 {
-								ifidx := *(*uint32)(unsafe.Pointer(&attr[unix.SizeofRtAttr]))
-								reqPeerLock.Lock()
-								if reqPeer == nil {
-									reqPeerLock.Unlock()
-									break
-								}
-								pePtr, ok := reqPeer[hdr.Seq]
-								reqPeerLock.Unlock()
-								if !ok {
-									break
-								}
-								pePtr.peer.Lock()
-								if &pePtr.peer.endpoint != pePtr.endpoint {
-									pePtr.peer.Unlock()
-									break
-								}
-								if uint32(pePtr.peer.endpoint.(*NativeEndpoint).src4().ifindex) == ifidx {
-									pePtr.peer.Unlock()
-									break
-								}
-								pePtr.peer.endpoint.(*NativeEndpoint).ClearSrc()
-								pePtr.peer.Unlock()
-							}
-							attr = attr[attrhdr.Len:]
-						}
-					}
-					break
-				}
-				reqPeerLock.Lock()
-				reqPeer = make(map[uint32]peerEndpointPtr)
-				reqPeerLock.Unlock()
-				go func() {
-					device.peers.RLock()
-					i := uint32(1)
-					for _, peer := range device.peers.keyMap {
-						peer.RLock()
-						if peer.endpoint == nil || peer.endpoint.(*NativeEndpoint) == nil {
-							peer.RUnlock()
-							continue
-						}
-						if peer.endpoint.(*NativeEndpoint).isV6 || peer.endpoint.(*NativeEndpoint).src4().ifindex == 0 {
-							peer.RUnlock()
-							break
-						}
-						nlmsg := struct {
-							hdr     unix.NlMsghdr
-							msg     unix.RtMsg
-							dsthdr  unix.RtAttr
-							dst     [4]byte
-							srchdr  unix.RtAttr
-							src     [4]byte
-							markhdr unix.RtAttr
-							mark    uint32
-						}{
-							unix.NlMsghdr{
-								Type:  uint16(unix.RTM_GETROUTE),
-								Flags: unix.NLM_F_REQUEST,
-								Seq:   i,
-							},
-							unix.RtMsg{
-								Family:  unix.AF_INET,
-								Dst_len: 32,
-								Src_len: 32,
-							},
-							unix.RtAttr{
-								Len:  8,
-								Type: unix.RTA_DST,
-							},
-							peer.endpoint.(*NativeEndpoint).dst4().Addr,
-							unix.RtAttr{
-								Len:  8,
-								Type: unix.RTA_SRC,
-							},
-							peer.endpoint.(*NativeEndpoint).src4().src,
-							unix.RtAttr{
-								Len:  8,
-								Type: unix.RTA_MARK,
-							},
-							uint32(bind.lastMark),
-						}
-						nlmsg.hdr.Len = uint32(unsafe.Sizeof(nlmsg))
-						reqPeerLock.Lock()
-						reqPeer[i] = peerEndpointPtr{
-							peer:     peer,
-							endpoint: &peer.endpoint,
-						}
-						reqPeerLock.Unlock()
-						peer.RUnlock()
-						i++
-						_, err := bind.netlinkCancel.Write((*[unsafe.Sizeof(nlmsg)]byte)(unsafe.Pointer(&nlmsg))[:])
-						if err != nil {
-							break
-						}
-					}
-					device.peers.RUnlock()
-				}()
-			}
-			remain = remain[hdr.Len:]
-		}
-	}
-}

device/constants.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -12,8 +12,8 @@ import (
 /* Specification constants */
 
 const (
-	RekeyAfterMessages      = (1 << 64) - (1 << 16) - 1
-	RejectAfterMessages     = (1 << 64) - (1 << 4) - 1
+	RekeyAfterMessages      = (1 << 60)
+	RejectAfterMessages     = (1 << 64) - (1 << 13) - 1
 	RekeyAfterTime          = time.Second * 120
 	RekeyAttemptTime        = time.Second * 90
 	RekeyTimeout            = time.Second * 5
@@ -35,7 +35,6 @@ const (
 /* Implementation constants */
 
 const (
-	UnderLoadQueueSize = QueueHandshakeSize / 8
 	UnderLoadAfterTime = time.Second // how long does the device remain under load after detected
 	MaxPeers           = 1 << 16     // maximum number of configured peers
 )

device/cookie.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/cookie_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/device.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -11,37 +11,39 @@ import (
 	"sync/atomic"
 	"time"
 
+	"golang.zx2c4.com/wireguard/conn"
 	"golang.zx2c4.com/wireguard/ratelimiter"
+	"golang.zx2c4.com/wireguard/rwcancel"
 	"golang.zx2c4.com/wireguard/tun"
 )
 
-const (
-	DeviceRoutineNumberPerCPU     = 3
-	DeviceRoutineNumberAdditional = 2
-)
-
 type Device struct {
-	isUp     AtomicBool // device is (going) up
-	isClosed AtomicBool // device is closed? (acting as guard)
-	log      *Logger
-
-	// synchronized resources (locks acquired in order)
-
 	state struct {
-		starting sync.WaitGroup
+		// state holds the device's state. It is accessed atomically.
+		// Use the device.deviceState method to read it.
+		// device.deviceState does not acquire the mutex, so it captures only a snapshot.
+		// During state transitions, the state variable is updated before the device itself.
+		// The state is thus either the current state of the device or
+		// the intended future state of the device.
+		// For example, while executing a call to Up, state will be deviceStateUp.
+		// There is no guarantee that that intended future state of the device
+		// will become the actual state; Up can fail.
+		// The device can also change state multiple times between time of check and time of use.
+		// Unsynchronized uses of state must therefore be advisory/best-effort only.
+		state uint32 // actually a deviceState, but typed uint32 for convenience
+		// stopping blocks until all inputs to Device have been closed.
 		stopping sync.WaitGroup
+		// mu protects state changes.
 		sync.Mutex
-		changing AtomicBool
-		current  bool
 	}
 
 	net struct {
-		starting sync.WaitGroup
 		stopping sync.WaitGroup
 		sync.RWMutex
-		bind   Bind   // bind interface
-		port   uint16 // listening port
-		fwmark uint32 // mark value (0 = disabled)
+		bind          conn.Bind // bind interface
+		netlinkCancel *rwcancel.RWCancel
+		port          uint16 // listening port
+		fwmark        uint32 // mark value (0 = disabled)
 	}
 
 	staticIdentity struct {
@@ -50,154 +52,171 @@ type Device struct {
 		publicKey  NoisePublicKey
 	}
 
-	peers struct {
-		sync.RWMutex
-		keyMap map[NoisePublicKey]*Peer
+	rate struct {
+		underLoadUntil int64
+		limiter        ratelimiter.Ratelimiter
 	}
 
-	// unprotected / "self-synchronising resources"
+	peers struct {
+		sync.RWMutex // protects keyMap
+		keyMap       map[NoisePublicKey]*Peer
+	}
 
 	allowedips    AllowedIPs
 	indexTable    IndexTable
 	cookieChecker CookieChecker
 
-	rate struct {
-		underLoadUntil atomic.Value
-		limiter        ratelimiter.Ratelimiter
-	}
-
 	pool struct {
-		messageBufferPool        *sync.Pool
-		messageBufferReuseChan   chan *[MaxMessageSize]byte
-		inboundElementPool       *sync.Pool
-		inboundElementReuseChan  chan *QueueInboundElement
-		outboundElementPool      *sync.Pool
-		outboundElementReuseChan chan *QueueOutboundElement
+		messageBuffers   *WaitPool
+		inboundElements  *WaitPool
+		outboundElements *WaitPool
 	}
 
 	queue struct {
-		encryption chan *QueueOutboundElement
-		decryption chan *QueueInboundElement
-		handshake  chan QueueHandshakeElement
-	}
-
-	signals struct {
-		stop chan struct{}
+		encryption *outboundQueue
+		decryption *inboundQueue
+		handshake  *handshakeQueue
 	}
 
 	tun struct {
 		device tun.Device
 		mtu    int32
 	}
+
+	ipcMutex sync.RWMutex
+	closed   chan struct{}
+	log      *Logger
 }
 
-/* Converts the peer into a "zombie", which remains in the peer map,
- * but processes no packets and does not exists in the routing table.
- *
- * Must hold device.peers.Mutex
- */
-func unsafeRemovePeer(device *Device, peer *Peer, key NoisePublicKey) {
+// deviceState represents the state of a Device.
+// There are three states: down, up, closed.
+// Transitions:
+//
+//   down -----+
+//     ↑↓      ↓
+//     up -> closed
+//
+type deviceState uint32
 
+//go:generate go run golang.org/x/tools/cmd/stringer -type deviceState -trimprefix=deviceState
+const (
+	deviceStateDown deviceState = iota
+	deviceStateUp
+	deviceStateClosed
+)
+
+// deviceState returns device.state.state as a deviceState
+// See those docs for how to interpret this value.
+func (device *Device) deviceState() deviceState {
+	return deviceState(atomic.LoadUint32(&device.state.state))
+}
+
+// isClosed reports whether the device is closed (or is closing).
+// See device.state.state comments for how to interpret this value.
+func (device *Device) isClosed() bool {
+	return device.deviceState() == deviceStateClosed
+}
+
+// isUp reports whether the device is up (or is attempting to come up).
+// See device.state.state comments for how to interpret this value.
+func (device *Device) isUp() bool {
+	return device.deviceState() == deviceStateUp
+}
+
+// Must hold device.peers.Lock()
+func removePeerLocked(device *Device, peer *Peer, key NoisePublicKey) {
 	// stop routing and processing of packets
-
 	device.allowedips.RemoveByPeer(peer)
 	peer.Stop()
 
 	// remove from peer map
-
 	delete(device.peers.keyMap, key)
 }
 
-func deviceUpdateState(device *Device) {
-
-	// check if state already being updated (guard)
-
-	if device.state.changing.Swap(true) {
-		return
-	}
-
-	// compare to current state of device
-
+// changeState attempts to change the device state to match want.
+func (device *Device) changeState(want deviceState) (err error) {
 	device.state.Lock()
-
-	newIsUp := device.isUp.Get()
-
-	if newIsUp == device.state.current {
-		device.state.changing.Set(false)
-		device.state.Unlock()
-		return
+	defer device.state.Unlock()
+	old := device.deviceState()
+	if old == deviceStateClosed {
+		// once closed, always closed
+		device.log.Verbosef("Interface closed, ignored requested state %s", want)
+		return nil
 	}
-
-	// change state of device
-
-	switch newIsUp {
-	case true:
-		if err := device.BindUpdate(); err != nil {
-			device.log.Error.Printf("Unable to update bind: %v\n", err)
-			device.isUp.Set(false)
+	switch want {
+	case old:
+		return nil
+	case deviceStateUp:
+		atomic.StoreUint32(&device.state.state, uint32(deviceStateUp))
+		err = device.upLocked()
+		if err == nil {
 			break
 		}
-		device.peers.RLock()
-		for _, peer := range device.peers.keyMap {
-			peer.Start()
-			if peer.persistentKeepaliveInterval > 0 {
-				peer.SendKeepalive()
-			}
+		fallthrough // up failed; bring the device all the way back down
+	case deviceStateDown:
+		atomic.StoreUint32(&device.state.state, uint32(deviceStateDown))
+		errDown := device.downLocked()
+		if err == nil {
+			err = errDown
 		}
-		device.peers.RUnlock()
-
-	case false:
-		device.BindClose()
-		device.peers.RLock()
-		for _, peer := range device.peers.keyMap {
-			peer.Stop()
-		}
-		device.peers.RUnlock()
 	}
-
-	// update state variables
-
-	device.state.current = newIsUp
-	device.state.changing.Set(false)
-	device.state.Unlock()
-
-	// check for state change in the mean time
-
-	deviceUpdateState(device)
+	device.log.Verbosef("Interface state was %s, requested %s, now %s", old, want, device.deviceState())
+	return
 }
 
-func (device *Device) Up() {
-
-	// closed device cannot be brought up
-
-	if device.isClosed.Get() {
-		return
+// upLocked attempts to bring the device up and reports whether it succeeded.
+// The caller must hold device.state.mu and is responsible for updating device.state.state.
+func (device *Device) upLocked() error {
+	if err := device.BindUpdate(); err != nil {
+		device.log.Errorf("Unable to update bind: %v", err)
+		return err
 	}
 
-	device.isUp.Set(true)
-	deviceUpdateState(device)
+	device.peers.RLock()
+	for _, peer := range device.peers.keyMap {
+		peer.Start()
+		if atomic.LoadUint32(&peer.persistentKeepaliveInterval) > 0 {
+			peer.SendKeepalive()
+		}
+	}
+	device.peers.RUnlock()
+	return nil
 }
 
-func (device *Device) Down() {
-	device.isUp.Set(false)
-	deviceUpdateState(device)
+// downLocked attempts to bring the device down.
+// The caller must hold device.state.mu and is responsible for updating device.state.state.
+func (device *Device) downLocked() error {
+	err := device.BindClose()
+	if err != nil {
+		device.log.Errorf("Bind close failed: %v", err)
+	}
+
+	device.peers.RLock()
+	for _, peer := range device.peers.keyMap {
+		peer.Stop()
+	}
+	device.peers.RUnlock()
+	return err
+}
+
+func (device *Device) Up() error {
+	return device.changeState(deviceStateUp)
+}
+
+func (device *Device) Down() error {
+	return device.changeState(deviceStateDown)
 }
 
 func (device *Device) IsUnderLoad() bool {
-
 	// check if currently under load
-
 	now := time.Now()
-	underLoad := len(device.queue.handshake) >= UnderLoadQueueSize
+	underLoad := len(device.queue.handshake.c) >= QueueHandshakeSize/8
 	if underLoad {
-		device.rate.underLoadUntil.Store(now.Add(UnderLoadAfterTime))
+		atomic.StoreInt64(&device.rate.underLoadUntil, now.Add(UnderLoadAfterTime).UnixNano())
 		return true
 	}
-
 	// check if recently under load
-
-	until := device.rate.underLoadUntil.Load().(time.Time)
-	return until.After(now)
+	return atomic.LoadInt64(&device.rate.underLoadUntil) > now.UnixNano()
 }
 
 func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
@@ -224,7 +243,9 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
 	publicKey := sk.publicKey()
 	for key, peer := range device.peers.keyMap {
 		if peer.handshake.remoteStatic.Equals(publicKey) {
-			unsafeRemovePeer(device, peer, key)
+			peer.handshake.mutex.RUnlock()
+			removePeerLocked(device, peer, key)
+			peer.handshake.mutex.RLock()
 		}
 	}
 
@@ -236,23 +257,11 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
 
 	// do static-static DH pre-computations
 
-	rmKey := device.staticIdentity.privateKey.IsZero()
-
 	expiredPeers := make([]*Peer, 0, len(device.peers.keyMap))
-	for key, peer := range device.peers.keyMap {
+	for _, peer := range device.peers.keyMap {
 		handshake := &peer.handshake
-
-		if rmKey {
-			handshake.precomputedStaticStatic = [NoisePublicKeySize]byte{}
-		} else {
-			handshake.precomputedStaticStatic = device.staticIdentity.privateKey.sharedSecret(handshake.remoteStatic)
-		}
-
-		if isZero(handshake.precomputedStaticStatic[:]) {
-			unsafeRemovePeer(device, peer, key)
-		} else {
-			expiredPeers = append(expiredPeers, peer)
-		}
+		handshake.precomputedStaticStatic = device.staticIdentity.privateKey.sharedSecret(handshake.remoteStatic)
+		expiredPeers = append(expiredPeers, peer)
 	}
 
 	for _, peer := range lockedPeers {
@@ -265,65 +274,46 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
 	return nil
 }
 
-func NewDevice(tunDevice tun.Device, logger *Logger) *Device {
+func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device {
 	device := new(Device)
-
-	device.isUp.Set(false)
-	device.isClosed.Set(false)
-
+	device.state.state = uint32(deviceStateDown)
+	device.closed = make(chan struct{})
 	device.log = logger
-
+	device.net.bind = bind
 	device.tun.device = tunDevice
 	mtu, err := device.tun.device.MTU()
 	if err != nil {
-		logger.Error.Println("Trouble determining MTU, assuming default:", err)
+		device.log.Errorf("Trouble determining MTU, assuming default: %v", err)
 		mtu = DefaultMTU
 	}
 	device.tun.mtu = int32(mtu)
-
 	device.peers.keyMap = make(map[NoisePublicKey]*Peer)
-
 	device.rate.limiter.Init()
-	device.rate.underLoadUntil.Store(time.Time{})
-
 	device.indexTable.Init()
-	device.allowedips.Reset()
-
 	device.PopulatePools()
 
 	// create queues
 
-	device.queue.handshake = make(chan QueueHandshakeElement, QueueHandshakeSize)
-	device.queue.encryption = make(chan *QueueOutboundElement, QueueOutboundSize)
-	device.queue.decryption = make(chan *QueueInboundElement, QueueInboundSize)
-
-	// prepare signals
-
-	device.signals.stop = make(chan struct{})
-
-	// prepare net
-
-	device.net.port = 0
-	device.net.bind = nil
+	device.queue.handshake = newHandshakeQueue()
+	device.queue.encryption = newOutboundQueue()
+	device.queue.decryption = newInboundQueue()
 
 	// start workers
 
 	cpus := runtime.NumCPU()
-	device.state.starting.Wait()
 	device.state.stopping.Wait()
-	device.state.stopping.Add(DeviceRoutineNumberPerCPU*cpus + DeviceRoutineNumberAdditional)
-	device.state.starting.Add(DeviceRoutineNumberPerCPU*cpus + DeviceRoutineNumberAdditional)
-	for i := 0; i < cpus; i += 1 {
-		go device.RoutineEncryption()
-		go device.RoutineDecryption()
-		go device.RoutineHandshake()
+	device.queue.encryption.wg.Add(cpus) // One for each RoutineHandshake
+	for i := 0; i < cpus; i++ {
+		go device.RoutineEncryption(i + 1)
+		go device.RoutineDecryption(i + 1)
+		go device.RoutineHandshake(i + 1)
 	}
 
+	device.state.stopping.Add(1)      // RoutineReadFromTUN
+	device.queue.encryption.wg.Add(1) // RoutineReadFromTUN
 	go device.RoutineReadFromTUN()
 	go device.RoutineTUNEventReader()
 
-	device.state.starting.Wait()
-
 	return device
 }
 
@@ -341,7 +331,7 @@ func (device *Device) RemovePeer(key NoisePublicKey) {
 
 	peer, ok := device.peers.keyMap[key]
 	if ok {
-		unsafeRemovePeer(device, peer, key)
+		removePeerLocked(device, peer, key)
 	}
 }
 
@@ -350,67 +340,48 @@ func (device *Device) RemoveAllPeers() {
 	defer device.peers.Unlock()
 
 	for key, peer := range device.peers.keyMap {
-		unsafeRemovePeer(device, peer, key)
+		removePeerLocked(device, peer, key)
 	}
 
 	device.peers.keyMap = make(map[NoisePublicKey]*Peer)
 }
 
-func (device *Device) FlushPacketQueues() {
-	for {
-		select {
-		case elem, ok := <-device.queue.decryption:
-			if ok {
-				elem.Drop()
-			}
-		case elem, ok := <-device.queue.encryption:
-			if ok {
-				elem.Drop()
-			}
-		case <-device.queue.handshake:
-		default:
-			return
-		}
-	}
-
-}
-
 func (device *Device) Close() {
-	if device.isClosed.Swap(true) {
-		return
-	}
-
-	device.state.starting.Wait()
-
-	device.log.Info.Println("Device closing")
-	device.state.changing.Set(true)
 	device.state.Lock()
 	defer device.state.Unlock()
+	if device.isClosed() {
+		return
+	}
+	atomic.StoreUint32(&device.state.state, uint32(deviceStateClosed))
+	device.log.Verbosef("Device closing")
 
 	device.tun.device.Close()
-	device.BindClose()
-
-	device.isUp.Set(false)
-
-	close(device.signals.stop)
+	device.downLocked()
 
+	// Remove peers before closing queues,
+	// because peers assume that queues are active.
 	device.RemoveAllPeers()
 
+	// We kept a reference to the encryption and decryption queues,
+	// in case we started any new peers that might write to them.
+	// No new peers are coming; we are done with these queues.
+	device.queue.encryption.wg.Done()
+	device.queue.decryption.wg.Done()
+	device.queue.handshake.wg.Done()
 	device.state.stopping.Wait()
-	device.FlushPacketQueues()
 
 	device.rate.limiter.Close()
 
-	device.state.changing.Set(false)
-	device.log.Info.Println("Interface closed")
+	device.log.Verbosef("Device closed")
+	close(device.closed)
 }
 
 func (device *Device) Wait() chan struct{} {
-	return device.signals.stop
+	return device.closed
 }
 
 func (device *Device) SendKeepalivesToPeersWithCurrentKeypair() {
-	if device.isClosed.Get() {
+	if !device.isUp() {
 		return
 	}
 
@@ -425,3 +396,123 @@ func (device *Device) SendKeepalivesToPeersWithCurrentKeypair() {
 	}
 	device.peers.RUnlock()
 }
+
+// closeBindLocked closes the device's net.bind.
+// The caller must hold the net mutex.
+func closeBindLocked(device *Device) error {
+	var err error
+	netc := &device.net
+	if netc.netlinkCancel != nil {
+		netc.netlinkCancel.Cancel()
+	}
+	if netc.bind != nil {
+		err = netc.bind.Close()
+	}
+	netc.stopping.Wait()
+	return err
+}
+
+func (device *Device) Bind() conn.Bind {
+	device.net.Lock()
+	defer device.net.Unlock()
+	return device.net.bind
+}
+
+func (device *Device) BindSetMark(mark uint32) error {
+	device.net.Lock()
+	defer device.net.Unlock()
+
+	// check if modified
+	if device.net.fwmark == mark {
+		return nil
+	}
+
+	// update fwmark on existing bind
+	device.net.fwmark = mark
+	if device.isUp() && device.net.bind != nil {
+		if err := device.net.bind.SetMark(mark); err != nil {
+			return err
+		}
+	}
+
+	// clear cached source addresses
+	device.peers.RLock()
+	for _, peer := range device.peers.keyMap {
+		peer.Lock()
+		defer peer.Unlock()
+		if peer.endpoint != nil {
+			peer.endpoint.ClearSrc()
+		}
+	}
+	device.peers.RUnlock()
+
+	return nil
+}
+
+func (device *Device) BindUpdate() error {
+	device.net.Lock()
+	defer device.net.Unlock()
+
+	// close existing sockets
+	if err := closeBindLocked(device); err != nil {
+		return err
+	}
+
+	// open new sockets
+	if !device.isUp() {
+		return nil
+	}
+
+	// bind to new port
+	var err error
+	var recvFns []conn.ReceiveFunc
+	netc := &device.net
+	recvFns, netc.port, err = netc.bind.Open(netc.port)
+	if err != nil {
+		netc.port = 0
+		return err
+	}
+	netc.netlinkCancel, err = device.startRouteListener(netc.bind)
+	if err != nil {
+		netc.bind.Close()
+		netc.port = 0
+		return err
+	}
+
+	// set fwmark
+	if netc.fwmark != 0 {
+		err = netc.bind.SetMark(netc.fwmark)
+		if err != nil {
+			return err
+		}
+	}
+
+	// clear cached source addresses
+	device.peers.RLock()
+	for _, peer := range device.peers.keyMap {
+		peer.Lock()
+		defer peer.Unlock()
+		if peer.endpoint != nil {
+			peer.endpoint.ClearSrc()
+		}
+	}
+	device.peers.RUnlock()
+
+	// start receiving routines
+	device.net.stopping.Add(len(recvFns))
+	device.queue.decryption.wg.Add(len(recvFns)) // each RoutineReceiveIncoming goroutine writes to device.queue.decryption
+	device.queue.handshake.wg.Add(len(recvFns))  // each RoutineReceiveIncoming goroutine writes to device.queue.handshake
+	for _, fn := range recvFns {
+		go device.RoutineReceiveIncoming(fn)
+	}
+
+	device.log.Verbosef("UDP bind has been updated")
+	return nil
+}
+
+func (device *Device) BindClose() error {
+	device.net.Lock()
+	err := closeBindLocked(device)
+	device.net.Unlock()
+	return err
+}

device/device_test.go

@@ -1,68 +1,407 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
-/* Create two device instances and simulate full WireGuard interaction
- * without network dependencies
- */
-
 import (
 	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
+
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/conn/bindtest"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 )
 
-func TestDevice(t *testing.T) {
-
-	// prepare tun devices for generating traffic
-
-	tun1 := newDummyTUN("tun1")
-	tun2 := newDummyTUN("tun2")
-
-	_ = tun1
-	_ = tun2
-
-	// prepare endpoints
-
-	end1, err := CreateDummyEndpoint()
-	if err != nil {
-		t.Error("failed to create endpoint:", err.Error())
+// uapiCfg returns a string that contains cfg formatted use with IpcSet.
+// cfg is a series of alternating key/value strings.
+// uapiCfg exists because editors and humans like to insert
+// whitespace into configs, which can cause failures, some of which are silent.
+// For example, a leading blank newline causes the remainder
+// of the config to be silently ignored.
+func uapiCfg(cfg ...string) string {
+	if len(cfg)%2 != 0 {
+		panic("odd number of args to uapiReader")
 	}
-
-	end2, err := CreateDummyEndpoint()
-	if err != nil {
-		t.Error("failed to create endpoint:", err.Error())
+	buf := new(bytes.Buffer)
+	for i, s := range cfg {
+		buf.WriteString(s)
+		sep := byte('\n')
+		if i%2 == 0 {
+			sep = '='
+		}
+		buf.WriteByte(sep)
 	}
-
-	_ = end1
-	_ = end2
-
-	// create binds
-
+	return buf.String()
 }
 
-func randDevice(t *testing.T) *Device {
-	sk, err := newPrivateKey()
+// genConfigs generates a pair of configs that connect to each other.
+// The configs use distinct, probably-usable ports.
+func genConfigs(tb testing.TB) (cfgs [2]string, endpointCfgs [2]string) {
+	var key1, key2 NoisePrivateKey
+	_, err := rand.Read(key1[:])
 	if err != nil {
-		t.Fatal(err)
+		tb.Errorf("unable to generate private key random bytes: %v", err)
 	}
-	tun := newDummyTUN("dummy")
-	logger := NewLogger(LogLevelError, "")
-	device := NewDevice(tun, logger)
-	device.SetPrivateKey(sk)
-	return device
+	_, err = rand.Read(key2[:])
+	if err != nil {
+		tb.Errorf("unable to generate private key random bytes: %v", err)
+	}
+	pub1, pub2 := key1.publicKey(), key2.publicKey()
+
+	cfgs[0] = uapiCfg(
+		"private_key", hex.EncodeToString(key1[:]),
+		"listen_port", "0",
+		"replace_peers", "true",
+		"public_key", hex.EncodeToString(pub2[:]),
+		"protocol_version", "1",
+		"replace_allowed_ips", "true",
+		"allowed_ip", "1.0.0.2/32",
+	)
+	endpointCfgs[0] = uapiCfg(
+		"public_key", hex.EncodeToString(pub2[:]),
+		"endpoint", "127.0.0.1:%d",
+	)
+	cfgs[1] = uapiCfg(
+		"private_key", hex.EncodeToString(key2[:]),
+		"listen_port", "0",
+		"replace_peers", "true",
+		"public_key", hex.EncodeToString(pub1[:]),
+		"protocol_version", "1",
+		"replace_allowed_ips", "true",
+		"allowed_ip", "1.0.0.1/32",
+	)
+	endpointCfgs[1] = uapiCfg(
+		"public_key", hex.EncodeToString(pub1[:]),
+		"endpoint", "127.0.0.1:%d",
+	)
+	return
 }
 
-func assertNil(t *testing.T, err error) {
+// A testPair is a pair of testPeers.
+type testPair [2]testPeer
+
+// A testPeer is a peer used for testing.
+type testPeer struct {
+	tun *tuntest.ChannelTUN
+	dev *Device
+	ip  net.IP
+}
+
+type SendDirection bool
+
+const (
+	Ping SendDirection = true
+	Pong SendDirection = false
+)
+
+func (d SendDirection) String() string {
+	if d == Ping {
+		return "ping"
+	}
+	return "pong"
+}
+
+func (pair *testPair) Send(tb testing.TB, ping SendDirection, done chan struct{}) {
+	tb.Helper()
+	p0, p1 := pair[0], pair[1]
+	if !ping {
+		// pong is the new ping
+		p0, p1 = p1, p0
+	}
+	msg := tuntest.Ping(p0.ip, p1.ip)
+	p1.tun.Outbound <- msg
+	timer := time.NewTimer(5 * time.Second)
+	defer timer.Stop()
+	var err error
+	select {
+	case msgRecv := <-p0.tun.Inbound:
+		if !bytes.Equal(msg, msgRecv) {
+			err = fmt.Errorf("%s did not transit correctly", ping)
+		}
+	case <-timer.C:
+		err = fmt.Errorf("%s did not transit", ping)
+	case <-done:
+	}
 	if err != nil {
-		t.Fatal(err)
+		// The error may have occurred because the test is done.
+		select {
+		case <-done:
+			return
+		default:
+		}
+		// Real error.
+		tb.Error(err)
 	}
 }
 
-func assertEqual(t *testing.T, a, b []byte) {
-	if !bytes.Equal(a, b) {
-		t.Fatal(a, "!=", b)
+// genTestPair creates a testPair.
+func genTestPair(tb testing.TB, realSocket bool) (pair testPair) {
+	cfg, endpointCfg := genConfigs(tb)
+	var binds [2]conn.Bind
+	if realSocket {
+		binds[0], binds[1] = conn.NewDefaultBind(), conn.NewDefaultBind()
+	} else {
+		binds = bindtest.NewChannelBinds()
+	}
+	// Bring up a ChannelTun for each config.
+	for i := range pair {
+		p := &pair[i]
+		p.tun = tuntest.NewChannelTUN()
+		p.ip = net.IPv4(1, 0, 0, byte(i+1))
+		level := LogLevelVerbose
+		if _, ok := tb.(*testing.B); ok && !testing.Verbose() {
+			level = LogLevelError
+		}
+		p.dev = NewDevice(p.tun.TUN(), binds[i], NewLogger(level, fmt.Sprintf("dev%d: ", i)))
+		if err := p.dev.IpcSet(cfg[i]); err != nil {
+			tb.Errorf("failed to configure device %d: %v", i, err)
+			p.dev.Close()
+			continue
+		}
+		if err := p.dev.Up(); err != nil {
+			tb.Errorf("failed to bring up device %d: %v", i, err)
+			p.dev.Close()
+			continue
+		}
+		endpointCfg[i^1] = fmt.Sprintf(endpointCfg[i^1], p.dev.net.port)
+	}
+	for i := range pair {
+		p := &pair[i]
+		if err := p.dev.IpcSet(endpointCfg[i]); err != nil {
+			tb.Errorf("failed to configure device endpoint %d: %v", i, err)
+			p.dev.Close()
+			continue
+		}
+		// The device is ready. Close it when the test completes.
+		tb.Cleanup(p.dev.Close)
+	}
+	return
+}
+
+func TestTwoDevicePing(t *testing.T) {
+	goroutineLeakCheck(t)
+	pair := genTestPair(t, true)
+	t.Run("ping 1.0.0.1", func(t *testing.T) {
+		pair.Send(t, Ping, nil)
+	})
+	t.Run("ping 1.0.0.2", func(t *testing.T) {
+		pair.Send(t, Pong, nil)
+	})
+}
+
+func TestUpDown(t *testing.T) {
+	goroutineLeakCheck(t)
+	const itrials = 50
+	const otrials = 10
+
+	for n := 0; n < otrials; n++ {
+		pair := genTestPair(t, false)
+		for i := range pair {
+			for k := range pair[i].dev.peers.keyMap {
+				pair[i].dev.IpcSet(fmt.Sprintf("public_key=%s\npersistent_keepalive_interval=1\n", hex.EncodeToString(k[:])))
+			}
+		}
+		var wg sync.WaitGroup
+		wg.Add(len(pair))
+		for i := range pair {
+			go func(d *Device) {
+				defer wg.Done()
+				for i := 0; i < itrials; i++ {
+					if err := d.Up(); err != nil {
+						t.Errorf("failed up bring up device: %v", err)
+					}
+					time.Sleep(time.Duration(rand.Intn(int(time.Nanosecond * (0x10000 - 1)))))
+					if err := d.Down(); err != nil {
+						t.Errorf("failed to bring down device: %v", err)
+					}
+					time.Sleep(time.Duration(rand.Intn(int(time.Nanosecond * (0x10000 - 1)))))
+				}
+			}(pair[i].dev)
+		}
+		wg.Wait()
+		for i := range pair {
+			pair[i].dev.Up()
+			pair[i].dev.Close()
+		}
 	}
 }
+
+// TestConcurrencySafety does other things concurrently with tunnel use.
+// It is intended to be used with the race detector to catch data races.
+func TestConcurrencySafety(t *testing.T) {
+	pair := genTestPair(t, true)
+	done := make(chan struct{})
+
+	const warmupIters = 10
+	var warmup sync.WaitGroup
+	warmup.Add(warmupIters)
+	go func() {
+		// Send data continuously back and forth until we're done.
+		// Note that we may continue to attempt to send data
+		// even after done is closed.
+		i := warmupIters
+		for ping := Ping; ; ping = !ping {
+			pair.Send(t, ping, done)
+			select {
+			case <-done:
+				return
+			default:
+			}
+			if i > 0 {
+				warmup.Done()
+				i--
+			}
+		}
+	}()
+	warmup.Wait()
+
+	applyCfg := func(cfg string) {
+		err := pair[0].dev.IpcSet(cfg)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Change persistent_keepalive_interval concurrently with tunnel use.
+	t.Run("persistentKeepaliveInterval", func(t *testing.T) {
+		var pub NoisePublicKey
+		for key := range pair[0].dev.peers.keyMap {
+			pub = key
+			break
+		}
+		cfg := uapiCfg(
+			"public_key", hex.EncodeToString(pub[:]),
+			"persistent_keepalive_interval", "1",
+		)
+		for i := 0; i < 1000; i++ {
+			applyCfg(cfg)
+		}
+	})
+
+	// Change private keys concurrently with tunnel use.
+	t.Run("privateKey", func(t *testing.T) {
+		bad := uapiCfg("private_key", "7777777777777777777777777777777777777777777777777777777777777777")
+		good := uapiCfg("private_key", hex.EncodeToString(pair[0].dev.staticIdentity.privateKey[:]))
+		// Set iters to a large number like 1000 to flush out data races quickly.
+		// Don't leave it large. That can cause logical races
+		// in which the handshake is interleaved with key changes
+		// such that the private key appears to be unchanging but
+		// other state gets reset, which can cause handshake failures like
+		// "Received packet with invalid mac1".
+		const iters = 1
+		for i := 0; i < iters; i++ {
+			applyCfg(bad)
+			applyCfg(good)
+		}
+	})
+
+	close(done)
+}
+
+func BenchmarkLatency(b *testing.B) {
+	pair := genTestPair(b, true)
+
+	// Establish a connection.
+	pair.Send(b, Ping, nil)
+	pair.Send(b, Pong, nil)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pair.Send(b, Ping, nil)
+		pair.Send(b, Pong, nil)
+	}
+}
+
+func BenchmarkThroughput(b *testing.B) {
+	pair := genTestPair(b, true)
+
+	// Establish a connection.
+	pair.Send(b, Ping, nil)
+	pair.Send(b, Pong, nil)
+
+	// Measure how long it takes to receive b.N packets,
+	// starting when we receive the first packet.
+	var recv uint64
+	var elapsed time.Duration
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		var start time.Time
+		for {
+			<-pair[0].tun.Inbound
+			new := atomic.AddUint64(&recv, 1)
+			if new == 1 {
+				start = time.Now()
+			}
+			// Careful! Don't change this to else if; b.N can be equal to 1.
+			if new == uint64(b.N) {
+				elapsed = time.Since(start)
+				return
+			}
+		}
+	}()
+
+	// Send packets as fast as we can until we've received enough.
+	ping := tuntest.Ping(pair[0].ip, pair[1].ip)
+	pingc := pair[1].tun.Outbound
+	var sent uint64
+	for atomic.LoadUint64(&recv) != uint64(b.N) {
+		sent++
+		pingc <- ping
+	}
+	wg.Wait()
+
+	b.ReportMetric(float64(elapsed)/float64(b.N), "ns/op")
+	b.ReportMetric(1-float64(b.N)/float64(sent), "packet-loss")
+}
+
+func BenchmarkUAPIGet(b *testing.B) {
+	pair := genTestPair(b, true)
+	pair.Send(b, Ping, nil)
+	pair.Send(b, Pong, nil)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pair[0].dev.IpcGetOperation(io.Discard)
+	}
+}
+
+func goroutineLeakCheck(t *testing.T) {
+	goroutines := func() (int, []byte) {
+		p := pprof.Lookup("goroutine")
+		b := new(bytes.Buffer)
+		p.WriteTo(b, 1)
+		return p.Count(), b.Bytes()
+	}
+
+	startGoroutines, startStacks := goroutines()
+	t.Cleanup(func() {
+		if t.Failed() {
+			return
+		}
+		// Give goroutines time to exit, if they need it.
+		for i := 0; i < 10000; i++ {
+			if runtime.NumGoroutine() <= startGoroutines {
+				return
+			}
+			time.Sleep(1 * time.Millisecond)
+		}
+		endGoroutines, endStacks := goroutines()
+		t.Logf("starting stacks:\n%s\n", startStacks)
+		t.Logf("ending stacks:\n%s\n", endStacks)
+		t.Fatalf("expected %d goroutines, got %d, leak?", startGoroutines, endGoroutines)
+	})
+}

device/devicestate_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type deviceState -trimprefix=deviceState"; DO NOT EDIT.
+
+package device
+
+import "strconv"
+
+const _deviceState_name = "DownUpClosed"
+
+var _deviceState_index = [...]uint8{0, 4, 6, 12}
+
+func (i deviceState) String() string {
+	if i >= deviceState(len(_deviceState_index)-1) {
+		return "deviceState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _deviceState_name[_deviceState_index[i]:_deviceState_index[i+1]]
+}

device/endpoint_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/indextable.go

@@ -1,14 +1,14 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
 	"crypto/rand"
+	"encoding/binary"
 	"sync"
-	"unsafe"
 )
 
 type IndexTableEntry struct {
@@ -25,7 +25,8 @@ type IndexTable struct {
 func randUint32() (uint32, error) {
 	var integer [4]byte
 	_, err := rand.Read(integer[:])
-	return *(*uint32)(unsafe.Pointer(&integer[0])), err
+	// Arbitrary endianness; both are intrinsified by the Go compiler.
+	return binary.LittleEndian.Uint32(integer[:]), err
 }
 
 func (table *IndexTable) Init() {

device/ip.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/kdf_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/keypair.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -8,7 +8,9 @@ package device
 import (
 	"crypto/cipher"
 	"sync"
+	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"golang.zx2c4.com/wireguard/replay"
 )
@@ -21,10 +23,10 @@ import (
  */
 
 type Keypair struct {
-	sendNonce    uint64
+	sendNonce    uint64 // accessed atomically
 	send         cipher.AEAD
 	receive      cipher.AEAD
-	replayFilter replay.ReplayFilter
+	replayFilter replay.Filter
 	isInitiator  bool
 	created      time.Time
 	localIndex   uint32
@@ -38,6 +40,14 @@ type Keypairs struct {
 	next     *Keypair
 }
 
+func (kp *Keypairs) storeNext(next *Keypair) {
+	atomic.StorePointer((*unsafe.Pointer)((unsafe.Pointer)(&kp.next)), (unsafe.Pointer)(next))
+}
+
+func (kp *Keypairs) loadNext() *Keypair {
+	return (*Keypair)(atomic.LoadPointer((*unsafe.Pointer)((unsafe.Pointer)(&kp.next))))
+}
+
 func (kp *Keypairs) Current() *Keypair {
 	kp.RLock()
 	defer kp.RUnlock()

device/logger.go

@@ -1,59 +1,48 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
-	"io"
-	"io/ioutil"
 	"log"
 	"os"
 )
 
+// A Logger provides logging for a Device.
+// The functions are Printf-style functions.
+// They must be safe for concurrent use.
+// They do not require a trailing newline in the format.
+// If nil, that level of logging will be silent.
+type Logger struct {
+	Verbosef func(format string, args ...interface{})
+	Errorf   func(format string, args ...interface{})
+}
+
+// Log levels for use with NewLogger.
 const (
 	LogLevelSilent = iota
 	LogLevelError
-	LogLevelInfo
-	LogLevelDebug
+	LogLevelVerbose
 )
 
-type Logger struct {
-	Debug *log.Logger
-	Info  *log.Logger
-	Error *log.Logger
-}
+// Function for use in Logger for discarding logged lines.
+func DiscardLogf(format string, args ...interface{}) {}
 
+// NewLogger constructs a Logger that writes to stdout.
+// It logs at the specified log level and above.
+// It decorates log lines with the log level, date, time, and prepend.
 func NewLogger(level int, prepend string) *Logger {
-	output := os.Stdout
-	logger := new(Logger)
-
-	logErr, logInfo, logDebug := func() (io.Writer, io.Writer, io.Writer) {
-		if level >= LogLevelDebug {
-			return output, output, output
-		}
-		if level >= LogLevelInfo {
-			return output, output, ioutil.Discard
-		}
-		if level >= LogLevelError {
-			return output, ioutil.Discard, ioutil.Discard
-		}
-		return ioutil.Discard, ioutil.Discard, ioutil.Discard
-	}()
-
-	logger.Debug = log.New(logDebug,
-		"DEBUG: "+prepend,
-		log.Ldate|log.Ltime,
-	)
-
-	logger.Info = log.New(logInfo,
-		"INFO: "+prepend,
-		log.Ldate|log.Ltime,
-	)
-	logger.Error = log.New(logErr,
-		"ERROR: "+prepend,
-		log.Ldate|log.Ltime,
-	)
+	logger := &Logger{DiscardLogf, DiscardLogf}
+	logf := func(prefix string) func(string, ...interface{}) {
+		return log.New(os.Stdout, prefix+": "+prepend, log.Ldate|log.Ltime).Printf
+	}
+	if level >= LogLevelVerbose {
+		logger.Verbosef = logf("DEBUG")
+	}
+	if level >= LogLevelError {
+		logger.Errorf = logf("ERROR")
+	}
 	return logger
 }

device/mark_default.go

@@ -1,12 +0,0 @@
-// +build !linux,!openbsd,!freebsd
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-func (bind *nativeBind) SetMark(mark uint32) error {
-	return nil
-}

device/misc.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -39,10 +39,3 @@ func (a *AtomicBool) Set(val bool) {
 	}
 	atomic.StoreInt32(&a.int32, flag)
 }
-
-func min(a, b uint) uint {
-	if a > b {
-		return b
-	}
-	return a
-}

device/mobilequirks.go

@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+func (device *Device) DisableSomeRoamingForBrokenMobileSemantics() {
+	device.peers.RLock()
+	for _, peer := range device.peers.keyMap {
+		peer.Lock()
+		peer.disableRoaming = peer.endpoint != nil
+		peer.Unlock()
+	}
+	device.peers.RUnlock()
+}

device/noise-helpers.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/noise-protocol.go

@@ -1,29 +1,50 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
 	"errors"
+	"fmt"
 	"sync"
 	"time"
 
 	"golang.org/x/crypto/blake2s"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/poly1305"
+
 	"golang.zx2c4.com/wireguard/tai64n"
 )
 
+type handshakeState int
+
 const (
-	HandshakeZeroed = iota
-	HandshakeInitiationCreated
-	HandshakeInitiationConsumed
-	HandshakeResponseCreated
-	HandshakeResponseConsumed
+	handshakeZeroed = handshakeState(iota)
+	handshakeInitiationCreated
+	handshakeInitiationConsumed
+	handshakeResponseCreated
+	handshakeResponseConsumed
 )
 
+func (hs handshakeState) String() string {
+	switch hs {
+	case handshakeZeroed:
+		return "handshakeZeroed"
+	case handshakeInitiationCreated:
+		return "handshakeInitiationCreated"
+	case handshakeInitiationConsumed:
+		return "handshakeInitiationConsumed"
+	case handshakeResponseCreated:
+		return "handshakeResponseCreated"
+	case handshakeResponseConsumed:
+		return "handshakeResponseConsumed"
+	default:
+		return fmt.Sprintf("Handshake(UNKNOWN:%d)", int(hs))
+	}
+}
+
 const (
 	NoiseConstruction = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"
 	WGIdentifier      = "WireGuard v1 zx2c4 Jason@zx2c4.com"
@@ -39,13 +60,13 @@ const (
 )
 
 const (
-	MessageInitiationSize      = 148                                           // size of handshake initation message
+	MessageInitiationSize      = 148                                           // size of handshake initiation message
 	MessageResponseSize        = 92                                            // size of response message
 	MessageCookieReplySize     = 64                                            // size of cookie reply message
-	MessageTransportHeaderSize = 16                                            // size of data preceeding content in transport message
+	MessageTransportHeaderSize = 16                                            // size of data preceding content in transport message
 	MessageTransportSize       = MessageTransportHeaderSize + poly1305.TagSize // size of empty transport
 	MessageKeepaliveSize       = MessageTransportSize                          // size of keepalive
-	MessageHandshakeSize       = MessageInitiationSize                         // size of largest handshake releated message
+	MessageHandshakeSize       = MessageInitiationSize                         // size of largest handshake related message
 )
 
 const (
@@ -95,11 +116,11 @@ type MessageCookieReply struct {
 }
 
 type Handshake struct {
-	state                     int
+	state                     handshakeState
 	mutex                     sync.RWMutex
 	hash                      [blake2s.Size]byte       // hash value
 	chainKey                  [blake2s.Size]byte       // chain key
-	presharedKey              NoiseSymmetricKey        // psk
+	presharedKey              NoisePresharedKey        // psk
 	localEphemeral            NoisePrivateKey          // ephemeral secret key
 	localIndex                uint32                   // used to clear hash-table
 	remoteIndex               uint32                   // index for sending
@@ -135,7 +156,7 @@ func (h *Handshake) Clear() {
 	setZero(h.chainKey[:])
 	setZero(h.hash[:])
 	h.localIndex = 0
-	h.state = HandshakeZeroed
+	h.state = handshakeZeroed
 }
 
 func (h *Handshake) mixHash(data []byte) {
@@ -154,6 +175,7 @@ func init() {
 }
 
 func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, error) {
+	var errZeroECDHResult = errors.New("ECDH returned all zeros")
 
 	device.staticIdentity.RLock()
 	defer device.staticIdentity.RUnlock()
@@ -162,12 +184,7 @@ func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, e
 	handshake.mutex.Lock()
 	defer handshake.mutex.Unlock()
 
-	if isZero(handshake.precomputedStaticStatic[:]) {
-		return nil, errors.New("static shared secret is zero")
-	}
-
 	// create ephemeral key
-
 	var err error
 	handshake.hash = InitialHash
 	handshake.chainKey = InitialChainKey
@@ -176,59 +193,56 @@ func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, e
 		return nil, err
 	}
 
-	// assign index
-
-	device.indexTable.Delete(handshake.localIndex)
-	handshake.localIndex, err = device.indexTable.NewIndexForHandshake(peer, handshake)
-
-	if err != nil {
-		return nil, err
-	}
-
 	handshake.mixHash(handshake.remoteStatic[:])
 
 	msg := MessageInitiation{
 		Type:      MessageInitiationType,
 		Ephemeral: handshake.localEphemeral.publicKey(),
-		Sender:    handshake.localIndex,
 	}
 
 	handshake.mixKey(msg.Ephemeral[:])
 	handshake.mixHash(msg.Ephemeral[:])
 
 	// encrypt static key
-
-	func() {
-		var key [chacha20poly1305.KeySize]byte
-		ss := handshake.localEphemeral.sharedSecret(handshake.remoteStatic)
-		KDF2(
-			&handshake.chainKey,
-			&key,
-			handshake.chainKey[:],
-			ss[:],
-		)
-		aead, _ := chacha20poly1305.New(key[:])
-		aead.Seal(msg.Static[:0], ZeroNonce[:], device.staticIdentity.publicKey[:], handshake.hash[:])
-	}()
+	ss := handshake.localEphemeral.sharedSecret(handshake.remoteStatic)
+	if isZero(ss[:]) {
+		return nil, errZeroECDHResult
+	}
+	var key [chacha20poly1305.KeySize]byte
+	KDF2(
+		&handshake.chainKey,
+		&key,
+		handshake.chainKey[:],
+		ss[:],
+	)
+	aead, _ := chacha20poly1305.New(key[:])
+	aead.Seal(msg.Static[:0], ZeroNonce[:], device.staticIdentity.publicKey[:], handshake.hash[:])
 	handshake.mixHash(msg.Static[:])
 
 	// encrypt timestamp
-
+	if isZero(handshake.precomputedStaticStatic[:]) {
+		return nil, errZeroECDHResult
+	}
+	KDF2(
+		&handshake.chainKey,
+		&key,
+		handshake.chainKey[:],
+		handshake.precomputedStaticStatic[:],
+	)
 	timestamp := tai64n.Now()
-	func() {
-		var key [chacha20poly1305.KeySize]byte
-		KDF2(
-			&handshake.chainKey,
-			&key,
-			handshake.chainKey[:],
-			handshake.precomputedStaticStatic[:],
-		)
-		aead, _ := chacha20poly1305.New(key[:])
-		aead.Seal(msg.Timestamp[:0], ZeroNonce[:], timestamp[:], handshake.hash[:])
-	}()
+	aead, _ = chacha20poly1305.New(key[:])
+	aead.Seal(msg.Timestamp[:0], ZeroNonce[:], timestamp[:], handshake.hash[:])
+
+	// assign index
+	device.indexTable.Delete(handshake.localIndex)
+	msg.Sender, err = device.indexTable.NewIndexForHandshake(peer, handshake)
+	if err != nil {
+		return nil, err
+	}
+	handshake.localIndex = msg.Sender
 
 	handshake.mixHash(msg.Timestamp[:])
-	handshake.state = HandshakeInitiationCreated
+	handshake.state = handshakeInitiationCreated
 	return &msg, nil
 }
 
@@ -250,16 +264,16 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer {
 	mixKey(&chainKey, &InitialChainKey, msg.Ephemeral[:])
 
 	// decrypt static key
-
 	var err error
 	var peerPK NoisePublicKey
-	func() {
-		var key [chacha20poly1305.KeySize]byte
-		ss := device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
-		KDF2(&chainKey, &key, chainKey[:], ss[:])
-		aead, _ := chacha20poly1305.New(key[:])
-		_, err = aead.Open(peerPK[:0], ZeroNonce[:], msg.Static[:], hash[:])
-	}()
+	var key [chacha20poly1305.KeySize]byte
+	ss := device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
+	if isZero(ss[:]) {
+		return nil
+	}
+	KDF2(&chainKey, &key, chainKey[:], ss[:])
+	aead, _ := chacha20poly1305.New(key[:])
+	_, err = aead.Open(peerPK[:0], ZeroNonce[:], msg.Static[:], hash[:])
 	if err != nil {
 		return nil
 	}
@@ -273,23 +287,24 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer {
 	}
 
 	handshake := &peer.handshake
-	if isZero(handshake.precomputedStaticStatic[:]) {
-		return nil
-	}
 
 	// verify identity
 
 	var timestamp tai64n.Timestamp
-	var key [chacha20poly1305.KeySize]byte
 
 	handshake.mutex.RLock()
+
+	if isZero(handshake.precomputedStaticStatic[:]) {
+		handshake.mutex.RUnlock()
+		return nil
+	}
 	KDF2(
 		&chainKey,
 		&key,
 		chainKey[:],
 		handshake.precomputedStaticStatic[:],
 	)
-	aead, _ := chacha20poly1305.New(key[:])
+	aead, _ = chacha20poly1305.New(key[:])
 	_, err = aead.Open(timestamp[:0], ZeroNonce[:], msg.Timestamp[:], hash[:])
 	if err != nil {
 		handshake.mutex.RUnlock()
@@ -299,11 +314,15 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer {
 
 	// protect against replay & flood
 
-	var ok bool
-	ok = timestamp.After(handshake.lastTimestamp)
-	ok = ok && time.Since(handshake.lastInitiationConsumption) > HandshakeInitationRate
+	replay := !timestamp.After(handshake.lastTimestamp)
+	flood := time.Since(handshake.lastInitiationConsumption) <= HandshakeInitationRate
 	handshake.mutex.RUnlock()
-	if !ok {
+	if replay {
+		device.log.Verbosef("%v - ConsumeMessageInitiation: handshake replay @ %v", peer, timestamp)
+		return nil
+	}
+	if flood {
+		device.log.Verbosef("%v - ConsumeMessageInitiation: handshake flood", peer)
 		return nil
 	}
 
@@ -315,9 +334,14 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer {
 	handshake.chainKey = chainKey
 	handshake.remoteIndex = msg.Sender
 	handshake.remoteEphemeral = msg.Ephemeral
-	handshake.lastTimestamp = timestamp
-	handshake.lastInitiationConsumption = time.Now()
-	handshake.state = HandshakeInitiationConsumed
+	if timestamp.After(handshake.lastTimestamp) {
+		handshake.lastTimestamp = timestamp
+	}
+	now := time.Now()
+	if now.After(handshake.lastInitiationConsumption) {
+		handshake.lastInitiationConsumption = now
+	}
+	handshake.state = handshakeInitiationConsumed
 
 	handshake.mutex.Unlock()
 
@@ -332,7 +356,7 @@ func (device *Device) CreateMessageResponse(peer *Peer) (*MessageResponse, error
 	handshake.mutex.Lock()
 	defer handshake.mutex.Unlock()
 
-	if handshake.state != HandshakeInitiationConsumed {
+	if handshake.state != handshakeInitiationConsumed {
 		return nil, errors.New("handshake initiation must be consumed first")
 	}
 
@@ -388,7 +412,7 @@ func (device *Device) CreateMessageResponse(peer *Peer) (*MessageResponse, error
 		handshake.mixHash(msg.Empty[:])
 	}()
 
-	handshake.state = HandshakeResponseCreated
+	handshake.state = handshakeResponseCreated
 
 	return &msg, nil
 }
@@ -418,7 +442,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 		handshake.mutex.RLock()
 		defer handshake.mutex.RUnlock()
 
-		if handshake.state != HandshakeInitiationCreated {
+		if handshake.state != handshakeInitiationCreated {
 			return false
 		}
 
@@ -479,7 +503,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 	handshake.hash = hash
 	handshake.chainKey = chainKey
 	handshake.remoteIndex = msg.Sender
-	handshake.state = HandshakeResponseConsumed
+	handshake.state = handshakeResponseConsumed
 
 	handshake.mutex.Unlock()
 
@@ -504,7 +528,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 	var sendKey [chacha20poly1305.KeySize]byte
 	var recvKey [chacha20poly1305.KeySize]byte
 
-	if handshake.state == HandshakeResponseConsumed {
+	if handshake.state == handshakeResponseConsumed {
 		KDF2(
 			&sendKey,
 			&recvKey,
@@ -512,7 +536,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 			nil,
 		)
 		isInitiator = true
-	} else if handshake.state == HandshakeResponseCreated {
+	} else if handshake.state == handshakeResponseCreated {
 		KDF2(
 			&recvKey,
 			&sendKey,
@@ -521,7 +545,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 		)
 		isInitiator = false
 	} else {
-		return errors.New("invalid state for keypair derivation")
+		return fmt.Errorf("invalid state for keypair derivation: %v", handshake.state)
 	}
 
 	// zero handshake
@@ -529,7 +553,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 	setZero(handshake.chainKey[:])
 	setZero(handshake.hash[:]) // Doesn't necessarily need to be zeroed. Could be used for something interesting down the line.
 	setZero(handshake.localEphemeral[:])
-	peer.handshake.state = HandshakeZeroed
+	peer.handshake.state = handshakeZeroed
 
 	// create AEAD instances
 
@@ -541,8 +565,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 	setZero(recvKey[:])
 
 	keypair.created = time.Now()
-	keypair.sendNonce = 0
-	keypair.replayFilter.Init()
+	keypair.replayFilter.Reset()
 	keypair.isInitiator = isInitiator
 	keypair.localIndex = peer.handshake.localIndex
 	keypair.remoteIndex = peer.handshake.remoteIndex
@@ -559,12 +582,12 @@ func (peer *Peer) BeginSymmetricSession() error {
 	defer keypairs.Unlock()
 
 	previous := keypairs.previous
-	next := keypairs.next
+	next := keypairs.loadNext()
 	current := keypairs.current
 
 	if isInitiator {
 		if next != nil {
-			keypairs.next = nil
+			keypairs.storeNext(nil)
 			keypairs.previous = next
 			device.DeleteKeypair(current)
 		} else {
@@ -573,7 +596,7 @@ func (peer *Peer) BeginSymmetricSession() error {
 		device.DeleteKeypair(previous)
 		keypairs.current = keypair
 	} else {
-		keypairs.next = keypair
+		keypairs.storeNext(keypair)
 		device.DeleteKeypair(next)
 		keypairs.previous = nil
 		device.DeleteKeypair(previous)
@@ -584,18 +607,19 @@ func (peer *Peer) BeginSymmetricSession() error {
 
 func (peer *Peer) ReceivedWithKeypair(receivedKeypair *Keypair) bool {
 	keypairs := &peer.keypairs
-	if keypairs.next != receivedKeypair {
+
+	if keypairs.loadNext() != receivedKeypair {
 		return false
 	}
 	keypairs.Lock()
 	defer keypairs.Unlock()
-	if keypairs.next != receivedKeypair {
+	if keypairs.loadNext() != receivedKeypair {
 		return false
 	}
 	old := keypairs.previous
 	keypairs.previous = keypairs.current
 	peer.device.DeleteKeypair(old)
-	keypairs.current = keypairs.next
-	keypairs.next = nil
+	keypairs.current = keypairs.loadNext()
+	keypairs.storeNext(nil)
 	return true
 }

device/noise-types.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -9,19 +9,18 @@ import (
 	"crypto/subtle"
 	"encoding/hex"
 	"errors"
-
-	"golang.org/x/crypto/chacha20poly1305"
 )
 
 const (
-	NoisePublicKeySize  = 32
-	NoisePrivateKeySize = 32
+	NoisePublicKeySize    = 32
+	NoisePrivateKeySize   = 32
+	NoisePresharedKeySize = 32
 )
 
 type (
 	NoisePublicKey    [NoisePublicKeySize]byte
 	NoisePrivateKey   [NoisePrivateKeySize]byte
-	NoiseSymmetricKey [chacha20poly1305.KeySize]byte
+	NoisePresharedKey [NoisePresharedKeySize]byte
 	NoiseNonce        uint64 // padded to 12-bytes
 )
 
@@ -52,18 +51,19 @@ func (key *NoisePrivateKey) FromHex(src string) (err error) {
 	return
 }
 
-func (key NoisePrivateKey) ToHex() string {
-	return hex.EncodeToString(key[:])
+func (key *NoisePrivateKey) FromMaybeZeroHex(src string) (err error) {
+	err = loadExactHex(key[:], src)
+	if key.IsZero() {
+		return
+	}
+	key.clamp()
+	return
 }
 
 func (key *NoisePublicKey) FromHex(src string) error {
 	return loadExactHex(key[:], src)
 }
 
-func (key NoisePublicKey) ToHex() string {
-	return hex.EncodeToString(key[:])
-}
-
 func (key NoisePublicKey) IsZero() bool {
 	var zero NoisePublicKey
 	return key.Equals(zero)
@@ -73,10 +73,6 @@ func (key NoisePublicKey) Equals(tar NoisePublicKey) bool {
 	return subtle.ConstantTimeCompare(key[:], tar[:]) == 1
 }
 
-func (key *NoiseSymmetricKey) FromHex(src string) error {
+func (key *NoisePresharedKey) FromHex(src string) error {
 	return loadExactHex(key[:], src)
 }
-
-func (key NoiseSymmetricKey) ToHex() string {
-	return hex.EncodeToString(key[:])
-}

device/noise_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -9,6 +9,9 @@ import (
 	"bytes"
 	"encoding/binary"
 	"testing"
+
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 )
 
 func TestCurveWrappers(t *testing.T) {
@@ -29,6 +32,30 @@ func TestCurveWrappers(t *testing.T) {
 	}
 }
 
+func randDevice(t *testing.T) *Device {
+	sk, err := newPrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	tun := tuntest.NewChannelTUN()
+	logger := NewLogger(LogLevelError, "")
+	device := NewDevice(tun.TUN(), conn.NewDefaultBind(), logger)
+	device.SetPrivateKey(sk)
+	return device
+}
+
+func assertNil(t *testing.T, err error) {
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func assertEqual(t *testing.T, a, b []byte) {
+	if !bytes.Equal(a, b) {
+		t.Fatal(a, "!=", b)
+	}
+}
+
 func TestNoiseHandshake(t *testing.T) {
 	dev1 := randDevice(t)
 	dev2 := randDevice(t)
@@ -36,8 +63,14 @@ func TestNoiseHandshake(t *testing.T) {
 	defer dev1.Close()
 	defer dev2.Close()
 
-	peer1, _ := dev2.NewPeer(dev1.staticIdentity.privateKey.publicKey())
-	peer2, _ := dev1.NewPeer(dev2.staticIdentity.privateKey.publicKey())
+	peer1, err := dev2.NewPeer(dev1.staticIdentity.privateKey.publicKey())
+	if err != nil {
+		t.Fatal(err)
+	}
+	peer2, err := dev1.NewPeer(dev2.staticIdentity.privateKey.publicKey())
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	assertEqual(
 		t,
@@ -113,7 +146,7 @@ func TestNoiseHandshake(t *testing.T) {
 		t.Fatal("failed to derive keypair for peer 2", err)
 	}
 
-	key1 := peer1.keypairs.next
+	key1 := peer1.keypairs.loadNext()
 	key2 := peer2.keypairs.current
 
 	// encrypting / decryption test

device/peer.go

@@ -1,39 +1,42 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
-	"encoding/base64"
+	"container/list"
 	"errors"
-	"fmt"
 	"sync"
 	"sync/atomic"
 	"time"
-)
 
-const (
-	PeerRoutineNumber = 3
+	"golang.zx2c4.com/wireguard/conn"
 )
 
 type Peer struct {
-	isRunning                   AtomicBool
-	sync.RWMutex                // Mostly protects endpoint, but is generally taken whenever we modify peer
-	keypairs                    Keypairs
-	handshake                   Handshake
-	device                      *Device
-	endpoint                    Endpoint
-	persistentKeepaliveInterval uint16
+	isRunning    AtomicBool
+	sync.RWMutex // Mostly protects endpoint, but is generally taken whenever we modify peer
+	keypairs     Keypairs
+	handshake    Handshake
+	device       *Device
+	endpoint     conn.Endpoint
+	stopping     sync.WaitGroup // routines pending stop
 
-	// This must be 64-bit aligned, so make sure the above members come out to even alignment and pad accordingly
+	// These fields are accessed with atomic operations, which must be
+	// 64-bit aligned even on 32-bit platforms. Go guarantees that an
+	// allocated struct will be 64-bit aligned. So we place
+	// atomically-accessed fields up front, so that they can share in
+	// this alignment before smaller fields throw it off.
 	stats struct {
 		txBytes           uint64 // bytes send to peer (endpoint)
 		rxBytes           uint64 // bytes received from peer
 		lastHandshakeNano int64  // nano seconds since epoch
 	}
 
+	disableRoaming bool
+
 	timers struct {
 		retransmitHandshake     *Timer
 		sendKeepalive           *Timer
@@ -45,35 +48,27 @@ type Peer struct {
 		sentLastMinuteHandshake AtomicBool
 	}
 
-	signals struct {
-		newKeypairArrived chan struct{}
-		flushNonceQueue   chan struct{}
+	state struct {
+		sync.Mutex // protects against concurrent Start/Stop
 	}
 
 	queue struct {
-		nonce                           chan *QueueOutboundElement // nonce / pre-handshake queue
-		outbound                        chan *QueueOutboundElement // sequential ordering of work
-		inbound                         chan *QueueInboundElement  // sequential ordering of work
-		packetInNonceQueueIsAwaitingKey AtomicBool
+		staged   chan *QueueOutboundElement // staged packets before a handshake is available
+		outbound *autodrainingOutboundQueue // sequential ordering of udp transmission
+		inbound  *autodrainingInboundQueue  // sequential ordering of tun writing
 	}
 
-	routines struct {
-		sync.Mutex                // held when stopping / starting routines
-		starting   sync.WaitGroup // routines pending start
-		stopping   sync.WaitGroup // routines pending stop
-		stop       chan struct{}  // size 0, stop all go routines in peer
-	}
-
-	cookieGenerator CookieGenerator
+	cookieGenerator             CookieGenerator
+	trieEntries                 list.List
+	persistentKeepaliveInterval uint32 // accessed atomically
 }
 
 func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
-	if device.isClosed.Get() {
+	if device.isClosed() {
 		return nil, errors.New("device closed")
 	}
 
 	// lock resources
-
 	device.staticIdentity.RLock()
 	defer device.staticIdentity.RUnlock()
 
@@ -81,52 +76,43 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 	defer device.peers.Unlock()
 
 	// check if over limit
-
 	if len(device.peers.keyMap) >= MaxPeers {
 		return nil, errors.New("too many peers")
 	}
 
 	// create peer
-
 	peer := new(Peer)
 	peer.Lock()
 	defer peer.Unlock()
 
 	peer.cookieGenerator.Init(pk)
 	peer.device = device
-	peer.isRunning.Set(false)
+	peer.queue.outbound = newAutodrainingOutboundQueue(device)
+	peer.queue.inbound = newAutodrainingInboundQueue(device)
+	peer.queue.staged = make(chan *QueueOutboundElement, QueueStagedSize)
 
 	// map public key
-
 	_, ok := device.peers.keyMap[pk]
 	if ok {
 		return nil, errors.New("adding existing peer")
 	}
 
 	// pre-compute DH
-
 	handshake := &peer.handshake
 	handshake.mutex.Lock()
 	handshake.precomputedStaticStatic = device.staticIdentity.privateKey.sharedSecret(pk)
-	ssIsZero := isZero(handshake.precomputedStaticStatic[:])
 	handshake.remoteStatic = pk
 	handshake.mutex.Unlock()
 
 	// reset endpoint
-
 	peer.endpoint = nil
 
-	// conditionally add
-
-	if !ssIsZero {
-		device.peers.keyMap[pk] = peer
-	} else {
-		return nil, nil
-	}
+	// add
+	device.peers.keyMap[pk] = peer
 
 	// start peer
-
-	if peer.device.isUp.Get() {
+	peer.timersInit()
+	if peer.device.isUp() {
 		peer.Start()
 	}
 
@@ -137,8 +123,8 @@ func (peer *Peer) SendBuffer(buffer []byte) error {
 	peer.device.net.RLock()
 	defer peer.device.net.RUnlock()
 
-	if peer.device.net.bind == nil {
-		return errors.New("no bind")
+	if peer.device.isClosed() {
+		return nil
 	}
 
 	peer.RLock()
@@ -156,60 +142,65 @@ func (peer *Peer) SendBuffer(buffer []byte) error {
 }
 
 func (peer *Peer) String() string {
-	base64Key := base64.StdEncoding.EncodeToString(peer.handshake.remoteStatic[:])
-	abbreviatedKey := "invalid"
-	if len(base64Key) == 44 {
-		abbreviatedKey = base64Key[0:4] + "…" + base64Key[39:43]
+	// The awful goo that follows is identical to:
+	//
+	//   base64Key := base64.StdEncoding.EncodeToString(peer.handshake.remoteStatic[:])
+	//   abbreviatedKey := base64Key[0:4] + "…" + base64Key[39:43]
+	//   return fmt.Sprintf("peer(%s)", abbreviatedKey)
+	//
+	// except that it is considerably more efficient.
+	src := peer.handshake.remoteStatic
+	b64 := func(input byte) byte {
+		return input + 'A' + byte(((25-int(input))>>8)&6) - byte(((51-int(input))>>8)&75) - byte(((61-int(input))>>8)&15) + byte(((62-int(input))>>8)&3)
 	}
-	return fmt.Sprintf("peer(%s)", abbreviatedKey)
+	b := []byte("peer(____…____)")
+	const first = len("peer(")
+	const second = len("peer(____…")
+	b[first+0] = b64((src[0] >> 2) & 63)
+	b[first+1] = b64(((src[0] << 4) | (src[1] >> 4)) & 63)
+	b[first+2] = b64(((src[1] << 2) | (src[2] >> 6)) & 63)
+	b[first+3] = b64(src[2] & 63)
+	b[second+0] = b64(src[29] & 63)
+	b[second+1] = b64((src[30] >> 2) & 63)
+	b[second+2] = b64(((src[30] << 4) | (src[31] >> 4)) & 63)
+	b[second+3] = b64((src[31] << 2) & 63)
+	return string(b)
 }
 
 func (peer *Peer) Start() {
-
 	// should never start a peer on a closed device
-
-	if peer.device.isClosed.Get() {
+	if peer.device.isClosed() {
 		return
 	}
 
 	// prevent simultaneous start/stop operations
-
-	peer.routines.Lock()
-	defer peer.routines.Unlock()
+	peer.state.Lock()
+	defer peer.state.Unlock()
 
 	if peer.isRunning.Get() {
 		return
 	}
 
 	device := peer.device
-	device.log.Debug.Println(peer, "- Starting...")
+	device.log.Verbosef("%v - Starting", peer)
 
 	// reset routine state
+	peer.stopping.Wait()
+	peer.stopping.Add(2)
 
-	peer.routines.starting.Wait()
-	peer.routines.stopping.Wait()
-	peer.routines.stop = make(chan struct{})
-	peer.routines.starting.Add(PeerRoutineNumber)
-	peer.routines.stopping.Add(PeerRoutineNumber)
-
-	// prepare queues
-
-	peer.queue.nonce = make(chan *QueueOutboundElement, QueueOutboundSize)
-	peer.queue.outbound = make(chan *QueueOutboundElement, QueueOutboundSize)
-	peer.queue.inbound = make(chan *QueueInboundElement, QueueInboundSize)
-
-	peer.timersInit()
+	peer.handshake.mutex.Lock()
 	peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second))
-	peer.signals.newKeypairArrived = make(chan struct{}, 1)
-	peer.signals.flushNonceQueue = make(chan struct{}, 1)
+	peer.handshake.mutex.Unlock()
 
-	// wait for routines to start
+	peer.device.queue.encryption.wg.Add(1) // keep encryption queue open for our writes
 
-	go peer.RoutineNonce()
+	peer.timersStart()
+
+	device.flushInboundQueue(peer.queue.inbound)
+	device.flushOutboundQueue(peer.queue.outbound)
 	go peer.RoutineSequentialSender()
 	go peer.RoutineSequentialReceiver()
 
-	peer.routines.starting.Wait()
 	peer.isRunning.Set(true)
 }
 
@@ -222,10 +213,10 @@ func (peer *Peer) ZeroAndFlushAll() {
 	keypairs.Lock()
 	device.DeleteKeypair(keypairs.previous)
 	device.DeleteKeypair(keypairs.current)
-	device.DeleteKeypair(keypairs.next)
+	device.DeleteKeypair(keypairs.loadNext())
 	keypairs.previous = nil
 	keypairs.current = nil
-	keypairs.next = nil
+	keypairs.storeNext(nil)
 	keypairs.Unlock()
 
 	// clear handshake state
@@ -236,7 +227,7 @@ func (peer *Peer) ZeroAndFlushAll() {
 	handshake.Clear()
 	handshake.mutex.Unlock()
 
-	peer.FlushNonceQueue()
+	peer.FlushStagedPackets()
 }
 
 func (peer *Peer) ExpireCurrentKeypairs() {
@@ -244,55 +235,43 @@ func (peer *Peer) ExpireCurrentKeypairs() {
 	handshake.mutex.Lock()
 	peer.device.indexTable.Delete(handshake.localIndex)
 	handshake.Clear()
-	handshake.mutex.Unlock()
 	peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second))
+	handshake.mutex.Unlock()
 
 	keypairs := &peer.keypairs
 	keypairs.Lock()
 	if keypairs.current != nil {
-		keypairs.current.sendNonce = RejectAfterMessages
+		atomic.StoreUint64(&keypairs.current.sendNonce, RejectAfterMessages)
 	}
 	if keypairs.next != nil {
-		keypairs.next.sendNonce = RejectAfterMessages
+		next := keypairs.loadNext()
+		atomic.StoreUint64(&next.sendNonce, RejectAfterMessages)
 	}
 	keypairs.Unlock()
 }
 
 func (peer *Peer) Stop() {
-
-	// prevent simultaneous start/stop operations
+	peer.state.Lock()
+	defer peer.state.Unlock()
 
 	if !peer.isRunning.Swap(false) {
 		return
 	}
 
-	peer.routines.starting.Wait()
-
-	peer.routines.Lock()
-	defer peer.routines.Unlock()
-
-	peer.device.log.Debug.Println(peer, "- Stopping...")
+	peer.device.log.Verbosef("%v - Stopping", peer)
 
 	peer.timersStop()
-
-	// stop & wait for ongoing peer routines
-
-	close(peer.routines.stop)
-	peer.routines.stopping.Wait()
-
-	// close queues
-
-	close(peer.queue.nonce)
-	close(peer.queue.outbound)
-	close(peer.queue.inbound)
+	// Signal that RoutineSequentialSender and RoutineSequentialReceiver should exit.
+	peer.queue.inbound.c <- nil
+	peer.queue.outbound.c <- nil
+	peer.stopping.Wait()
+	peer.device.queue.encryption.wg.Done() // no more writes to encryption queue from us
 
 	peer.ZeroAndFlushAll()
 }
 
-var RoamingDisabled bool
-
-func (peer *Peer) SetEndpointFromPacket(endpoint Endpoint) {
-	if RoamingDisabled {
+func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) {
+	if peer.disableRoaming {
 		return
 	}
 	peer.Lock()

device/pools.go

@@ -1,89 +1,84 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
-import "sync"
+import (
+	"sync"
+	"sync/atomic"
+)
+
+type WaitPool struct {
+	pool  sync.Pool
+	cond  sync.Cond
+	lock  sync.Mutex
+	count uint32
+	max   uint32
+}
+
+func NewWaitPool(max uint32, new func() interface{}) *WaitPool {
+	p := &WaitPool{pool: sync.Pool{New: new}, max: max}
+	p.cond = sync.Cond{L: &p.lock}
+	return p
+}
+
+func (p *WaitPool) Get() interface{} {
+	if p.max != 0 {
+		p.lock.Lock()
+		for atomic.LoadUint32(&p.count) >= p.max {
+			p.cond.Wait()
+		}
+		atomic.AddUint32(&p.count, 1)
+		p.lock.Unlock()
+	}
+	return p.pool.Get()
+}
+
+func (p *WaitPool) Put(x interface{}) {
+	p.pool.Put(x)
+	if p.max == 0 {
+		return
+	}
+	atomic.AddUint32(&p.count, ^uint32(0))
+	p.cond.Signal()
+}
 
 func (device *Device) PopulatePools() {
-	if PreallocatedBuffersPerPool == 0 {
-		device.pool.messageBufferPool = &sync.Pool{
-			New: func() interface{} {
-				return new([MaxMessageSize]byte)
-			},
-		}
-		device.pool.inboundElementPool = &sync.Pool{
-			New: func() interface{} {
-				return new(QueueInboundElement)
-			},
-		}
-		device.pool.outboundElementPool = &sync.Pool{
-			New: func() interface{} {
-				return new(QueueOutboundElement)
-			},
-		}
-	} else {
-		device.pool.messageBufferReuseChan = make(chan *[MaxMessageSize]byte, PreallocatedBuffersPerPool)
-		for i := 0; i < PreallocatedBuffersPerPool; i += 1 {
-			device.pool.messageBufferReuseChan <- new([MaxMessageSize]byte)
-		}
-		device.pool.inboundElementReuseChan = make(chan *QueueInboundElement, PreallocatedBuffersPerPool)
-		for i := 0; i < PreallocatedBuffersPerPool; i += 1 {
-			device.pool.inboundElementReuseChan <- new(QueueInboundElement)
-		}
-		device.pool.outboundElementReuseChan = make(chan *QueueOutboundElement, PreallocatedBuffersPerPool)
-		for i := 0; i < PreallocatedBuffersPerPool; i += 1 {
-			device.pool.outboundElementReuseChan <- new(QueueOutboundElement)
-		}
-	}
+	device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() interface{} {
+		return new([MaxMessageSize]byte)
+	})
+	device.pool.inboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() interface{} {
+		return new(QueueInboundElement)
+	})
+	device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() interface{} {
+		return new(QueueOutboundElement)
+	})
 }
 
 func (device *Device) GetMessageBuffer() *[MaxMessageSize]byte {
-	if PreallocatedBuffersPerPool == 0 {
-		return device.pool.messageBufferPool.Get().(*[MaxMessageSize]byte)
-	} else {
-		return <-device.pool.messageBufferReuseChan
-	}
+	return device.pool.messageBuffers.Get().(*[MaxMessageSize]byte)
 }
 
 func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) {
-	if PreallocatedBuffersPerPool == 0 {
-		device.pool.messageBufferPool.Put(msg)
-	} else {
-		device.pool.messageBufferReuseChan <- msg
-	}
+	device.pool.messageBuffers.Put(msg)
 }
 
 func (device *Device) GetInboundElement() *QueueInboundElement {
-	if PreallocatedBuffersPerPool == 0 {
-		return device.pool.inboundElementPool.Get().(*QueueInboundElement)
-	} else {
-		return <-device.pool.inboundElementReuseChan
-	}
+	return device.pool.inboundElements.Get().(*QueueInboundElement)
 }
 
-func (device *Device) PutInboundElement(msg *QueueInboundElement) {
-	if PreallocatedBuffersPerPool == 0 {
-		device.pool.inboundElementPool.Put(msg)
-	} else {
-		device.pool.inboundElementReuseChan <- msg
-	}
+func (device *Device) PutInboundElement(elem *QueueInboundElement) {
+	elem.clearPointers()
+	device.pool.inboundElements.Put(elem)
 }
 
 func (device *Device) GetOutboundElement() *QueueOutboundElement {
-	if PreallocatedBuffersPerPool == 0 {
-		return device.pool.outboundElementPool.Get().(*QueueOutboundElement)
-	} else {
-		return <-device.pool.outboundElementReuseChan
-	}
+	return device.pool.outboundElements.Get().(*QueueOutboundElement)
 }
 
-func (device *Device) PutOutboundElement(msg *QueueOutboundElement) {
-	if PreallocatedBuffersPerPool == 0 {
-		device.pool.outboundElementPool.Put(msg)
-	} else {
-		device.pool.outboundElementReuseChan <- msg
-	}
+func (device *Device) PutOutboundElement(elem *QueueOutboundElement) {
+	elem.clearPointers()
+	device.pool.outboundElements.Put(elem)
 }

device/pools_test.go

@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"math/rand"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestWaitPool(t *testing.T) {
+	t.Skip("Currently disabled")
+	var wg sync.WaitGroup
+	trials := int32(100000)
+	if raceEnabled {
+		// This test can be very slow with -race.
+		trials /= 10
+	}
+	workers := runtime.NumCPU() + 2
+	if workers-4 <= 0 {
+		t.Skip("Not enough cores")
+	}
+	p := NewWaitPool(uint32(workers-4), func() interface{} { return make([]byte, 16) })
+	wg.Add(workers)
+	max := uint32(0)
+	updateMax := func() {
+		count := atomic.LoadUint32(&p.count)
+		if count > p.max {
+			t.Errorf("count (%d) > max (%d)", count, p.max)
+		}
+		for {
+			old := atomic.LoadUint32(&max)
+			if count <= old {
+				break
+			}
+			if atomic.CompareAndSwapUint32(&max, old, count) {
+				break
+			}
+		}
+	}
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for atomic.AddInt32(&trials, -1) > 0 {
+				updateMax()
+				x := p.Get()
+				updateMax()
+				time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
+				updateMax()
+				p.Put(x)
+				updateMax()
+			}
+		}()
+	}
+	wg.Wait()
+	if max != p.max {
+		t.Errorf("Actual maximum count (%d) != ideal maximum count (%d)", max, p.max)
+	}
+}
+
+func BenchmarkWaitPool(b *testing.B) {
+	var wg sync.WaitGroup
+	trials := int32(b.N)
+	workers := runtime.NumCPU() + 2
+	if workers-4 <= 0 {
+		b.Skip("Not enough cores")
+	}
+	p := NewWaitPool(uint32(workers-4), func() interface{} { return make([]byte, 16) })
+	wg.Add(workers)
+	b.ResetTimer()
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
+			for atomic.AddInt32(&trials, -1) > 0 {
+				x := p.Get()
+				time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond)
+				p.Put(x)
+			}
+		}()
+	}
+	wg.Wait()
+}

device/queueconstants_android.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -8,6 +8,7 @@ package device
 /* Reduce memory consumption for Android */
 
 const (
+	QueueStagedSize            = 128
 	QueueOutboundSize          = 1024
 	QueueInboundSize           = 1024
 	QueueHandshakeSize         = 1024

device/queueconstants_default.go

@@ -1,13 +1,14 @@
-// +build !android,!ios
+//go:build !android && !ios && !windows
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 const (
+	QueueStagedSize            = 128
 	QueueOutboundSize          = 1024
 	QueueInboundSize           = 1024
 	QueueHandshakeSize         = 1024

device/queueconstants_ios.go

@@ -1,18 +1,21 @@
-// +build ios
+//go:build ios
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
-/* Fit within memory limits for iOS's Network Extension API, which has stricter requirements */
-
-const (
-	QueueOutboundSize          = 1024
-	QueueInboundSize           = 1024
-	QueueHandshakeSize         = 1024
-	MaxSegmentSize             = 1700
-	PreallocatedBuffersPerPool = 1024
+// Fit within memory limits for iOS's Network Extension API, which has stricter requirements.
+// These are vars instead of consts, because heavier network extensions might want to reduce
+// them further.
+var (
+	QueueStagedSize                   = 128
+	QueueOutboundSize                 = 1024
+	QueueInboundSize                  = 1024
+	QueueHandshakeSize                = 1024
+	PreallocatedBuffersPerPool uint32 = 1024
 )
+
+const MaxSegmentSize = 1700

device/queueconstants_windows.go

@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+const (
+	QueueStagedSize            = 128
+	QueueOutboundSize          = 1024
+	QueueInboundSize           = 1024
+	QueueHandshakeSize         = 1024
+	MaxSegmentSize             = 2048 - 32 // largest possible UDP datagram
+	PreallocatedBuffersPerPool = 0         // Disable and allow for infinite memory growth
+)

device/race_disabled_test.go

@@ -0,0 +1,10 @@
+//go:build !race
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+const raceEnabled = false

device/race_enabled_test.go

@@ -0,0 +1,10 @@
+//go:build race
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+const raceEnabled = true

device/receive.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -8,8 +8,8 @@ package device
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"net"
-	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -17,57 +17,35 @@ import (
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
+
+	"golang.zx2c4.com/wireguard/conn"
 )
 
 type QueueHandshakeElement struct {
 	msgType  uint32
 	packet   []byte
-	endpoint Endpoint
+	endpoint conn.Endpoint
 	buffer   *[MaxMessageSize]byte
 }
 
 type QueueInboundElement struct {
-	dropped int32
 	sync.Mutex
 	buffer   *[MaxMessageSize]byte
 	packet   []byte
 	counter  uint64
 	keypair  *Keypair
-	endpoint Endpoint
+	endpoint conn.Endpoint
 }
 
-func (elem *QueueInboundElement) Drop() {
-	atomic.StoreInt32(&elem.dropped, AtomicTrue)
-}
-
-func (elem *QueueInboundElement) IsDropped() bool {
-	return atomic.LoadInt32(&elem.dropped) == AtomicTrue
-}
-
-func (device *Device) addToInboundAndDecryptionQueues(inboundQueue chan *QueueInboundElement, decryptionQueue chan *QueueInboundElement, element *QueueInboundElement) bool {
-	select {
-	case inboundQueue <- element:
-		select {
-		case decryptionQueue <- element:
-			return true
-		default:
-			element.Drop()
-			element.Unlock()
-			return false
-		}
-	default:
-		device.PutInboundElement(element)
-		return false
-	}
-}
-
-func (device *Device) addToHandshakeQueue(queue chan QueueHandshakeElement, element QueueHandshakeElement) bool {
-	select {
-	case queue <- element:
-		return true
-	default:
-		return false
-	}
+// clearPointers clears elem fields that contain pointers.
+// This makes the garbage collector's life easier and
+// avoids accidentally keeping other objects around unnecessarily.
+// It also reduces the possible collateral damage from use-after-free bugs.
+func (elem *QueueInboundElement) clearPointers() {
+	elem.buffer = nil
+	elem.packet = nil
+	elem.keypair = nil
+	elem.endpoint = nil
 }
 
 /* Called when a new authenticated message has been received
@@ -90,44 +68,49 @@ func (peer *Peer) keepKeyFreshReceiving() {
  * Every time the bind is updated a new routine is started for
  * IPv4 and IPv6 (separately)
  */
-func (device *Device) RoutineReceiveIncoming(IP int, bind Bind) {
-
-	logDebug := device.log.Debug
+func (device *Device) RoutineReceiveIncoming(recv conn.ReceiveFunc) {
+	recvName := recv.PrettyName()
 	defer func() {
-		logDebug.Println("Routine: receive incoming IPv" + strconv.Itoa(IP) + " - stopped")
+		device.log.Verbosef("Routine: receive incoming %s - stopped", recvName)
+		device.queue.decryption.wg.Done()
+		device.queue.handshake.wg.Done()
 		device.net.stopping.Done()
 	}()
 
-	logDebug.Println("Routine: receive incoming IPv" + strconv.Itoa(IP) + " - started")
-	device.net.starting.Done()
+	device.log.Verbosef("Routine: receive incoming %s - started", recvName)
 
 	// receive datagrams until conn is closed
 
 	buffer := device.GetMessageBuffer()
 
 	var (
-		err      error
-		size     int
-		endpoint Endpoint
+		err         error
+		size        int
+		endpoint    conn.Endpoint
+		deathSpiral int
 	)
 
 	for {
-
-		// read next datagram
-
-		switch IP {
-		case ipv4.Version:
-			size, endpoint, err = bind.ReceiveIPv4(buffer[:])
-		case ipv6.Version:
-			size, endpoint, err = bind.ReceiveIPv6(buffer[:])
-		default:
-			panic("invalid IP version")
-		}
+		size, endpoint, err = recv(buffer[:])
 
 		if err != nil {
 			device.PutMessageBuffer(buffer)
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			device.log.Verbosef("Failed to receive %s packet: %v", recvName, err)
+			if neterr, ok := err.(net.Error); ok && !neterr.Temporary() {
+				return
+			}
+			if deathSpiral < 10 {
+				deathSpiral++
+				time.Sleep(time.Second / 3)
+				buffer = device.GetMessageBuffer()
+				continue
+			}
 			return
 		}
+		deathSpiral = 0
 
 		if size < MinMessageSize {
 			continue
@@ -175,20 +158,19 @@ func (device *Device) RoutineReceiveIncoming(IP int, bind Bind) {
 			elem.packet = packet
 			elem.buffer = buffer
 			elem.keypair = keypair
-			elem.dropped = AtomicFalse
 			elem.endpoint = endpoint
 			elem.counter = 0
 			elem.Mutex = sync.Mutex{}
 			elem.Lock()
 
 			// add to decryption queues
-
 			if peer.isRunning.Get() {
-				if device.addToInboundAndDecryptionQueues(peer.queue.inbound, device.queue.decryption, elem) {
-					buffer = device.GetMessageBuffer()
-				}
+				peer.queue.inbound.c <- elem
+				device.queue.decryption.c <- elem
+				buffer = device.GetMessageBuffer()
+			} else {
+				device.PutInboundElement(elem)
 			}
-
 			continue
 
 		// otherwise it is a fixed size & handshake related packet
@@ -203,127 +185,63 @@ func (device *Device) RoutineReceiveIncoming(IP int, bind Bind) {
 			okay = len(packet) == MessageCookieReplySize
 
 		default:
-			logDebug.Println("Received message with unknown type")
+			device.log.Verbosef("Received message with unknown type")
 		}
 
 		if okay {
-			if (device.addToHandshakeQueue(
-				device.queue.handshake,
-				QueueHandshakeElement{
-					msgType:  msgType,
-					buffer:   buffer,
-					packet:   packet,
-					endpoint: endpoint,
-				},
-			)) {
+			select {
+			case device.queue.handshake.c <- QueueHandshakeElement{
+				msgType:  msgType,
+				buffer:   buffer,
+				packet:   packet,
+				endpoint: endpoint,
+			}:
 				buffer = device.GetMessageBuffer()
+			default:
 			}
 		}
 	}
 }
 
-func (device *Device) RoutineDecryption() {
-
+func (device *Device) RoutineDecryption(id int) {
 	var nonce [chacha20poly1305.NonceSize]byte
 
-	logDebug := device.log.Debug
-	defer func() {
-		logDebug.Println("Routine: decryption worker - stopped")
-		device.state.stopping.Done()
-	}()
-	logDebug.Println("Routine: decryption worker - started")
-	device.state.starting.Done()
+	defer device.log.Verbosef("Routine: decryption worker %d - stopped", id)
+	device.log.Verbosef("Routine: decryption worker %d - started", id)
 
-	for {
-		select {
-		case <-device.signals.stop:
-			return
+	for elem := range device.queue.decryption.c {
+		// split message into fields
+		counter := elem.packet[MessageTransportOffsetCounter:MessageTransportOffsetContent]
+		content := elem.packet[MessageTransportOffsetContent:]
 
-		case elem, ok := <-device.queue.decryption:
-
-			if !ok {
-				return
-			}
-
-			// check if dropped
-
-			if elem.IsDropped() {
-				continue
-			}
-
-			// split message into fields
-
-			counter := elem.packet[MessageTransportOffsetCounter:MessageTransportOffsetContent]
-			content := elem.packet[MessageTransportOffsetContent:]
-
-			// expand nonce
-
-			nonce[0x4] = counter[0x0]
-			nonce[0x5] = counter[0x1]
-			nonce[0x6] = counter[0x2]
-			nonce[0x7] = counter[0x3]
-
-			nonce[0x8] = counter[0x4]
-			nonce[0x9] = counter[0x5]
-			nonce[0xa] = counter[0x6]
-			nonce[0xb] = counter[0x7]
-
-			// decrypt and release to consumer
-
-			var err error
-			elem.counter = binary.LittleEndian.Uint64(counter)
-			elem.packet, err = elem.keypair.receive.Open(
-				content[:0],
-				nonce[:],
-				content,
-				nil,
-			)
-			if err != nil {
-				elem.Drop()
-				device.PutMessageBuffer(elem.buffer)
-			}
-			elem.Unlock()
+		// decrypt and release to consumer
+		var err error
+		elem.counter = binary.LittleEndian.Uint64(counter)
+		// copy counter to nonce
+		binary.LittleEndian.PutUint64(nonce[0x4:0xc], elem.counter)
+		elem.packet, err = elem.keypair.receive.Open(
+			content[:0],
+			nonce[:],
+			content,
+			nil,
+		)
+		if err != nil {
+			elem.packet = nil
 		}
+		elem.Unlock()
 	}
 }
 
 /* Handles incoming packets related to handshake
  */
-func (device *Device) RoutineHandshake() {
-
-	logInfo := device.log.Info
-	logError := device.log.Error
-	logDebug := device.log.Debug
-
-	var elem QueueHandshakeElement
-	var ok bool
-
+func (device *Device) RoutineHandshake(id int) {
 	defer func() {
-		logDebug.Println("Routine: handshake worker - stopped")
-		device.state.stopping.Done()
-		if elem.buffer != nil {
-			device.PutMessageBuffer(elem.buffer)
-		}
+		device.log.Verbosef("Routine: handshake worker %d - stopped", id)
+		device.queue.encryption.wg.Done()
 	}()
+	device.log.Verbosef("Routine: handshake worker %d - started", id)
 
-	logDebug.Println("Routine: handshake worker - started")
-	device.state.starting.Done()
-
-	for {
-		if elem.buffer != nil {
-			device.PutMessageBuffer(elem.buffer)
-			elem.buffer = nil
-		}
-
-		select {
-		case elem, ok = <-device.queue.handshake:
-		case <-device.signals.stop:
-			return
-		}
-
-		if !ok {
-			return
-		}
+	for elem := range device.queue.handshake.c {
 
 		// handle cookie fields and ratelimiting
 
@@ -337,8 +255,8 @@ func (device *Device) RoutineHandshake() {
 			reader := bytes.NewReader(elem.packet)
 			err := binary.Read(reader, binary.LittleEndian, &reply)
 			if err != nil {
-				logDebug.Println("Failed to decode cookie reply")
-				return
+				device.log.Verbosef("Failed to decode cookie reply")
+				goto skip
 			}
 
 			// lookup peer from index
@@ -346,27 +264,27 @@ func (device *Device) RoutineHandshake() {
 			entry := device.indexTable.Lookup(reply.Receiver)
 
 			if entry.peer == nil {
-				continue
+				goto skip
 			}
 
 			// consume reply
 
 			if peer := entry.peer; peer.isRunning.Get() {
-				logDebug.Println("Receiving cookie response from ", elem.endpoint.DstToString())
+				device.log.Verbosef("Receiving cookie response from %s", elem.endpoint.DstToString())
 				if !peer.cookieGenerator.ConsumeReply(&reply) {
-					logDebug.Println("Could not decrypt invalid cookie response")
+					device.log.Verbosef("Could not decrypt invalid cookie response")
 				}
 			}
 
-			continue
+			goto skip
 
 		case MessageInitiationType, MessageResponseType:
 
 			// check mac fields and maybe ratelimit
 
 			if !device.cookieChecker.CheckMAC1(elem.packet) {
-				logDebug.Println("Received packet with invalid mac1")
-				continue
+				device.log.Verbosef("Received packet with invalid mac1")
+				goto skip
 			}
 
 			// endpoints destination address is the source of the datagram
@@ -377,19 +295,19 @@ func (device *Device) RoutineHandshake() {
 
 				if !device.cookieChecker.CheckMAC2(elem.packet, elem.endpoint.DstToBytes()) {
 					device.SendHandshakeCookie(&elem)
-					continue
+					goto skip
 				}
 
 				// check ratelimiter
 
 				if !device.rate.limiter.Allow(elem.endpoint.DstIP()) {
-					continue
+					goto skip
 				}
 			}
 
 		default:
-			logError.Println("Invalid packet ended up in the handshake queue")
-			continue
+			device.log.Errorf("Invalid packet ended up in the handshake queue")
+			goto skip
 		}
 
 		// handle handshake initiation/response content
@@ -403,19 +321,16 @@ func (device *Device) RoutineHandshake() {
 			reader := bytes.NewReader(elem.packet)
 			err := binary.Read(reader, binary.LittleEndian, &msg)
 			if err != nil {
-				logError.Println("Failed to decode initiation message")
-				continue
+				device.log.Errorf("Failed to decode initiation message")
+				goto skip
 			}
 
 			// consume initiation
 
 			peer := device.ConsumeMessageInitiation(&msg)
 			if peer == nil {
-				logInfo.Println(
-					"Received invalid initiation message from",
-					elem.endpoint.DstToString(),
-				)
-				continue
+				device.log.Verbosef("Received invalid initiation message from %s", elem.endpoint.DstToString())
+				goto skip
 			}
 
 			// update timers
@@ -426,7 +341,7 @@ func (device *Device) RoutineHandshake() {
 			// update endpoint
 			peer.SetEndpointFromPacket(elem.endpoint)
 
-			logDebug.Println(peer, "- Received handshake initiation")
+			device.log.Verbosef("%v - Received handshake initiation", peer)
 			atomic.AddUint64(&peer.stats.rxBytes, uint64(len(elem.packet)))
 
 			peer.SendHandshakeResponse()
@@ -439,25 +354,22 @@ func (device *Device) RoutineHandshake() {
 			reader := bytes.NewReader(elem.packet)
 			err := binary.Read(reader, binary.LittleEndian, &msg)
 			if err != nil {
-				logError.Println("Failed to decode response message")
-				continue
+				device.log.Errorf("Failed to decode response message")
+				goto skip
 			}
 
 			// consume response
 
 			peer := device.ConsumeMessageResponse(&msg)
 			if peer == nil {
-				logInfo.Println(
-					"Received invalid response message from",
-					elem.endpoint.DstToString(),
-				)
-				continue
+				device.log.Verbosef("Received invalid response message from %s", elem.endpoint.DstToString())
+				goto skip
 			}
 
 			// update endpoint
 			peer.SetEndpointFromPacket(elem.endpoint)
 
-			logDebug.Println(peer, "- Received handshake response")
+			device.log.Verbosef("%v - Received handshake response", peer)
 			atomic.AddUint64(&peer.stats.rxBytes, uint64(len(elem.packet)))
 
 			// update timers
@@ -470,88 +382,46 @@ func (device *Device) RoutineHandshake() {
 			err = peer.BeginSymmetricSession()
 
 			if err != nil {
-				logError.Println(peer, "- Failed to derive keypair:", err)
-				continue
+				device.log.Errorf("%v - Failed to derive keypair: %v", peer, err)
+				goto skip
 			}
 
 			peer.timersSessionDerived()
 			peer.timersHandshakeComplete()
 			peer.SendKeepalive()
-			select {
-			case peer.signals.newKeypairArrived <- struct{}{}:
-			default:
-			}
 		}
+	skip:
+		device.PutMessageBuffer(elem.buffer)
 	}
 }
 
 func (peer *Peer) RoutineSequentialReceiver() {
-
 	device := peer.device
-	logInfo := device.log.Info
-	logError := device.log.Error
-	logDebug := device.log.Debug
-
-	var elem *QueueInboundElement
-
 	defer func() {
-		logDebug.Println(peer, "- Routine: sequential receiver - stopped")
-		peer.routines.stopping.Done()
-		if elem != nil {
-			if !elem.IsDropped() {
-				device.PutMessageBuffer(elem.buffer)
-			}
-			device.PutInboundElement(elem)
-		}
+		device.log.Verbosef("%v - Routine: sequential receiver - stopped", peer)
+		peer.stopping.Done()
 	}()
+	device.log.Verbosef("%v - Routine: sequential receiver - started", peer)
 
-	logDebug.Println(peer, "- Routine: sequential receiver - started")
-
-	peer.routines.starting.Done()
-
-	for {
-		if elem != nil {
-			if !elem.IsDropped() {
-				device.PutMessageBuffer(elem.buffer)
-			}
-			device.PutInboundElement(elem)
-			elem = nil
-		}
-
-		var elemOk bool
-		select {
-		case <-peer.routines.stop:
+	for elem := range peer.queue.inbound.c {
+		if elem == nil {
 			return
-		case elem, elemOk = <-peer.queue.inbound:
-			if !elemOk {
-				return
-			}
 		}
-
-		// wait for decryption
-
+		var err error
 		elem.Lock()
-
-		if elem.IsDropped() {
-			continue
+		if elem.packet == nil {
+			// decryption failed
+			goto skip
 		}
 
-		// check for replay
-
 		if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) {
-			continue
+			goto skip
 		}
 
-		// update endpoint
 		peer.SetEndpointFromPacket(elem.endpoint)
-
-		// check if using new keypair
 		if peer.ReceivedWithKeypair(elem.keypair) {
 			peer.timersHandshakeComplete()
-			select {
-			case peer.signals.newKeypairArrived <- struct{}{}:
-			default:
-			}
+			peer.SendStagedPackets()
 		}
 
 		peer.keepKeyFreshReceiving()
@@ -559,89 +429,63 @@ func (peer *Peer) RoutineSequentialReceiver() {
 		peer.timersAnyAuthenticatedPacketReceived()
 		atomic.AddUint64(&peer.stats.rxBytes, uint64(len(elem.packet)+MinMessageSize))
 
-		// check for keepalive
-
 		if len(elem.packet) == 0 {
-			logDebug.Println(peer, "- Receiving keepalive packet")
-			continue
+			device.log.Verbosef("%v - Receiving keepalive packet", peer)
+			goto skip
 		}
 		peer.timersDataReceived()
 
-		// verify source and strip padding
-
 		switch elem.packet[0] >> 4 {
 		case ipv4.Version:
-
-			// strip padding
-
 			if len(elem.packet) < ipv4.HeaderLen {
-				continue
+				goto skip
 			}
-
 			field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]
 			length := binary.BigEndian.Uint16(field)
 			if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen {
-				continue
+				goto skip
 			}
-
 			elem.packet = elem.packet[:length]
-
-			// verify IPv4 source
-
 			src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]
-			if device.allowedips.LookupIPv4(src) != peer {
-				logInfo.Println(
-					"IPv4 packet with disallowed source address from",
-					peer,
-				)
-				continue
+			if device.allowedips.Lookup(src) != peer {
+				device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer)
+				goto skip
 			}
 
 		case ipv6.Version:
-
-			// strip padding
-
 			if len(elem.packet) < ipv6.HeaderLen {
-				continue
+				goto skip
 			}
-
 			field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2]
 			length := binary.BigEndian.Uint16(field)
 			length += ipv6.HeaderLen
 			if int(length) > len(elem.packet) {
-				continue
+				goto skip
 			}
-
 			elem.packet = elem.packet[:length]
-
-			// verify IPv6 source
-
 			src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]
-			if device.allowedips.LookupIPv6(src) != peer {
-				logInfo.Println(
-					"IPv6 packet with disallowed source address from",
-					peer,
-				)
-				continue
+			if device.allowedips.Lookup(src) != peer {
+				device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer)
+				goto skip
 			}
 
 		default:
-			logInfo.Println("Packet with invalid IP version from", peer)
-			continue
+			device.log.Verbosef("Packet with invalid IP version from %v", peer)
+			goto skip
 		}
 
-		// write to tun device
-
-		offset := MessageTransportOffsetContent
-		_, err := device.tun.device.Write(elem.buffer[:offset+len(elem.packet)], offset)
-		if len(peer.queue.inbound) == 0 {
+		_, err = device.tun.device.Write(elem.buffer[:MessageTransportOffsetContent+len(elem.packet)], MessageTransportOffsetContent)
+		if err != nil && !device.isClosed() {
+			device.log.Errorf("Failed to write packet to TUN device: %v", err)
+		}
+		if len(peer.queue.inbound.c) == 0 {
 			err = device.tun.device.Flush()
 			if err != nil {
-				peer.device.log.Error.Printf("Unable to flush packets: %v", err)
+				peer.device.log.Errorf("Unable to flush packets: %v", err)
 			}
 		}
-		if err != nil && !device.isClosed.Get() {
-			logError.Println("Failed to write packet to TUN device:", err)
-		}
+	skip:
+		device.PutMessageBuffer(elem.buffer)
+		device.PutInboundElement(elem)
 	}
 }

device/send.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -8,7 +8,9 @@ package device
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"net"
+	"os"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -43,7 +45,6 @@ import (
  */
 
 type QueueOutboundElement struct {
-	dropped int32
 	sync.Mutex
 	buffer  *[MaxMessageSize]byte // slice holding the packet data
 	packet  []byte                // slice of "buffer" (always!)
@@ -54,73 +55,38 @@ type QueueOutboundElement struct {
 
 func (device *Device) NewOutboundElement() *QueueOutboundElement {
 	elem := device.GetOutboundElement()
-	elem.dropped = AtomicFalse
 	elem.buffer = device.GetMessageBuffer()
 	elem.Mutex = sync.Mutex{}
 	elem.nonce = 0
-	elem.keypair = nil
-	elem.peer = nil
+	// keypair and peer were cleared (if necessary) by clearPointers.
 	return elem
 }
 
-func (elem *QueueOutboundElement) Drop() {
-	atomic.StoreInt32(&elem.dropped, AtomicTrue)
-}
-
-func (elem *QueueOutboundElement) IsDropped() bool {
-	return atomic.LoadInt32(&elem.dropped) == AtomicTrue
-}
-
-func addToNonceQueue(queue chan *QueueOutboundElement, element *QueueOutboundElement, device *Device) {
-	for {
-		select {
-		case queue <- element:
-			return
-		default:
-			select {
-			case old := <-queue:
-				device.PutMessageBuffer(old.buffer)
-				device.PutOutboundElement(old)
-			default:
-			}
-		}
-	}
-}
-
-func addToOutboundAndEncryptionQueues(outboundQueue chan *QueueOutboundElement, encryptionQueue chan *QueueOutboundElement, element *QueueOutboundElement) {
-	select {
-	case outboundQueue <- element:
-		select {
-		case encryptionQueue <- element:
-			return
-		default:
-			element.Drop()
-			element.peer.device.PutMessageBuffer(element.buffer)
-			element.Unlock()
-		}
-	default:
-		element.peer.device.PutMessageBuffer(element.buffer)
-		element.peer.device.PutOutboundElement(element)
-	}
+// clearPointers clears elem fields that contain pointers.
+// This makes the garbage collector's life easier and
+// avoids accidentally keeping other objects around unnecessarily.
+// It also reduces the possible collateral damage from use-after-free bugs.
+func (elem *QueueOutboundElement) clearPointers() {
+	elem.buffer = nil
+	elem.packet = nil
+	elem.keypair = nil
+	elem.peer = nil
 }
 
 /* Queues a keepalive if no packets are queued for peer
  */
-func (peer *Peer) SendKeepalive() bool {
-	if len(peer.queue.nonce) != 0 || peer.queue.packetInNonceQueueIsAwaitingKey.Get() || !peer.isRunning.Get() {
-		return false
-	}
-	elem := peer.device.NewOutboundElement()
-	elem.packet = nil
-	select {
-	case peer.queue.nonce <- elem:
-		peer.device.log.Debug.Println(peer, "- Sending keepalive packet")
-		return true
-	default:
-		peer.device.PutMessageBuffer(elem.buffer)
-		peer.device.PutOutboundElement(elem)
-		return false
+func (peer *Peer) SendKeepalive() {
+	if len(peer.queue.staged) == 0 && peer.isRunning.Get() {
+		elem := peer.device.NewOutboundElement()
+		select {
+		case peer.queue.staged <- elem:
+			peer.device.log.Verbosef("%v - Sending keepalive packet", peer)
+		default:
+			peer.device.PutMessageBuffer(elem.buffer)
+			peer.device.PutOutboundElement(elem)
+		}
 	}
+	peer.SendStagedPackets()
 }
 
 func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
@@ -143,11 +109,11 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 	peer.handshake.lastSentHandshake = time.Now()
 	peer.handshake.mutex.Unlock()
 
-	peer.device.log.Debug.Println(peer, "- Sending handshake initiation")
+	peer.device.log.Verbosef("%v - Sending handshake initiation", peer)
 
 	msg, err := peer.device.CreateMessageInitiation(peer)
 	if err != nil {
-		peer.device.log.Error.Println(peer, "- Failed to create initiation message:", err)
+		peer.device.log.Errorf("%v - Failed to create initiation message: %v", peer, err)
 		return err
 	}
 
@@ -162,7 +128,7 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 
 	err = peer.SendBuffer(packet)
 	if err != nil {
-		peer.device.log.Error.Println(peer, "- Failed to send handshake initiation", err)
+		peer.device.log.Errorf("%v - Failed to send handshake initiation: %v", peer, err)
 	}
 	peer.timersHandshakeInitiated()
 
@@ -174,11 +140,11 @@ func (peer *Peer) SendHandshakeResponse() error {
 	peer.handshake.lastSentHandshake = time.Now()
 	peer.handshake.mutex.Unlock()
 
-	peer.device.log.Debug.Println(peer, "- Sending handshake response")
+	peer.device.log.Verbosef("%v - Sending handshake response", peer)
 
 	response, err := peer.device.CreateMessageResponse(peer)
 	if err != nil {
-		peer.device.log.Error.Println(peer, "- Failed to create response message:", err)
+		peer.device.log.Errorf("%v - Failed to create response message: %v", peer, err)
 		return err
 	}
 
@@ -190,7 +156,7 @@ func (peer *Peer) SendHandshakeResponse() error {
 
 	err = peer.BeginSymmetricSession()
 	if err != nil {
-		peer.device.log.Error.Println(peer, "- Failed to derive keypair:", err)
+		peer.device.log.Errorf("%v - Failed to derive keypair: %v", peer, err)
 		return err
 	}
 
@@ -200,19 +166,18 @@ func (peer *Peer) SendHandshakeResponse() error {
 
 	err = peer.SendBuffer(packet)
 	if err != nil {
-		peer.device.log.Error.Println(peer, "- Failed to send handshake response", err)
+		peer.device.log.Errorf("%v - Failed to send handshake response: %v", peer, err)
 	}
 	return err
 }
 
 func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement) error {
-
-	device.log.Debug.Println("Sending cookie response for denied handshake message for", initiatingElem.endpoint.DstToString())
+	device.log.Verbosef("Sending cookie response for denied handshake message for %v", initiatingElem.endpoint.DstToString())
 
 	sender := binary.LittleEndian.Uint32(initiatingElem.packet[4:8])
 	reply, err := device.cookieChecker.CreateReply(initiatingElem.packet, sender, initiatingElem.endpoint.DstToBytes())
 	if err != nil {
-		device.log.Error.Println("Failed to create cookie reply:", err)
+		device.log.Errorf("Failed to create cookie reply: %v", err)
 		return err
 	}
 
@@ -220,10 +185,7 @@ func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement)
 	writer := bytes.NewBuffer(buff[:0])
 	binary.Write(writer, binary.LittleEndian, reply)
 	device.net.bind.Send(writer.Bytes(), initiatingElem.endpoint)
-	if err != nil {
-		device.log.Error.Println("Failed to send cookie reply:", err)
-	}
-	return err
+	return nil
 }
 
 func (peer *Peer) keepKeyFreshSending() {
@@ -238,22 +200,18 @@ func (peer *Peer) keepKeyFreshSending() {
 }
 
 /* Reads packets from the TUN and inserts
- * into nonce queue for peer
+ * into staged queue for peer
  *
  * Obs. Single instance per TUN device
  */
 func (device *Device) RoutineReadFromTUN() {
-
-	logDebug := device.log.Debug
-	logError := device.log.Error
-
 	defer func() {
-		logDebug.Println("Routine: TUN reader - stopped")
+		device.log.Verbosef("Routine: TUN reader - stopped")
 		device.state.stopping.Done()
+		device.queue.encryption.wg.Done()
 	}()
 
-	logDebug.Println("Routine: TUN reader - started")
-	device.state.starting.Done()
+	device.log.Verbosef("Routine: TUN reader - started")
 
 	var elem *QueueOutboundElement
 
@@ -270,9 +228,11 @@ func (device *Device) RoutineReadFromTUN() {
 		size, err := device.tun.device.Read(elem.buffer[:], offset)
 
 		if err != nil {
-			if !device.isClosed.Get() {
-				logError.Println("Failed to read packet from TUN device:", err)
-				device.Close()
+			if !device.isClosed() {
+				if !errors.Is(err, os.ErrClosed) {
+					device.log.Errorf("Failed to read packet from TUN device: %v", err)
+				}
+				go device.Close()
 			}
 			device.PutMessageBuffer(elem.buffer)
 			device.PutOutboundElement(elem)
@@ -294,250 +254,151 @@ func (device *Device) RoutineReadFromTUN() {
 				continue
 			}
 			dst := elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]
-			peer = device.allowedips.LookupIPv4(dst)
+			peer = device.allowedips.Lookup(dst)
 
 		case ipv6.Version:
 			if len(elem.packet) < ipv6.HeaderLen {
 				continue
 			}
 			dst := elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]
-			peer = device.allowedips.LookupIPv6(dst)
+			peer = device.allowedips.Lookup(dst)
 
 		default:
-			logDebug.Println("Received packet with unknown IP version")
+			device.log.Verbosef("Received packet with unknown IP version")
 		}
 
 		if peer == nil {
 			continue
 		}
-
-		// insert into nonce/pre-handshake queue
-
 		if peer.isRunning.Get() {
-			if peer.queue.packetInNonceQueueIsAwaitingKey.Get() {
-				peer.SendHandshakeInitiation(false)
-			}
-			addToNonceQueue(peer.queue.nonce, elem, device)
+			peer.StagePacket(elem)
 			elem = nil
+			peer.SendStagedPackets()
 		}
 	}
 }
 
-func (peer *Peer) FlushNonceQueue() {
-	select {
-	case peer.signals.flushNonceQueue <- struct{}{}:
-	default:
+func (peer *Peer) StagePacket(elem *QueueOutboundElement) {
+	for {
+		select {
+		case peer.queue.staged <- elem:
+			return
+		default:
+		}
+		select {
+		case tooOld := <-peer.queue.staged:
+			peer.device.PutMessageBuffer(tooOld.buffer)
+			peer.device.PutOutboundElement(tooOld)
+		default:
+		}
 	}
 }
 
-/* Queues packets when there is no handshake.
- * Then assigns nonces to packets sequentially
- * and creates "work" structs for workers
- *
- * Obs. A single instance per peer
- */
-func (peer *Peer) RoutineNonce() {
-	var keypair *Keypair
-
-	device := peer.device
-	logDebug := device.log.Debug
-
-	flush := func() {
-		for {
-			select {
-			case elem := <-peer.queue.nonce:
-				device.PutMessageBuffer(elem.buffer)
-				device.PutOutboundElement(elem)
-			default:
-				return
-			}
-		}
+func (peer *Peer) SendStagedPackets() {
+top:
+	if len(peer.queue.staged) == 0 || !peer.device.isUp() {
+		return
 	}
 
-	defer func() {
-		flush()
-		logDebug.Println(peer, "- Routine: nonce worker - stopped")
-		peer.queue.packetInNonceQueueIsAwaitingKey.Set(false)
-		peer.routines.stopping.Done()
-	}()
-
-	peer.routines.starting.Done()
-	logDebug.Println(peer, "- Routine: nonce worker - started")
+	keypair := peer.keypairs.Current()
+	if keypair == nil || atomic.LoadUint64(&keypair.sendNonce) >= RejectAfterMessages || time.Since(keypair.created) >= RejectAfterTime {
+		peer.SendHandshakeInitiation(false)
+		return
+	}
 
 	for {
-	NextPacket:
-		peer.queue.packetInNonceQueueIsAwaitingKey.Set(false)
-
 		select {
-		case <-peer.routines.stop:
-			return
-
-		case <-peer.signals.flushNonceQueue:
-			flush()
-			goto NextPacket
-
-		case elem, ok := <-peer.queue.nonce:
-
-			if !ok {
-				return
-			}
-
-			// make sure to always pick the newest key
-
-			for {
-
-				// check validity of newest key pair
-
-				keypair = peer.keypairs.Current()
-				if keypair != nil && keypair.sendNonce < RejectAfterMessages {
-					if time.Since(keypair.created) < RejectAfterTime {
-						break
-					}
-				}
-				peer.queue.packetInNonceQueueIsAwaitingKey.Set(true)
-
-				// no suitable key pair, request for new handshake
-
-				select {
-				case <-peer.signals.newKeypairArrived:
-				default:
-				}
-
-				peer.SendHandshakeInitiation(false)
-
-				// wait for key to be established
-
-				logDebug.Println(peer, "- Awaiting keypair")
-
-				select {
-				case <-peer.signals.newKeypairArrived:
-					logDebug.Println(peer, "- Obtained awaited keypair")
-
-				case <-peer.signals.flushNonceQueue:
-					device.PutMessageBuffer(elem.buffer)
-					device.PutOutboundElement(elem)
-					flush()
-					goto NextPacket
-
-				case <-peer.routines.stop:
-					device.PutMessageBuffer(elem.buffer)
-					device.PutOutboundElement(elem)
-					return
-				}
-			}
-			peer.queue.packetInNonceQueueIsAwaitingKey.Set(false)
-
-			// populate work element
-
+		case elem := <-peer.queue.staged:
 			elem.peer = peer
 			elem.nonce = atomic.AddUint64(&keypair.sendNonce, 1) - 1
-
-			// double check in case of race condition added by future code
-
 			if elem.nonce >= RejectAfterMessages {
 				atomic.StoreUint64(&keypair.sendNonce, RejectAfterMessages)
-				device.PutMessageBuffer(elem.buffer)
-				device.PutOutboundElement(elem)
-				goto NextPacket
+				peer.StagePacket(elem) // XXX: Out of order, but we can't front-load go chans
+				goto top
 			}
 
 			elem.keypair = keypair
-			elem.dropped = AtomicFalse
 			elem.Lock()
 
 			// add to parallel and sequential queue
-			addToOutboundAndEncryptionQueues(peer.queue.outbound, device.queue.encryption, elem)
+			if peer.isRunning.Get() {
+				peer.queue.outbound.c <- elem
+				peer.device.queue.encryption.c <- elem
+			} else {
+				peer.device.PutMessageBuffer(elem.buffer)
+				peer.device.PutOutboundElement(elem)
+			}
+		default:
+			return
 		}
 	}
 }
 
+func (peer *Peer) FlushStagedPackets() {
+	for {
+		select {
+		case elem := <-peer.queue.staged:
+			peer.device.PutMessageBuffer(elem.buffer)
+			peer.device.PutOutboundElement(elem)
+		default:
+			return
+		}
+	}
+}
+
+func calculatePaddingSize(packetSize, mtu int) int {
+	lastUnit := packetSize
+	if mtu == 0 {
+		return ((lastUnit + PaddingMultiple - 1) & ^(PaddingMultiple - 1)) - lastUnit
+	}
+	if lastUnit > mtu {
+		lastUnit %= mtu
+	}
+	paddedSize := ((lastUnit + PaddingMultiple - 1) & ^(PaddingMultiple - 1))
+	if paddedSize > mtu {
+		paddedSize = mtu
+	}
+	return paddedSize - lastUnit
+}
+
 /* Encrypts the elements in the queue
  * and marks them for sequential consumption (by releasing the mutex)
  *
  * Obs. One instance per core
  */
-func (device *Device) RoutineEncryption() {
-
+func (device *Device) RoutineEncryption(id int) {
+	var paddingZeros [PaddingMultiple]byte
 	var nonce [chacha20poly1305.NonceSize]byte
 
-	logDebug := device.log.Debug
+	defer device.log.Verbosef("Routine: encryption worker %d - stopped", id)
+	device.log.Verbosef("Routine: encryption worker %d - started", id)
 
-	defer func() {
-		for {
-			select {
-			case elem, ok := <-device.queue.encryption:
-				if ok && !elem.IsDropped() {
-					elem.Drop()
-					device.PutMessageBuffer(elem.buffer)
-					elem.Unlock()
-				}
-			default:
-				goto out
-			}
-		}
-	out:
-		logDebug.Println("Routine: encryption worker - stopped")
-		device.state.stopping.Done()
-	}()
+	for elem := range device.queue.encryption.c {
+		// populate header fields
+		header := elem.buffer[:MessageTransportHeaderSize]
 
-	logDebug.Println("Routine: encryption worker - started")
-	device.state.starting.Done()
+		fieldType := header[0:4]
+		fieldReceiver := header[4:8]
+		fieldNonce := header[8:16]
 
-	for {
+		binary.LittleEndian.PutUint32(fieldType, MessageTransportType)
+		binary.LittleEndian.PutUint32(fieldReceiver, elem.keypair.remoteIndex)
+		binary.LittleEndian.PutUint64(fieldNonce, elem.nonce)
 
-		// fetch next element
+		// pad content to multiple of 16
+		paddingSize := calculatePaddingSize(len(elem.packet), int(atomic.LoadInt32(&device.tun.mtu)))
+		elem.packet = append(elem.packet, paddingZeros[:paddingSize]...)
 
-		select {
-		case <-device.signals.stop:
-			return
+		// encrypt content and release to consumer
 
-		case elem, ok := <-device.queue.encryption:
-
-			if !ok {
-				return
-			}
-
-			// check if dropped
-
-			if elem.IsDropped() {
-				continue
-			}
-
-			// populate header fields
-
-			header := elem.buffer[:MessageTransportHeaderSize]
-
-			fieldType := header[0:4]
-			fieldReceiver := header[4:8]
-			fieldNonce := header[8:16]
-
-			binary.LittleEndian.PutUint32(fieldType, MessageTransportType)
-			binary.LittleEndian.PutUint32(fieldReceiver, elem.keypair.remoteIndex)
-			binary.LittleEndian.PutUint64(fieldNonce, elem.nonce)
-
-			// pad content to multiple of 16
-
-			mtu := int(atomic.LoadInt32(&device.tun.mtu))
-			lastUnit := len(elem.packet) % mtu
-			paddedSize := (lastUnit + PaddingMultiple - 1) & ^(PaddingMultiple - 1)
-			if paddedSize > mtu {
-				paddedSize = mtu
-			}
-			for i := len(elem.packet); i < paddedSize; i++ {
-				elem.packet = append(elem.packet, 0)
-			}
-
-			// encrypt content and release to consumer
-
-			binary.LittleEndian.PutUint64(nonce[4:], elem.nonce)
-			elem.packet = elem.keypair.send.Seal(
-				header,
-				nonce[:],
-				elem.packet,
-				nil,
-			)
-			elem.Unlock()
-		}
+		binary.LittleEndian.PutUint64(nonce[4:], elem.nonce)
+		elem.packet = elem.keypair.send.Seal(
+			header,
+			nonce[:],
+			elem.packet,
+			nil,
+		)
+		elem.Unlock()
 	}
 }
 
@@ -547,71 +408,46 @@ func (device *Device) RoutineEncryption() {
  * The routine terminates then the outbound queue is closed.
  */
 func (peer *Peer) RoutineSequentialSender() {
-
 	device := peer.device
-
-	logDebug := device.log.Debug
-	logError := device.log.Error
-
 	defer func() {
-		for {
-			select {
-			case elem, ok := <-peer.queue.outbound:
-				if ok {
-					if !elem.IsDropped() {
-						device.PutMessageBuffer(elem.buffer)
-						elem.Drop()
-					}
-					device.PutOutboundElement(elem)
-				}
-			default:
-				goto out
-			}
-		}
-	out:
-		logDebug.Println(peer, "- Routine: sequential sender - stopped")
-		peer.routines.stopping.Done()
+		defer device.log.Verbosef("%v - Routine: sequential sender - stopped", peer)
+		peer.stopping.Done()
 	}()
+	device.log.Verbosef("%v - Routine: sequential sender - started", peer)
 
-	logDebug.Println(peer, "- Routine: sequential sender - started")
-
-	peer.routines.starting.Done()
-
-	for {
-		select {
-
-		case <-peer.routines.stop:
+	for elem := range peer.queue.outbound.c {
+		if elem == nil {
 			return
-
-		case elem, ok := <-peer.queue.outbound:
-
-			if !ok {
-				return
-			}
-
-			elem.Lock()
-			if elem.IsDropped() {
-				device.PutOutboundElement(elem)
-				continue
-			}
-
-			peer.timersAnyAuthenticatedPacketTraversal()
-			peer.timersAnyAuthenticatedPacketSent()
-
-			// send message and return buffer to pool
-
-			err := peer.SendBuffer(elem.packet)
-			if len(elem.packet) != MessageKeepaliveSize {
-				peer.timersDataSent()
-			}
+		}
+		elem.Lock()
+		if !peer.isRunning.Get() {
+			// peer has been stopped; return re-usable elems to the shared pool.
+			// This is an optimization only. It is possible for the peer to be stopped
+			// immediately after this check, in which case, elem will get processed.
+			// The timers and SendBuffer code are resilient to a few stragglers.
+			// TODO: rework peer shutdown order to ensure
+			// that we never accidentally keep timers alive longer than necessary.
 			device.PutMessageBuffer(elem.buffer)
 			device.PutOutboundElement(elem)
-			if err != nil {
-				logError.Println(peer, "- Failed to send data packet", err)
-				continue
-			}
-
-			peer.keepKeyFreshSending()
+			continue
 		}
+
+		peer.timersAnyAuthenticatedPacketTraversal()
+		peer.timersAnyAuthenticatedPacketSent()
+
+		// send message and return buffer to pool
+
+		err := peer.SendBuffer(elem.packet)
+		if len(elem.packet) != MessageKeepaliveSize {
+			peer.timersDataSent()
+		}
+		device.PutMessageBuffer(elem.buffer)
+		device.PutOutboundElement(elem)
+		if err != nil {
+			device.log.Errorf("%v - Failed to send data packet: %v", peer, err)
+			continue
+		}
+
+		peer.keepKeyFreshSending()
 	}
 }

device/sticky_default.go

@@ -0,0 +1,12 @@
+//go:build !linux
+
+package device
+
+import (
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/rwcancel"
+)
+
+func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) {
+	return nil, nil
+}

device/sticky_linux.go

@@ -0,0 +1,221 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ *
+ * This implements userspace semantics of "sticky sockets", modeled after
+ * WireGuard's kernelspace implementation. This is more or less a straight port
+ * of the sticky-sockets.c example code:
+ * https://git.zx2c4.com/WireGuard/tree/contrib/examples/sticky-sockets/sticky-sockets.c
+ *
+ * Currently there is no way to achieve this within the net package:
+ * See e.g. https://github.com/golang/go/issues/17930
+ * So this code is remains platform dependent.
+ */
+
+package device
+
+import (
+	"sync"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/rwcancel"
+)
+
+func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) {
+	if _, ok := bind.(*conn.LinuxSocketBind); !ok {
+		return nil, nil
+	}
+
+	netlinkSock, err := createNetlinkRouteSocket()
+	if err != nil {
+		return nil, err
+	}
+	netlinkCancel, err := rwcancel.NewRWCancel(netlinkSock)
+	if err != nil {
+		unix.Close(netlinkSock)
+		return nil, err
+	}
+
+	go device.routineRouteListener(bind, netlinkSock, netlinkCancel)
+
+	return netlinkCancel, nil
+}
+
+func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) {
+	type peerEndpointPtr struct {
+		peer     *Peer
+		endpoint *conn.Endpoint
+	}
+	var reqPeer map[uint32]peerEndpointPtr
+	var reqPeerLock sync.Mutex
+
+	defer netlinkCancel.Close()
+	defer unix.Close(netlinkSock)
+
+	for msg := make([]byte, 1<<16); ; {
+		var err error
+		var msgn int
+		for {
+			msgn, _, _, _, err = unix.Recvmsg(netlinkSock, msg[:], nil, 0)
+			if err == nil || !rwcancel.RetryAfterError(err) {
+				break
+			}
+			if !netlinkCancel.ReadyRead() {
+				return
+			}
+		}
+		if err != nil {
+			return
+		}
+
+		for remain := msg[:msgn]; len(remain) >= unix.SizeofNlMsghdr; {
+
+			hdr := *(*unix.NlMsghdr)(unsafe.Pointer(&remain[0]))
+
+			if uint(hdr.Len) > uint(len(remain)) {
+				break
+			}
+
+			switch hdr.Type {
+			case unix.RTM_NEWROUTE, unix.RTM_DELROUTE:
+				if hdr.Seq <= MaxPeers && hdr.Seq > 0 {
+					if uint(len(remain)) < uint(hdr.Len) {
+						break
+					}
+					if hdr.Len > unix.SizeofNlMsghdr+unix.SizeofRtMsg {
+						attr := remain[unix.SizeofNlMsghdr+unix.SizeofRtMsg:]
+						for {
+							if uint(len(attr)) < uint(unix.SizeofRtAttr) {
+								break
+							}
+							attrhdr := *(*unix.RtAttr)(unsafe.Pointer(&attr[0]))
+							if attrhdr.Len < unix.SizeofRtAttr || uint(len(attr)) < uint(attrhdr.Len) {
+								break
+							}
+							if attrhdr.Type == unix.RTA_OIF && attrhdr.Len == unix.SizeofRtAttr+4 {
+								ifidx := *(*uint32)(unsafe.Pointer(&attr[unix.SizeofRtAttr]))
+								reqPeerLock.Lock()
+								if reqPeer == nil {
+									reqPeerLock.Unlock()
+									break
+								}
+								pePtr, ok := reqPeer[hdr.Seq]
+								reqPeerLock.Unlock()
+								if !ok {
+									break
+								}
+								pePtr.peer.Lock()
+								if &pePtr.peer.endpoint != pePtr.endpoint {
+									pePtr.peer.Unlock()
+									break
+								}
+								if uint32(pePtr.peer.endpoint.(*conn.LinuxSocketEndpoint).Src4().Ifindex) == ifidx {
+									pePtr.peer.Unlock()
+									break
+								}
+								pePtr.peer.endpoint.(*conn.LinuxSocketEndpoint).ClearSrc()
+								pePtr.peer.Unlock()
+							}
+							attr = attr[attrhdr.Len:]
+						}
+					}
+					break
+				}
+				reqPeerLock.Lock()
+				reqPeer = make(map[uint32]peerEndpointPtr)
+				reqPeerLock.Unlock()
+				go func() {
+					device.peers.RLock()
+					i := uint32(1)
+					for _, peer := range device.peers.keyMap {
+						peer.RLock()
+						if peer.endpoint == nil {
+							peer.RUnlock()
+							continue
+						}
+						nativeEP, _ := peer.endpoint.(*conn.LinuxSocketEndpoint)
+						if nativeEP == nil {
+							peer.RUnlock()
+							continue
+						}
+						if nativeEP.IsV6() || nativeEP.Src4().Ifindex == 0 {
+							peer.RUnlock()
+							break
+						}
+						nlmsg := struct {
+							hdr     unix.NlMsghdr
+							msg     unix.RtMsg
+							dsthdr  unix.RtAttr
+							dst     [4]byte
+							srchdr  unix.RtAttr
+							src     [4]byte
+							markhdr unix.RtAttr
+							mark    uint32
+						}{
+							unix.NlMsghdr{
+								Type:  uint16(unix.RTM_GETROUTE),
+								Flags: unix.NLM_F_REQUEST,
+								Seq:   i,
+							},
+							unix.RtMsg{
+								Family:  unix.AF_INET,
+								Dst_len: 32,
+								Src_len: 32,
+							},
+							unix.RtAttr{
+								Len:  8,
+								Type: unix.RTA_DST,
+							},
+							nativeEP.Dst4().Addr,
+							unix.RtAttr{
+								Len:  8,
+								Type: unix.RTA_SRC,
+							},
+							nativeEP.Src4().Src,
+							unix.RtAttr{
+								Len:  8,
+								Type: unix.RTA_MARK,
+							},
+							device.net.fwmark,
+						}
+						nlmsg.hdr.Len = uint32(unsafe.Sizeof(nlmsg))
+						reqPeerLock.Lock()
+						reqPeer[i] = peerEndpointPtr{
+							peer:     peer,
+							endpoint: &peer.endpoint,
+						}
+						reqPeerLock.Unlock()
+						peer.RUnlock()
+						i++
+						_, err := netlinkCancel.Write((*[unsafe.Sizeof(nlmsg)]byte)(unsafe.Pointer(&nlmsg))[:])
+						if err != nil {
+							break
+						}
+					}
+					device.peers.RUnlock()
+				}()
+			}
+			remain = remain[hdr.Len:]
+		}
+	}
+}
+
+func createNetlinkRouteSocket() (int, error) {
+	sock, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_ROUTE)
+	if err != nil {
+		return -1, err
+	}
+	saddr := &unix.SockaddrNetlink{
+		Family: unix.AF_NETLINK,
+		Groups: unix.RTMGRP_IPV4_ROUTE,
+	}
+	err = unix.Bind(sock, saddr)
+	if err != nil {
+		unix.Close(sock)
+		return -1, err
+	}
+	return sock, nil
+}

device/timers.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  *
  * This is based heavily on timers.c from the kernel implementation.
  */
@@ -14,10 +14,8 @@ import (
 	"time"
 )
 
-/* This Timer structure and related functions should roughly copy the interface of
- * the Linux kernel's struct timer_list.
- */
-
+// A Timer manages time-based aspects of the WireGuard protocol.
+// Timer roughly copies the interface of the Linux kernel's struct timer_list.
 type Timer struct {
 	*time.Timer
 	modifyingLock sync.RWMutex
@@ -29,18 +27,17 @@ func (peer *Peer) NewTimer(expirationFunction func(*Peer)) *Timer {
 	timer := &Timer{}
 	timer.Timer = time.AfterFunc(time.Hour, func() {
 		timer.runningLock.Lock()
+		defer timer.runningLock.Unlock()
 
 		timer.modifyingLock.Lock()
 		if !timer.isPending {
 			timer.modifyingLock.Unlock()
-			timer.runningLock.Unlock()
 			return
 		}
 		timer.isPending = false
 		timer.modifyingLock.Unlock()
 
 		expirationFunction(peer)
-		timer.runningLock.Unlock()
 	})
 	timer.Stop()
 	return timer
@@ -74,12 +71,12 @@ func (timer *Timer) IsPending() bool {
 }
 
 func (peer *Peer) timersActive() bool {
-	return peer.isRunning.Get() && peer.device != nil && peer.device.isUp.Get() && len(peer.device.peers.keyMap) > 0
+	return peer.isRunning.Get() && peer.device != nil && peer.device.isUp()
 }
 
 func expiredRetransmitHandshake(peer *Peer) {
 	if atomic.LoadUint32(&peer.timers.handshakeAttempts) > MaxTimerHandshakes {
-		peer.device.log.Debug.Printf("%s - Handshake did not complete after %d attempts, giving up\n", peer, MaxTimerHandshakes+2)
+		peer.device.log.Verbosef("%s - Handshake did not complete after %d attempts, giving up", peer, MaxTimerHandshakes+2)
 
 		if peer.timersActive() {
 			peer.timers.sendKeepalive.Del()
@@ -88,7 +85,7 @@ func expiredRetransmitHandshake(peer *Peer) {
 		/* We drop all packets without a keypair and don't try again,
 		 * if we try unsuccessfully for too long to make a handshake.
 		 */
-		peer.FlushNonceQueue()
+		peer.FlushStagedPackets()
 
 		/* We set a timer for destroying any residue that might be left
 		 * of a partial exchange.
@@ -98,7 +95,7 @@ func expiredRetransmitHandshake(peer *Peer) {
 		}
 	} else {
 		atomic.AddUint32(&peer.timers.handshakeAttempts, 1)
-		peer.device.log.Debug.Printf("%s - Handshake did not complete after %d seconds, retrying (try %d)\n", peer, int(RekeyTimeout.Seconds()), atomic.LoadUint32(&peer.timers.handshakeAttempts)+1)
+		peer.device.log.Verbosef("%s - Handshake did not complete after %d seconds, retrying (try %d)", peer, int(RekeyTimeout.Seconds()), atomic.LoadUint32(&peer.timers.handshakeAttempts)+1)
 
 		/* We clear the endpoint address src address, in case this is the cause of trouble. */
 		peer.Lock()
@@ -122,7 +119,7 @@ func expiredSendKeepalive(peer *Peer) {
 }
 
 func expiredNewHandshake(peer *Peer) {
-	peer.device.log.Debug.Printf("%s - Retrying handshake because we stopped hearing back after %d seconds\n", peer, int((KeepaliveTimeout + RekeyTimeout).Seconds()))
+	peer.device.log.Verbosef("%s - Retrying handshake because we stopped hearing back after %d seconds", peer, int((KeepaliveTimeout + RekeyTimeout).Seconds()))
 	/* We clear the endpoint address src address, in case this is the cause of trouble. */
 	peer.Lock()
 	if peer.endpoint != nil {
@@ -134,12 +131,12 @@ func expiredNewHandshake(peer *Peer) {
 }
 
 func expiredZeroKeyMaterial(peer *Peer) {
-	peer.device.log.Debug.Printf("%s - Removing all keys, since we haven't received a new one in %d seconds\n", peer, int((RejectAfterTime * 3).Seconds()))
+	peer.device.log.Verbosef("%s - Removing all keys, since we haven't received a new one in %d seconds", peer, int((RejectAfterTime * 3).Seconds()))
 	peer.ZeroAndFlushAll()
 }
 
 func expiredPersistentKeepalive(peer *Peer) {
-	if peer.persistentKeepaliveInterval > 0 {
+	if atomic.LoadUint32(&peer.persistentKeepaliveInterval) > 0 {
 		peer.SendKeepalive()
 	}
 }
@@ -202,8 +199,9 @@ func (peer *Peer) timersSessionDerived() {
 
 /* Should be called before a packet with authentication -- keepalive, data, or handshake -- is sent, or after one is received. */
 func (peer *Peer) timersAnyAuthenticatedPacketTraversal() {
-	if peer.persistentKeepaliveInterval > 0 && peer.timersActive() {
-		peer.timers.persistentKeepalive.Mod(time.Duration(peer.persistentKeepaliveInterval) * time.Second)
+	keepalive := atomic.LoadUint32(&peer.persistentKeepaliveInterval)
+	if keepalive > 0 && peer.timersActive() {
+		peer.timers.persistentKeepalive.Mod(time.Duration(keepalive) * time.Second)
 	}
 }
 
@@ -213,6 +211,9 @@ func (peer *Peer) timersInit() {
 	peer.timers.newHandshake = peer.NewTimer(expiredNewHandshake)
 	peer.timers.zeroKeyMaterial = peer.NewTimer(expiredZeroKeyMaterial)
 	peer.timers.persistentKeepalive = peer.NewTimer(expiredPersistentKeepalive)
+}
+
+func (peer *Peer) timersStart() {
 	atomic.StoreUint32(&peer.timers.handshakeAttempts, 0)
 	peer.timers.sentLastMinuteHandshake.Set(false)
 	peer.timers.needAnotherKeepalive.Set(false)

device/tun.go

@@ -1,11 +1,12 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
+	"fmt"
 	"sync/atomic"
 
 	"golang.zx2c4.com/wireguard/tun"
@@ -14,43 +15,40 @@ import (
 const DefaultMTU = 1420
 
 func (device *Device) RoutineTUNEventReader() {
-	setUp := false
-	logDebug := device.log.Debug
-	logInfo := device.log.Info
-	logError := device.log.Error
-
-	logDebug.Println("Routine: event worker - started")
-	device.state.starting.Done()
+	device.log.Verbosef("Routine: event worker - started")
 
 	for event := range device.tun.device.Events() {
 		if event&tun.EventMTUUpdate != 0 {
 			mtu, err := device.tun.device.MTU()
-			old := atomic.LoadInt32(&device.tun.mtu)
 			if err != nil {
-				logError.Println("Failed to load updated MTU of device:", err)
-			} else if int(old) != mtu {
-				if mtu+MessageTransportSize > MaxMessageSize {
-					logInfo.Println("MTU updated:", mtu, "(too large)")
-				} else {
-					logInfo.Println("MTU updated:", mtu)
-				}
-				atomic.StoreInt32(&device.tun.mtu, int32(mtu))
+				device.log.Errorf("Failed to load updated MTU of device: %v", err)
+				continue
+			}
+			if mtu < 0 {
+				device.log.Errorf("MTU not updated to negative value: %v", mtu)
+				continue
+			}
+			var tooLarge string
+			if mtu > MaxContentSize {
+				tooLarge = fmt.Sprintf(" (too large, capped at %v)", MaxContentSize)
+				mtu = MaxContentSize
+			}
+			old := atomic.SwapInt32(&device.tun.mtu, int32(mtu))
+			if int(old) != mtu {
+				device.log.Verbosef("MTU updated: %v%s", mtu, tooLarge)
 			}
 		}
 
-		if event&tun.EventUp != 0 && !setUp {
-			logInfo.Println("Interface set up")
-			setUp = true
+		if event&tun.EventUp != 0 {
+			device.log.Verbosef("Interface up requested")
 			device.Up()
 		}
 
-		if event&tun.EventDown != 0 && setUp {
-			logInfo.Println("Interface set down")
-			setUp = false
+		if event&tun.EventDown != 0 {
+			device.log.Verbosef("Interface down requested")
 			device.Down()
 		}
 	}
 
-	logDebug.Println("Routine: event worker - stopped")
-	device.state.stopping.Done()
+	device.log.Verbosef("Routine: event worker - stopped")
 }

device/tun_test.go

@@ -1,56 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-import (
-	"errors"
-	"os"
-
-	"golang.zx2c4.com/wireguard/tun"
-)
-
-// newDummyTUN creates a dummy TUN device with the specified name.
-func newDummyTUN(name string) tun.Device {
-	return &dummyTUN{
-		name:    name,
-		packets: make(chan []byte, 100),
-		events:  make(chan tun.Event, 10),
-	}
-}
-
-// A dummyTUN is a tun.Device which is used in unit tests.
-type dummyTUN struct {
-	name    string
-	mtu     int
-	packets chan []byte
-	events  chan tun.Event
-}
-
-func (d *dummyTUN) Events() chan tun.Event { return d.events }
-func (*dummyTUN) File() *os.File           { return nil }
-func (*dummyTUN) Flush() error             { return nil }
-func (d *dummyTUN) MTU() (int, error)      { return d.mtu, nil }
-func (d *dummyTUN) Name() (string, error)  { return d.name, nil }
-
-func (d *dummyTUN) Close() error {
-	close(d.events)
-	close(d.packets)
-	return nil
-}
-
-func (d *dummyTUN) Read(b []byte, offset int) (int, error) {
-	buf, ok := <-d.packets
-	if !ok {
-		return 0, errors.New("device closed")
-	}
-	copy(b[offset:], buf)
-	return len(buf), nil
-}
-
-func (d *dummyTUN) Write(b []byte, offset int) (int, error) {
-	d.packets <- b[offset:]
-	return len(b), nil
-}

device/uapi.go

@@ -1,17 +1,20 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package device
 
 import (
 	"bufio"
+	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -19,21 +22,53 @@ import (
 )
 
 type IPCError struct {
-	int64
+	code int64 // error code
+	err  error // underlying/wrapped error
 }
 
 func (s IPCError) Error() string {
-	return fmt.Sprintf("IPC error: %d", s.int64)
+	return fmt.Sprintf("IPC error %d: %v", s.code, s.err)
+}
+
+func (s IPCError) Unwrap() error {
+	return s.err
 }
 
 func (s IPCError) ErrorCode() int64 {
-	return s.int64
+	return s.code
 }
 
-func (device *Device) IpcGetOperation(socket *bufio.Writer) *IPCError {
-	lines := make([]string, 0, 100)
-	send := func(line string) {
-		lines = append(lines, line)
+func ipcErrorf(code int64, msg string, args ...interface{}) *IPCError {
+	return &IPCError{code: code, err: fmt.Errorf(msg, args...)}
+}
+
+var byteBufferPool = &sync.Pool{
+	New: func() interface{} { return new(bytes.Buffer) },
+}
+
+// IpcGetOperation implements the WireGuard configuration protocol "get" operation.
+// See https://www.wireguard.com/xplatform/#configuration-protocol for details.
+func (device *Device) IpcGetOperation(w io.Writer) error {
+	device.ipcMutex.RLock()
+	defer device.ipcMutex.RUnlock()
+
+	buf := byteBufferPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer byteBufferPool.Put(buf)
+	sendf := func(format string, args ...interface{}) {
+		fmt.Fprintf(buf, format, args...)
+		buf.WriteByte('\n')
+	}
+	keyf := func(prefix string, key *[32]byte) {
+		buf.Grow(len(key)*2 + 2 + len(prefix))
+		buf.WriteString(prefix)
+		buf.WriteByte('=')
+		const hex = "0123456789abcdef"
+		for i := 0; i < len(key); i++ {
+			buf.WriteByte(hex[key[i]>>4])
+			buf.WriteByte(hex[key[i]&0xf])
+		}
+		buf.WriteByte('\n')
 	}
 
 	func() {
@@ -52,15 +87,15 @@ func (device *Device) IpcGetOperation(socket *bufio.Writer) *IPCError {
 		// serialize device related values
 
 		if !device.staticIdentity.privateKey.IsZero() {
-			send("private_key=" + device.staticIdentity.privateKey.ToHex())
+			keyf("private_key", (*[32]byte)(&device.staticIdentity.privateKey))
 		}
 
 		if device.net.port != 0 {
-			send(fmt.Sprintf("listen_port=%d", device.net.port))
+			sendf("listen_port=%d", device.net.port)
 		}
 
 		if device.net.fwmark != 0 {
-			send(fmt.Sprintf("fwmark=%d", device.net.fwmark))
+			sendf("fwmark=%d", device.net.fwmark)
 		}
 
 		// serialize each peer state
@@ -69,320 +104,308 @@ func (device *Device) IpcGetOperation(socket *bufio.Writer) *IPCError {
 			peer.RLock()
 			defer peer.RUnlock()
 
-			send("public_key=" + peer.handshake.remoteStatic.ToHex())
-			send("preshared_key=" + peer.handshake.presharedKey.ToHex())
-			send("protocol_version=1")
+			keyf("public_key", (*[32]byte)(&peer.handshake.remoteStatic))
+			keyf("preshared_key", (*[32]byte)(&peer.handshake.presharedKey))
+			sendf("protocol_version=1")
 			if peer.endpoint != nil {
-				send("endpoint=" + peer.endpoint.DstToString())
+				sendf("endpoint=%s", peer.endpoint.DstToString())
 			}
 
 			nano := atomic.LoadInt64(&peer.stats.lastHandshakeNano)
 			secs := nano / time.Second.Nanoseconds()
 			nano %= time.Second.Nanoseconds()
 
-			send(fmt.Sprintf("last_handshake_time_sec=%d", secs))
-			send(fmt.Sprintf("last_handshake_time_nsec=%d", nano))
-			send(fmt.Sprintf("tx_bytes=%d", atomic.LoadUint64(&peer.stats.txBytes)))
-			send(fmt.Sprintf("rx_bytes=%d", atomic.LoadUint64(&peer.stats.rxBytes)))
-			send(fmt.Sprintf("persistent_keepalive_interval=%d", peer.persistentKeepaliveInterval))
-
-			for _, ip := range device.allowedips.EntriesForPeer(peer) {
-				send("allowed_ip=" + ip.String())
-			}
+			sendf("last_handshake_time_sec=%d", secs)
+			sendf("last_handshake_time_nsec=%d", nano)
+			sendf("tx_bytes=%d", atomic.LoadUint64(&peer.stats.txBytes))
+			sendf("rx_bytes=%d", atomic.LoadUint64(&peer.stats.rxBytes))
+			sendf("persistent_keepalive_interval=%d", atomic.LoadUint32(&peer.persistentKeepaliveInterval))
 
+			device.allowedips.EntriesForPeer(peer, func(ip net.IP, cidr uint8) bool {
+				sendf("allowed_ip=%s/%d", ip.String(), cidr)
+				return true
+			})
 		}
 	}()
 
 	// send lines (does not require resource locks)
-
-	for _, line := range lines {
-		_, err := socket.WriteString(line + "\n")
-		if err != nil {
-			return &IPCError{ipc.IpcErrorIO}
-		}
+	if _, err := w.Write(buf.Bytes()); err != nil {
+		return ipcErrorf(ipc.IpcErrorIO, "failed to write output: %w", err)
 	}
 
 	return nil
 }
 
-func (device *Device) IpcSetOperation(socket *bufio.Reader) *IPCError {
-	scanner := bufio.NewScanner(socket)
-	logError := device.log.Error
-	logDebug := device.log.Debug
+// IpcSetOperation implements the WireGuard configuration protocol "set" operation.
+// See https://www.wireguard.com/xplatform/#configuration-protocol for details.
+func (device *Device) IpcSetOperation(r io.Reader) (err error) {
+	device.ipcMutex.Lock()
+	defer device.ipcMutex.Unlock()
 
-	var peer *Peer
+	defer func() {
+		if err != nil {
+			device.log.Errorf("%v", err)
+		}
+	}()
 
-	dummy := false
+	peer := new(ipcSetPeer)
 	deviceConfig := true
 
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
-
-		// parse line
-
 		line := scanner.Text()
 		if line == "" {
+			// Blank line means terminate operation.
 			return nil
 		}
 		parts := strings.Split(line, "=")
 		if len(parts) != 2 {
-			return &IPCError{ipc.IpcErrorProtocol}
+			return ipcErrorf(ipc.IpcErrorProtocol, "failed to parse line %q, found %d =-separated parts, want 2", line, len(parts))
 		}
 		key := parts[0]
 		value := parts[1]
 
-		/* device configuration */
-
-		if deviceConfig {
-
-			switch key {
-			case "private_key":
-				var sk NoisePrivateKey
-				err := sk.FromHex(value)
-				if err != nil {
-					logError.Println("Failed to set private_key:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-				logDebug.Println("UAPI: Updating private key")
-				device.SetPrivateKey(sk)
-
-			case "listen_port":
-
-				// parse port number
-
-				port, err := strconv.ParseUint(value, 10, 16)
-				if err != nil {
-					logError.Println("Failed to parse listen_port:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				// update port and rebind
-
-				logDebug.Println("UAPI: Updating listen port")
-
-				device.net.Lock()
-				device.net.port = uint16(port)
-				device.net.Unlock()
-
-				if err := device.BindUpdate(); err != nil {
-					logError.Println("Failed to set listen_port:", err)
-					return &IPCError{ipc.IpcErrorPortInUse}
-				}
-
-			case "fwmark":
-
-				// parse fwmark field
-
-				fwmark, err := func() (uint32, error) {
-					if value == "" {
-						return 0, nil
-					}
-					mark, err := strconv.ParseUint(value, 10, 32)
-					return uint32(mark), err
-				}()
-
-				if err != nil {
-					logError.Println("Invalid fwmark", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				logDebug.Println("UAPI: Updating fwmark")
-
-				if err := device.BindSetMark(uint32(fwmark)); err != nil {
-					logError.Println("Failed to update fwmark:", err)
-					return &IPCError{ipc.IpcErrorPortInUse}
-				}
-
-			case "public_key":
-				// switch to peer configuration
-				logDebug.Println("UAPI: Transition to peer configuration")
+		if key == "public_key" {
+			if deviceConfig {
 				deviceConfig = false
-
-			case "replace_peers":
-				if value != "true" {
-					logError.Println("Failed to set replace_peers, invalid value:", value)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-				logDebug.Println("UAPI: Removing all peers")
-				device.RemoveAllPeers()
-
-			default:
-				logError.Println("Invalid UAPI device key:", key)
-				return &IPCError{ipc.IpcErrorInvalid}
 			}
+			peer.handlePostConfig()
+			// Load/create the peer we are now configuring.
+			err := device.handlePublicKeyLine(peer, value)
+			if err != nil {
+				return err
+			}
+			continue
 		}
 
-		/* peer configuration */
-
-		if !deviceConfig {
-
-			switch key {
-
-			case "public_key":
-				var publicKey NoisePublicKey
-				err := publicKey.FromHex(value)
-				if err != nil {
-					logError.Println("Failed to get peer by public key:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				// ignore peer with public key of device
-
-				device.staticIdentity.RLock()
-				dummy = device.staticIdentity.publicKey.Equals(publicKey)
-				device.staticIdentity.RUnlock()
-
-				if dummy {
-					peer = &Peer{}
-				} else {
-					peer = device.LookupPeer(publicKey)
-				}
-
-				if peer == nil {
-					peer, err = device.NewPeer(publicKey)
-					if err != nil {
-						logError.Println("Failed to create new peer:", err)
-						return &IPCError{ipc.IpcErrorInvalid}
-					}
-					if peer == nil {
-						dummy = true
-						peer = &Peer{}
-					} else {
-						logDebug.Println(peer, "- UAPI: Created")
-					}
-				}
-
-			case "remove":
-
-				// remove currently selected peer from device
-
-				if value != "true" {
-					logError.Println("Failed to set remove, invalid value:", value)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-				if !dummy {
-					logDebug.Println(peer, "- UAPI: Removing")
-					device.RemovePeer(peer.handshake.remoteStatic)
-				}
-				peer = &Peer{}
-				dummy = true
-
-			case "preshared_key":
-
-				// update PSK
-
-				logDebug.Println(peer, "- UAPI: Updating preshared key")
-
-				peer.handshake.mutex.Lock()
-				err := peer.handshake.presharedKey.FromHex(value)
-				peer.handshake.mutex.Unlock()
-
-				if err != nil {
-					logError.Println("Failed to set preshared key:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-			case "endpoint":
-
-				// set endpoint destination
-
-				logDebug.Println(peer, "- UAPI: Updating endpoint")
-
-				err := func() error {
-					peer.Lock()
-					defer peer.Unlock()
-					endpoint, err := CreateEndpoint(value)
-					if err != nil {
-						return err
-					}
-					peer.endpoint = endpoint
-					return nil
-				}()
-
-				if err != nil {
-					logError.Println("Failed to set endpoint:", err, ":", value)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-			case "persistent_keepalive_interval":
-
-				// update persistent keepalive interval
-
-				logDebug.Println(peer, "- UAPI: Updating persistent keepalive interval")
-
-				secs, err := strconv.ParseUint(value, 10, 16)
-				if err != nil {
-					logError.Println("Failed to set persistent keepalive interval:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				old := peer.persistentKeepaliveInterval
-				peer.persistentKeepaliveInterval = uint16(secs)
-
-				// send immediate keepalive if we're turning it on and before it wasn't on
-
-				if old == 0 && secs != 0 {
-					if err != nil {
-						logError.Println("Failed to get tun device status:", err)
-						return &IPCError{ipc.IpcErrorIO}
-					}
-					if device.isUp.Get() && !dummy {
-						peer.SendKeepalive()
-					}
-				}
-
-			case "replace_allowed_ips":
-
-				logDebug.Println(peer, "- UAPI: Removing all allowedips")
-
-				if value != "true" {
-					logError.Println("Failed to replace allowedips, invalid value:", value)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				if dummy {
-					continue
-				}
-
-				device.allowedips.RemoveByPeer(peer)
-
-			case "allowed_ip":
-
-				logDebug.Println(peer, "- UAPI: Adding allowedip")
-
-				_, network, err := net.ParseCIDR(value)
-				if err != nil {
-					logError.Println("Failed to set allowed ip:", err)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-				if dummy {
-					continue
-				}
-
-				ones, _ := network.Mask.Size()
-				device.allowedips.Insert(network.IP, uint(ones), peer)
-
-			case "protocol_version":
-
-				if value != "1" {
-					logError.Println("Invalid protocol version:", value)
-					return &IPCError{ipc.IpcErrorInvalid}
-				}
-
-			default:
-				logError.Println("Invalid UAPI peer key:", key)
-				return &IPCError{ipc.IpcErrorInvalid}
-			}
+		var err error
+		if deviceConfig {
+			err = device.handleDeviceLine(key, value)
+		} else {
+			err = device.handlePeerLine(peer, key, value)
 		}
+		if err != nil {
+			return err
+		}
+	}
+	peer.handlePostConfig()
+
+	if err := scanner.Err(); err != nil {
+		return ipcErrorf(ipc.IpcErrorIO, "failed to read input: %w", err)
+	}
+	return nil
+}
+
+func (device *Device) handleDeviceLine(key, value string) error {
+	switch key {
+	case "private_key":
+		var sk NoisePrivateKey
+		err := sk.FromMaybeZeroHex(value)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set private_key: %w", err)
+		}
+		device.log.Verbosef("UAPI: Updating private key")
+		device.SetPrivateKey(sk)
+
+	case "listen_port":
+		port, err := strconv.ParseUint(value, 10, 16)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to parse listen_port: %w", err)
+		}
+
+		// update port and rebind
+		device.log.Verbosef("UAPI: Updating listen port")
+
+		device.net.Lock()
+		device.net.port = uint16(port)
+		device.net.Unlock()
+
+		if err := device.BindUpdate(); err != nil {
+			return ipcErrorf(ipc.IpcErrorPortInUse, "failed to set listen_port: %w", err)
+		}
+
+	case "fwmark":
+		mark, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "invalid fwmark: %w", err)
+		}
+
+		device.log.Verbosef("UAPI: Updating fwmark")
+		if err := device.BindSetMark(uint32(mark)); err != nil {
+			return ipcErrorf(ipc.IpcErrorPortInUse, "failed to update fwmark: %w", err)
+		}
+
+	case "replace_peers":
+		if value != "true" {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set replace_peers, invalid value: %v", value)
+		}
+		device.log.Verbosef("UAPI: Removing all peers")
+		device.RemoveAllPeers()
+
+	default:
+		return ipcErrorf(ipc.IpcErrorInvalid, "invalid UAPI device key: %v", key)
 	}
 
 	return nil
 }
 
+// An ipcSetPeer is the current state of an IPC set operation on a peer.
+type ipcSetPeer struct {
+	*Peer        // Peer is the current peer being operated on
+	dummy   bool // dummy reports whether this peer is a temporary, placeholder peer
+	created bool // new reports whether this is a newly created peer
+}
+
+func (peer *ipcSetPeer) handlePostConfig() {
+	if peer.Peer != nil && !peer.dummy && peer.Peer.device.isUp() {
+		peer.SendStagedPackets()
+	}
+}
+
+func (device *Device) handlePublicKeyLine(peer *ipcSetPeer, value string) error {
+	// Load/create the peer we are configuring.
+	var publicKey NoisePublicKey
+	err := publicKey.FromHex(value)
+	if err != nil {
+		return ipcErrorf(ipc.IpcErrorInvalid, "failed to get peer by public key: %w", err)
+	}
+
+	// Ignore peer with the same public key as this device.
+	device.staticIdentity.RLock()
+	peer.dummy = device.staticIdentity.publicKey.Equals(publicKey)
+	device.staticIdentity.RUnlock()
+
+	if peer.dummy {
+		peer.Peer = &Peer{}
+	} else {
+		peer.Peer = device.LookupPeer(publicKey)
+	}
+
+	peer.created = peer.Peer == nil
+	if peer.created {
+		peer.Peer, err = device.NewPeer(publicKey)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to create new peer: %w", err)
+		}
+		device.log.Verbosef("%v - UAPI: Created", peer.Peer)
+	}
+	return nil
+}
+
+func (device *Device) handlePeerLine(peer *ipcSetPeer, key, value string) error {
+	switch key {
+	case "update_only":
+		// allow disabling of creation
+		if value != "true" {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set update only, invalid value: %v", value)
+		}
+		if peer.created && !peer.dummy {
+			device.RemovePeer(peer.handshake.remoteStatic)
+			peer.Peer = &Peer{}
+			peer.dummy = true
+		}
+
+	case "remove":
+		// remove currently selected peer from device
+		if value != "true" {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set remove, invalid value: %v", value)
+		}
+		if !peer.dummy {
+			device.log.Verbosef("%v - UAPI: Removing", peer.Peer)
+			device.RemovePeer(peer.handshake.remoteStatic)
+		}
+		peer.Peer = &Peer{}
+		peer.dummy = true
+
+	case "preshared_key":
+		device.log.Verbosef("%v - UAPI: Updating preshared key", peer.Peer)
+
+		peer.handshake.mutex.Lock()
+		err := peer.handshake.presharedKey.FromHex(value)
+		peer.handshake.mutex.Unlock()
+
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set preshared key: %w", err)
+		}
+
+	case "endpoint":
+		device.log.Verbosef("%v - UAPI: Updating endpoint", peer.Peer)
+		endpoint, err := device.net.bind.ParseEndpoint(value)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set endpoint %v: %w", value, err)
+		}
+		peer.Lock()
+		defer peer.Unlock()
+		peer.endpoint = endpoint
+
+	case "persistent_keepalive_interval":
+		device.log.Verbosef("%v - UAPI: Updating persistent keepalive interval", peer.Peer)
+
+		secs, err := strconv.ParseUint(value, 10, 16)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set persistent keepalive interval: %w", err)
+		}
+
+		old := atomic.SwapUint32(&peer.persistentKeepaliveInterval, uint32(secs))
+
+		// Send immediate keepalive if we're turning it on and before it wasn't on.
+		if old == 0 && secs != 0 {
+			if err != nil {
+				return ipcErrorf(ipc.IpcErrorIO, "failed to get tun device status: %w", err)
+			}
+			if device.isUp() && !peer.dummy {
+				peer.SendKeepalive()
+			}
+		}
+
+	case "replace_allowed_ips":
+		device.log.Verbosef("%v - UAPI: Removing all allowedips", peer.Peer)
+		if value != "true" {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to replace allowedips, invalid value: %v", value)
+		}
+		if peer.dummy {
+			return nil
+		}
+		device.allowedips.RemoveByPeer(peer.Peer)
+
+	case "allowed_ip":
+		device.log.Verbosef("%v - UAPI: Adding allowedip", peer.Peer)
+
+		_, network, err := net.ParseCIDR(value)
+		if err != nil {
+			return ipcErrorf(ipc.IpcErrorInvalid, "failed to set allowed ip: %w", err)
+		}
+		if peer.dummy {
+			return nil
+		}
+		ones, _ := network.Mask.Size()
+		device.allowedips.Insert(network.IP, uint8(ones), peer.Peer)
+
+	case "protocol_version":
+		if value != "1" {
+			return ipcErrorf(ipc.IpcErrorInvalid, "invalid protocol version: %v", value)
+		}
+
+	default:
+		return ipcErrorf(ipc.IpcErrorInvalid, "invalid UAPI peer key: %v", key)
+	}
+
+	return nil
+}
+
+func (device *Device) IpcGet() (string, error) {
+	buf := new(strings.Builder)
+	if err := device.IpcGetOperation(buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func (device *Device) IpcSet(uapiConf string) error {
+	return device.IpcSetOperation(strings.NewReader(uapiConf))
+}
+
 func (device *Device) IpcHandle(socket net.Conn) {
-
-	// create buffered read/writer
-
 	defer socket.Close()
 
 	buffered := func(s io.ReadWriter) *bufio.ReadWriter {
@@ -391,35 +414,44 @@ func (device *Device) IpcHandle(socket net.Conn) {
 		return bufio.NewReadWriter(reader, writer)
 	}(socket)
 
-	defer buffered.Flush()
+	for {
+		op, err := buffered.ReadString('\n')
+		if err != nil {
+			return
+		}
 
-	op, err := buffered.ReadString('\n')
-	if err != nil {
-		return
-	}
+		// handle operation
+		switch op {
+		case "set=1\n":
+			err = device.IpcSetOperation(buffered.Reader)
+		case "get=1\n":
+			var nextByte byte
+			nextByte, err = buffered.ReadByte()
+			if err != nil {
+				return
+			}
+			if nextByte != '\n' {
+				err = ipcErrorf(ipc.IpcErrorInvalid, "trailing character in UAPI get: %q", nextByte)
+				break
+			}
+			err = device.IpcGetOperation(buffered.Writer)
+		default:
+			device.log.Errorf("invalid UAPI operation: %v", op)
+			return
+		}
 
-	// handle operation
-
-	var status *IPCError
-
-	switch op {
-	case "set=1\n":
-		status = device.IpcSetOperation(buffered.Reader)
-
-	case "get=1\n":
-		status = device.IpcGetOperation(buffered.Writer)
-
-	default:
-		device.log.Error.Println("Invalid UAPI operation:", op)
-		return
-	}
-
-	// write status
-
-	if status != nil {
-		device.log.Error.Println(status)
-		fmt.Fprintf(buffered, "errno=%d\n\n", status.ErrorCode())
-	} else {
-		fmt.Fprintf(buffered, "errno=0\n\n")
+		// write status
+		var status *IPCError
+		if err != nil && !errors.As(err, &status) {
+			// shouldn't happen
+			status = ipcErrorf(ipc.IpcErrorUnknown, "other UAPI error: %w", err)
+		}
+		if status != nil {
+			device.log.Errorf("%v", status)
+			fmt.Fprintf(buffered, "errno=%d\n\n", status.ErrorCode())
+		} else {
+			fmt.Fprintf(buffered, "errno=0\n\n")
+		}
+		buffered.Flush()
 	}
 }

device/version.go

@@ -1,3 +0,0 @@
-package device
-
-const WireGuardGoVersion = "0.0.20190805"

donotuseon_linux.go

@@ -1,15 +0,0 @@
-// +build !android
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package main
-
-const DoNotUseThisProgramOnLinux = UseTheKernelModuleInstead
-
-// --------------------------------------------------------
-// Do not use this on Linux. Instead use the kernel module.
-// See wireguard.com/install for more information.
-// --------------------------------------------------------

format_test.go

@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2021 WireGuard LLC. All Rights Reserved.
+ */
+package main
+
+import (
+	"bytes"
+	"go/format"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+)
+
+func TestFormatting(t *testing.T) {
+	var wg sync.WaitGroup
+	filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			t.Errorf("unable to walk %s: %v", path, err)
+			return nil
+		}
+		if d.IsDir() || filepath.Ext(path) != ".go" {
+			return nil
+		}
+		wg.Add(1)
+		go func(path string) {
+			defer wg.Done()
+			src, err := os.ReadFile(path)
+			if err != nil {
+				t.Errorf("unable to read %s: %v", path, err)
+				return
+			}
+			if runtime.GOOS == "windows" {
+				src = bytes.ReplaceAll(src, []byte{'\r', '\n'}, []byte{'\n'})
+			}
+			formatted, err := format.Source(src)
+			if err != nil {
+				t.Errorf("unable to format %s: %v", path, err)
+				return
+			}
+			if !bytes.Equal(src, formatted) {
+				t.Errorf("unformatted code: %s", path)
+			}
+		}(path)
+		return nil
+	})
+	wg.Wait()
+}

go.mod

@@ -1,9 +1,9 @@
 module golang.zx2c4.com/wireguard
 
-go 1.12
+go 1.17
 
 require (
-	golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/sys v0.0.0-20190618155005-516e3c20635f
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/net v0.0.0-20210927181540-4e4d966f7476
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6
 )

go.sum

@@ -1,11 +1,14 @@
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k=
-golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190618155005-516e3c20635f h1:dHNZYIYdq2QuU6w73vZ/DzesPbVlZVYZTtTZmrnsbQ8=
-golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 h1:s5hu7bTnLKswvidgtqc4GwsW83m9LZu8UAqzmWOZtI4=
+golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

ipc/uapi_bsd.go

@@ -1,33 +1,21 @@
-// +build darwin freebsd openbsd
+//go:build darwin || freebsd || openbsd
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package ipc
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"os"
-	"path"
 	"unsafe"
 
 	"golang.org/x/sys/unix"
 )
 
-var socketDirectory = "/var/run/wireguard"
-
-const (
-	IpcErrorIO        = -int64(unix.EIO)
-	IpcErrorProtocol  = -int64(unix.EPROTO)
-	IpcErrorInvalid   = -int64(unix.EINVAL)
-	IpcErrorPortInUse = -int64(unix.EADDRINUSE)
-	socketName        = "%s.sock"
-)
-
 type UAPIListener struct {
 	listener net.Listener // unix socket listener
 	connNew  chan net.Conn
@@ -84,10 +72,7 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) {
 		unixListener.SetUnlinkOnClose(true)
 	}
 
-	socketPath := path.Join(
-		socketDirectory,
-		fmt.Sprintf(socketName, name),
-	)
+	socketPath := sockPath(name)
 
 	// watch for deletion of socket
 
@@ -146,58 +131,3 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) {
 
 	return uapi, nil
 }
-
-func UAPIOpen(name string) (*os.File, error) {
-
-	// check if path exist
-
-	err := os.MkdirAll(socketDirectory, 0755)
-	if err != nil && !os.IsExist(err) {
-		return nil, err
-	}
-
-	// open UNIX socket
-
-	socketPath := path.Join(
-		socketDirectory,
-		fmt.Sprintf(socketName, name),
-	)
-
-	addr, err := net.ResolveUnixAddr("unix", socketPath)
-	if err != nil {
-		return nil, err
-	}
-
-	oldUmask := unix.Umask(0077)
-	listener, err := func() (*net.UnixListener, error) {
-
-		// initial connection attempt
-
-		listener, err := net.ListenUnix("unix", addr)
-		if err == nil {
-			return listener, nil
-		}
-
-		// check if socket already active
-
-		_, err = net.Dial("unix", socketPath)
-		if err == nil {
-			return nil, errors.New("unix socket in use")
-		}
-
-		// cleanup & attempt again
-
-		err = os.Remove(socketPath)
-		if err != nil {
-			return nil, err
-		}
-		return net.ListenUnix("unix", addr)
-	}()
-	unix.Umask(oldUmask)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return listener.File()
-}

ipc/uapi_linux.go

@@ -1,31 +1,18 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package ipc
 
 import (
-	"errors"
-	"fmt"
 	"net"
 	"os"
-	"path"
 
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/rwcancel"
 )
 
-var socketDirectory = "/var/run/wireguard"
-
-const (
-	IpcErrorIO        = -int64(unix.EIO)
-	IpcErrorProtocol  = -int64(unix.EPROTO)
-	IpcErrorInvalid   = -int64(unix.EINVAL)
-	IpcErrorPortInUse = -int64(unix.EADDRINUSE)
-	socketName        = "%s.sock"
-)
-
 type UAPIListener struct {
 	listener        net.Listener // unix socket listener
 	connNew         chan net.Conn
@@ -84,10 +71,7 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) {
 
 	// watch for deletion of socket
 
-	socketPath := path.Join(
-		socketDirectory,
-		fmt.Sprintf(socketName, name),
-	)
+	socketPath := sockPath(name)
 
 	uapi.inotifyFd, err = unix.InotifyInit()
 	if err != nil {
@@ -115,6 +99,7 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) {
 	go func(l *UAPIListener) {
 		var buff [0]byte
 		for {
+			defer uapi.inotifyRWCancel.Close()
 			// start with lstat to avoid race condition
 			if _, err := os.Lstat(socketPath); os.IsNotExist(err) {
 				l.connErr <- err
@@ -143,58 +128,3 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) {
 
 	return uapi, nil
 }
-
-func UAPIOpen(name string) (*os.File, error) {
-
-	// check if path exist
-
-	err := os.MkdirAll(socketDirectory, 0755)
-	if err != nil && !os.IsExist(err) {
-		return nil, err
-	}
-
-	// open UNIX socket
-
-	socketPath := path.Join(
-		socketDirectory,
-		fmt.Sprintf(socketName, name),
-	)
-
-	addr, err := net.ResolveUnixAddr("unix", socketPath)
-	if err != nil {
-		return nil, err
-	}
-
-	oldUmask := unix.Umask(0077)
-	listener, err := func() (*net.UnixListener, error) {
-
-		// initial connection attempt
-
-		listener, err := net.ListenUnix("unix", addr)
-		if err == nil {
-			return listener, nil
-		}
-
-		// check if socket already active
-
-		_, err = net.Dial("unix", socketPath)
-		if err == nil {
-			return nil, errors.New("unix socket in use")
-		}
-
-		// cleanup & attempt again
-
-		err = os.Remove(socketPath)
-		if err != nil {
-			return nil, err
-		}
-		return net.ListenUnix("unix", addr)
-	}()
-	unix.Umask(oldUmask)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return listener.File()
-}

ipc/uapi_unix.go

@@ -0,0 +1,66 @@
+//go:build linux || darwin || freebsd || openbsd
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package ipc
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	IpcErrorIO        = -int64(unix.EIO)
+	IpcErrorProtocol  = -int64(unix.EPROTO)
+	IpcErrorInvalid   = -int64(unix.EINVAL)
+	IpcErrorPortInUse = -int64(unix.EADDRINUSE)
+	IpcErrorUnknown   = -55 // ENOANO
+)
+
+// socketDirectory is variable because it is modified by a linker
+// flag in wireguard-android.
+var socketDirectory = "/var/run/wireguard"
+
+func sockPath(iface string) string {
+	return fmt.Sprintf("%s/%s.sock", socketDirectory, iface)
+}
+
+func UAPIOpen(name string) (*os.File, error) {
+	if err := os.MkdirAll(socketDirectory, 0755); err != nil {
+		return nil, err
+	}
+
+	socketPath := sockPath(name)
+	addr, err := net.ResolveUnixAddr("unix", socketPath)
+	if err != nil {
+		return nil, err
+	}
+
+	oldUmask := unix.Umask(0077)
+	defer unix.Umask(oldUmask)
+
+	listener, err := net.ListenUnix("unix", addr)
+	if err == nil {
+		return listener.File()
+	}
+
+	// Test socket, if not in use cleanup and try again.
+	if _, err := net.Dial("unix", socketPath); err == nil {
+		return nil, errors.New("unix socket in use")
+	}
+	if err := os.Remove(socketPath); err != nil {
+		return nil, err
+	}
+	listener, err = net.ListenUnix("unix", addr)
+	if err != nil {
+		return nil, err
+	}
+	return listener.File()
+}

ipc/uapi_windows.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package ipc
@@ -8,6 +8,8 @@ package ipc
 import (
 	"net"
 
+	"golang.org/x/sys/windows"
+
 	"golang.zx2c4.com/wireguard/ipc/winpipe"
 )
 
@@ -17,6 +19,7 @@ const (
 	IpcErrorProtocol  = -int64(71)
 	IpcErrorInvalid   = -int64(22)
 	IpcErrorPortInUse = -int64(98)
+	IpcErrorUnknown   = -int64(55)
 )
 
 type UAPIListener struct {
@@ -47,14 +50,21 @@ func (l *UAPIListener) Addr() net.Addr {
 	return l.listener.Addr()
 }
 
-/* SDDL_DEVOBJ_SYS_ALL from the WDK */
-var UAPISecurityDescriptor = "O:SYD:P(A;;GA;;;SY)"
+var UAPISecurityDescriptor *windows.SECURITY_DESCRIPTOR
+
+func init() {
+	var err error
+	UAPISecurityDescriptor, err = windows.SecurityDescriptorFromString("O:SYD:P(A;;GA;;;SY)(A;;GA;;;BA)S:(ML;;NWNRNX;;;HI)")
+	if err != nil {
+		panic(err)
+	}
+}
 
 func UAPIListen(name string) (net.Listener, error) {
-	config := winpipe.PipeConfig{
+	config := winpipe.ListenConfig{
 		SecurityDescriptor: UAPISecurityDescriptor,
 	}
-	listener, err := winpipe.ListenPipe("\\\\.\\pipe\\WireGuard\\"+name, &config)
+	listener, err := winpipe.Listen(`\\.\pipe\ProtectedPrefix\Administrators\WireGuard\`+name, &config)
 	if err != nil {
 		return nil, err
 	}

ipc/winpipe/file.go

@@ -1,61 +1,29 @@
-// +build windows
+//go:build windows
 
 /* SPDX-License-Identifier: MIT
  *
  * Copyright (C) 2005 Microsoft
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
+
 package winpipe
 
 import (
-	"errors"
 	"io"
+	"os"
 	"runtime"
 	"sync"
 	"sync/atomic"
-	"syscall"
 	"time"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
 )
 
-//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
-//sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort
-//sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
-//sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
-//sys wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult
-
-type atomicBool int32
-
-func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
-func (b *atomicBool) setFalse()   { atomic.StoreInt32((*int32)(b), 0) }
-func (b *atomicBool) setTrue()    { atomic.StoreInt32((*int32)(b), 1) }
-func (b *atomicBool) swap(new bool) bool {
-	var newInt int32
-	if new {
-		newInt = 1
-	}
-	return atomic.SwapInt32((*int32)(b), newInt) == 1
-}
-
-const (
-	cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
-	cFILE_SKIP_SET_EVENT_ON_HANDLE        = 2
-)
-
-var (
-	ErrFileClosed = errors.New("file has already been closed")
-	ErrTimeout    = &timeoutError{}
-)
-
-type timeoutError struct{}
-
-func (e *timeoutError) Error() string   { return "i/o timeout" }
-func (e *timeoutError) Timeout() bool   { return true }
-func (e *timeoutError) Temporary() bool { return true }
-
 type timeoutChan chan struct{}
 
 var ioInitOnce sync.Once
-var ioCompletionPort syscall.Handle
+var ioCompletionPort windows.Handle
 
 // ioResult contains the result of an asynchronous IO operation
 type ioResult struct {
@@ -65,12 +33,12 @@ type ioResult struct {
 
 // ioOperation represents an outstanding asynchronous Win32 IO
 type ioOperation struct {
-	o  syscall.Overlapped
+	o  windows.Overlapped
 	ch chan ioResult
 }
 
 func initIo() {
-	h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
+	h, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0)
 	if err != nil {
 		panic(err)
 	}
@@ -78,13 +46,13 @@ func initIo() {
 	go ioCompletionProcessor(h)
 }
 
-// win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
+// file implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
 // It takes ownership of this handle and will close it if it is garbage collected.
-type win32File struct {
-	handle        syscall.Handle
+type file struct {
+	handle        windows.Handle
 	wg            sync.WaitGroup
 	wgLock        sync.RWMutex
-	closing       atomicBool
+	closing       uint32 // used as atomic boolean
 	socket        bool
 	readDeadline  deadlineHandler
 	writeDeadline deadlineHandler
@@ -95,18 +63,18 @@ type deadlineHandler struct {
 	channel     timeoutChan
 	channelLock sync.RWMutex
 	timer       *time.Timer
-	timedout    atomicBool
+	timedout    uint32 // used as atomic boolean
 }
 
-// makeWin32File makes a new win32File from an existing file handle
-func makeWin32File(h syscall.Handle) (*win32File, error) {
-	f := &win32File{handle: h}
+// makeFile makes a new file from an existing file handle
+func makeFile(h windows.Handle) (*file, error) {
+	f := &file{handle: h}
 	ioInitOnce.Do(initIo)
-	_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
+	_, err := windows.CreateIoCompletionPort(h, ioCompletionPort, 0, 0)
 	if err != nil {
 		return nil, err
 	}
-	err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
+	err = windows.SetFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
 	if err != nil {
 		return nil, err
 	}
@@ -115,40 +83,36 @@ func makeWin32File(h syscall.Handle) (*win32File, error) {
 	return f, nil
 }
 
-func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
-	return makeWin32File(h)
-}
-
 // closeHandle closes the resources associated with a Win32 handle
-func (f *win32File) closeHandle() {
+func (f *file) closeHandle() {
 	f.wgLock.Lock()
 	// Atomically set that we are closing, releasing the resources only once.
-	if !f.closing.swap(true) {
+	if atomic.SwapUint32(&f.closing, 1) == 0 {
 		f.wgLock.Unlock()
 		// cancel all IO and wait for it to complete
-		cancelIoEx(f.handle, nil)
+		windows.CancelIoEx(f.handle, nil)
 		f.wg.Wait()
 		// at this point, no new IO can start
-		syscall.Close(f.handle)
+		windows.Close(f.handle)
 		f.handle = 0
 	} else {
 		f.wgLock.Unlock()
 	}
 }
 
-// Close closes a win32File.
-func (f *win32File) Close() error {
+// Close closes a file.
+func (f *file) Close() error {
 	f.closeHandle()
 	return nil
 }
 
 // prepareIo prepares for a new IO operation.
 // The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
-func (f *win32File) prepareIo() (*ioOperation, error) {
+func (f *file) prepareIo() (*ioOperation, error) {
 	f.wgLock.RLock()
-	if f.closing.isSet() {
+	if atomic.LoadUint32(&f.closing) == 1 {
 		f.wgLock.RUnlock()
-		return nil, ErrFileClosed
+		return nil, os.ErrClosed
 	}
 	f.wg.Add(1)
 	f.wgLock.RUnlock()
@@ -158,12 +122,12 @@ func (f *win32File) prepareIo() (*ioOperation, error) {
 }
 
 // ioCompletionProcessor processes completed async IOs forever
-func ioCompletionProcessor(h syscall.Handle) {
+func ioCompletionProcessor(h windows.Handle) {
 	for {
 		var bytes uint32
 		var key uintptr
 		var op *ioOperation
-		err := getQueuedCompletionStatus(h, &bytes, &key, &op, syscall.INFINITE)
+		err := windows.GetQueuedCompletionStatus(h, &bytes, &key, (**windows.Overlapped)(unsafe.Pointer(&op)), windows.INFINITE)
 		if op == nil {
 			panic(err)
 		}
@@ -173,13 +137,13 @@ func ioCompletionProcessor(h syscall.Handle) {
 
 // asyncIo processes the return value from ReadFile or WriteFile, blocking until
 // the operation has actually completed.
-func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
-	if err != syscall.ERROR_IO_PENDING {
+func (f *file) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
+	if err != windows.ERROR_IO_PENDING {
 		return int(bytes), err
 	}
 
-	if f.closing.isSet() {
-		cancelIoEx(f.handle, &c.o)
+	if atomic.LoadUint32(&f.closing) == 1 {
+		windows.CancelIoEx(f.handle, &c.o)
 	}
 
 	var timeout timeoutChan
@@ -193,21 +157,21 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
 	select {
 	case r = <-c.ch:
 		err = r.err
-		if err == syscall.ERROR_OPERATION_ABORTED {
-			if f.closing.isSet() {
-				err = ErrFileClosed
+		if err == windows.ERROR_OPERATION_ABORTED {
+			if atomic.LoadUint32(&f.closing) == 1 {
+				err = os.ErrClosed
 			}
 		} else if err != nil && f.socket {
 			// err is from Win32. Query the overlapped structure to get the winsock error.
 			var bytes, flags uint32
-			err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
+			err = windows.WSAGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
 		}
 	case <-timeout:
-		cancelIoEx(f.handle, &c.o)
+		windows.CancelIoEx(f.handle, &c.o)
 		r = <-c.ch
 		err = r.err
-		if err == syscall.ERROR_OPERATION_ABORTED {
-			err = ErrTimeout
+		if err == windows.ERROR_OPERATION_ABORTED {
+			err = os.ErrDeadlineExceeded
 		}
 	}
 
@@ -219,26 +183,26 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
 }
 
 // Read reads from a file handle.
-func (f *win32File) Read(b []byte) (int, error) {
+func (f *file) Read(b []byte) (int, error) {
 	c, err := f.prepareIo()
 	if err != nil {
 		return 0, err
 	}
 	defer f.wg.Done()
 
-	if f.readDeadline.timedout.isSet() {
-		return 0, ErrTimeout
+	if atomic.LoadUint32(&f.readDeadline.timedout) == 1 {
+		return 0, os.ErrDeadlineExceeded
 	}
 
 	var bytes uint32
-	err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
+	err = windows.ReadFile(f.handle, b, &bytes, &c.o)
 	n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
 	runtime.KeepAlive(b)
 
 	// Handle EOF conditions.
 	if err == nil && n == 0 && len(b) != 0 {
 		return 0, io.EOF
-	} else if err == syscall.ERROR_BROKEN_PIPE {
+	} else if err == windows.ERROR_BROKEN_PIPE {
 		return 0, io.EOF
 	} else {
 		return n, err
@@ -246,37 +210,37 @@ func (f *win32File) Read(b []byte) (int, error) {
 }
 
 // Write writes to a file handle.
-func (f *win32File) Write(b []byte) (int, error) {
+func (f *file) Write(b []byte) (int, error) {
 	c, err := f.prepareIo()
 	if err != nil {
 		return 0, err
 	}
 	defer f.wg.Done()
 
-	if f.writeDeadline.timedout.isSet() {
-		return 0, ErrTimeout
+	if atomic.LoadUint32(&f.writeDeadline.timedout) == 1 {
+		return 0, os.ErrDeadlineExceeded
 	}
 
 	var bytes uint32
-	err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
+	err = windows.WriteFile(f.handle, b, &bytes, &c.o)
 	n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
 	runtime.KeepAlive(b)
 	return n, err
 }
 
-func (f *win32File) SetReadDeadline(deadline time.Time) error {
+func (f *file) SetReadDeadline(deadline time.Time) error {
 	return f.readDeadline.set(deadline)
 }
 
-func (f *win32File) SetWriteDeadline(deadline time.Time) error {
+func (f *file) SetWriteDeadline(deadline time.Time) error {
 	return f.writeDeadline.set(deadline)
 }
 
-func (f *win32File) Flush() error {
-	return syscall.FlushFileBuffers(f.handle)
+func (f *file) Flush() error {
+	return windows.FlushFileBuffers(f.handle)
 }
 
-func (f *win32File) Fd() uintptr {
+func (f *file) Fd() uintptr {
 	return uintptr(f.handle)
 }
 
@@ -290,7 +254,7 @@ func (d *deadlineHandler) set(deadline time.Time) error {
 		}
 		d.timer = nil
 	}
-	d.timedout.setFalse()
+	atomic.StoreUint32(&d.timedout, 0)
 
 	select {
 	case <-d.channel:
@@ -305,7 +269,7 @@ func (d *deadlineHandler) set(deadline time.Time) error {
 	}
 
 	timeoutIO := func() {
-		d.timedout.setTrue()
+		atomic.StoreUint32(&d.timedout, 1)
 		close(d.channel)
 	}
 

ipc/winpipe/mksyscall.go

@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2005 Microsoft
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package winpipe
-
-//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go pipe.go sd.go file.go

ipc/winpipe/pipe.go

@@ -1,516 +0,0 @@
-// +build windows
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2005 Microsoft
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package winpipe
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"runtime"
-	"syscall"
-	"time"
-	"unsafe"
-)
-
-//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
-//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error)  [failretval==syscall.InvalidHandle] = CreateNamedPipeW
-//sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW
-//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
-//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
-//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
-//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) = ntdll.NtCreateNamedPipeFile
-//sys rtlNtStatusToDosError(status ntstatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
-//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) = ntdll.RtlDosPathNameToNtPathName_U
-//sys rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) = ntdll.RtlDefaultNpAcl
-
-type ioStatusBlock struct {
-	Status, Information uintptr
-}
-
-type objectAttributes struct {
-	Length             uintptr
-	RootDirectory      uintptr
-	ObjectName         *unicodeString
-	Attributes         uintptr
-	SecurityDescriptor *securityDescriptor
-	SecurityQoS        uintptr
-}
-
-type unicodeString struct {
-	Length        uint16
-	MaximumLength uint16
-	Buffer        uintptr
-}
-
-type securityDescriptor struct {
-	Revision byte
-	Sbz1     byte
-	Control  uint16
-	Owner    uintptr
-	Group    uintptr
-	Sacl     uintptr
-	Dacl     uintptr
-}
-
-type ntstatus int32
-
-func (status ntstatus) Err() error {
-	if status >= 0 {
-		return nil
-	}
-	return rtlNtStatusToDosError(status)
-}
-
-const (
-	cERROR_PIPE_BUSY      = syscall.Errno(231)
-	cERROR_NO_DATA        = syscall.Errno(232)
-	cERROR_PIPE_CONNECTED = syscall.Errno(535)
-	cERROR_SEM_TIMEOUT    = syscall.Errno(121)
-
-	cSECURITY_SQOS_PRESENT = 0x100000
-	cSECURITY_ANONYMOUS    = 0
-
-	cPIPE_TYPE_MESSAGE = 4
-
-	cPIPE_READMODE_MESSAGE = 2
-
-	cFILE_OPEN   = 1
-	cFILE_CREATE = 2
-
-	cFILE_PIPE_MESSAGE_TYPE          = 1
-	cFILE_PIPE_REJECT_REMOTE_CLIENTS = 2
-
-	cSE_DACL_PRESENT = 4
-)
-
-var (
-	// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
-	// This error should match net.errClosing since docker takes a dependency on its text.
-	ErrPipeListenerClosed = errors.New("use of closed network connection")
-
-	errPipeWriteClosed = errors.New("pipe has been closed for write")
-)
-
-type win32Pipe struct {
-	*win32File
-	path string
-}
-
-type win32MessageBytePipe struct {
-	win32Pipe
-	writeClosed bool
-	readEOF     bool
-}
-
-type pipeAddress string
-
-func (f *win32Pipe) LocalAddr() net.Addr {
-	return pipeAddress(f.path)
-}
-
-func (f *win32Pipe) RemoteAddr() net.Addr {
-	return pipeAddress(f.path)
-}
-
-func (f *win32Pipe) SetDeadline(t time.Time) error {
-	f.SetReadDeadline(t)
-	f.SetWriteDeadline(t)
-	return nil
-}
-
-// CloseWrite closes the write side of a message pipe in byte mode.
-func (f *win32MessageBytePipe) CloseWrite() error {
-	if f.writeClosed {
-		return errPipeWriteClosed
-	}
-	err := f.win32File.Flush()
-	if err != nil {
-		return err
-	}
-	_, err = f.win32File.Write(nil)
-	if err != nil {
-		return err
-	}
-	f.writeClosed = true
-	return nil
-}
-
-// Write writes bytes to a message pipe in byte mode. Zero-byte writes are ignored, since
-// they are used to implement CloseWrite().
-func (f *win32MessageBytePipe) Write(b []byte) (int, error) {
-	if f.writeClosed {
-		return 0, errPipeWriteClosed
-	}
-	if len(b) == 0 {
-		return 0, nil
-	}
-	return f.win32File.Write(b)
-}
-
-// Read reads bytes from a message pipe in byte mode. A read of a zero-byte message on a message
-// mode pipe will return io.EOF, as will all subsequent reads.
-func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
-	if f.readEOF {
-		return 0, io.EOF
-	}
-	n, err := f.win32File.Read(b)
-	if err == io.EOF {
-		// If this was the result of a zero-byte read, then
-		// it is possible that the read was due to a zero-size
-		// message. Since we are simulating CloseWrite with a
-		// zero-byte message, ensure that all future Read() calls
-		// also return EOF.
-		f.readEOF = true
-	} else if err == syscall.ERROR_MORE_DATA {
-		// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
-		// and the message still has more bytes. Treat this as a success, since
-		// this package presents all named pipes as byte streams.
-		err = nil
-	}
-	return n, err
-}
-
-func (s pipeAddress) Network() string {
-	return "pipe"
-}
-
-func (s pipeAddress) String() string {
-	return string(s)
-}
-
-// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
-func tryDialPipe(ctx context.Context, path *string) (syscall.Handle, error) {
-	for {
-		select {
-		case <-ctx.Done():
-			return syscall.Handle(0), ctx.Err()
-		default:
-			h, err := createFile(*path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
-			if err == nil {
-				return h, nil
-			}
-			if err != cERROR_PIPE_BUSY {
-				return h, &os.PathError{Err: err, Op: "open", Path: *path}
-			}
-			// Wait 10 msec and try again. This is a rather simplistic
-			// view, as we always try each 10 milliseconds.
-			time.Sleep(time.Millisecond * 10)
-		}
-	}
-}
-
-// DialPipe connects to a named pipe by path, timing out if the connection
-// takes longer than the specified duration. If timeout is nil, then we use
-// a default timeout of 2 seconds.  (We do not use WaitNamedPipe.)
-func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
-	var absTimeout time.Time
-	if timeout != nil {
-		absTimeout = time.Now().Add(*timeout)
-	} else {
-		absTimeout = time.Now().Add(time.Second * 2)
-	}
-	ctx, _ := context.WithDeadline(context.Background(), absTimeout)
-	conn, err := DialPipeContext(ctx, path)
-	if err == context.DeadlineExceeded {
-		return nil, ErrTimeout
-	}
-	return conn, err
-}
-
-// DialPipeContext attempts to connect to a named pipe by `path` until `ctx`
-// cancellation or timeout.
-func DialPipeContext(ctx context.Context, path string) (net.Conn, error) {
-	var err error
-	var h syscall.Handle
-	h, err = tryDialPipe(ctx, &path)
-	if err != nil {
-		return nil, err
-	}
-
-	var flags uint32
-	err = getNamedPipeInfo(h, &flags, nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := makeWin32File(h)
-	if err != nil {
-		syscall.Close(h)
-		return nil, err
-	}
-
-	// If the pipe is in message mode, return a message byte pipe, which
-	// supports CloseWrite().
-	if flags&cPIPE_TYPE_MESSAGE != 0 {
-		return &win32MessageBytePipe{
-			win32Pipe: win32Pipe{win32File: f, path: path},
-		}, nil
-	}
-	return &win32Pipe{win32File: f, path: path}, nil
-}
-
-type acceptResponse struct {
-	f   *win32File
-	err error
-}
-
-type win32PipeListener struct {
-	firstHandle syscall.Handle
-	path        string
-	config      PipeConfig
-	acceptCh    chan (chan acceptResponse)
-	closeCh     chan int
-	doneCh      chan int
-}
-
-func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (syscall.Handle, error) {
-	path16, err := syscall.UTF16FromString(path)
-	if err != nil {
-		return 0, &os.PathError{Op: "open", Path: path, Err: err}
-	}
-
-	var oa objectAttributes
-	oa.Length = unsafe.Sizeof(oa)
-
-	var ntPath unicodeString
-	if err := rtlDosPathNameToNtPathName(&path16[0], &ntPath, 0, 0).Err(); err != nil {
-		return 0, &os.PathError{Op: "open", Path: path, Err: err}
-	}
-	defer localFree(ntPath.Buffer)
-	oa.ObjectName = &ntPath
-
-	// The security descriptor is only needed for the first pipe.
-	if first {
-		if sd != nil {
-			len := uint32(len(sd))
-			sdb := localAlloc(0, len)
-			defer localFree(sdb)
-			copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
-			oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
-		} else {
-			// Construct the default named pipe security descriptor.
-			var dacl uintptr
-			if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
-				return 0, fmt.Errorf("getting default named pipe ACL: %s", err)
-			}
-			defer localFree(dacl)
-
-			sdb := &securityDescriptor{
-				Revision: 1,
-				Control:  cSE_DACL_PRESENT,
-				Dacl:     dacl,
-			}
-			oa.SecurityDescriptor = sdb
-		}
-	}
-
-	typ := uint32(cFILE_PIPE_REJECT_REMOTE_CLIENTS)
-	if c.MessageMode {
-		typ |= cFILE_PIPE_MESSAGE_TYPE
-	}
-
-	disposition := uint32(cFILE_OPEN)
-	access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE)
-	if first {
-		disposition = cFILE_CREATE
-		// By not asking for read or write access, the named pipe file system
-		// will put this pipe into an initially disconnected state, blocking
-		// client connections until the next call with first == false.
-		access = syscall.SYNCHRONIZE
-	}
-
-	timeout := int64(-50 * 10000) // 50ms
-
-	var (
-		h    syscall.Handle
-		iosb ioStatusBlock
-	)
-	err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout).Err()
-	if err != nil {
-		return 0, &os.PathError{Op: "open", Path: path, Err: err}
-	}
-
-	runtime.KeepAlive(ntPath)
-	return h, nil
-}
-
-func (l *win32PipeListener) makeServerPipe() (*win32File, error) {
-	h, err := makeServerPipeHandle(l.path, nil, &l.config, false)
-	if err != nil {
-		return nil, err
-	}
-	f, err := makeWin32File(h)
-	if err != nil {
-		syscall.Close(h)
-		return nil, err
-	}
-	return f, nil
-}
-
-func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
-	p, err := l.makeServerPipe()
-	if err != nil {
-		return nil, err
-	}
-
-	// Wait for the client to connect.
-	ch := make(chan error)
-	go func(p *win32File) {
-		ch <- connectPipe(p)
-	}(p)
-
-	select {
-	case err = <-ch:
-		if err != nil {
-			p.Close()
-			p = nil
-		}
-	case <-l.closeCh:
-		// Abort the connect request by closing the handle.
-		p.Close()
-		p = nil
-		err = <-ch
-		if err == nil || err == ErrFileClosed {
-			err = ErrPipeListenerClosed
-		}
-	}
-	return p, err
-}
-
-func (l *win32PipeListener) listenerRoutine() {
-	closed := false
-	for !closed {
-		select {
-		case <-l.closeCh:
-			closed = true
-		case responseCh := <-l.acceptCh:
-			var (
-				p   *win32File
-				err error
-			)
-			for {
-				p, err = l.makeConnectedServerPipe()
-				// If the connection was immediately closed by the client, try
-				// again.
-				if err != cERROR_NO_DATA {
-					break
-				}
-			}
-			responseCh <- acceptResponse{p, err}
-			closed = err == ErrPipeListenerClosed
-		}
-	}
-	syscall.Close(l.firstHandle)
-	l.firstHandle = 0
-	// Notify Close() and Accept() callers that the handle has been closed.
-	close(l.doneCh)
-}
-
-// PipeConfig contain configuration for the pipe listener.
-type PipeConfig struct {
-	// SecurityDescriptor contains a Windows security descriptor in SDDL format.
-	SecurityDescriptor string
-
-	// MessageMode determines whether the pipe is in byte or message mode. In either
-	// case the pipe is read in byte mode by default. The only practical difference in
-	// this implementation is that CloseWrite() is only supported for message mode pipes;
-	// CloseWrite() is implemented as a zero-byte write, but zero-byte writes are only
-	// transferred to the reader (and returned as io.EOF in this implementation)
-	// when the pipe is in message mode.
-	MessageMode bool
-
-	// InputBufferSize specifies the size the input buffer, in bytes.
-	InputBufferSize int32
-
-	// OutputBufferSize specifies the size the input buffer, in bytes.
-	OutputBufferSize int32
-}
-
-// ListenPipe creates a listener on a Windows named pipe path, e.g. \\.\pipe\mypipe.
-// The pipe must not already exist.
-func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
-	var (
-		sd  []byte
-		err error
-	)
-	if c == nil {
-		c = &PipeConfig{}
-	}
-	if c.SecurityDescriptor != "" {
-		sd, err = SddlToSecurityDescriptor(c.SecurityDescriptor)
-		if err != nil {
-			return nil, err
-		}
-	}
-	h, err := makeServerPipeHandle(path, sd, c, true)
-	if err != nil {
-		return nil, err
-	}
-	l := &win32PipeListener{
-		firstHandle: h,
-		path:        path,
-		config:      *c,
-		acceptCh:    make(chan (chan acceptResponse)),
-		closeCh:     make(chan int),
-		doneCh:      make(chan int),
-	}
-	go l.listenerRoutine()
-	return l, nil
-}
-
-func connectPipe(p *win32File) error {
-	c, err := p.prepareIo()
-	if err != nil {
-		return err
-	}
-	defer p.wg.Done()
-
-	err = connectNamedPipe(p.handle, &c.o)
-	_, err = p.asyncIo(c, nil, 0, err)
-	if err != nil && err != cERROR_PIPE_CONNECTED {
-		return err
-	}
-	return nil
-}
-
-func (l *win32PipeListener) Accept() (net.Conn, error) {
-	ch := make(chan acceptResponse)
-	select {
-	case l.acceptCh <- ch:
-		response := <-ch
-		err := response.err
-		if err != nil {
-			return nil, err
-		}
-		if l.config.MessageMode {
-			return &win32MessageBytePipe{
-				win32Pipe: win32Pipe{win32File: response.f, path: l.path},
-			}, nil
-		}
-		return &win32Pipe{win32File: response.f, path: l.path}, nil
-	case <-l.doneCh:
-		return nil, ErrPipeListenerClosed
-	}
-}
-
-func (l *win32PipeListener) Close() error {
-	select {
-	case l.closeCh <- 1:
-		<-l.doneCh
-	case <-l.doneCh:
-	}
-	return nil
-}
-
-func (l *win32PipeListener) Addr() net.Addr {
-	return pipeAddress(l.path)
-}

ipc/winpipe/sd.go

@@ -1,29 +0,0 @@
-// +build windows
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2005 Microsoft
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package winpipe
-
-import (
-	"unsafe"
-)
-
-//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
-//sys localFree(mem uintptr) = LocalFree
-//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
-
-func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
-	var sdBuffer uintptr
-	err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer localFree(sdBuffer)
-	sd := make([]byte, getSecurityDescriptorLength(sdBuffer))
-	copy(sd, (*[0xffff]byte)(unsafe.Pointer(sdBuffer))[:len(sd)])
-	return sd, nil
-}

ipc/winpipe/winpipe.go

@@ -0,0 +1,474 @@
+//go:build windows
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2005 Microsoft
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+// Package winpipe implements a net.Conn and net.Listener around Windows named pipes.
+package winpipe
+
+import (
+	"context"
+	"io"
+	"net"
+	"os"
+	"runtime"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+type pipe struct {
+	*file
+	path string
+}
+
+type messageBytePipe struct {
+	pipe
+	writeClosed bool
+	readEOF     bool
+}
+
+type pipeAddress string
+
+func (f *pipe) LocalAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *pipe) RemoteAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *pipe) SetDeadline(t time.Time) error {
+	f.SetReadDeadline(t)
+	f.SetWriteDeadline(t)
+	return nil
+}
+
+// CloseWrite closes the write side of a message pipe in byte mode.
+func (f *messageBytePipe) CloseWrite() error {
+	if f.writeClosed {
+		return io.ErrClosedPipe
+	}
+	err := f.file.Flush()
+	if err != nil {
+		return err
+	}
+	_, err = f.file.Write(nil)
+	if err != nil {
+		return err
+	}
+	f.writeClosed = true
+	return nil
+}
+
+// Write writes bytes to a message pipe in byte mode. Zero-byte writes are ignored, since
+// they are used to implement CloseWrite.
+func (f *messageBytePipe) Write(b []byte) (int, error) {
+	if f.writeClosed {
+		return 0, io.ErrClosedPipe
+	}
+	if len(b) == 0 {
+		return 0, nil
+	}
+	return f.file.Write(b)
+}
+
+// Read reads bytes from a message pipe in byte mode. A read of a zero-byte message on a message
+// mode pipe will return io.EOF, as will all subsequent reads.
+func (f *messageBytePipe) Read(b []byte) (int, error) {
+	if f.readEOF {
+		return 0, io.EOF
+	}
+	n, err := f.file.Read(b)
+	if err == io.EOF {
+		// If this was the result of a zero-byte read, then
+		// it is possible that the read was due to a zero-size
+		// message. Since we are simulating CloseWrite with a
+		// zero-byte message, ensure that all future Read calls
+		// also return EOF.
+		f.readEOF = true
+	} else if err == windows.ERROR_MORE_DATA {
+		// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
+		// and the message still has more bytes. Treat this as a success, since
+		// this package presents all named pipes as byte streams.
+		err = nil
+	}
+	return n, err
+}
+
+func (f *pipe) Handle() windows.Handle {
+	return f.handle
+}
+
+func (s pipeAddress) Network() string {
+	return "pipe"
+}
+
+func (s pipeAddress) String() string {
+	return string(s)
+}
+
+// tryDialPipe attempts to dial the specified pipe until cancellation or timeout.
+func tryDialPipe(ctx context.Context, path *string) (windows.Handle, error) {
+	for {
+		select {
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		default:
+			path16, err := windows.UTF16PtrFromString(*path)
+			if err != nil {
+				return 0, err
+			}
+			h, err := windows.CreateFile(path16, windows.GENERIC_READ|windows.GENERIC_WRITE, 0, nil, windows.OPEN_EXISTING, windows.FILE_FLAG_OVERLAPPED|windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS, 0)
+			if err == nil {
+				return h, nil
+			}
+			if err != windows.ERROR_PIPE_BUSY {
+				return h, &os.PathError{Err: err, Op: "open", Path: *path}
+			}
+			// Wait 10 msec and try again. This is a rather simplistic
+			// view, as we always try each 10 milliseconds.
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+}
+
+// DialConfig exposes various options for use in Dial and DialContext.
+type DialConfig struct {
+	ExpectedOwner *windows.SID // If non-nil, the pipe is verified to be owned by this SID.
+}
+
+// Dial connects to the specified named pipe by path, timing out if the connection
+// takes longer than the specified duration. If timeout is nil, then we use
+// a default timeout of 2 seconds.
+func Dial(path string, timeout *time.Duration, config *DialConfig) (net.Conn, error) {
+	var absTimeout time.Time
+	if timeout != nil {
+		absTimeout = time.Now().Add(*timeout)
+	} else {
+		absTimeout = time.Now().Add(2 * time.Second)
+	}
+	ctx, _ := context.WithDeadline(context.Background(), absTimeout)
+	conn, err := DialContext(ctx, path, config)
+	if err == context.DeadlineExceeded {
+		return nil, os.ErrDeadlineExceeded
+	}
+	return conn, err
+}
+
+// DialContext attempts to connect to the specified named pipe by path
+// cancellation or timeout.
+func DialContext(ctx context.Context, path string, config *DialConfig) (net.Conn, error) {
+	if config == nil {
+		config = &DialConfig{}
+	}
+	var err error
+	var h windows.Handle
+	h, err = tryDialPipe(ctx, &path)
+	if err != nil {
+		return nil, err
+	}
+
+	if config.ExpectedOwner != nil {
+		sd, err := windows.GetSecurityInfo(h, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION)
+		if err != nil {
+			windows.Close(h)
+			return nil, err
+		}
+		realOwner, _, err := sd.Owner()
+		if err != nil {
+			windows.Close(h)
+			return nil, err
+		}
+		if !realOwner.Equals(config.ExpectedOwner) {
+			windows.Close(h)
+			return nil, windows.ERROR_ACCESS_DENIED
+		}
+	}
+
+	var flags uint32
+	err = windows.GetNamedPipeInfo(h, &flags, nil, nil, nil)
+	if err != nil {
+		windows.Close(h)
+		return nil, err
+	}
+
+	f, err := makeFile(h)
+	if err != nil {
+		windows.Close(h)
+		return nil, err
+	}
+
+	// If the pipe is in message mode, return a message byte pipe, which
+	// supports CloseWrite.
+	if flags&windows.PIPE_TYPE_MESSAGE != 0 {
+		return &messageBytePipe{
+			pipe: pipe{file: f, path: path},
+		}, nil
+	}
+	return &pipe{file: f, path: path}, nil
+}
+
+type acceptResponse struct {
+	f   *file
+	err error
+}
+
+type pipeListener struct {
+	firstHandle windows.Handle
+	path        string
+	config      ListenConfig
+	acceptCh    chan (chan acceptResponse)
+	closeCh     chan int
+	doneCh      chan int
+}
+
+func makeServerPipeHandle(path string, sd *windows.SECURITY_DESCRIPTOR, c *ListenConfig, first bool) (windows.Handle, error) {
+	path16, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		return 0, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+
+	var oa windows.OBJECT_ATTRIBUTES
+	oa.Length = uint32(unsafe.Sizeof(oa))
+
+	var ntPath windows.NTUnicodeString
+	if err := windows.RtlDosPathNameToNtPathName(path16, &ntPath, nil, nil); err != nil {
+		if ntstatus, ok := err.(windows.NTStatus); ok {
+			err = ntstatus.Errno()
+		}
+		return 0, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+	defer windows.LocalFree(windows.Handle(unsafe.Pointer(ntPath.Buffer)))
+	oa.ObjectName = &ntPath
+
+	// The security descriptor is only needed for the first pipe.
+	if first {
+		if sd != nil {
+			oa.SecurityDescriptor = sd
+		} else {
+			// Construct the default named pipe security descriptor.
+			var acl *windows.ACL
+			if err := windows.RtlDefaultNpAcl(&acl); err != nil {
+				return 0, err
+			}
+			defer windows.LocalFree(windows.Handle(unsafe.Pointer(acl)))
+			sd, err := windows.NewSecurityDescriptor()
+			if err != nil {
+				return 0, err
+			}
+			if err = sd.SetDACL(acl, true, false); err != nil {
+				return 0, err
+			}
+			oa.SecurityDescriptor = sd
+		}
+	}
+
+	typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
+	if c.MessageMode {
+		typ |= windows.FILE_PIPE_MESSAGE_TYPE
+	}
+
+	disposition := uint32(windows.FILE_OPEN)
+	access := uint32(windows.GENERIC_READ | windows.GENERIC_WRITE | windows.SYNCHRONIZE)
+	if first {
+		disposition = windows.FILE_CREATE
+		// By not asking for read or write access, the named pipe file system
+		// will put this pipe into an initially disconnected state, blocking
+		// client connections until the next call with first == false.
+		access = windows.SYNCHRONIZE
+	}
+
+	timeout := int64(-50 * 10000) // 50ms
+
+	var (
+		h    windows.Handle
+		iosb windows.IO_STATUS_BLOCK
+	)
+	err = windows.NtCreateNamedPipeFile(&h, access, &oa, &iosb, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout)
+	if err != nil {
+		if ntstatus, ok := err.(windows.NTStatus); ok {
+			err = ntstatus.Errno()
+		}
+		return 0, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+
+	runtime.KeepAlive(ntPath)
+	return h, nil
+}
+
+func (l *pipeListener) makeServerPipe() (*file, error) {
+	h, err := makeServerPipeHandle(l.path, nil, &l.config, false)
+	if err != nil {
+		return nil, err
+	}
+	f, err := makeFile(h)
+	if err != nil {
+		windows.Close(h)
+		return nil, err
+	}
+	return f, nil
+}
+
+func (l *pipeListener) makeConnectedServerPipe() (*file, error) {
+	p, err := l.makeServerPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	// Wait for the client to connect.
+	ch := make(chan error)
+	go func(p *file) {
+		ch <- connectPipe(p)
+	}(p)
+
+	select {
+	case err = <-ch:
+		if err != nil {
+			p.Close()
+			p = nil
+		}
+	case <-l.closeCh:
+		// Abort the connect request by closing the handle.
+		p.Close()
+		p = nil
+		err = <-ch
+		if err == nil || err == os.ErrClosed {
+			err = net.ErrClosed
+		}
+	}
+	return p, err
+}
+
+func (l *pipeListener) listenerRoutine() {
+	closed := false
+	for !closed {
+		select {
+		case <-l.closeCh:
+			closed = true
+		case responseCh := <-l.acceptCh:
+			var (
+				p   *file
+				err error
+			)
+			for {
+				p, err = l.makeConnectedServerPipe()
+				// If the connection was immediately closed by the client, try
+				// again.
+				if err != windows.ERROR_NO_DATA {
+					break
+				}
+			}
+			responseCh <- acceptResponse{p, err}
+			closed = err == net.ErrClosed
+		}
+	}
+	windows.Close(l.firstHandle)
+	l.firstHandle = 0
+	// Notify Close and Accept callers that the handle has been closed.
+	close(l.doneCh)
+}
+
+// ListenConfig contains configuration for the pipe listener.
+type ListenConfig struct {
+	// SecurityDescriptor contains a Windows security descriptor. If nil, the default from RtlDefaultNpAcl is used.
+	SecurityDescriptor *windows.SECURITY_DESCRIPTOR
+
+	// MessageMode determines whether the pipe is in byte or message mode. In either
+	// case the pipe is read in byte mode by default. The only practical difference in
+	// this implementation is that CloseWrite is only supported for message mode pipes;
+	// CloseWrite is implemented as a zero-byte write, but zero-byte writes are only
+	// transferred to the reader (and returned as io.EOF in this implementation)
+	// when the pipe is in message mode.
+	MessageMode bool
+
+	// InputBufferSize specifies the initial size of the input buffer, in bytes, which the OS will grow as needed.
+	InputBufferSize int32
+
+	// OutputBufferSize specifies the initial size of the output buffer, in bytes, which the OS will grow as needed.
+	OutputBufferSize int32
+}
+
+// Listen creates a listener on a Windows named pipe path,such as \\.\pipe\mypipe.
+// The pipe must not already exist.
+func Listen(path string, c *ListenConfig) (net.Listener, error) {
+	if c == nil {
+		c = &ListenConfig{}
+	}
+	h, err := makeServerPipeHandle(path, c.SecurityDescriptor, c, true)
+	if err != nil {
+		return nil, err
+	}
+	l := &pipeListener{
+		firstHandle: h,
+		path:        path,
+		config:      *c,
+		acceptCh:    make(chan (chan acceptResponse)),
+		closeCh:     make(chan int),
+		doneCh:      make(chan int),
+	}
+	// The first connection is swallowed on Windows 7 & 8, so synthesize it.
+	if maj, _, _ := windows.RtlGetNtVersionNumbers(); maj <= 8 {
+		path16, err := windows.UTF16PtrFromString(path)
+		if err == nil {
+			h, err = windows.CreateFile(path16, 0, 0, nil, windows.OPEN_EXISTING, windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS, 0)
+			if err == nil {
+				windows.CloseHandle(h)
+			}
+		}
+	}
+	go l.listenerRoutine()
+	return l, nil
+}
+
+func connectPipe(p *file) error {
+	c, err := p.prepareIo()
+	if err != nil {
+		return err
+	}
+	defer p.wg.Done()
+
+	err = windows.ConnectNamedPipe(p.handle, &c.o)
+	_, err = p.asyncIo(c, nil, 0, err)
+	if err != nil && err != windows.ERROR_PIPE_CONNECTED {
+		return err
+	}
+	return nil
+}
+
+func (l *pipeListener) Accept() (net.Conn, error) {
+	ch := make(chan acceptResponse)
+	select {
+	case l.acceptCh <- ch:
+		response := <-ch
+		err := response.err
+		if err != nil {
+			return nil, err
+		}
+		if l.config.MessageMode {
+			return &messageBytePipe{
+				pipe: pipe{file: response.f, path: l.path},
+			}, nil
+		}
+		return &pipe{file: response.f, path: l.path}, nil
+	case <-l.doneCh:
+		return nil, net.ErrClosed
+	}
+}
+
+func (l *pipeListener) Close() error {
+	select {
+	case l.closeCh <- 1:
+		<-l.doneCh
+	case <-l.doneCh:
+	}
+	return nil
+}
+
+func (l *pipeListener) Addr() net.Addr {
+	return pipeAddress(l.path)
+}

ipc/winpipe/winpipe_test.go

@@ -0,0 +1,660 @@
+//go:build windows
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2005 Microsoft
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package winpipe_test
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/ipc/winpipe"
+)
+
+func randomPipePath() string {
+	guid, err := windows.GenerateGUID()
+	if err != nil {
+		panic(err)
+	}
+	return `\\.\PIPE\go-winpipe-test-` + guid.String()
+}
+
+func TestPingPong(t *testing.T) {
+	const (
+		ping = 42
+		pong = 24
+	)
+	pipePath := randomPipePath()
+	listener, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatalf("unable to listen on pipe: %v", err)
+	}
+	defer listener.Close()
+	go func() {
+		incoming, err := listener.Accept()
+		if err != nil {
+			t.Fatalf("unable to accept pipe connection: %v", err)
+		}
+		defer incoming.Close()
+		var data [1]byte
+		_, err = incoming.Read(data[:])
+		if err != nil {
+			t.Fatalf("unable to read ping from pipe: %v", err)
+		}
+		if data[0] != ping {
+			t.Fatalf("expected ping, got %d", data[0])
+		}
+		data[0] = pong
+		_, err = incoming.Write(data[:])
+		if err != nil {
+			t.Fatalf("unable to write pong to pipe: %v", err)
+		}
+	}()
+	client, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		t.Fatalf("unable to dial pipe: %v", err)
+	}
+	defer client.Close()
+	var data [1]byte
+	data[0] = ping
+	_, err = client.Write(data[:])
+	if err != nil {
+		t.Fatalf("unable to write ping to pipe: %v", err)
+	}
+	_, err = client.Read(data[:])
+	if err != nil {
+		t.Fatalf("unable to read pong from pipe: %v", err)
+	}
+	if data[0] != pong {
+		t.Fatalf("expected pong, got %d", data[0])
+	}
+}
+
+func TestDialUnknownFailsImmediately(t *testing.T) {
+	_, err := winpipe.Dial(randomPipePath(), nil, nil)
+	if !errors.Is(err, syscall.ENOENT) {
+		t.Fatalf("expected ENOENT got %v", err)
+	}
+}
+
+func TestDialListenerTimesOut(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	d := 10 * time.Millisecond
+	_, err = winpipe.Dial(pipePath, &d, nil)
+	if err != os.ErrDeadlineExceeded {
+		t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err)
+	}
+}
+
+func TestDialContextListenerTimesOut(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	d := 10 * time.Millisecond
+	ctx, _ := context.WithTimeout(context.Background(), d)
+	_, err = winpipe.DialContext(ctx, pipePath, nil)
+	if err != context.DeadlineExceeded {
+		t.Fatalf("expected context.DeadlineExceeded, got %v", err)
+	}
+}
+
+func TestDialListenerGetsCancelled(t *testing.T) {
+	pipePath := randomPipePath()
+	ctx, cancel := context.WithCancel(context.Background())
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ch := make(chan error)
+	defer l.Close()
+	go func(ctx context.Context, ch chan error) {
+		_, err := winpipe.DialContext(ctx, pipePath, nil)
+		ch <- err
+	}(ctx, ch)
+	time.Sleep(time.Millisecond * 30)
+	cancel()
+	err = <-ch
+	if err != context.Canceled {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+}
+
+func TestDialAccessDeniedWithRestrictedSD(t *testing.T) {
+	if windows.NewLazySystemDLL("ntdll.dll").NewProc("wine_get_version").Find() == nil {
+		t.Skip("dacls on named pipes are broken on wine")
+	}
+	pipePath := randomPipePath()
+	sd, _ := windows.SecurityDescriptorFromString("D:")
+	c := winpipe.ListenConfig{
+		SecurityDescriptor: sd,
+	}
+	l, err := winpipe.Listen(pipePath, &c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	_, err = winpipe.Dial(pipePath, nil, nil)
+	if !errors.Is(err, windows.ERROR_ACCESS_DENIED) {
+		t.Fatalf("expected ERROR_ACCESS_DENIED, got %v", err)
+	}
+}
+
+func getConnection(cfg *winpipe.ListenConfig) (client net.Conn, server net.Conn, err error) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, cfg)
+	if err != nil {
+		return
+	}
+	defer l.Close()
+
+	type response struct {
+		c   net.Conn
+		err error
+	}
+	ch := make(chan response)
+	go func() {
+		c, err := l.Accept()
+		ch <- response{c, err}
+	}()
+
+	c, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		return
+	}
+
+	r := <-ch
+	if err = r.err; err != nil {
+		c.Close()
+		return
+	}
+
+	client = c
+	server = r.c
+	return
+}
+
+func TestReadTimeout(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+
+	c.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+
+	buf := make([]byte, 10)
+	_, err = c.Read(buf)
+	if err != os.ErrDeadlineExceeded {
+		t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err)
+	}
+}
+
+func server(l net.Listener, ch chan int) {
+	c, err := l.Accept()
+	if err != nil {
+		panic(err)
+	}
+	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+	s, err := rw.ReadString('\n')
+	if err != nil {
+		panic(err)
+	}
+	_, err = rw.WriteString("got " + s)
+	if err != nil {
+		panic(err)
+	}
+	err = rw.Flush()
+	if err != nil {
+		panic(err)
+	}
+	c.Close()
+	ch <- 1
+}
+
+func TestFullListenDialReadWrite(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	ch := make(chan int)
+	go server(l, ch)
+
+	c, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+	_, err = rw.WriteString("hello world\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = rw.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := rw.ReadString('\n')
+	if err != nil {
+		t.Fatal(err)
+	}
+	ms := "got hello world\n"
+	if s != ms {
+		t.Errorf("expected '%s', got '%s'", ms, s)
+	}
+
+	<-ch
+}
+
+func TestCloseAbortsListen(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch := make(chan error)
+	go func() {
+		_, err := l.Accept()
+		ch <- err
+	}()
+
+	time.Sleep(30 * time.Millisecond)
+	l.Close()
+
+	err = <-ch
+	if err != net.ErrClosed {
+		t.Fatalf("expected net.ErrClosed, got %v", err)
+	}
+}
+
+func ensureEOFOnClose(t *testing.T, r io.Reader, w io.Closer) {
+	b := make([]byte, 10)
+	w.Close()
+	n, err := r.Read(b)
+	if n > 0 {
+		t.Errorf("unexpected byte count %d", n)
+	}
+	if err != io.EOF {
+		t.Errorf("expected EOF: %v", err)
+	}
+}
+
+func TestCloseClientEOFServer(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+	ensureEOFOnClose(t, c, s)
+}
+
+func TestCloseServerEOFClient(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+	ensureEOFOnClose(t, s, c)
+}
+
+func TestCloseWriteEOF(t *testing.T) {
+	cfg := &winpipe.ListenConfig{
+		MessageMode: true,
+	}
+	c, s, err := getConnection(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+
+	type closeWriter interface {
+		CloseWrite() error
+	}
+
+	err = c.(closeWriter).CloseWrite()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := make([]byte, 10)
+	_, err = s.Read(b)
+	if err != io.EOF {
+		t.Fatal(err)
+	}
+}
+
+func TestAcceptAfterCloseFails(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	l.Close()
+	_, err = l.Accept()
+	if err != net.ErrClosed {
+		t.Fatalf("expected net.ErrClosed, got %v", err)
+	}
+}
+
+func TestDialTimesOutByDefault(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	_, err = winpipe.Dial(pipePath, nil, nil)
+	if err != os.ErrDeadlineExceeded {
+		t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err)
+	}
+}
+
+func TestTimeoutPendingRead(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	serverDone := make(chan struct{})
+
+	go func() {
+		s, err := l.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(1 * time.Second)
+		s.Close()
+		close(serverDone)
+	}()
+
+	client, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	clientErr := make(chan error)
+	go func() {
+		buf := make([]byte, 10)
+		_, err = client.Read(buf)
+		clientErr <- err
+	}()
+
+	time.Sleep(100 * time.Millisecond) // make *sure* the pipe is reading before we set the deadline
+	client.SetReadDeadline(time.Unix(1, 0))
+
+	select {
+	case err = <-clientErr:
+		if err != os.ErrDeadlineExceeded {
+			t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err)
+		}
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("timed out while waiting for read to cancel")
+		<-clientErr
+	}
+	<-serverDone
+}
+
+func TestTimeoutPendingWrite(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	serverDone := make(chan struct{})
+
+	go func() {
+		s, err := l.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(1 * time.Second)
+		s.Close()
+		close(serverDone)
+	}()
+
+	client, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	clientErr := make(chan error)
+	go func() {
+		_, err = client.Write([]byte("this should timeout"))
+		clientErr <- err
+	}()
+
+	time.Sleep(100 * time.Millisecond) // make *sure* the pipe is writing before we set the deadline
+	client.SetWriteDeadline(time.Unix(1, 0))
+
+	select {
+	case err = <-clientErr:
+		if err != os.ErrDeadlineExceeded {
+			t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err)
+		}
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("timed out while waiting for write to cancel")
+		<-clientErr
+	}
+	<-serverDone
+}
+
+type CloseWriter interface {
+	CloseWrite() error
+}
+
+func TestEchoWithMessaging(t *testing.T) {
+	c := winpipe.ListenConfig{
+		MessageMode:      true,  // Use message mode so that CloseWrite() is supported
+		InputBufferSize:  65536, // Use 64KB buffers to improve performance
+		OutputBufferSize: 65536,
+	}
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, &c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	listenerDone := make(chan bool)
+	clientDone := make(chan bool)
+	go func() {
+		// server echo
+		conn, e := l.Accept()
+		if e != nil {
+			t.Fatal(e)
+		}
+		defer conn.Close()
+
+		time.Sleep(500 * time.Millisecond) // make *sure* we don't begin to read before eof signal is sent
+		io.Copy(conn, conn)
+		conn.(CloseWriter).CloseWrite()
+		close(listenerDone)
+	}()
+	timeout := 1 * time.Second
+	client, err := winpipe.Dial(pipePath, &timeout, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	go func() {
+		// client read back
+		bytes := make([]byte, 2)
+		n, e := client.Read(bytes)
+		if e != nil {
+			t.Fatal(e)
+		}
+		if n != 2 {
+			t.Fatalf("expected 2 bytes, got %v", n)
+		}
+		close(clientDone)
+	}()
+
+	payload := make([]byte, 2)
+	payload[0] = 0
+	payload[1] = 1
+
+	n, err := client.Write(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("expected 2 bytes, got %v", n)
+	}
+	client.(CloseWriter).CloseWrite()
+	<-listenerDone
+	<-clientDone
+}
+
+func TestConnectRace(t *testing.T) {
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	go func() {
+		for {
+			s, err := l.Accept()
+			if err == net.ErrClosed {
+				return
+			}
+
+			if err != nil {
+				t.Fatal(err)
+			}
+			s.Close()
+		}
+	}()
+
+	for i := 0; i < 1000; i++ {
+		c, err := winpipe.Dial(pipePath, nil, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		c.Close()
+	}
+}
+
+func TestMessageReadMode(t *testing.T) {
+	if maj, _, _ := windows.RtlGetNtVersionNumbers(); maj <= 8 {
+		t.Skipf("Skipping on Windows %d", maj)
+	}
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	pipePath := randomPipePath()
+	l, err := winpipe.Listen(pipePath, &winpipe.ListenConfig{MessageMode: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	msg := ([]byte)("hello world")
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		s, err := l.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = s.Write(msg)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.Close()
+	}()
+
+	c, err := winpipe.Dial(pipePath, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	mode := uint32(windows.PIPE_READMODE_MESSAGE)
+	err = windows.SetNamedPipeHandleState(c.(interface{ Handle() windows.Handle }).Handle(), &mode, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch := make([]byte, 1)
+	var vmsg []byte
+	for {
+		n, err := c.Read(ch)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n != 1 {
+			t.Fatalf("expected 1, got %d", n)
+		}
+		vmsg = append(vmsg, ch[0])
+	}
+	if !bytes.Equal(msg, vmsg) {
+		t.Fatalf("expected %s, got %s", msg, vmsg)
+	}
+}
+
+func TestListenConnectRace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping long race test")
+	}
+	pipePath := randomPipePath()
+	for i := 0; i < 50 && !t.Failed(); i++ {
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			c, err := winpipe.Dial(pipePath, nil, nil)
+			if err == nil {
+				c.Close()
+			}
+			wg.Done()
+		}()
+		s, err := winpipe.Listen(pipePath, nil)
+		if err != nil {
+			t.Error(i, err)
+		} else {
+			s.Close()
+		}
+		wg.Wait()
+	}
+}

ipc/winpipe/zsyscall_windows.go

@@ -1,274 +0,0 @@
-// Code generated by 'go generate'; DO NOT EDIT.
-
-package winpipe
-
-import (
-	"syscall"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-var _ unsafe.Pointer
-
-// Do the interface allocations only once for common
-// Errno values.
-const (
-	errnoERROR_IO_PENDING = 997
-)
-
-var (
-	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
-)
-
-// errnoErr returns common boxed Errno values, to prevent
-// allocations at runtime.
-func errnoErr(e syscall.Errno) error {
-	switch e {
-	case 0:
-		return nil
-	case errnoERROR_IO_PENDING:
-		return errERROR_IO_PENDING
-	}
-	// TODO: add more here, after collecting data on the common
-	// error values see on Windows. (perhaps when running
-	// all.bat?)
-	return e
-}
-
-var (
-	modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
-	modntdll    = windows.NewLazySystemDLL("ntdll.dll")
-	modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
-	modws2_32   = windows.NewLazySystemDLL("ws2_32.dll")
-
-	procConnectNamedPipe                                     = modkernel32.NewProc("ConnectNamedPipe")
-	procCreateNamedPipeW                                     = modkernel32.NewProc("CreateNamedPipeW")
-	procCreateFileW                                          = modkernel32.NewProc("CreateFileW")
-	procGetNamedPipeInfo                                     = modkernel32.NewProc("GetNamedPipeInfo")
-	procGetNamedPipeHandleStateW                             = modkernel32.NewProc("GetNamedPipeHandleStateW")
-	procLocalAlloc                                           = modkernel32.NewProc("LocalAlloc")
-	procNtCreateNamedPipeFile                                = modntdll.NewProc("NtCreateNamedPipeFile")
-	procRtlNtStatusToDosErrorNoTeb                           = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
-	procRtlDosPathNameToNtPathName_U                         = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
-	procRtlDefaultNpAcl                                      = modntdll.NewProc("RtlDefaultNpAcl")
-	procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
-	procLocalFree                                            = modkernel32.NewProc("LocalFree")
-	procGetSecurityDescriptorLength                          = modadvapi32.NewProc("GetSecurityDescriptorLength")
-	procCancelIoEx                                           = modkernel32.NewProc("CancelIoEx")
-	procCreateIoCompletionPort                               = modkernel32.NewProc("CreateIoCompletionPort")
-	procGetQueuedCompletionStatus                            = modkernel32.NewProc("GetQueuedCompletionStatus")
-	procSetFileCompletionNotificationModes                   = modkernel32.NewProc("SetFileCompletionNotificationModes")
-	procWSAGetOverlappedResult                               = modws2_32.NewProc("WSAGetOverlappedResult")
-)
-
-func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) {
-	r1, _, e1 := syscall.Syscall(procConnectNamedPipe.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(o)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(name)
-	if err != nil {
-		return
-	}
-	return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa)
-}
-
-func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall9(procCreateNamedPipeW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)), 0)
-	handle = syscall.Handle(r0)
-	if handle == syscall.InvalidHandle {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(name)
-	if err != nil {
-		return
-	}
-	return _createFile(_p0, access, mode, sa, createmode, attrs, templatefile)
-}
-
-func _createFile(name *uint16, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0)
-	handle = syscall.Handle(r0)
-	if handle == syscall.InvalidHandle {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procGetNamedPipeInfo.Addr(), 5, uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
-	r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func localAlloc(uFlags uint32, length uint32) (ptr uintptr) {
-	r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(uFlags), uintptr(length), 0)
-	ptr = uintptr(r0)
-	return
-}
-
-func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
-	r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
-	status = ntstatus(r0)
-	return
-}
-
-func rtlNtStatusToDosError(status ntstatus) (winerr error) {
-	r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
-	if r0 != 0 {
-		winerr = syscall.Errno(r0)
-	}
-	return
-}
-
-func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
-	r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
-	status = ntstatus(r0)
-	return
-}
-
-func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
-	r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
-	status = ntstatus(r0)
-	return
-}
-
-func convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) {
-	var _p0 *uint16
-	_p0, err = syscall.UTF16PtrFromString(str)
-	if err != nil {
-		return
-	}
-	return _convertStringSecurityDescriptorToSecurityDescriptor(_p0, revision, sd, size)
-}
-
-func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision uint32, sd *uintptr, size *uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procConvertStringSecurityDescriptorToSecurityDescriptorW.Addr(), 4, uintptr(unsafe.Pointer(str)), uintptr(revision), uintptr(unsafe.Pointer(sd)), uintptr(unsafe.Pointer(size)), 0, 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func localFree(mem uintptr) {
-	syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0)
-	return
-}
-
-func getSecurityDescriptorLength(sd uintptr) (len uint32) {
-	r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
-	len = uint32(r0)
-	return
-}
-
-func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) {
-	r1, _, e1 := syscall.Syscall(procCancelIoEx.Addr(), 2, uintptr(file), uintptr(unsafe.Pointer(o)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) {
-	r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0)
-	newport = syscall.Handle(r0)
-	if newport == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) {
-	r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) {
-	r1, _, e1 := syscall.Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(h), uintptr(flags), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}
-
-func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) {
-	var _p0 uint32
-	if wait {
-		_p0 = 1
-	} else {
-		_p0 = 0
-	}
-	r1, _, e1 := syscall.Syscall6(procWSAGetOverlappedResult.Addr(), 5, uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)), 0)
-	if r1 == 0 {
-		if e1 != 0 {
-			err = errnoErr(e1)
-		} else {
-			err = syscall.EINVAL
-		}
-	}
-	return
-}

main.go

@@ -1,8 +1,8 @@
-// +build !windows
+//go:build !windows
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package main
@@ -15,6 +15,7 @@ import (
 	"strconv"
 	"syscall"
 
+	"golang.zx2c4.com/wireguard/conn"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/ipc"
 	"golang.zx2c4.com/wireguard/tun"
@@ -32,51 +33,38 @@ const (
 )
 
 func printUsage() {
-	fmt.Printf("usage:\n")
-	fmt.Printf("%s [-f/--foreground] INTERFACE-NAME\n", os.Args[0])
+	fmt.Printf("Usage: %s [-f/--foreground] INTERFACE-NAME\n", os.Args[0])
 }
 
 func warning() {
-	if runtime.GOOS != "linux" || os.Getenv(ENV_WG_PROCESS_FOREGROUND) == "1" {
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd":
+		if os.Getenv(ENV_WG_PROCESS_FOREGROUND) == "1" {
+			return
+		}
+	default:
 		return
 	}
-	shouldQuit := os.Getenv("WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD") != "1"
 
-	fmt.Fprintln(os.Stderr, "WARNING WARNING WARNING WARNING WARNING WARNING WARNING")
-	fmt.Fprintln(os.Stderr, "W                                                     G")
-	fmt.Fprintln(os.Stderr, "W   You are running this software on a Linux kernel,  G")
-	fmt.Fprintln(os.Stderr, "W   which is probably unnecessary and foolish. This   G")
-	fmt.Fprintln(os.Stderr, "W   is because the Linux kernel has built-in first    G")
-	fmt.Fprintln(os.Stderr, "W   class support for WireGuard, and this support is  G")
-	fmt.Fprintln(os.Stderr, "W   much more refined than this slower userspace      G")
-	fmt.Fprintln(os.Stderr, "W   implementation. For more information on           G")
-	fmt.Fprintln(os.Stderr, "W   installing the kernel module, please visit:       G")
-	fmt.Fprintln(os.Stderr, "W           https://www.wireguard.com/install         G")
-	if shouldQuit {
-		fmt.Fprintln(os.Stderr, "W                                                     G")
-		fmt.Fprintln(os.Stderr, "W   If you still want to use this program, against    G")
-		fmt.Fprintln(os.Stderr, "W   the advice here, please first export this         G")
-		fmt.Fprintln(os.Stderr, "W   environment variable:                             G")
-		fmt.Fprintln(os.Stderr, "W   WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD=1    G")
-	}
-	fmt.Fprintln(os.Stderr, "W                                                     G")
-	fmt.Fprintln(os.Stderr, "WARNING WARNING WARNING WARNING WARNING WARNING WARNING")
-
-	if shouldQuit {
-		os.Exit(1)
-	}
+	fmt.Fprintln(os.Stderr, "┌──────────────────────────────────────────────────────┐")
+	fmt.Fprintln(os.Stderr, "│                                                      │")
+	fmt.Fprintln(os.Stderr, "│   Running wireguard-go is not required because this  │")
+	fmt.Fprintln(os.Stderr, "│   kernel has first class support for WireGuard. For  │")
+	fmt.Fprintln(os.Stderr, "│   information on installing the kernel module,       │")
+	fmt.Fprintln(os.Stderr, "│   please visit:                                      │")
+	fmt.Fprintln(os.Stderr, "│         https://www.wireguard.com/install/           │")
+	fmt.Fprintln(os.Stderr, "│                                                      │")
+	fmt.Fprintln(os.Stderr, "└──────────────────────────────────────────────────────┘")
 }
 
 func main() {
 	if len(os.Args) == 2 && os.Args[1] == "--version" {
-		fmt.Printf("wireguard-go v%s\n\nUserspace WireGuard daemon for %s-%s.\nInformation available at https://www.wireguard.com.\nCopyright (C) Jason A. Donenfeld <Jason@zx2c4.com>.\n", device.WireGuardGoVersion, runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("wireguard-go v%s\n\nUserspace WireGuard daemon for %s-%s.\nInformation available at https://www.wireguard.com.\nCopyright (C) Jason A. Donenfeld <Jason@zx2c4.com>.\n", Version, runtime.GOOS, runtime.GOARCH)
 		return
 	}
 
 	warning()
 
-	// parse arguments
-
 	var foreground bool
 	var interfaceName string
 	if len(os.Args) < 2 || len(os.Args) > 3 {
@@ -111,16 +99,14 @@ func main() {
 
 	logLevel := func() int {
 		switch os.Getenv("LOG_LEVEL") {
-		case "debug":
-			return device.LogLevelDebug
-		case "info":
-			return device.LogLevelInfo
+		case "verbose", "debug":
+			return device.LogLevelVerbose
 		case "error":
 			return device.LogLevelError
 		case "silent":
 			return device.LogLevelSilent
 		}
-		return device.LogLevelInfo
+		return device.LogLevelError
 	}()
 
 	// open TUN device (or use supplied fd)
@@ -159,12 +145,10 @@ func main() {
 		fmt.Sprintf("(%s) ", interfaceName),
 	)
 
-	logger.Info.Println("Starting wireguard-go version", device.WireGuardGoVersion)
-
-	logger.Debug.Println("Debug log enabled")
+	logger.Verbosef("Starting wireguard-go version %s", Version)
 
 	if err != nil {
-		logger.Error.Println("Failed to create TUN device:", err)
+		logger.Errorf("Failed to create TUN device: %v", err)
 		os.Exit(ExitSetupFailed)
 	}
 
@@ -187,7 +171,7 @@ func main() {
 	}()
 
 	if err != nil {
-		logger.Error.Println("UAPI listen error:", err)
+		logger.Errorf("UAPI listen error: %v", err)
 		os.Exit(ExitSetupFailed)
 		return
 	}
@@ -222,7 +206,7 @@ func main() {
 
 		path, err := os.Executable()
 		if err != nil {
-			logger.Error.Println("Failed to determine executable:", err)
+			logger.Errorf("Failed to determine executable: %v", err)
 			os.Exit(ExitSetupFailed)
 		}
 
@@ -232,23 +216,23 @@ func main() {
 			attr,
 		)
 		if err != nil {
-			logger.Error.Println("Failed to daemonize:", err)
+			logger.Errorf("Failed to daemonize: %v", err)
 			os.Exit(ExitSetupFailed)
 		}
 		process.Release()
 		return
 	}
 
-	device := device.NewDevice(tun, logger)
+	device := device.NewDevice(tun, conn.NewDefaultBind(), logger)
 
-	logger.Info.Println("Device started")
+	logger.Verbosef("Device started")
 
 	errs := make(chan error)
 	term := make(chan os.Signal, 1)
 
 	uapi, err := ipc.UAPIListen(interfaceName, fileUAPI)
 	if err != nil {
-		logger.Error.Println("Failed to listen on uapi socket:", err)
+		logger.Errorf("Failed to listen on uapi socket: %v", err)
 		os.Exit(ExitSetupFailed)
 	}
 
@@ -263,7 +247,7 @@ func main() {
 		}
 	}()
 
-	logger.Info.Println("UAPI listener started")
+	logger.Verbosef("UAPI listener started")
 
 	// wait for program to terminate
 
@@ -281,5 +265,5 @@ func main() {
 	uapi.Close()
 	device.Close()
 
-	logger.Info.Println("Shutting down")
+	logger.Verbosef("Shutting down")
 }

main_windows.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package main
@@ -11,6 +11,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"golang.zx2c4.com/wireguard/conn"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/ipc"
 
@@ -31,30 +32,33 @@ func main() {
 	fmt.Fprintln(os.Stderr, "Warning: this is a test program for Windows, mainly used for debugging this Go package. For a real WireGuard for Windows client, the repo you want is <https://git.zx2c4.com/wireguard-windows/>, which includes this code as a module.")
 
 	logger := device.NewLogger(
-		device.LogLevelDebug,
+		device.LogLevelVerbose,
 		fmt.Sprintf("(%s) ", interfaceName),
 	)
-	logger.Info.Println("Starting wireguard-go version", device.WireGuardGoVersion)
-	logger.Debug.Println("Debug log enabled")
+	logger.Verbosef("Starting wireguard-go version %s", Version)
 
-	tun, err := tun.CreateTUN(interfaceName)
+	tun, err := tun.CreateTUN(interfaceName, 0)
 	if err == nil {
 		realInterfaceName, err2 := tun.Name()
 		if err2 == nil {
 			interfaceName = realInterfaceName
 		}
 	} else {
-		logger.Error.Println("Failed to create TUN device:", err)
+		logger.Errorf("Failed to create TUN device: %v", err)
 		os.Exit(ExitSetupFailed)
 	}
 
-	device := device.NewDevice(tun, logger)
-	device.Up()
-	logger.Info.Println("Device started")
+	device := device.NewDevice(tun, conn.NewDefaultBind(), logger)
+	err = device.Up()
+	if err != nil {
+		logger.Errorf("Failed to bring up device: %v", err)
+		os.Exit(ExitSetupFailed)
+	}
+	logger.Verbosef("Device started")
 
 	uapi, err := ipc.UAPIListen(interfaceName)
 	if err != nil {
-		logger.Error.Println("Failed to listen on uapi socket:", err)
+		logger.Errorf("Failed to listen on uapi socket: %v", err)
 		os.Exit(ExitSetupFailed)
 	}
 
@@ -71,7 +75,7 @@ func main() {
 			go device.IpcHandle(conn)
 		}
 	}()
-	logger.Info.Println("UAPI listener started")
+	logger.Verbosef("UAPI listener started")
 
 	// wait for program to terminate
 
@@ -90,5 +94,5 @@ func main() {
 	uapi.Close()
 	device.Close()
 
-	logger.Info.Println("Shutting down")
+	logger.Verbosef("Shutting down")
 }

ratelimiter/ratelimiter.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package ratelimiter
@@ -20,21 +20,23 @@ const (
 )
 
 type RatelimiterEntry struct {
-	sync.Mutex
+	mu       sync.Mutex
 	lastTime time.Time
 	tokens   int64
 }
 
 type Ratelimiter struct {
-	sync.RWMutex
-	stopReset chan struct{}
+	mu      sync.RWMutex
+	timeNow func() time.Time
+
+	stopReset chan struct{} // send to reset, close to stop
 	tableIPv4 map[[net.IPv4len]byte]*RatelimiterEntry
 	tableIPv6 map[[net.IPv6len]byte]*RatelimiterEntry
 }
 
 func (rate *Ratelimiter) Close() {
-	rate.Lock()
-	defer rate.Unlock()
+	rate.mu.Lock()
+	defer rate.mu.Unlock()
 
 	if rate.stopReset != nil {
 		close(rate.stopReset)
@@ -42,11 +44,14 @@ func (rate *Ratelimiter) Close() {
 }
 
 func (rate *Ratelimiter) Init() {
-	rate.Lock()
-	defer rate.Unlock()
+	rate.mu.Lock()
+	defer rate.mu.Unlock()
+
+	if rate.timeNow == nil {
+		rate.timeNow = time.Now
+	}
 
 	// stop any ongoing garbage collection routine
-
 	if rate.stopReset != nil {
 		close(rate.stopReset)
 	}
@@ -55,50 +60,52 @@ func (rate *Ratelimiter) Init() {
 	rate.tableIPv4 = make(map[[net.IPv4len]byte]*RatelimiterEntry)
 	rate.tableIPv6 = make(map[[net.IPv6len]byte]*RatelimiterEntry)
 
-	// start garbage collection routine
+	stopReset := rate.stopReset // store in case Init is called again.
 
+	// Start garbage collection routine.
 	go func() {
 		ticker := time.NewTicker(time.Second)
 		ticker.Stop()
 		for {
 			select {
-			case _, ok := <-rate.stopReset:
+			case _, ok := <-stopReset:
 				ticker.Stop()
-				if ok {
-					ticker = time.NewTicker(time.Second)
-				} else {
+				if !ok {
 					return
 				}
+				ticker = time.NewTicker(time.Second)
 			case <-ticker.C:
-				func() {
-					rate.Lock()
-					defer rate.Unlock()
-
-					for key, entry := range rate.tableIPv4 {
-						entry.Lock()
-						if time.Since(entry.lastTime) > garbageCollectTime {
-							delete(rate.tableIPv4, key)
-						}
-						entry.Unlock()
-					}
-
-					for key, entry := range rate.tableIPv6 {
-						entry.Lock()
-						if time.Since(entry.lastTime) > garbageCollectTime {
-							delete(rate.tableIPv6, key)
-						}
-						entry.Unlock()
-					}
-
-					if len(rate.tableIPv4) == 0 && len(rate.tableIPv6) == 0 {
-						ticker.Stop()
-					}
-				}()
+				if rate.cleanup() {
+					ticker.Stop()
+				}
 			}
 		}
 	}()
 }
 
+func (rate *Ratelimiter) cleanup() (empty bool) {
+	rate.mu.Lock()
+	defer rate.mu.Unlock()
+
+	for key, entry := range rate.tableIPv4 {
+		entry.mu.Lock()
+		if rate.timeNow().Sub(entry.lastTime) > garbageCollectTime {
+			delete(rate.tableIPv4, key)
+		}
+		entry.mu.Unlock()
+	}
+
+	for key, entry := range rate.tableIPv6 {
+		entry.mu.Lock()
+		if rate.timeNow().Sub(entry.lastTime) > garbageCollectTime {
+			delete(rate.tableIPv6, key)
+		}
+		entry.mu.Unlock()
+	}
+
+	return len(rate.tableIPv4) == 0 && len(rate.tableIPv6) == 0
+}
+
 func (rate *Ratelimiter) Allow(ip net.IP) bool {
 	var entry *RatelimiterEntry
 	var keyIPv4 [net.IPv4len]byte
@@ -109,7 +116,7 @@ func (rate *Ratelimiter) Allow(ip net.IP) bool {
 	IPv4 := ip.To4()
 	IPv6 := ip.To16()
 
-	rate.RLock()
+	rate.mu.RLock()
 
 	if IPv4 != nil {
 		copy(keyIPv4[:], IPv4)
@@ -119,15 +126,15 @@ func (rate *Ratelimiter) Allow(ip net.IP) bool {
 		entry = rate.tableIPv6[keyIPv6]
 	}
 
-	rate.RUnlock()
+	rate.mu.RUnlock()
 
 	// make new entry if not found
 
 	if entry == nil {
 		entry = new(RatelimiterEntry)
 		entry.tokens = maxTokens - packetCost
-		entry.lastTime = time.Now()
-		rate.Lock()
+		entry.lastTime = rate.timeNow()
+		rate.mu.Lock()
 		if IPv4 != nil {
 			rate.tableIPv4[keyIPv4] = entry
 			if len(rate.tableIPv4) == 1 && len(rate.tableIPv6) == 0 {
@@ -139,14 +146,14 @@ func (rate *Ratelimiter) Allow(ip net.IP) bool {
 				rate.stopReset <- struct{}{}
 			}
 		}
-		rate.Unlock()
+		rate.mu.Unlock()
 		return true
 	}
 
 	// add tokens to entry
 
-	entry.Lock()
-	now := time.Now()
+	entry.mu.Lock()
+	now := rate.timeNow()
 	entry.tokens += now.Sub(entry.lastTime).Nanoseconds()
 	entry.lastTime = now
 	if entry.tokens > maxTokens {
@@ -157,9 +164,9 @@ func (rate *Ratelimiter) Allow(ip net.IP) bool {
 
 	if entry.tokens > packetCost {
 		entry.tokens -= packetCost
-		entry.Unlock()
+		entry.mu.Unlock()
 		return true
 	}
-	entry.Unlock()
+	entry.mu.Unlock()
 	return false
 }

ratelimiter/ratelimiter_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package ratelimiter
@@ -11,22 +11,21 @@ import (
 	"time"
 )
 
-type RatelimiterResult struct {
+type result struct {
 	allowed bool
 	text    string
 	wait    time.Duration
 }
 
 func TestRatelimiter(t *testing.T) {
+	var rate Ratelimiter
+	var expectedResults []result
 
-	var ratelimiter Ratelimiter
-	var expectedResults []RatelimiterResult
-
-	Nano := func(nano int64) time.Duration {
+	nano := func(nano int64) time.Duration {
 		return time.Nanosecond * time.Duration(nano)
 	}
 
-	Add := func(res RatelimiterResult) {
+	add := func(res result) {
 		expectedResults = append(
 			expectedResults,
 			res,
@@ -34,40 +33,40 @@ func TestRatelimiter(t *testing.T) {
 	}
 
 	for i := 0; i < packetsBurstable; i++ {
-		Add(RatelimiterResult{
+		add(result{
 			allowed: true,
-			text:    "inital burst",
+			text:    "initial burst",
 		})
 	}
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: false,
 		text:    "after burst",
 	})
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: true,
-		wait:    Nano(time.Second.Nanoseconds() / packetsPerSecond),
+		wait:    nano(time.Second.Nanoseconds() / packetsPerSecond),
 		text:    "filling tokens for single packet",
 	})
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: false,
 		text:    "not having refilled enough",
 	})
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: true,
-		wait:    2 * (Nano(time.Second.Nanoseconds() / packetsPerSecond)),
+		wait:    2 * (nano(time.Second.Nanoseconds() / packetsPerSecond)),
 		text:    "filling tokens for two packet burst",
 	})
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: true,
 		text:    "second packet in 2 packet burst",
 	})
 
-	Add(RatelimiterResult{
+	add(result{
 		allowed: false,
 		text:    "packet following 2 packet burst",
 	})
@@ -89,14 +88,31 @@ func TestRatelimiter(t *testing.T) {
 		net.ParseIP("3f0e:54a2:f5b4:cd19:a21d:58e1:3746:84c4"),
 	}
 
-	ratelimiter.Init()
+	now := time.Now()
+	rate.timeNow = func() time.Time {
+		return now
+	}
+	defer func() {
+		// Lock to avoid data race with cleanup goroutine from Init.
+		rate.mu.Lock()
+		defer rate.mu.Unlock()
+
+		rate.timeNow = time.Now
+	}()
+	timeSleep := func(d time.Duration) {
+		now = now.Add(d + 1)
+		rate.cleanup()
+	}
+
+	rate.Init()
+	defer rate.Close()
 
 	for i, res := range expectedResults {
-		time.Sleep(res.wait)
+		timeSleep(res.wait)
 		for _, ip := range ips {
-			allowed := ratelimiter.Allow(ip)
+			allowed := rate.Allow(ip)
 			if allowed != res.allowed {
-				t.Fatal("Test failed for", ip.String(), ", on:", i, "(", res.text, ")", "expected:", res.allowed, "got:", allowed)
+				t.Fatalf("%d: %s: rate.Allow(%q)=%v, want %v", i, res.text, ip, allowed, res.allowed)
 			}
 		}
 	}

replay/replay.go

@@ -1,83 +1,62 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
+// Package replay implements an efficient anti-replay algorithm as specified in RFC 6479.
 package replay
 
-/* Implementation of RFC6479
- * https://tools.ietf.org/html/rfc6479
- *
- * The implementation is not safe for concurrent use!
- */
+type block uint64
 
 const (
-	// See: https://golang.org/src/math/big/arith.go
-	_Wordm       = ^uintptr(0)
-	_WordLogSize = _Wordm>>8&1 + _Wordm>>16&1 + _Wordm>>32&1
-	_WordSize    = 1 << _WordLogSize
+	blockBitLog = 6                // 1<<6 == 64 bits
+	blockBits   = 1 << blockBitLog // must be power of 2
+	ringBlocks  = 1 << 7           // must be power of 2
+	windowSize  = (ringBlocks - 1) * blockBits
+	blockMask   = ringBlocks - 1
+	bitMask     = blockBits - 1
 )
 
-const (
-	CounterRedundantBitsLog = _WordLogSize + 3
-	CounterRedundantBits    = _WordSize * 8
-	CounterBitsTotal        = 2048
-	CounterWindowSize       = uint64(CounterBitsTotal - CounterRedundantBits)
-)
-
-const (
-	BacktrackWords = CounterBitsTotal / _WordSize
-)
-
-func minUint64(a uint64, b uint64) uint64 {
-	if a > b {
-		return b
-	}
-	return a
+// A Filter rejects replayed messages by checking if message counter value is
+// within a sliding window of previously received messages.
+// The zero value for Filter is an empty filter ready to use.
+// Filters are unsafe for concurrent use.
+type Filter struct {
+	last uint64
+	ring [ringBlocks]block
 }
 
-type ReplayFilter struct {
-	counter   uint64
-	backtrack [BacktrackWords]uintptr
+// Reset resets the filter to empty state.
+func (f *Filter) Reset() {
+	f.last = 0
+	f.ring[0] = 0
 }
 
-func (filter *ReplayFilter) Init() {
-	filter.counter = 0
-	filter.backtrack[0] = 0
-}
-
-func (filter *ReplayFilter) ValidateCounter(counter uint64, limit uint64) bool {
+// ValidateCounter checks if the counter should be accepted.
+// Overlimit counters (>= limit) are always rejected.
+func (f *Filter) ValidateCounter(counter uint64, limit uint64) bool {
 	if counter >= limit {
 		return false
 	}
-
-	indexWord := counter >> CounterRedundantBitsLog
-
-	if counter > filter.counter {
-
-		// move window forward
-
-		current := filter.counter >> CounterRedundantBitsLog
-		diff := minUint64(indexWord-current, BacktrackWords)
-		for i := uint64(1); i <= diff; i++ {
-			filter.backtrack[(current+i)%BacktrackWords] = 0
+	indexBlock := counter >> blockBitLog
+	if counter > f.last { // move window forward
+		current := f.last >> blockBitLog
+		diff := indexBlock - current
+		if diff > ringBlocks {
+			diff = ringBlocks // cap diff to clear the whole ring
 		}
-		filter.counter = counter
-
-	} else if filter.counter-counter > CounterWindowSize {
-
-		// behind current window
-
+		for i := current + 1; i <= current+diff; i++ {
+			f.ring[i&blockMask] = 0
+		}
+		f.last = counter
+	} else if f.last-counter > windowSize { // behind current window
 		return false
 	}
-
-	indexWord %= BacktrackWords
-	indexBit := counter & uint64(CounterRedundantBits-1)
-
 	// check and set bit
-
-	oldValue := filter.backtrack[indexWord]
-	newValue := oldValue | (1 << indexBit)
-	filter.backtrack[indexWord] = newValue
-	return oldValue != newValue
+	indexBlock &= blockMask
+	indexBit := counter & bitMask
+	old := f.ring[indexBlock]
+	new := old | 1<<indexBit
+	f.ring[indexBlock] = new
+	return old != new
 }

replay/replay_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package replay
@@ -14,22 +14,22 @@ import (
  *
  */
 
-const RejectAfterMessages = (1 << 64) - (1 << 4) - 1
+const RejectAfterMessages = 1<<64 - 1<<13 - 1
 
 func TestReplay(t *testing.T) {
-	var filter ReplayFilter
+	var filter Filter
 
-	T_LIM := CounterWindowSize + 1
+	const T_LIM = windowSize + 1
 
 	testNumber := 0
-	T := func(n uint64, v bool) {
+	T := func(n uint64, expected bool) {
 		testNumber++
-		if filter.ValidateCounter(n, RejectAfterMessages) != v {
-			t.Fatal("Test", testNumber, "failed", n, v)
+		if filter.ValidateCounter(n, RejectAfterMessages) != expected {
+			t.Fatal("Test", testNumber, "failed", n, expected)
 		}
 	}
 
-	filter.Init()
+	filter.Reset()
 
 	T(0, true)                      /*  1 */
 	T(1, true)                      /*  2 */
@@ -67,53 +67,53 @@ func TestReplay(t *testing.T) {
 	T(0, false)                     /* 34 */
 
 	t.Log("Bulk test 1")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := uint64(1); i <= CounterWindowSize; i++ {
+	for i := uint64(1); i <= windowSize; i++ {
 		T(i, true)
 	}
 	T(0, true)
 	T(0, false)
 
 	t.Log("Bulk test 2")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := uint64(2); i <= CounterWindowSize+1; i++ {
+	for i := uint64(2); i <= windowSize+1; i++ {
 		T(i, true)
 	}
 	T(1, true)
 	T(0, false)
 
 	t.Log("Bulk test 3")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := CounterWindowSize + 1; i > 0; i-- {
+	for i := uint64(windowSize + 1); i > 0; i-- {
 		T(i, true)
 	}
 
 	t.Log("Bulk test 4")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := CounterWindowSize + 2; i > 1; i-- {
+	for i := uint64(windowSize + 2); i > 1; i-- {
 		T(i, true)
 	}
 	T(0, false)
 
 	t.Log("Bulk test 5")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := CounterWindowSize; i > 0; i-- {
+	for i := uint64(windowSize); i > 0; i-- {
 		T(i, true)
 	}
-	T(CounterWindowSize+1, true)
+	T(windowSize+1, true)
 	T(0, false)
 
 	t.Log("Bulk test 6")
-	filter.Init()
+	filter.Reset()
 	testNumber = 0
-	for i := CounterWindowSize; i > 0; i-- {
+	for i := uint64(windowSize); i > 0; i-- {
 		T(i, true)
 	}
 	T(0, true)
-	T(CounterWindowSize+1, true)
+	T(windowSize+1, true)
 }

rwcancel/fdset.go

@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package rwcancel
-
-import "golang.org/x/sys/unix"
-
-type fdSet struct {
-	unix.FdSet
-}
-
-func (fdset *fdSet) set(i int) {
-	bits := 32 << (^uint(0) >> 63)
-	fdset.Bits[i/bits] |= 1 << uint(i%bits)
-}
-
-func (fdset *fdSet) check(i int) bool {
-	bits := 32 << (^uint(0) >> 63)
-	return (fdset.Bits[i/bits] & (1 << uint(i%bits))) != 0
-}

rwcancel/rwcancel.go

@@ -1,8 +1,12 @@
+//go:build !windows
+
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
+// Package rwcancel implements cancelable read/write operations on
+// a file descriptor.
 package rwcancel
 
 import (
@@ -13,13 +17,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func max(a, b int) int {
-	if a > b {
-		return a
-	}
-	return b
-}
-
 type RWCancel struct {
 	fd            int
 	closingReader *os.File
@@ -42,47 +39,47 @@ func NewRWCancel(fd int) (*RWCancel, error) {
 }
 
 func RetryAfterError(err error) bool {
-	if pe, ok := err.(*os.PathError); ok {
-		err = pe.Err
-	}
-	if errno, ok := err.(syscall.Errno); ok {
-		switch errno {
-		case syscall.EAGAIN, syscall.EINTR:
-			return true
-		}
-
-	}
-	return false
+	return errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR)
 }
 
 func (rw *RWCancel) ReadyRead() bool {
-	closeFd := int(rw.closingReader.Fd())
-	fdset := fdSet{}
-	fdset.set(rw.fd)
-	fdset.set(closeFd)
-	err := unixSelect(max(rw.fd, closeFd)+1, &fdset.FdSet, nil, nil, nil)
+	closeFd := int32(rw.closingReader.Fd())
+
+	pollFds := []unix.PollFd{{Fd: int32(rw.fd), Events: unix.POLLIN}, {Fd: closeFd, Events: unix.POLLIN}}
+	var err error
+	for {
+		_, err = unix.Poll(pollFds, -1)
+		if err == nil || !RetryAfterError(err) {
+			break
+		}
+	}
 	if err != nil {
 		return false
 	}
-	if fdset.check(closeFd) {
+	if pollFds[1].Revents != 0 {
 		return false
 	}
-	return fdset.check(rw.fd)
+	return pollFds[0].Revents != 0
 }
 
 func (rw *RWCancel) ReadyWrite() bool {
-	closeFd := int(rw.closingReader.Fd())
-	fdset := fdSet{}
-	fdset.set(rw.fd)
-	fdset.set(closeFd)
-	err := unixSelect(max(rw.fd, closeFd)+1, nil, &fdset.FdSet, nil, nil)
+	closeFd := int32(rw.closingReader.Fd())
+	pollFds := []unix.PollFd{{Fd: int32(rw.fd), Events: unix.POLLOUT}, {Fd: closeFd, Events: unix.POLLOUT}}
+	var err error
+	for {
+		_, err = unix.Poll(pollFds, -1)
+		if err == nil || !RetryAfterError(err) {
+			break
+		}
+	}
 	if err != nil {
 		return false
 	}
-	if fdset.check(closeFd) {
+
+	if pollFds[1].Revents != 0 {
 		return false
 	}
-	return fdset.check(rw.fd)
+	return pollFds[0].Revents != 0
 }
 
 func (rw *RWCancel) Read(p []byte) (n int, err error) {
@@ -92,7 +89,7 @@ func (rw *RWCancel) Read(p []byte) (n int, err error) {
 			return n, err
 		}
 		if !rw.ReadyRead() {
-			return 0, errors.New("fd closed")
+			return 0, os.ErrClosed
 		}
 	}
 }
@@ -104,7 +101,7 @@ func (rw *RWCancel) Write(p []byte) (n int, err error) {
 			return n, err
 		}
 		if !rw.ReadyWrite() {
-			return 0, errors.New("fd closed")
+			return 0, os.ErrClosed
 		}
 	}
 }
@@ -113,3 +110,8 @@ func (rw *RWCancel) Cancel() (err error) {
 	_, err = rw.closingWriter.Write([]byte{0})
 	return
 }
+
+func (rw *RWCancel) Close() {
+	rw.closingReader.Close()
+	rw.closingWriter.Close()
+}

rwcancel/rwcancel_windows.go

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: MIT
+
+package rwcancel
+
+type RWCancel struct {
+}
+
+func (*RWCancel) Cancel() {}

rwcancel/select_default.go

@@ -1,14 +0,0 @@
-// +build !linux
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package rwcancel
-
-import "golang.org/x/sys/unix"
-
-func unixSelect(nfd int, r *unix.FdSet, w *unix.FdSet, e *unix.FdSet, timeout *unix.Timeval) error {
-	return unix.Select(nfd, r, w, e, timeout)
-}

rwcancel/select_linux.go

@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
- */
-
-package rwcancel
-
-import "golang.org/x/sys/unix"
-
-func unixSelect(nfd int, r *unix.FdSet, w *unix.FdSet, e *unix.FdSet, timeout *unix.Timeval) (err error) {
-	_, err = unix.Select(nfd, r, w, e, timeout)
-	return
-}

tai64n/tai64n.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package tai64n
@@ -17,16 +17,23 @@ const whitenerMask = uint32(0x1000000 - 1)
 
 type Timestamp [TimestampSize]byte
 
-func Now() Timestamp {
+func stamp(t time.Time) Timestamp {
 	var tai64n Timestamp
-	now := time.Now()
-	secs := base + uint64(now.Unix())
-	nano := uint32(now.Nanosecond()) &^ whitenerMask
+	secs := base + uint64(t.Unix())
+	nano := uint32(t.Nanosecond()) &^ whitenerMask
 	binary.BigEndian.PutUint64(tai64n[:], secs)
 	binary.BigEndian.PutUint32(tai64n[8:], nano)
 	return tai64n
 }
 
+func Now() Timestamp {
+	return stamp(time.Now())
+}
+
 func (t1 Timestamp) After(t2 Timestamp) bool {
 	return bytes.Compare(t1[:], t2[:]) > 0
 }
+
+func (t Timestamp) String() string {
+	return time.Unix(int64(binary.BigEndian.Uint64(t[:8])-base), int64(binary.BigEndian.Uint32(t[8:12]))).String()
+}

tai64n/tai64n_test.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package tai64n
@@ -10,21 +10,31 @@ import (
 	"time"
 )
 
-/* Testing the essential property of the timestamp
- * as used by WireGuard.
- */
+// Test that timestamps are monotonic as required by Wireguard and that
+// nanosecond-level information is whitened to prevent side channel attacks.
 func TestMonotonic(t *testing.T) {
-	old := Now()
-	for i := 0; i < 50; i++ {
-		next := Now()
-		if next.After(old) {
-			t.Error("Whitening insufficient")
-		}
-		time.Sleep(time.Duration(whitenerMask)/time.Nanosecond + 1)
-		next = Now()
-		if !next.After(old) {
-			t.Error("Not monotonically increasing on whitened nano-second scale")
-		}
-		old = next
+	startTime := time.Unix(0, 123456789) // a nontrivial bit pattern
+	// Whitening should reduce timestamp granularity
+	// to more than 10 but fewer than 20 milliseconds.
+	tests := []struct {
+		name      string
+		t1, t2    time.Time
+		wantAfter bool
+	}{
+		{"after_10_ns", startTime, startTime.Add(10 * time.Nanosecond), false},
+		{"after_10_us", startTime, startTime.Add(10 * time.Microsecond), false},
+		{"after_1_ms", startTime, startTime.Add(time.Millisecond), false},
+		{"after_10_ms", startTime, startTime.Add(10 * time.Millisecond), false},
+		{"after_20_ms", startTime, startTime.Add(20 * time.Millisecond), true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts1, ts2 := stamp(tt.t1), stamp(tt.t2)
+			got := ts2.After(ts1)
+			if got != tt.wantAfter {
+				t.Errorf("after = %v; want %v", got, tt.wantAfter)
+			}
+		})
 	}
 }

tests/netns.sh

@@ -36,7 +36,7 @@ netns0="wg-test-$$-0"
 netns1="wg-test-$$-1"
 netns2="wg-test-$$-2"
 program=$1
-export LOG_LEVEL="info"
+export LOG_LEVEL="verbose"
 
 pretty() { echo -e "\x1b[32m\x1b[1m[+] ${1:+NS$1: }${2}\x1b[0m" >&3; }
 pp() { pretty "" "$*"; "$@"; }

tun/alignment_windows_test.go

@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package tun
+
+import (
+	"reflect"
+	"testing"
+	"unsafe"
+)
+
+func checkAlignment(t *testing.T, name string, offset uintptr) {
+	t.Helper()
+	if offset%8 != 0 {
+		t.Errorf("offset of %q within struct is %d bytes, which does not align to 64-bit word boundaries (missing %d bytes). Atomic operations will crash on 32-bit systems.", name, offset, 8-(offset%8))
+	}
+}
+
+// TestRateJugglerAlignment checks that atomically-accessed fields are
+// aligned to 64-bit boundaries, as required by the atomic package.
+//
+// Unfortunately, violating this rule on 32-bit platforms results in a
+// hard segfault at runtime.
+func TestRateJugglerAlignment(t *testing.T) {
+	var r rateJuggler
+
+	typ := reflect.TypeOf(&r).Elem()
+	t.Logf("Peer type size: %d, with fields:", typ.Size())
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)",
+			field.Name,
+			field.Offset,
+			field.Type.Size(),
+			field.Type.Align(),
+		)
+	}
+
+	checkAlignment(t, "rateJuggler.current", unsafe.Offsetof(r.current))
+	checkAlignment(t, "rateJuggler.nextByteCount", unsafe.Offsetof(r.nextByteCount))
+	checkAlignment(t, "rateJuggler.nextStartTime", unsafe.Offsetof(r.nextStartTime))
+}
+
+// TestNativeTunAlignment checks that atomically-accessed fields are
+// aligned to 64-bit boundaries, as required by the atomic package.
+//
+// Unfortunately, violating this rule on 32-bit platforms results in a
+// hard segfault at runtime.
+func TestNativeTunAlignment(t *testing.T) {
+	var tun NativeTun
+
+	typ := reflect.TypeOf(&tun).Elem()
+	t.Logf("Peer type size: %d, with fields:", typ.Size())
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)",
+			field.Name,
+			field.Offset,
+			field.Type.Size(),
+			field.Type.Align(),
+		)
+	}
+
+	checkAlignment(t, "NativeTun.rate", unsafe.Offsetof(tun.rate))
+}

tun/netstack/examples/http_client.go

@@ -0,0 +1,54 @@
+//go:build ignore
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package main
+
+import (
+	"io"
+	"log"
+	"net"
+	"net/http"
+
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun/netstack"
+)
+
+func main() {
+	tun, tnet, err := netstack.CreateNetTUN(
+		[]net.IP{net.ParseIP("192.168.4.29")},
+		[]net.IP{net.ParseIP("8.8.8.8")},
+		1420)
+	if err != nil {
+		log.Panic(err)
+	}
+	dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelVerbose, ""))
+	dev.IpcSet(`private_key=a8dac1d8a70a751f0f699fb14ba1cff7b79cf4fbd8f09f44c6e6a90d0369604f
+public_key=25123c5dcd3328ff645e4f2a3fce0d754400d3887a0cb7c56f0267e20fbf3c5b
+endpoint=163.172.161.0:12912
+allowed_ip=0.0.0.0/0
+`)
+	err = dev.Up()
+	if err != nil {
+		log.Panic(err)
+	}
+
+	client := http.Client{
+		Transport: &http.Transport{
+			DialContext: tnet.DialContext,
+		},
+	}
+	resp, err := client.Get("https://www.zx2c4.com/ip")
+	if err != nil {
+		log.Panic(err)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Panic(err)
+	}
+	log.Println(string(body))
+}

tun/netstack/examples/http_server.go

@@ -0,0 +1,50 @@
+//go:build ignore
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package main
+
+import (
+	"io"
+	"log"
+	"net"
+	"net/http"
+
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun/netstack"
+)
+
+func main() {
+	tun, tnet, err := netstack.CreateNetTUN(
+		[]net.IP{net.ParseIP("192.168.4.29")},
+		[]net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("8.8.4.4")},
+		1420,
+	)
+	if err != nil {
+		log.Panic(err)
+	}
+	dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelVerbose, ""))
+	dev.IpcSet(`private_key=a8dac1d8a70a751f0f699fb14ba1cff7b79cf4fbd8f09f44c6e6a90d0369604f
+public_key=25123c5dcd3328ff645e4f2a3fce0d754400d3887a0cb7c56f0267e20fbf3c5b
+endpoint=163.172.161.0:12912
+allowed_ip=0.0.0.0/0
+persistent_keepalive_interval=25
+`)
+	dev.Up()
+	listener, err := tnet.ListenTCP(&net.TCPAddr{Port: 80})
+	if err != nil {
+		log.Panicln(err)
+	}
+	http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+		log.Printf("> %s - %s - %s", request.RemoteAddr, request.URL.String(), request.UserAgent())
+		io.WriteString(writer, "Hello from userspace TCP!")
+	})
+	err = http.Serve(listener, nil)
+	if err != nil {
+		log.Panicln(err)
+	}
+}

tun/netstack/go.mod

@@ -0,0 +1,11 @@
+module golang.zx2c4.com/wireguard/tun/netstack
+
+go 1.16
+
+require (
+	golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6
+	golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20210424170727-c9db4b7aaa22
+	gvisor.dev/gvisor v0.0.0-20210506004418-fbfeba3024f0
+)

tun/netstack/go.sum

@@ -0,0 +1,605 @@
+bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
+github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
+github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
+github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
+github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
+github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
+github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/containerd/typeurl v0.0.0-20200205145503-b45ef1f1f737/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.4.2-0.20191028175130-9e7d5ac5ea55/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210115211752-39141e76b647/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 h1:0PC75Fz/kyMGhL0e1QnypqK2kQMqKt9csD1GnMJR+Zk=
+golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wireguard v0.0.0-20210424170727-c9db4b7aaa22 h1:ytS28bw9HtZVDRMDxviC6ryCJuccw+zXhh04u2IRWJw=
+golang.zx2c4.com/wireguard v0.0.0-20210424170727-c9db4b7aaa22/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.36.0-dev.0.20210208035533-9280052d3665/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.25.1-0.20201020201750-d3470999428b/go.mod h1:hFxJC2f0epmp1elRCiEGJTKAWbwxZ2nvqZdHl3FQXCY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gvisor.dev/gvisor v0.0.0-20210506004418-fbfeba3024f0 h1:Ny53d6x76f7EgvYe7pi8SQcIzdRM69y1ckQ8rRtT6ww=
+gvisor.dev/gvisor v0.0.0-20210506004418-fbfeba3024f0/go.mod h1:ucHEMlckp+S/YzKEpwwAyGBhAh807Wxq/8Erc6gFxCE=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.1.1/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
+k8s.io/api v0.16.13/go.mod h1:QWu8UWSTiuQZMMeYjwLs6ILu5O74qKSJ0c+4vrchDxs=
+k8s.io/apimachinery v0.16.13/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ=
+k8s.io/apimachinery v0.16.14-rc.0/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ=
+k8s.io/client-go v0.16.13/go.mod h1:UKvVT4cajC2iN7DCjLgT0KVY/cbY6DGdUCyRiIfws5M=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

tun/netstack/tun.go

@@ -0,0 +1,818 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2019-2021 WireGuard LLC. All Rights Reserved.
+ */
+
+package netstack
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.zx2c4.com/wireguard/tun"
+
+	"golang.org/x/net/dns/dnsmessage"
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/adapters/gonet"
+	"gvisor.dev/gvisor/pkg/tcpip/buffer"
+	"gvisor.dev/gvisor/pkg/tcpip/header"
+	"gvisor.dev/gvisor/pkg/tcpip/network/ipv4"
+	"gvisor.dev/gvisor/pkg/tcpip/network/ipv6"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
+	"gvisor.dev/gvisor/pkg/tcpip/transport/tcp"
+	"gvisor.dev/gvisor/pkg/tcpip/transport/udp"
+)
+
+type netTun struct {
+	stack          *stack.Stack
+	dispatcher     stack.NetworkDispatcher
+	events         chan tun.Event
+	incomingPacket chan buffer.VectorisedView
+	mtu            int
+	dnsServers     []net.IP
+	hasV4, hasV6   bool
+}
+type endpoint netTun
+type Net netTun
+
+func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
+	e.dispatcher = dispatcher
+}
+
+func (e *endpoint) IsAttached() bool {
+	return e.dispatcher != nil
+}
+
+func (e *endpoint) MTU() uint32 {
+	mtu, err := (*netTun)(e).MTU()
+	if err != nil {
+		panic(err)
+	}
+	return uint32(mtu)
+}
+
+func (*endpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return stack.CapabilityNone
+}
+
+func (*endpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+
+func (*endpoint) LinkAddress() tcpip.LinkAddress {
+	return ""
+}
+
+func (*endpoint) Wait() {}
+
+func (e *endpoint) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	e.incomingPacket <- buffer.NewVectorisedView(pkt.Size(), pkt.Views())
+	return nil
+}
+
+func (e *endpoint) WritePackets(stack.RouteInfo, stack.PacketBufferList, tcpip.NetworkProtocolNumber) (int, tcpip.Error) {
+	panic("not implemented")
+}
+
+func (*endpoint) ARPHardwareType() header.ARPHardwareType {
+	return header.ARPHardwareNone
+}
+
+func (e *endpoint) AddHeader(tcpip.LinkAddress, tcpip.LinkAddress, tcpip.NetworkProtocolNumber, *stack.PacketBuffer) {
+}
+
+func CreateNetTUN(localAddresses, dnsServers []net.IP, mtu int) (tun.Device, *Net, error) {
+	opts := stack.Options{
+		NetworkProtocols:   []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
+		TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol},
+		HandleLocal:        true,
+	}
+	dev := &netTun{
+		stack:          stack.New(opts),
+		events:         make(chan tun.Event, 10),
+		incomingPacket: make(chan buffer.VectorisedView),
+		dnsServers:     dnsServers,
+		mtu:            mtu,
+	}
+	tcpipErr := dev.stack.CreateNIC(1, (*endpoint)(dev))
+	if tcpipErr != nil {
+		return nil, nil, fmt.Errorf("CreateNIC: %v", tcpipErr)
+	}
+	for _, ip := range localAddresses {
+		if ip4 := ip.To4(); ip4 != nil {
+			tcpipErr = dev.stack.AddAddress(1, ipv4.ProtocolNumber, tcpip.Address(ip4))
+			if tcpipErr != nil {
+				return nil, nil, fmt.Errorf("AddAddress(%v): %v", ip4, tcpipErr)
+			}
+			dev.hasV4 = true
+		} else {
+			tcpipErr = dev.stack.AddAddress(1, ipv6.ProtocolNumber, tcpip.Address(ip))
+			if tcpipErr != nil {
+				return nil, nil, fmt.Errorf("AddAddress(%v): %v", ip4, tcpipErr)
+			}
+			dev.hasV6 = true
+		}
+	}
+	if dev.hasV4 {
+		dev.stack.AddRoute(tcpip.Route{Destination: header.IPv4EmptySubnet, NIC: 1})
+	}
+	if dev.hasV6 {
+		dev.stack.AddRoute(tcpip.Route{Destination: header.IPv6EmptySubnet, NIC: 1})
+	}
+
+	dev.events <- tun.EventUp
+	return dev, (*Net)(dev), nil
+}
+
+func (tun *netTun) Name() (string, error) {
+	return "go", nil
+}
+
+func (tun *netTun) File() *os.File {
+	return nil
+}
+
+func (tun *netTun) Events() chan tun.Event {
+	return tun.events
+}
+
+func (tun *netTun) Read(buf []byte, offset int) (int, error) {
+	view, ok := <-tun.incomingPacket
+	if !ok {
+		return 0, os.ErrClosed
+	}
+	return view.Read(buf[offset:])
+}
+
+func (tun *netTun) Write(buf []byte, offset int) (int, error) {
+	packet := buf[offset:]
+	if len(packet) == 0 {
+		return 0, nil
+	}
+
+	pkb := stack.NewPacketBuffer(stack.PacketBufferOptions{Data: buffer.NewVectorisedView(len(packet), []buffer.View{buffer.NewViewFromBytes(packet)})})
+	switch packet[0] >> 4 {
+	case 4:
+		tun.dispatcher.DeliverNetworkPacket("", "", ipv4.ProtocolNumber, pkb)
+	case 6:
+		tun.dispatcher.DeliverNetworkPacket("", "", ipv6.ProtocolNumber, pkb)
+	}
+
+	return len(buf), nil
+}
+
+func (tun *netTun) Flush() error {
+	return nil
+}
+
+func (tun *netTun) Close() error {
+	tun.stack.RemoveNIC(1)
+
+	if tun.events != nil {
+		close(tun.events)
+	}
+	if tun.incomingPacket != nil {
+		close(tun.incomingPacket)
+	}
+	return nil
+}
+
+func (tun *netTun) MTU() (int, error) {
+	return tun.mtu, nil
+}
+
+func convertToFullAddr(ip net.IP, port int) (tcpip.FullAddress, tcpip.NetworkProtocolNumber) {
+	if ip4 := ip.To4(); ip4 != nil {
+		return tcpip.FullAddress{
+			NIC:  1,
+			Addr: tcpip.Address(ip4),
+			Port: uint16(port),
+		}, ipv4.ProtocolNumber
+	} else {
+		return tcpip.FullAddress{
+			NIC:  1,
+			Addr: tcpip.Address(ip),
+			Port: uint16(port),
+		}, ipv6.ProtocolNumber
+	}
+}
+
+func (net *Net) DialContextTCP(ctx context.Context, addr *net.TCPAddr) (*gonet.TCPConn, error) {
+	if addr == nil {
+		panic("todo: deal with auto addr semantics for nil addr")
+	}
+	fa, pn := convertToFullAddr(addr.IP, addr.Port)
+	return gonet.DialContextTCP(ctx, net.stack, fa, pn)
+}
+
+func (net *Net) DialTCP(addr *net.TCPAddr) (*gonet.TCPConn, error) {
+	if addr == nil {
+		panic("todo: deal with auto addr semantics for nil addr")
+	}
+	fa, pn := convertToFullAddr(addr.IP, addr.Port)
+	return gonet.DialTCP(net.stack, fa, pn)
+}
+
+func (net *Net) ListenTCP(addr *net.TCPAddr) (*gonet.TCPListener, error) {
+	if addr == nil {
+		panic("todo: deal with auto addr semantics for nil addr")
+	}
+	fa, pn := convertToFullAddr(addr.IP, addr.Port)
+	return gonet.ListenTCP(net.stack, fa, pn)
+}
+
+func (net *Net) DialUDP(laddr, raddr *net.UDPAddr) (*gonet.UDPConn, error) {
+	var lfa, rfa *tcpip.FullAddress
+	var pn tcpip.NetworkProtocolNumber
+	if laddr != nil {
+		var addr tcpip.FullAddress
+		addr, pn = convertToFullAddr(laddr.IP, laddr.Port)
+		lfa = &addr
+	}
+	if raddr != nil {
+		var addr tcpip.FullAddress
+		addr, pn = convertToFullAddr(raddr.IP, raddr.Port)
+		rfa = &addr
+	}
+	return gonet.DialUDP(net.stack, lfa, rfa, pn)
+}
+
+var (
+	errNoSuchHost                   = errors.New("no such host")
+	errLameReferral                 = errors.New("lame referral")
+	errCannotUnmarshalDNSMessage    = errors.New("cannot unmarshal DNS message")
+	errCannotMarshalDNSMessage      = errors.New("cannot marshal DNS message")
+	errServerMisbehaving            = errors.New("server misbehaving")
+	errInvalidDNSResponse           = errors.New("invalid DNS response")
+	errNoAnswerFromDNSServer        = errors.New("no answer from DNS server")
+	errServerTemporarilyMisbehaving = errors.New("server misbehaving")
+	errCanceled                     = errors.New("operation was canceled")
+	errTimeout                      = errors.New("i/o timeout")
+	errNumericPort                  = errors.New("port must be numeric")
+	errNoSuitableAddress            = errors.New("no suitable address found")
+	errMissingAddress               = errors.New("missing address")
+)
+
+func (net *Net) LookupHost(host string) (addrs []string, err error) {
+	return net.LookupContextHost(context.Background(), host)
+}
+
+func isDomainName(s string) bool {
+	l := len(s)
+	if l == 0 || l > 254 || l == 254 && s[l-1] != '.' {
+		return false
+	}
+	last := byte('.')
+	nonNumeric := false
+	partlen := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch {
+		default:
+			return false
+		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_':
+			nonNumeric = true
+			partlen++
+		case '0' <= c && c <= '9':
+			partlen++
+		case c == '-':
+			if last == '.' {
+				return false
+			}
+			partlen++
+			nonNumeric = true
+		case c == '.':
+			if last == '.' || last == '-' {
+				return false
+			}
+			if partlen > 63 || partlen == 0 {
+				return false
+			}
+			partlen = 0
+		}
+		last = c
+	}
+	if last == '-' || partlen > 63 {
+		return false
+	}
+	return nonNumeric
+}
+
+func randU16() uint16 {
+	var b [2]byte
+	_, err := rand.Read(b[:])
+	if err != nil {
+		panic(err)
+	}
+	return binary.LittleEndian.Uint16(b[:])
+}
+
+func newRequest(q dnsmessage.Question) (id uint16, udpReq, tcpReq []byte, err error) {
+	id = randU16()
+	b := dnsmessage.NewBuilder(make([]byte, 2, 514), dnsmessage.Header{ID: id, RecursionDesired: true})
+	b.EnableCompression()
+	if err := b.StartQuestions(); err != nil {
+		return 0, nil, nil, err
+	}
+	if err := b.Question(q); err != nil {
+		return 0, nil, nil, err
+	}
+	tcpReq, err = b.Finish()
+	udpReq = tcpReq[2:]
+	l := len(tcpReq) - 2
+	tcpReq[0] = byte(l >> 8)
+	tcpReq[1] = byte(l)
+	return id, udpReq, tcpReq, err
+}
+
+func equalASCIIName(x, y dnsmessage.Name) bool {
+	if x.Length != y.Length {
+		return false
+	}
+	for i := 0; i < int(x.Length); i++ {
+		a := x.Data[i]
+		b := y.Data[i]
+		if 'A' <= a && a <= 'Z' {
+			a += 0x20
+		}
+		if 'A' <= b && b <= 'Z' {
+			b += 0x20
+		}
+		if a != b {
+			return false
+		}
+	}
+	return true
+}
+
+func checkResponse(reqID uint16, reqQues dnsmessage.Question, respHdr dnsmessage.Header, respQues dnsmessage.Question) bool {
+	if !respHdr.Response {
+		return false
+	}
+	if reqID != respHdr.ID {
+		return false
+	}
+	if reqQues.Type != respQues.Type || reqQues.Class != respQues.Class || !equalASCIIName(reqQues.Name, respQues.Name) {
+		return false
+	}
+	return true
+}
+
+func dnsPacketRoundTrip(c net.Conn, id uint16, query dnsmessage.Question, b []byte) (dnsmessage.Parser, dnsmessage.Header, error) {
+	if _, err := c.Write(b); err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, err
+	}
+	b = make([]byte, 512)
+	for {
+		n, err := c.Read(b)
+		if err != nil {
+			return dnsmessage.Parser{}, dnsmessage.Header{}, err
+		}
+		var p dnsmessage.Parser
+		h, err := p.Start(b[:n])
+		if err != nil {
+			continue
+		}
+		q, err := p.Question()
+		if err != nil || !checkResponse(id, query, h, q) {
+			continue
+		}
+		return p, h, nil
+	}
+}
+
+func dnsStreamRoundTrip(c net.Conn, id uint16, query dnsmessage.Question, b []byte) (dnsmessage.Parser, dnsmessage.Header, error) {
+	if _, err := c.Write(b); err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, err
+	}
+	b = make([]byte, 1280)
+	if _, err := io.ReadFull(c, b[:2]); err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, err
+	}
+	l := int(b[0])<<8 | int(b[1])
+	if l > len(b) {
+		b = make([]byte, l)
+	}
+	n, err := io.ReadFull(c, b[:l])
+	if err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, err
+	}
+	var p dnsmessage.Parser
+	h, err := p.Start(b[:n])
+	if err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage
+	}
+	q, err := p.Question()
+	if err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage
+	}
+	if !checkResponse(id, query, h, q) {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse
+	}
+	return p, h, nil
+}
+
+func (tnet *Net) exchange(ctx context.Context, server net.IP, q dnsmessage.Question, timeout time.Duration) (dnsmessage.Parser, dnsmessage.Header, error) {
+	q.Class = dnsmessage.ClassINET
+	id, udpReq, tcpReq, err := newRequest(q)
+	if err != nil {
+		return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotMarshalDNSMessage
+	}
+
+	for _, useUDP := range []bool{true, false} {
+		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout))
+		defer cancel()
+
+		var c net.Conn
+		var err error
+		if useUDP {
+			c, err = tnet.DialUDP(nil, &net.UDPAddr{IP: server, Port: 53})
+		} else {
+			c, err = tnet.DialContextTCP(ctx, &net.TCPAddr{IP: server, Port: 53})
+		}
+
+		if err != nil {
+			return dnsmessage.Parser{}, dnsmessage.Header{}, err
+		}
+		if d, ok := ctx.Deadline(); ok && !d.IsZero() {
+			c.SetDeadline(d)
+		}
+		var p dnsmessage.Parser
+		var h dnsmessage.Header
+		if useUDP {
+			p, h, err = dnsPacketRoundTrip(c, id, q, udpReq)
+		} else {
+			p, h, err = dnsStreamRoundTrip(c, id, q, tcpReq)
+		}
+		c.Close()
+		if err != nil {
+			if err == context.Canceled {
+				err = errCanceled
+			} else if err == context.DeadlineExceeded {
+				err = errTimeout
+			}
+			return dnsmessage.Parser{}, dnsmessage.Header{}, err
+		}
+		if err := p.SkipQuestion(); err != dnsmessage.ErrSectionDone {
+			return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse
+		}
+		if h.Truncated {
+			continue
+		}
+		return p, h, nil
+	}
+	return dnsmessage.Parser{}, dnsmessage.Header{}, errNoAnswerFromDNSServer
+}
+
+func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header) error {
+	if h.RCode == dnsmessage.RCodeNameError {
+		return errNoSuchHost
+	}
+	_, err := p.AnswerHeader()
+	if err != nil && err != dnsmessage.ErrSectionDone {
+		return errCannotUnmarshalDNSMessage
+	}
+	if h.RCode == dnsmessage.RCodeSuccess && !h.Authoritative && !h.RecursionAvailable && err == dnsmessage.ErrSectionDone {
+		return errLameReferral
+	}
+	if h.RCode != dnsmessage.RCodeSuccess && h.RCode != dnsmessage.RCodeNameError {
+		if h.RCode == dnsmessage.RCodeServerFailure {
+			return errServerTemporarilyMisbehaving
+		}
+		return errServerMisbehaving
+	}
+	return nil
+}
+
+func skipToAnswer(p *dnsmessage.Parser, qtype dnsmessage.Type) error {
+	for {
+		h, err := p.AnswerHeader()
+		if err == dnsmessage.ErrSectionDone {
+			return errNoSuchHost
+		}
+		if err != nil {
+			return errCannotUnmarshalDNSMessage
+		}
+		if h.Type == qtype {
+			return nil
+		}
+		if err := p.SkipAnswer(); err != nil {
+			return errCannotUnmarshalDNSMessage
+		}
+	}
+}
+
+func (tnet *Net) tryOneName(ctx context.Context, name string, qtype dnsmessage.Type) (dnsmessage.Parser, string, error) {
+	var lastErr error
+
+	n, err := dnsmessage.NewName(name)
+	if err != nil {
+		return dnsmessage.Parser{}, "", errCannotMarshalDNSMessage
+	}
+	q := dnsmessage.Question{
+		Name:  n,
+		Type:  qtype,
+		Class: dnsmessage.ClassINET,
+	}
+
+	for i := 0; i < 2; i++ {
+		for _, server := range tnet.dnsServers {
+			p, h, err := tnet.exchange(ctx, server, q, time.Second*5)
+			if err != nil {
+				dnsErr := &net.DNSError{
+					Err:    err.Error(),
+					Name:   name,
+					Server: server.String(),
+				}
+				if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
+					dnsErr.IsTimeout = true
+				}
+				if _, ok := err.(*net.OpError); ok {
+					dnsErr.IsTemporary = true
+				}
+				lastErr = dnsErr
+				continue
+			}
+
+			if err := checkHeader(&p, h); err != nil {
+				dnsErr := &net.DNSError{
+					Err:    err.Error(),
+					Name:   name,
+					Server: server.String(),
+				}
+				if err == errServerTemporarilyMisbehaving {
+					dnsErr.IsTemporary = true
+				}
+				if err == errNoSuchHost {
+					dnsErr.IsNotFound = true
+					return p, server.String(), dnsErr
+				}
+				lastErr = dnsErr
+				continue
+			}
+
+			err = skipToAnswer(&p, qtype)
+			if err == nil {
+				return p, server.String(), nil
+			}
+			lastErr = &net.DNSError{
+				Err:    err.Error(),
+				Name:   name,
+				Server: server.String(),
+			}
+			if err == errNoSuchHost {
+				lastErr.(*net.DNSError).IsNotFound = true
+				return p, server.String(), lastErr
+			}
+		}
+	}
+	return dnsmessage.Parser{}, "", lastErr
+}
+
+func (tnet *Net) LookupContextHost(ctx context.Context, host string) ([]string, error) {
+	if host == "" || (!tnet.hasV6 && !tnet.hasV4) {
+		return nil, &net.DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true}
+	}
+	zlen := len(host)
+	if strings.IndexByte(host, ':') != -1 {
+		if zidx := strings.LastIndexByte(host, '%'); zidx != -1 {
+			zlen = zidx
+		}
+	}
+	if ip := net.ParseIP(host[:zlen]); ip != nil {
+		return []string{host[:zlen]}, nil
+	}
+
+	if !isDomainName(host) {
+		return nil, &net.DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true}
+	}
+	type result struct {
+		p      dnsmessage.Parser
+		server string
+		error
+	}
+	var addrsV4, addrsV6 []net.IP
+	lanes := 0
+	if tnet.hasV4 {
+		lanes++
+	}
+	if tnet.hasV6 {
+		lanes++
+	}
+	lane := make(chan result, lanes)
+	var lastErr error
+	if tnet.hasV4 {
+		go func() {
+			p, server, err := tnet.tryOneName(ctx, host+".", dnsmessage.TypeA)
+			lane <- result{p, server, err}
+		}()
+	}
+	if tnet.hasV6 {
+		go func() {
+			p, server, err := tnet.tryOneName(ctx, host+".", dnsmessage.TypeAAAA)
+			lane <- result{p, server, err}
+		}()
+	}
+	for l := 0; l < lanes; l++ {
+		result := <-lane
+		if result.error != nil {
+			if lastErr == nil {
+				lastErr = result.error
+			}
+			continue
+		}
+
+	loop:
+		for {
+			h, err := result.p.AnswerHeader()
+			if err != nil && err != dnsmessage.ErrSectionDone {
+				lastErr = &net.DNSError{
+					Err:    errCannotMarshalDNSMessage.Error(),
+					Name:   host,
+					Server: result.server,
+				}
+			}
+			if err != nil {
+				break
+			}
+			switch h.Type {
+			case dnsmessage.TypeA:
+				a, err := result.p.AResource()
+				if err != nil {
+					lastErr = &net.DNSError{
+						Err:    errCannotMarshalDNSMessage.Error(),
+						Name:   host,
+						Server: result.server,
+					}
+					break loop
+				}
+				addrsV4 = append(addrsV4, net.IP(a.A[:]))
+
+			case dnsmessage.TypeAAAA:
+				aaaa, err := result.p.AAAAResource()
+				if err != nil {
+					lastErr = &net.DNSError{
+						Err:    errCannotMarshalDNSMessage.Error(),
+						Name:   host,
+						Server: result.server,
+					}
+					break loop
+				}
+				addrsV6 = append(addrsV6, net.IP(aaaa.AAAA[:]))
+
+			default:
+				if err := result.p.SkipAnswer(); err != nil {
+					lastErr = &net.DNSError{
+						Err:    errCannotMarshalDNSMessage.Error(),
+						Name:   host,
+						Server: result.server,
+					}
+					break loop
+				}
+				continue
+			}
+		}
+	}
+	// We don't do RFC6724. Instead just put V6 addresess first if an IPv6 address is enabled
+	var addrs []net.IP
+	if tnet.hasV6 {
+		addrs = append(addrsV6, addrsV4...)
+	} else {
+		addrs = append(addrsV4, addrsV6...)
+	}
+
+	if len(addrs) == 0 && lastErr != nil {
+		return nil, lastErr
+	}
+	saddrs := make([]string, 0, len(addrs))
+	for _, ip := range addrs {
+		saddrs = append(saddrs, ip.String())
+	}
+	return saddrs, nil
+}
+
+func partialDeadline(now, deadline time.Time, addrsRemaining int) (time.Time, error) {
+	if deadline.IsZero() {
+		return deadline, nil
+	}
+	timeRemaining := deadline.Sub(now)
+	if timeRemaining <= 0 {
+		return time.Time{}, errTimeout
+	}
+	timeout := timeRemaining / time.Duration(addrsRemaining)
+	const saneMinimum = 2 * time.Second
+	if timeout < saneMinimum {
+		if timeRemaining < saneMinimum {
+			timeout = timeRemaining
+		} else {
+			timeout = saneMinimum
+		}
+	}
+	return now.Add(timeout), nil
+}
+
+func (tnet *Net) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	if ctx == nil {
+		panic("nil context")
+	}
+	var acceptV4, acceptV6, useUDP bool
+	if len(network) == 3 {
+		acceptV4 = true
+		acceptV6 = true
+	} else if len(network) == 4 {
+		acceptV4 = network[3] == '4'
+		acceptV6 = network[3] == '6'
+	}
+	if !acceptV4 && !acceptV6 {
+		return nil, &net.OpError{Op: "dial", Err: net.UnknownNetworkError(network)}
+	}
+	if network[:3] == "udp" {
+		useUDP = true
+	} else if network[:3] != "tcp" {
+		return nil, &net.OpError{Op: "dial", Err: net.UnknownNetworkError(network)}
+	}
+	host, sport, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, &net.OpError{Op: "dial", Err: err}
+	}
+	port, err := strconv.Atoi(sport)
+	if err != nil || port < 0 || port > 65535 {
+		return nil, &net.OpError{Op: "dial", Err: errNumericPort}
+	}
+	allAddr, err := tnet.LookupContextHost(ctx, host)
+	if err != nil {
+		return nil, &net.OpError{Op: "dial", Err: err}
+	}
+	var addrs []net.IP
+	for _, addr := range allAddr {
+		if strings.IndexByte(addr, ':') != -1 && acceptV6 {
+			addrs = append(addrs, net.ParseIP(addr))
+		} else if strings.IndexByte(addr, '.') != -1 && acceptV4 {
+			addrs = append(addrs, net.ParseIP(addr))
+		}
+	}
+	if len(addrs) == 0 && len(allAddr) != 0 {
+		return nil, &net.OpError{Op: "dial", Err: errNoSuitableAddress}
+	}
+
+	var firstErr error
+	for i, addr := range addrs {
+		select {
+		case <-ctx.Done():
+			err := ctx.Err()
+			if err == context.Canceled {
+				err = errCanceled
+			} else if err == context.DeadlineExceeded {
+				err = errTimeout
+			}
+			return nil, &net.OpError{Op: "dial", Err: err}
+		default:
+		}
+
+		dialCtx := ctx
+		if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
+			partialDeadline, err := partialDeadline(time.Now(), deadline, len(addrs)-i)
+			if err != nil {
+				if firstErr == nil {
+					firstErr = &net.OpError{Op: "dial", Err: err}
+				}
+				break
+			}
+			if partialDeadline.Before(deadline) {
+				var cancel context.CancelFunc
+				dialCtx, cancel = context.WithDeadline(ctx, partialDeadline)
+				defer cancel()
+			}
+		}
+
+		var c net.Conn
+		if useUDP {
+			c, err = tnet.DialUDP(nil, &net.UDPAddr{IP: addr, Port: port})
+		} else {
+			c, err = tnet.DialContextTCP(dialCtx, &net.TCPAddr{IP: addr, Port: port})
+		}
+		if err == nil {
+			return c, nil
+		}
+		if firstErr == nil {
+			firstErr = err
+		}
+	}
+	if firstErr == nil {
+		firstErr = &net.OpError{Op: "dial", Err: errMissingAddress}
+	}
+	return nil, firstErr
+}
+
+func (tnet *Net) Dial(network, address string) (net.Conn, error) {
+	return tnet.DialContext(context.Background(), network, address)
+}

tun/operateonfd.go

@@ -1,8 +1,8 @@
-// +build !windows
+//go:build darwin || freebsd
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package tun

tun/tun.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package tun

tun/tun_darwin.go

@@ -1,16 +1,18 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2019 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2021 WireGuard LLC. All Rights Reserved.
  */
 
 package tun
 
 import (
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
+	"sync"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"golang.org/x/net/ipv6"
@@ -19,28 +21,26 @@ import (
 
 const utunControlName = "com.apple.net.utun_control"
 
-// _CTLIOCGINFO value derived from /usr/include/sys/{kern_control,ioccom}.h
-const _CTLIOCGINFO = (0x40000000 | 0x80000000) | ((100 & 0x1fff) << 16) | uint32(byte('N'))<<8 | 3
-
-// sockaddr_ctl specifeid in /usr/include/sys/kern_control.h
-type sockaddrCtl struct {
-	scLen      uint8
-	scFamily   uint8
-	ssSysaddr  uint16
-	scID       uint32
-	scUnit     uint32
-	scReserved [5]uint32
-}
-
 type NativeTun struct {
 	name        string
 	tunFile     *os.File
 	events      chan Event
 	errors      chan error
 	routeSocket int
+	closeOnce   sync.Once
 }
 
-var sockaddrCtlSize uintptr = 32
+func retryInterfaceByIndex(index int) (iface *net.Interface, err error) {
+	for i := 0; i < 20; i++ {
+		iface, err = net.InterfaceByIndex(index)
+		if err != nil && errors.Is(err, syscall.ENOMEM) {
+			time.Sleep(time.Duration(i) * time.Second / 3)
+			continue
+		}
+		return iface, err
+	}
+	return nil, err
+}
 
 func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 	var (
@@ -74,7 +74,7 @@ func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 			continue
 		}
 
-		iface, err := net.InterfaceByIndex(ifindex)
+		iface, err := retryInterfaceByIndex(ifindex)
 		if err != nil {
 			tun.errors <- err
 			return
@@ -108,52 +108,32 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	}
 
 	fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, 2)
-
 	if err != nil {
 		return nil, err
 	}
 
-	var ctlInfo = &struct {
-		ctlID   uint32
-		ctlName [96]byte
-	}{}
-
-	copy(ctlInfo.ctlName[:], []byte(utunControlName))
-
-	_, _, errno := unix.Syscall(
-		unix.SYS_IOCTL,
-		uintptr(fd),
-		uintptr(_CTLIOCGINFO),
-		uintptr(unsafe.Pointer(ctlInfo)),
-	)
-
-	if errno != 0 {
-		return nil, fmt.Errorf("_CTLIOCGINFO: %v", errno)
-	}
-
-	sc := sockaddrCtl{
-		scLen:     uint8(sockaddrCtlSize),
-		scFamily:  unix.AF_SYSTEM,
-		ssSysaddr: 2,
-		scID:      ctlInfo.ctlID,
-		scUnit:    uint32(ifIndex) + 1,
-	}
-
-	scPointer := unsafe.Pointer(&sc)
-
-	_, _, errno = unix.RawSyscall(
-		unix.SYS_CONNECT,
-		uintptr(fd),
-		uintptr(scPointer),
-		uintptr(sockaddrCtlSize),
-	)
-
-	if errno != 0 {
-		return nil, fmt.Errorf("SYS_CONNECT: %v", errno)
-	}
-
-	err = syscall.SetNonblock(fd, true)
+	ctlInfo := &unix.CtlInfo{}
+	copy(ctlInfo.Name[:], []byte(utunControlName))
+	err = unix.IoctlCtlInfo(fd, ctlInfo)
 	if err != nil {
+		unix.Close(fd)
+		return nil, fmt.Errorf("IoctlGetCtlInfo: %w", err)
+	}
+
+	sc := &unix.SockaddrCtl{
+		ID:   ctlInfo.Id,
+		Unit: uint32(ifIndex) + 1,
+	}
+
+	err = unix.Connect(fd, sc)
+	if err != nil {
+		unix.Close(fd)
+		return nil, err
+	}
+
+	err = unix.SetNonblock(fd, true)
+	if err != nil {
+		unix.Close(fd)
 		return nil, err
 	}
 	tun, err := CreateTUNFromFile(os.NewFile(uintptr(fd), ""), mtu)
@@ -161,7 +141,7 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	if err == nil && name == "utun" {
 		fname := os.Getenv("WG_TUN_NAME_FILE")
 		if fname != "" {
-			ioutil.WriteFile(fname, []byte(tun.(*NativeTun).name+"\n"), 0400)
+			os.WriteFile(fname, []byte(tun.(*NativeTun).name+"\n"), 0400)
 		}
 	}
 
@@ -213,27 +193,19 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 }
 
 func (tun *NativeTun) Name() (string, error) {
-	var ifName struct {
-		name [16]byte
-	}
-	ifNameSize := uintptr(16)
-
-	var errno syscall.Errno
+	var err error
 	tun.operateOnFd(func(fd uintptr) {
-		_, _, errno = unix.Syscall6(
-			unix.SYS_GETSOCKOPT,
-			fd,
+		tun.name, err = unix.GetsockoptString(
+			int(fd),
 			2, /* #define SYSPROTO_CONTROL 2 */
 			2, /* #define UTUN_OPT_IFNAME 2 */
-			uintptr(unsafe.Pointer(&ifName)),
-			uintptr(unsafe.Pointer(&ifNameSize)), 0)
+		)
 	})
 
-	if errno != 0 {
-		return "", fmt.Errorf("SYS_GETSOCKOPT: %v", errno)
+	if err != nil {
+		return "", fmt.Errorf("GetSockoptString: %w", err)
 	}
 
-	tun.name = string(ifName.name[:ifNameSize-1])
 	return tun.name, nil
 }
 
@@ -288,14 +260,16 @@ func (tun *NativeTun) Flush() error {
 }
 
 func (tun *NativeTun) Close() error {
-	var err2 error
-	err1 := tun.tunFile.Close()
-	if tun.routeSocket != -1 {
-		unix.Shutdown(tun.routeSocket, unix.SHUT_RDWR)
-		err2 = unix.Close(tun.routeSocket)
-	} else if tun.events != nil {
-		close(tun.events)
-	}
+	var err1, err2 error
+	tun.closeOnce.Do(func() {
+		err1 = tun.tunFile.Close()
+		if tun.routeSocket != -1 {
+			unix.Shutdown(tun.routeSocket, unix.SHUT_RDWR)
+			err2 = unix.Close(tun.routeSocket)
+		} else if tun.events != nil {
+			close(tun.events)
+		}
+	})
 	if err1 != nil {
 		return err1
 	}
@@ -303,11 +277,6 @@ func (tun *NativeTun) Close() error {
 }
 
 func (tun *NativeTun) setMTU(n int) error {
-
-	// open datagram socket
-
-	var fd int
-
 	fd, err := unix.Socket(
 		unix.AF_INET,
 		unix.SOCK_DGRAM,
@@ -320,29 +289,18 @@ func (tun *NativeTun) setMTU(n int) error {
 
 	defer unix.Close(fd)
 
-	// do ioctl call
-
-	var ifr [32]byte
-	copy(ifr[:], tun.name)
-	*(*uint32)(unsafe.Pointer(&ifr[unix.IFNAMSIZ])) = uint32(n)
-	_, _, errno := unix.Syscall(
-		unix.SYS_IOCTL,
-		uintptr(fd),
-		uintptr(unix.SIOCSIFMTU),
-		uintptr(unsafe.Pointer(&ifr[0])),
-	)
-
-	if errno != 0 {
-		return fmt.Errorf("failed to set MTU on %s", tun.name)
+	var ifr unix.IfreqMTU
+	copy(ifr.Name[:], tun.name)
+	ifr.MTU = int32(n)
+	err = unix.IoctlSetIfreqMTU(fd, &ifr)
+	if err != nil {
+		return fmt.Errorf("failed to set MTU on %s: %w", tun.name, err)
 	}
 
 	return nil
 }
 
 func (tun *NativeTun) MTU() (int, error) {
-
-	// open datagram socket
-
 	fd, err := unix.Socket(
 		unix.AF_INET,
 		unix.SOCK_DGRAM,
@@ -355,19 +313,10 @@ func (tun *NativeTun) MTU() (int, error) {
 
 	defer unix.Close(fd)
 
-	// do ioctl call
-
-	var ifr [64]byte
-	copy(ifr[:], tun.name)
-	_, _, errno := unix.Syscall(
-		unix.SYS_IOCTL,
-		uintptr(fd),
-		uintptr(unix.SIOCGIFMTU),
-		uintptr(unsafe.Pointer(&ifr[0])),
-	)
-	if errno != 0 {
-		return 0, fmt.Errorf("failed to get MTU on %s", tun.name)
+	ifr, err := unix.IoctlGetIfreqMTU(fd, tun.name)
+	if err != nil {
+		return 0, fmt.Errorf("failed to get MTU on %s: %w", tun.name, err)
 	}
 
-	return int(*(*int32)(unsafe.Pointer(&ifr[16]))), nil
+	return int(ifr.MTU), nil
 }