//go:build linux
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/klog/v2"
"k8s.io/utils/exec"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
// mountNsPath is the default mount namespace of the host
mountNsPath = "/proc/1/ns/mnt"
// nsenterPath is the default nsenter command
nsenterPath = "nsenter"
)
// Nsenter is a type alias for backward compatibility
type Nsenter = NSEnter
// NSEnter is part of experimental support for running the kubelet
// in a container.
//
// NSEnter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch",
// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
// systemd is installed/enabled in the operating system.
//
// For more information about mount propagation modes, see:
//
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type NSEnter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
// Path to the host filesystem, typically "/rootfs". Used only for testing.
hostRootFsPath string
// Exec implementation
executor exec.Interface
}
// NewNsenter constructs a new instance of NSEnter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*NSEnter, error) {
ne := &NSEnter{
hostRootFsPath: hostRootFsPath,
executor: executor,
}
if err := ne.initPaths(); err != nil {
return nil, err
}
return ne, nil
}
func (ne *NSEnter) initPaths() error {
ne.paths = map[string]string{}
binaries := []string{
"mount",
"findmnt",
"umount",
"systemd-run",
"stat",
"touch",
"mkdir",
"sh",
"chmod",
"realpath",
}
// search for the required commands in other locations besides /usr/bin
for _, binary := range binaries {
// check for binary under the following directories
for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
continue
}
ne.paths[binary] = binPath
break
}
// systemd-run is optional, bailout if we don't find any of the other binaries
if ne.paths[binary] == "" && binary != "systemd-run" {
return fmt.Errorf("unable to find %v", binary)
}
}
return nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.Command(nsenterPath, fullArgs...)
}
// Command returns a command wrapped with nsenter
func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd {
return ne.Exec(cmd, args)
}
// CommandContext returns a CommandContext wrapped with nsenter
func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.CommandContext(ctx, nsenterPath, fullArgs...)
}
// LookPath returns a LookPath wrapped with nsenter
func (ne *NSEnter) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *NSEnter) AbsHostPath(command string) string {
path, ok := ne.paths[command]
if !ok {
return command
}
return path
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *NSEnter) SupportsSystemd() (string, bool) {
systemdRunPath, ok := ne.paths["systemd-run"]
return systemdRunPath, ok && systemdRunPath != ""
}
// EvalSymlinks returns the path name on the host after evaluating symlinks on the
// host.
// mustExist makes EvalSymlinks to return error when the path does not
// exist. When it's false, it evaluates symlinks of the existing part and
// blindly adds the non-existing part:
// pathname: /mnt/volume/non/existing/directory
//
// /mnt/volume exists
// non/existing/directory does not exist
//
// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
//
// /mnt/foo/non/existing/directory.
//
// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
func (ne *NSEnter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
var args []string
if mustExist {
// "realpath -e: all components of the path must exist"
args = []string{"-e", pathname}
} else {
// "realpath -m: no path components need exist or be a directory"
args = []string{"-m", pathname}
}
outBytes, err := ne.Exec("realpath", args).CombinedOutput()
if err != nil {
klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
return "", err
}
return strings.TrimSpace(string(outBytes)), nil
}
// KubeletPath returns the path name that can be accessed by containerized
// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
// before calling this function
func (ne *NSEnter) KubeletPath(pathname string) string {
return filepath.Join(ne.hostRootFsPath, pathname)
}
// NewFakeNsenter returns a NSEnter that does not run "nsenter --mount=... --",
// but runs everything in the same mount namespace as the unit test binary.
// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
// This fake NSEnter is enough for most operations, e.g. to resolve symlinks,
// but it's not enough to call /bin/mount - unit tests don't run as root.
func NewFakeNsenter(rootfsPath string) (*NSEnter, error) {
executor := &fakeExec{
rootfsPath: rootfsPath,
}
// prepare /rootfs/bin, usr/bin and usr/sbin
bin := filepath.Join(rootfsPath, "bin")
if err := os.Symlink("/bin", bin); err != nil {
return nil, err
}
usr := filepath.Join(rootfsPath, "usr")
if err := os.Mkdir(usr, 0755); err != nil {
return nil, err
}
usrbin := filepath.Join(usr, "bin")
if err := os.Symlink("/usr/bin", usrbin); err != nil {
return nil, err
}
usrsbin := filepath.Join(usr, "sbin")
if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
return nil, err
}
return NewNsenter(rootfsPath, executor)
}
type fakeExec struct {
rootfsPath string
}
func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
// This will intentionaly panic if NSEnter does not provide enough arguments.
realCmd := args[2]
realArgs := args[3:]
return exec.New().Command(realCmd, realArgs...)
}
func (fakeExec) LookPath(file string) (string, error) {
return "", errors.New("not implemented")
}
func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
var _ exec.Interface = fakeExec{}
var _ exec.Interface = &NSEnter{}
Codebase Browser
kubernetes