Skip to content

Commit

Permalink
Merge pull request #40 from krsacme/vhostuser_emptydir
Browse files Browse the repository at this point in the history 
Add support for vhostuser with empty dir support with libvirt
  • Loading branch information
@Billy99
Billy99 authored Apr 16, 2020
2 parents 323d722 + 44ad3f9 commit 673eeb0
Show file tree
Hide file tree
Showing 4 changed files with 185 additions and 71 deletions.
150 changes: 128 additions & 22 deletions cniovs/cniovs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@ import (
"errors"
"fmt"
"os"
"os/user"
"path/filepath"
"regexp"
"strconv"
"strings"

"golang.org/x/sys/unix"

v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
Expand All @@ -46,7 +51,10 @@ import (
//
// Constants
//
const defaultBridge = "br0"
const (
defaultBridge = "br0"
DefaultHostVhostuserBaseDir = "/var/lib/vhost_sockets/"
)

//
// Types
Expand All @@ -58,10 +66,11 @@ type CniOvs struct {
// API Functions
//
func (cniOvs CniOvs) AddOnHost(conf *types.NetConf,
args *skel.CmdArgs,
kubeClient kubernetes.Interface,
sharedDir string,
ipResult *current.Result) error {
args *skel.CmdArgs,
kubeClient kubernetes.Interface,
sharedDir string,
ipResult *current.Result) error {

var err error
var data OvsSavedData

Expand Down Expand Up @@ -130,11 +139,11 @@ func (cniOvs CniOvs) AddOnHost(conf *types.NetConf,
}

func (cniOvs CniOvs) AddOnContainer(conf *types.NetConf,
args *skel.CmdArgs,
kubeClient kubernetes.Interface,
sharedDir string,
pod *v1.Pod,
ipResult *current.Result) (*v1.Pod, error) {
args *skel.CmdArgs,
kubeClient kubernetes.Interface,
sharedDir string,
pod *v1.Pod,
ipResult *current.Result) (*v1.Pod, error) {
logging.Infof("OVS AddOnContainer: ENTER - Container %s Iface %s", args.ContainerID[:12], args.IfName)
return configdata.SaveRemoteConfig(conf, args, kubeClient, sharedDir, pod, ipResult)
}
Expand Down Expand Up @@ -215,20 +224,91 @@ func generateRandomMacAddress() string {
return macAddr
}

func addLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, sharedDir string, data *OvsSavedData) error {
func getShortSharedDir(sharedDir string) string {
// sun_path for unix domain socket has a array size of 108
// When the sharedDir path length greater than 89 (108 - 19)
// 19 is the possible vhostuser socke file name length "/abcdefghijkl-net99" (1 + 12 + 1 + 3 + 2)
if len(sharedDir) >= 89 && strings.Contains(sharedDir, "empty-dir") {
// Format - /var/lib/kubelet/pods/<podID>/volumes/kubernetes.io~empty-dir/shared-dir
parts := strings.Split(sharedDir, "/")
podID := parts[5]
newSharedDir := DefaultHostVhostuserBaseDir + podID
logging.Infof("getShortSharedDir: Short shared directory: %s", newSharedDir)
return newSharedDir
}
return sharedDir

}

func createSharedDir(sharedDir, oldSharedDir string) error {
var err error

_, err = os.Stat(sharedDir)
if os.IsNotExist(err) {
err = os.MkdirAll(sharedDir, 0750)
if err != nil {
logging.Errorf("createSharedDir: Failed to create dir (%s): %v", sharedDir, err)
return err
}

if strings.Contains(sharedDir, DefaultHostVhostuserBaseDir) {
logging.Debugf("createSharedDir: Mount from %s to %s", oldSharedDir, sharedDir)
err = unix.Mount(oldSharedDir, sharedDir, "", unix.MS_BIND, "")
if err != nil {
logging.Errorf("createSharedDir: Failed to bind mout: %s", err)
return err
}
}
return nil

}
return err
}

func setSharedDirGroup(sharedDir string, group string) error {
groupInfo, err := user.LookupGroup(group)
if err != nil {
return err
}

logging.Debugf("setSharedDirGroup: group %s's gid is %s", group, groupInfo.Gid)
gid, err := strconv.Atoi(groupInfo.Gid)
if err != nil {
return err
}

err = os.Chown(DefaultHostVhostuserBaseDir, -1, gid)
if err != nil {
return err
}

err = os.Chown(sharedDir, -1, gid)
if err != nil {
return err
}
return nil
}

func addLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, actualSharedDir string, data *OvsSavedData) error {
var err error
var vhostName string

if conf.HostConf.VhostConf.Socketfile == "" {
conf.HostConf.VhostConf.Socketfile = fmt.Sprintf("%s-%s", args.ContainerID[:12], args.IfName)
}

if _, err = os.Stat(sharedDir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(sharedDir, 0700); err != nil {
return err
}
} else {
sharedDir := getShortSharedDir(actualSharedDir)
err = createSharedDir(sharedDir, actualSharedDir)
if err != nil {
logging.Errorf("addLocalDeviceVhost: Failed to create shared dir group: %v", err)
return err
}

group := conf.HostConf.VhostConf.Group
if group != "" {
err = setSharedDirGroup(sharedDir, group)
if err != nil {
logging.Errorf("addLocalDeviceVhost: Failed to set shared dir group: %v", err)
return err
}
}
Expand All @@ -241,9 +321,9 @@ func addLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, sharedDir stri

// ovs-vsctl add-port
if vhostName, err = createVhostPort(sharedDir,
conf.HostConf.VhostConf.Socketfile,
clientMode,
conf.HostConf.BridgeConf.BridgeName); err == nil {
conf.HostConf.VhostConf.Socketfile,
clientMode,
conf.HostConf.BridgeConf.BridgeName); err == nil {
if vhostPortMac, err := getVhostPortMac(vhostName); err == nil {
data.VhostMac = vhostPortMac
} else {
Expand All @@ -259,9 +339,35 @@ func addLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, sharedDir stri
return err
}

func delLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, sharedDir string, data *OvsSavedData) error {
func delLocalDeviceVhost(conf *types.NetConf, args *skel.CmdArgs, actualSharedDir string, data *OvsSavedData) error {
sharedDir := getShortSharedDir(actualSharedDir)

// ovs-vsctl --if-exists del-port
if err := deleteVhostPort(data.Vhostname, conf.HostConf.BridgeConf.BridgeName); err == nil {
err := deleteVhostPort(data.Vhostname, conf.HostConf.BridgeConf.BridgeName)
if err != nil {
logging.Errorf("delLocalDeviceVhost: Failed to delete port: %v", err)
return err
}

// Check if sharedDir is a mount dir of EmptyDir
if strings.Contains(sharedDir, DefaultHostVhostuserBaseDir) {
logging.Debugf("delLocalDeviceVhost: Unmount shared directory: %v", sharedDir)
_, err = os.Stat(sharedDir)
if os.IsNotExist(err) {
logging.Errorf("delLocalDeviceVhost: shared directory %s does not existt to unmount", sharedDir)
return nil
}
err = unix.Unmount(sharedDir, 0)
if err != nil {
logging.Errorf("Failed to unmount dir: %v", err)
return err
}
err = os.Remove(sharedDir)
if err != nil {
logging.Errorf("Failed to remove dir: %v", err)
return err
}
} else {
folder, err := os.Open(sharedDir)
if err != nil {
return err
Expand Down
6 changes: 5 additions & 1 deletion cniovs/ovsctrl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@ func createVhostPort(sock_dir string, sock_name string, client bool, bridge_name

if client == true {
socketarg := "options:vhost-server-path=" + sock_dir
logging.Errorf("Additional string: %s", socketarg)
if sock_dir[len(sock_dir)-1] != '/' {
socketarg += "/"
}
socketarg += sock_name
logging.Debugf("Additional string: %s", socketarg)

args = append(args, socketarg)
}
Expand Down
57 changes: 30 additions & 27 deletions pkg/annotations/annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,55 +10,54 @@
package annotations

import (
"bytes"
"encoding/json"
"fmt"
"strings"
"bytes"
"io/ioutil"
"strings"

v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"

"github.com/go-logfmt/logfmt"

"github.com/intel/userspace-cni-network-plugin/logging"
"github.com/intel/userspace-cni-network-plugin/pkg/types"
"github.com/intel/userspace-cni-network-plugin/pkg/k8sclient"
"github.com/intel/userspace-cni-network-plugin/pkg/types"
)


// Annotation
// These structures are used to document the set of annotations used in
// the Userspace CNI pod spec to pass data from Admission Controller to
// the CNI and from the CNI to the Container.

// List of Annotations supported in the podSpec
const (
annotKeyNetwork = "k8s.v1.cni.cncf.io/networks"
annotKeyNetworkStatus = "k8s.v1.cni.cncf.io/networks-status"
annotKeyNetwork = "k8s.v1.cni.cncf.io/networks"
annotKeyNetworkStatus = "k8s.v1.cni.cncf.io/networks-status"
AnnotKeyUsrspConfigData = "userspace/configuration-data"
AnnotKeyUsrspMappedDir = "userspace/mapped-dir"
volMntKeySharedDir = "shared-dir"
AnnotKeyUsrspMappedDir = "userspace/mapped-dir"
volMntKeySharedDir = "shared-dir"

DefaultBaseCNIDir = "/var/lib/cni/usrspcni"
DefaultBaseCNIDir = "/var/lib/cni/usrspcni"
DefaultLocalCNIDir = "/var/lib/cni/usrspcni/data"

DefaultHostkubeletPodBaseDir = "/var/lib/kubelet/pods/"
DefaultHostEmptyDirVolumeName = "volumes/kubernetes.io~empty-dir/"
)



// Errors returned from this module
type NoSharedDirProvidedError struct {
message string
}

func (e *NoSharedDirProvidedError) Error() string { return string(e.message) }

type NoKubeClientProvidedError struct {
message string
}
func (e *NoKubeClientProvidedError) Error() string { return string(e.message) }


func (e *NoKubeClientProvidedError) Error() string { return string(e.message) }

func GetPodVolumeMountHostSharedDir(pod *v1.Pod) (string, error) {
var hostSharedDir string
Expand All @@ -71,7 +70,13 @@ func GetPodVolumeMountHostSharedDir(pod *v1.Pod) (string, error) {

for _, volumeMount := range pod.Spec.Volumes {
if volumeMount.Name == volMntKeySharedDir {
hostSharedDir = volumeMount.HostPath.Path
if volumeMount.HostPath != nil {
hostSharedDir = volumeMount.HostPath.Path
} else if volumeMount.EmptyDir != nil {
hostSharedDir = DefaultHostkubeletPodBaseDir + string(pod.UID) + "/" + DefaultHostEmptyDirVolumeName + volMntKeySharedDir
} else {
return hostSharedDir, &NoSharedDirProvidedError{"Error: Volume is invalid"}
}
break
}
}
Expand All @@ -86,7 +91,7 @@ func GetPodVolumeMountHostSharedDir(pod *v1.Pod) (string, error) {
func getPodVolumeMountHostMappedSharedDir(pod *v1.Pod) (string, error) {
var mappedSharedDir string

logging.Verbosef("getPodVolumeMountHostMappedSharedDir: type=%T Containers=%v", pod.Spec.Containers, pod.Spec.Containers)
logging.Verbosef("getPodVolumeMountHostMappedSharedDir: Containers=%v", pod.Spec.Containers)

if len(pod.Spec.Containers) == 0 {
return mappedSharedDir, &NoSharedDirProvidedError{"Error: No Containers. Need \"shared-dir\" in podSpec \"Volumes\""}
Expand All @@ -111,8 +116,8 @@ func getPodVolumeMountHostMappedSharedDir(pod *v1.Pod) (string, error) {
}

func WritePodAnnotation(kubeClient kubernetes.Interface,
pod *v1.Pod,
configData *types.ConfigurationData) (*v1.Pod, error) {
pod *v1.Pod,
configData *types.ConfigurationData) (*v1.Pod, error) {
var err error
var modifiedConfig bool
var modifiedMappedDir bool
Expand Down Expand Up @@ -162,12 +167,11 @@ func WritePodAnnotation(kubeClient kubernetes.Interface,
return pod, err
}


//
// Local Utility Functions
//
func setPodAnnotationMappedDir(pod *v1.Pod,
mappedDir string) (bool, error) {
mappedDir string) (bool, error) {
var modified bool

logging.Verbosef("SetPodAnnotationMappedDir: inputMappedDir=%s Annot - type=%T mappedDir=%v", mappedDir, pod.Annotations[AnnotKeyUsrspMappedDir], pod.Annotations[AnnotKeyUsrspMappedDir])
Expand Down Expand Up @@ -199,7 +203,7 @@ func setPodAnnotationMappedDir(pod *v1.Pod,
}

func setPodAnnotationConfigData(pod *v1.Pod,
configData *types.ConfigurationData) (bool, error) {
configData *types.ConfigurationData) (bool, error) {
var configDataStr []string
var modified bool

Expand All @@ -210,7 +214,7 @@ func setPodAnnotationConfigData(pod *v1.Pod,
pod.Annotations = make(map[string]string)
}

// Get current data, if any. The current data is a string in JSON format with
// Get current data, if any. The current data is a string in JSON format with
// data for multiple interfaces appended together. A given container can have
// multiple interfaces, added one at a time. So existing data may be empty if
// this is the first interface, or already contain data.
Expand Down Expand Up @@ -240,12 +244,11 @@ func setPodAnnotationConfigData(pod *v1.Pod,
}

func commitAnnotation(kubeClient kubernetes.Interface,
pod *v1.Pod) (*v1.Pod, error) {
pod *v1.Pod) (*v1.Pod, error) {
// Write the modified data back to the pod.
return k8sclient.WritePodAnnotation(kubeClient, pod)
}


//
// Container Access Functions
// These functions can be called from code running in a container. It reads
Expand Down Expand Up @@ -285,7 +288,7 @@ func GetFileAnnotationMappedDir() (string, error) {
return "", err
}

return string(rawData), err
return string(rawData), err
}

func GetFileAnnotationConfigData() ([]*types.ConfigurationData, error) {
Expand All @@ -308,7 +311,7 @@ func GetFileAnnotationConfigData() ([]*types.ConfigurationData, error) {
return nil, logging.Errorf("GetFileAnnotationConfigData: Invalid formatted JSON data")
}

return configDataList, err
return configDataList, err
}

//func GetFileAnnotationNetworksStatus() ([]*multusTypes.NetworkStatus, error) {
Expand All @@ -331,5 +334,5 @@ func GetFileAnnotationConfigData() ([]*types.ConfigurationData, error) {
// return nil, logging.Errorf("GetFileAnnotationNetworksStatus: Invalid formatted JSON data")
// }
//
// return networkStatusList, err
//}
// return networkStatusList, err
//}
Loading

0 comments on commit 673eeb0

Please sign in to comment.