diff --git a/bwm-cni/bwm-cni.go b/bwm-cni/bwm-cni.go new file mode 100644 index 0000000000000000000000000000000000000000..3ff0def788e30d4cf4bb31147d9cba4411305d6c --- /dev/null +++ b/bwm-cni/bwm-cni.go @@ -0,0 +1,203 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + CNI_PLUGIN_VERSION string = "0.3.1" +) + +// cniConf is whatever you expect your configuration json to be. This is whatever +// is passed in on stdin. Your plugin may wish to expose its functionality via +// runtime args, see CONVENTIONS.md in the CNI spec. +type cniConf struct { + types.NetConf + + // Add plugin-specific flags here + KubeConfig string `json:"kubeconfig,omitempty"` + Mode string `json:"mode,omitempty"` + EnableIpSec bool `json:"enableIpSec,omitempty"` +} + +func executeBwmcliCommand(param []string) error { + var err error + bwmcliPath := "/usr/bin/bwmcli" + + if _, err := os.Stat(bwmcliPath); os.IsNotExist(err) { + return fmt.Errorf("bwmcli可执行文件不存在于指定路径: %s", bwmcliPath) + } + + cmd := exec.Command(bwmcliPath, param...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("bwmcli执行失败: %v, stderr: %s", err, stderr.String()) + } + + return nil +} + +func cmdAdd(args *skel.CmdArgs) error { + var err error + logFile, err := os.OpenFile("/var/log/bwm-cni.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "打开日志文件失败: %v\n", err) + } + defer logFile.Close() + + cniConf, k8sConf, preResult, err := parseSkelArgs(args) + if err != nil { + fmt.Fprintf(logFile, "failed to parse config: %v\n", err) + if preResult == nil { + return err + } + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + podName := string(k8sConf.K8S_POD_NAME) + podNamespace := string(k8sConf.K8S_POD_NAMESPACE) + if podName == "" || podNamespace == "" { + fmt.Fprintf(logFile, "Not a kubernetes pod\n") + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + KubeConfig, err := findKubeConfig() + if err != nil { + KubeConfig = "/etc/cni/net.d/calico-kubeconfig" + } + client, err := CreateKubeClient(KubeConfig) + if err != nil { + err = fmt.Errorf("failed to get k8s client: %v", err) + fmt.Fprintf(logFile, "client :%v\n", err) + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + pod, err := client.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + fmt.Fprintf(logFile, "failed to get pod: %v", err) + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + annotations := pod.Annotations + for key, value := range annotations { + fmt.Fprintf(logFile, "Annotation Key: %s, Value: %s\n", key, value) + } + + // 获取特定的Annotation值 + specificValue, exists := annotations["cni.bwm-plugin.com/enable-feature"] + if exists && specificValue == "true" { + fmt.Fprintf(logFile, "Value of 'cni.bwm-plugin.com/enable-feature': %s\n", specificValue) + } else { + fmt.Fprintf(logFile, "Annotation 'cni.bwm-plugin.com/enable-feature' not found or is not true.") + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + podIP := removeCIDRSuffix(annotations["cni.projectcalico.org/podIP"]) + bandwidthIngress := annotations["cni.bwm-plugin.com/bandwidth-ingress"] + bandwidthEgress := annotations["cni.bwm-plugin.com/bandwidth-egress"] + + // 出方向带宽管理 + var hostSideVethIndex int + err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error { + link, err := netlink.LinkByName(args.IfName) + if err != nil { + return fmt.Errorf("failed to get link inside pod: %v", err) + } + + if link.Type() != "veth" { + return fmt.Errorf("interface %s is not a veth device", args.IfName) + } + + // 获取链路属性，其中包含对端接口的索引,对端索引存储在ParentIndex字段 + attrs := link.Attrs() + hostSideVethIndex = attrs.ParentIndex + fmt.Fprintf(logFile, "hostSideVethIndex: %v\n", hostSideVethIndex) + // 使能bwm带宽管理需要： + // 1.使能网卡 + // 2.设置对应cgroup + // 3.设置带宽 + // 此处仅设置内部网络网卡使能和设置带宽，设置对应cgroup无法在cni插件中完成 + params := []string{"-e", args.IfName} + err = executeBwmcliCommand(params) + if err != nil { + fmt.Fprintf(logFile, "警告: bwmcli执行失败: %v", err) + } + return nil + }) + if err != nil { + fmt.Fprintf(logFile, "WithNetNSPath: %v\n", err) + } + + if hostSideVethIndex == 0 { + return types.PrintResult(preResult, cniConf.CNIVersion) + } + + hostPeerLink, err := netlink.LinkByIndex(hostSideVethIndex) + if err != nil { + fmt.Fprintf(logFile, "failed to find host-side veth with index %d: %v", hostSideVethIndex, err) + return types.PrintResult(preResult, cniConf.CNIVersion) + } + hostIfaceName := hostPeerLink.Attrs().Name + fmt.Fprintf(logFile, "Found host-side veth: %s (index: %d)\n", hostIfaceName, hostSideVethIndex) + + // 入方向带宽管理，根据IP进行限流 + params := []string{"-E", hostIfaceName} + err = executeBwmcliCommand(params) + if err != nil { + fmt.Fprintf(logFile, "警告: bwmcli执行失败: %v", err) + } + params = []string{"-A", podIP} + err = executeBwmcliCommand(params) + if err != nil { + fmt.Fprintf(logFile, "警告: bwmcli执行失败: %v", err) + } + params = []string{"-S", "bandwidth", bandwidthIngress} + err = executeBwmcliCommand(params) + if err != nil { + fmt.Fprintf(logFile, "警告: bwmcli执行失败: %v", err) + } + + params = []string{"-s", "bandwidth", bandwidthEgress} + err = executeBwmcliCommand(params) + if err != nil { + fmt.Fprintf(logFile, "警告: bwmcli执行失败: %v", err) + } + return types.PrintResult(preResult, cniConf.CNIVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + // DEL操作暂时保持空实现 + return nil +} + +func main() { + funcs := skel.CNIFuncs{ + Add: cmdAdd, + Del: cmdDel, + } + err := skel.PluginMainFuncsWithError(funcs, version.All, + fmt.Sprintf("CNI plugin bwm-cni %v", CNI_PLUGIN_VERSION)) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + diff --git a/bwm-cni/go.mod b/bwm-cni/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..d5fa97e4295e52566161ec421ec97963cadd08fc --- /dev/null +++ b/bwm-cni/go.mod @@ -0,0 +1,54 @@ +module bwm-cni + +go 1.23.2 + +require ( + github.com/containernetworking/cni v1.3.0 + github.com/containernetworking/plugins v1.7.0 + github.com/vishvananda/netlink v1.3.1-0.20250303224720-0e7078ed04c8 + k8s.io/apimachinery v0.32.2 + k8s.io/client-go v0.32.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vishvananda/netns v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.32.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/bwm-cni/go.sum b/bwm-cni/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..d03f1634c0c53def53a5488858aa45f3a743da92 --- /dev/null +++ b/bwm-cni/go.sum @@ -0,0 +1,167 @@ +github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= +github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= +github.com/containernetworking/plugins v1.7.0 h1:mHCFD5unxtnziWnqNLg4pJ+pBKgctPI7yYAvsqloOEU= +github.com/containernetworking/plugins v1.7.0/go.mod h1:xuMdjuio+a1oVQsHKjr/mgzuZ24leAsqUYRnzGoXHy0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vishvananda/netlink v1.3.1-0.20250303224720-0e7078ed04c8 h1:Y4egeTrP7sccowz2GWTJVtHlwkZippgBTpUmMteFUWQ= +github.com/vishvananda/netlink v1.3.1-0.20250303224720-0e7078ed04c8/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= +k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= +k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/bwm-cni/kubeclient.go b/bwm-cni/kubeclient.go new file mode 100644 index 0000000000000000000000000000000000000000..3660aa09d4111baa0b3295ca44e8bcde37f01a7b --- /dev/null +++ b/bwm-cni/kubeclient.go @@ -0,0 +1,161 @@ + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + current "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/cni/pkg/version" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// K8sArgs parameter is used to transfer the k8s information transferred +// when the cni plugin is invoked. +// The field names need to match exact keys in kubelet args for unmarshalling +type k8sArgs struct { + types.CommonArgs + K8S_POD_NAME types.UnmarshallableString + K8S_POD_NAMESPACE types.UnmarshallableString +} + +// CreateKubeClient creates a kube client with the given kubeconfig file, if no kubeconfig specified, in cluster kubeconfig will be used. +// applyFuncs is optional, which can be used to tune client rest.Config +func CreateKubeClient(kubeConfig string, applyFuncs ...func(c *rest.Config)) (kubernetes.Interface, error) { + var restConfig *rest.Config + var err error + + if kubeConfig != "" { + restConfig, err = clientcmd.BuildConfigFromFlags("", kubeConfig) + } else { + restConfig, err = rest.InClusterConfig() + } + if err != nil { + return nil, err + } + + for _, fn := range applyFuncs { + fn(restConfig) + } + + return kubernetes.NewForConfig(restConfig) +} + +func parseSkelArgs(args *skel.CmdArgs) (*cniConf, *k8sArgs, *current.Result, error) { + cniConf := cniConf{} + if err := json.Unmarshal(args.StdinData, &cniConf); err != nil { + fmt.Fprintf(os.Stderr, "failed to unmarshal json: %v\n", err) + return nil, nil, nil, err + } + + result, err := getPrevCniResult(&cniConf) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to get cni result: %v\n", err) + return nil, nil, nil, err + } + + k8sCommonArgs := k8sArgs{} + if err := types.LoadArgs(args.Args, &k8sCommonArgs); err != nil { + fmt.Fprintf(os.Stderr, "failed to load k8s args: %v\n", err) + return nil, nil, result, err + } + return &cniConf, &k8sCommonArgs, result, nil +} + +func getPrevCniResult(conf *cniConf) (*current.Result, error) { + var err error + if conf.RawPrevResult == nil { + err = fmt.Errorf("bwm-cni can not use standalone") + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return nil, err + } + + prevResultBytes, err := json.Marshal(conf.RawPrevResult) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to serialize prev cni result: %v\n", err) + return nil, err + } + res, err := version.NewResult(conf.CNIVersion, prevResultBytes) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to parse prev result: %v\n", err) + return nil, err + } + cniv1PrevResult, err := current.NewResultFromResult(res) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to convert result to version %s: %v\n", current.ImplementedSpecVersion, err) + return nil, err + } + return cniv1PrevResult, nil +} + +// 查找kubeconfig文件的函数 +func findKubeConfig() (string, error) { + configDir := "/etc/cni/net.d/" + + // 读取目录内容 + entries, err := os.ReadDir(configDir) + if err != nil { + return "", fmt.Errorf("无法读取目录 %s: %v", configDir, err) + } + + // 优先查找常见的kubeconfig文件名模式 + preferredPatterns := []string{"*kubeconfig*", "*config*"} + + for _, pattern := range preferredPatterns { + matches, err := filepath.Glob(filepath.Join(configDir, pattern)) + if err != nil { + continue + } + + for _, match := range matches { + // 检查是否为文件且不是目录 + if info, err := os.Stat(match); err == nil && !info.IsDir() { + // 进一步验证文件内容是否为有效的kubeconfig + if isValidKubeConfig(match) { + return match, nil + } + } + } + } + + // 如果没有找到符合模式的文件，尝试目录中的所有文件 + for _, entry := range entries { + if !entry.IsDir() { + filePath := filepath.Join(configDir, entry.Name()) + if isValidKubeConfig(filePath) { + return filePath, nil + } + } + } + + return "", fmt.Errorf("在 %s 目录中未找到有效的kubeconfig文件", configDir) +} + +// 验证文件是否为有效的kubeconfig +func isValidKubeConfig(filePath string) bool { + // 跳过可能的备份文件或临时文件 + if strings.HasSuffix(filePath, "~") || strings.HasPrefix(filepath.Base(filePath), ".") { + return false + } + + // 尝试加载配置来验证 + _, err := clientcmd.LoadFromFile(filePath) + return err == nil +} + +func removeCIDRSuffix(ipWithCIDR string) string { + // 使用 Split 按 "/" 分割，取第一部分 + parts := strings.Split(ipWithCIDR, "/") + if len(parts) > 0 { + return parts[0] + } + return ipWithCIDR +} + diff --git a/bwm-cni/sample/iperf-client.yaml b/bwm-cni/sample/iperf-client.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2e6a06af8ed22c55f24ec866b913cf89230276f --- /dev/null +++ b/bwm-cni/sample/iperf-client.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: iperf-client +spec: + containers: + - name: iperf-client + image: networkstatic/iperf3 + imagePullPolicy: Never + command: ["/bin/sh"] + args: ["-c", "sleep 86400"] + restartPolicy: Never \ No newline at end of file diff --git a/bwm-cni/sample/iperf-server.yaml b/bwm-cni/sample/iperf-server.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5aa511f3914585af1580bd7e2e4fa18f75ca79 --- /dev/null +++ b/bwm-cni/sample/iperf-server.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: iperf-server +spec: + replicas: 1 + selector: + matchLabels: + app: iperf-server + template: + metadata: + labels: + app: iperf-server + annotations: + cni.bwm-plugin.com/enable-feature: "true" + cni.bwm-plugin.com/bandwidth-ingress: "20mb,100mb" + cni.bwm-plugin.com/bandwidth-egress: "20mb,100mb" + + spec: + containers: + - name: iperf-server + image: networkstatic/iperf3 + imagePullPolicy: Never + args: ["-s"] + ports: + - containerPort: 5201 + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: v1 +kind: Service +metadata: + name: iperf-server +spec: + selector: + app: iperf-server + ports: + - protocol: TCP + port: 5201 + targetPort: 5201 + type: ClusterIP