From 686a5785b52e19905371f121e126b8037df8626f Mon Sep 17 00:00:00 2001 From: Cyclinder Kuo Date: Wed, 8 Jan 2025 11:39:09 +0800 Subject: [PATCH] add flag --kubelet-root-dir --- cmd/sriovdp/main.go | 5 ++- cmd/sriovdp/manager.go | 6 ++- cmd/sriovdp/manager_test.go | 2 +- pkg/accelerator/accelDeviceProvider_test.go | 6 +-- pkg/accelerator/accelDevice_test.go | 12 +++--- pkg/auxnetdevice/auxNetDeviceProvider_test.go | 8 ++-- pkg/auxnetdevice/auxNetDevice_test.go | 2 +- pkg/devices/host_test.go | 10 ++--- pkg/factory/factory.go | 6 ++- pkg/factory/factory_test.go | 30 ++++++------- pkg/netdevice/netDeviceProvider_test.go | 6 +-- pkg/netdevice/netResourcePool_test.go | 4 +- pkg/netdevice/pciNetDevice_test.go | 6 +-- pkg/resources/pool_stub_test.go | 2 +- pkg/resources/server.go | 26 +++++++++--- pkg/resources/server_test.go | 42 +++++++------------ pkg/types/types.go | 5 ++- 17 files changed, 96 insertions(+), 82 deletions(-) diff --git a/cmd/sriovdp/main.go b/cmd/sriovdp/main.go index 93998c47e..5f0678143 100644 --- a/cmd/sriovdp/main.go +++ b/cmd/sriovdp/main.go @@ -24,11 +24,14 @@ import ( ) const ( - defaultConfig = "/etc/pcidp/config.json" + defaultConfig = "/etc/pcidp/config.json" + defaultKubeletDir = "/var/lib/kubelet" ) // flagInit parse command line flags func flagInit(cp *cliParams) { + flag.StringVar(&cp.kubeletRootDir, "kubelet-root-dir", defaultKubeletDir, + "kubelet root dir") flag.StringVar(&cp.configFile, "config-file", defaultConfig, "JSON device pool config file location") flag.StringVar(&cp.resourcePrefix, "resource-prefix", "intel.com", diff --git a/cmd/sriovdp/manager.go b/cmd/sriovdp/manager.go index c860b8620..f0ff0728b 100644 --- a/cmd/sriovdp/manager.go +++ b/cmd/sriovdp/manager.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "os" + "path" "github.com/golang/glog" "github.com/jaypipes/ghw" @@ -34,6 +35,7 @@ const ( // cliParams presents CLI parameters for SR-IOV Network Device Plugin type cliParams struct { + kubeletRootDir string configFile string resourcePrefix string useCdi bool @@ -52,14 +54,14 @@ type resourceManager struct { // newResourceManager initiates a new instance of resourceManager func newResourceManager(cp *cliParams) *resourceManager { - pluginWatchMode := utils.DetectPluginWatchMode(types.SockDir) + pluginWatchMode := utils.DetectPluginWatchMode(path.Join(cp.kubeletRootDir, types.PluginRegistry)) if pluginWatchMode { glog.Infof("Using Kubelet Plugin Registry Mode") } else { glog.Infof("Using Deprecated Device Plugin Registry Path") } - rf := factory.NewResourceFactory(cp.resourcePrefix, socketSuffix, pluginWatchMode, cp.useCdi) + rf := factory.NewResourceFactory(cp.resourcePrefix, socketSuffix, cp.kubeletRootDir, pluginWatchMode, cp.useCdi) dp := make(map[types.DeviceType]types.DeviceProvider) for k := range types.SupportedDevices { dp[k] = rf.GetDeviceProvider(k) diff --git a/cmd/sriovdp/manager_test.go b/cmd/sriovdp/manager_test.go index f5ca19727..e64fda8b1 100644 --- a/cmd/sriovdp/manager_test.go +++ b/cmd/sriovdp/manager_test.go @@ -443,7 +443,7 @@ var _ = Describe("Resource manager", func() { _ = os.Unsetenv("GHW_CHROOT") }() - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) rm := &resourceManager{ rFactory: rf, diff --git a/pkg/accelerator/accelDeviceProvider_test.go b/pkg/accelerator/accelDeviceProvider_test.go index b8098bec9..348abb0fe 100644 --- a/pkg/accelerator/accelDeviceProvider_test.go +++ b/pkg/accelerator/accelDeviceProvider_test.go @@ -92,7 +92,7 @@ var _ = Describe("AcceleratorProvider", func() { defer fs.Use()() - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := accelerator.NewAccelDeviceProvider(rf) config := &types.ResourceConfig{ DeviceType: types.AcceleratorType, @@ -148,7 +148,7 @@ var _ = Describe("AcceleratorProvider", func() { Describe("getting Filtered devices", func() { Context("using selectors", func() { It("should correctly filter devices", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := accelerator.NewAccelDeviceProvider(rf) all := make([]types.HostDevice, 5) mocked := make([]mocks.AccelDevice, 5) @@ -218,7 +218,7 @@ var _ = Describe("AcceleratorProvider", func() { Expect(actual).To(ConsistOf(matchingDevices)) }) It("should error if the selector index is out of bounds", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := accelerator.NewAccelDeviceProvider(rf) devs := make([]types.HostDevice, 0) diff --git a/pkg/accelerator/accelDevice_test.go b/pkg/accelerator/accelDevice_test.go index 26485a417..e49b0f88f 100644 --- a/pkg/accelerator/accelDevice_test.go +++ b/pkg/accelerator/accelDevice_test.go @@ -53,7 +53,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{} @@ -99,7 +99,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{} @@ -122,7 +122,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{} @@ -146,7 +146,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{ExcludeTopology: true} @@ -164,7 +164,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{} @@ -185,7 +185,7 @@ var _ = Describe("Accelerator", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn() config := &types.ResourceConfig{} diff --git a/pkg/auxnetdevice/auxNetDeviceProvider_test.go b/pkg/auxnetdevice/auxNetDeviceProvider_test.go index 8c2bde2a4..3dedd2664 100644 --- a/pkg/auxnetdevice/auxNetDeviceProvider_test.go +++ b/pkg/auxnetdevice/auxNetDeviceProvider_test.go @@ -37,7 +37,7 @@ import ( var _ = Describe("AuxNetDeviceProvider", func() { DescribeTable("validating configuration", func(rc *types.ResourceConfig, expected bool) { - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := auxnetdevice.NewAuxNetDeviceProvider(rf) actual := p.ValidConfig(rc) Expect(actual).To(Equal(expected)) @@ -133,7 +133,7 @@ var _ = Describe("AuxNetDeviceProvider", func() { On("GetAuxNetDevicesFromPci", "0000:02:00.0").Return([]string{}, nil) utils.SetSriovnetProviderInst(&fakeSriovnetProvider) - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := auxnetdevice.NewAuxNetDeviceProvider(rf) config := &types.ResourceConfig{ DeviceType: types.AuxNetDeviceType, @@ -181,7 +181,7 @@ var _ = Describe("AuxNetDeviceProvider", func() { Describe("getting Filtered devices", func() { Context("using selectors", func() { It("should correctly filter devices", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := auxnetdevice.NewAuxNetDeviceProvider(rf) all := make([]types.HostDevice, 5) mocked := make([]tmocks.AuxNetDevice, 5) @@ -265,7 +265,7 @@ var _ = Describe("AuxNetDeviceProvider", func() { Expect(actual).To(ConsistOf(matchingDevices)) }) It("should error if the selector index is out of bounds", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := auxnetdevice.NewAuxNetDeviceProvider(rf) devs := make([]types.HostDevice, 0) diff --git a/pkg/auxnetdevice/auxNetDevice_test.go b/pkg/auxnetdevice/auxNetDevice_test.go index 9ac172c29..ccec64240 100644 --- a/pkg/auxnetdevice/auxNetDevice_test.go +++ b/pkg/auxnetdevice/auxNetDevice_test.go @@ -73,7 +73,7 @@ var _ = Describe("AuxNetDevice", func() { On("GetNetDevicesFromAux", auxDevID).Return([]string{"eth0"}, nil) utils.SetSriovnetProviderInst(&fakeSriovnetProvider) - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDevice("0000:00:00.1") rc := &types.ResourceConfig{} diff --git a/pkg/devices/host_test.go b/pkg/devices/host_test.go index 2aad15b1b..7fa403bb6 100644 --- a/pkg/devices/host_test.go +++ b/pkg/devices/host_test.go @@ -48,7 +48,7 @@ var _ = Describe("HostDevice", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) pciAddr := "0000:00:00.1" in := newPciDeviceFn(pciAddr) rc := &types.ResourceConfig{} @@ -72,7 +72,7 @@ var _ = Describe("HostDevice", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) pciAddr := "0000:00:00.1" in := newPciDeviceFn(pciAddr) rc := &types.ResourceConfig{} @@ -100,7 +100,7 @@ var _ = Describe("HostDevice", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) pciAddr := "0000:00:00.1" in := newPciDeviceFn(pciAddr) rc := &types.ResourceConfig{} @@ -123,7 +123,7 @@ var _ = Describe("HostDevice", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) pciAddr := "0000:00:00.1" in := newPciDeviceFn(pciAddr) rc := &types.ResourceConfig{} @@ -147,7 +147,7 @@ var _ = Describe("HostDevice", func() { } defer fs.Use()() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) pciAddr := "0000:00:00.1" in := newPciDeviceFn(pciAddr) rc := &types.ResourceConfig{ExcludeTopology: true} diff --git a/pkg/factory/factory.go b/pkg/factory/factory.go index 3b3ad938b..2a0eedd21 100644 --- a/pkg/factory/factory.go +++ b/pkg/factory/factory.go @@ -30,6 +30,7 @@ import ( ) type resourceFactory struct { + kubeletRootDir string endPointPrefix string endPointSuffix string pluginWatch bool @@ -39,11 +40,12 @@ type resourceFactory struct { var instance *resourceFactory // NewResourceFactory returns an instance of Resource Server factory -func NewResourceFactory(prefix, suffix string, pluginWatch, useCdi bool) types.ResourceFactory { +func NewResourceFactory(prefix, suffix, kubeletRootDir string, pluginWatch, useCdi bool) types.ResourceFactory { if instance == nil { return &resourceFactory{ endPointPrefix: prefix, endPointSuffix: suffix, + kubeletRootDir: kubeletRootDir, pluginWatch: pluginWatch, useCdi: useCdi, } @@ -58,7 +60,7 @@ func (rf *resourceFactory) GetResourceServer(rp types.ResourcePool) (types.Resou if prefixOverride := rp.GetResourcePrefix(); prefixOverride != "" { prefix = prefixOverride } - return resources.NewResourceServer(prefix, rf.endPointSuffix, rf.pluginWatch, rf.useCdi, rp), nil + return resources.NewResourceServer(prefix, rf.endPointSuffix, rf.kubeletRootDir, rf.pluginWatch, rf.useCdi, rp), nil } return nil, fmt.Errorf("factory: unable to get resource pool object") } diff --git a/pkg/factory/factory_test.go b/pkg/factory/factory_test.go index 906c421b7..8cbc04be0 100644 --- a/pkg/factory/factory_test.go +++ b/pkg/factory/factory_test.go @@ -43,16 +43,16 @@ var _ = Describe("Factory", func() { Describe("getting factory instance", func() { Context("always", func() { It("should return the same instance", func() { - f0 := factory.NewResourceFactory("fake", "fake", true, false) + f0 := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) Expect(f0).NotTo(BeNil()) - f1 := factory.NewResourceFactory("fake", "fake", true, false) + f1 := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) Expect(f1).To(Equal(f0)) }) }) }) DescribeTable("getting info provider", func(name string, expected reflect.Type) { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := f.GetDefaultInfoProvider("fakePCIAddr", name) Expect(p).To(HaveLen(2)) // for all the providers except netdevice we expect 2 info providers Expect(reflect.TypeOf(p[1])).To(Equal(expected)) @@ -64,7 +64,7 @@ var _ = Describe("Factory", func() { ) Describe("getting info provider for generic netdevice", func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := f.GetDefaultInfoProvider("fakePCIAddr", "netdevice") Expect(p).To(HaveLen(1)) // for all the providers except netdevice we expect 2 info providers Expect(reflect.TypeOf(p[0])).To(Equal(reflect.TypeOf(infoprovider.NewGenericInfoProvider("fakePCIAddr")))) @@ -72,7 +72,7 @@ var _ = Describe("Factory", func() { DescribeTable("getting selector", func(selector string, shouldSucceed bool, expected reflect.Type) { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) v := []string{"val1", "val2", "val3"} s, e := f.GetSelector(selector, v) @@ -111,7 +111,7 @@ var _ = Describe("Factory", func() { devs []types.HostDevice ) BeforeEach(func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) devs = make([]types.HostDevice, 4) vendors := []string{"8086", "8086", "8086", "1234"} @@ -180,7 +180,7 @@ var _ = Describe("Factory", func() { DescribeTable("getting resource pool", func(selectorBytes []byte, hasDevices []string) { // create factory - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) // parse selector configuration & create resource config var selectors json.RawMessage @@ -310,7 +310,7 @@ var _ = Describe("Factory", func() { devs []types.HostDevice ) BeforeEach(func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) devs = make([]types.HostDevice, 4) vendors := []string{"8086", "8086", "8086", "8086"} codes := []string{"1111", "1111", "1111", "1111"} @@ -443,7 +443,7 @@ var _ = Describe("Factory", func() { devs []types.HostDevice ) BeforeEach(func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) devs = make([]types.HostDevice, 1) vendors := []string{"8086"} @@ -501,7 +501,7 @@ var _ = Describe("Factory", func() { devs []types.HostDevice ) BeforeEach(func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) devs = make([]types.HostDevice, 4) vendors := []string{"8086", "8086", "15b3", "15b3"} @@ -564,7 +564,7 @@ var _ = Describe("Factory", func() { }) DescribeTable("getting device provider", func(dt types.DeviceType, shouldSucceed bool) { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := f.GetDeviceProvider(dt) if shouldSucceed { Expect(p).NotTo(BeNil()) @@ -589,7 +589,7 @@ var _ = Describe("Factory", func() { Selectors: &s, } - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) _, e := f.GetDeviceFilter(rc) if shouldSucceed { @@ -612,7 +612,7 @@ var _ = Describe("Factory", func() { mockProvider.On("HasRdmaParam", mock.AnythingOfType("string"), mock.AnythingOfType("string")).Return(false, nil) utils.SetNetlinkProviderInst(mockProvider) - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) rs1 := f.GetRdmaSpec(types.NetDeviceType, "0000:00:00.1") rs2 := f.GetRdmaSpec(types.AcceleratorType, "0000:00:00.2") rs3 := f.GetRdmaSpec(types.AuxNetDeviceType, "foo.bar.3") @@ -633,7 +633,7 @@ var _ = Describe("Factory", func() { }) Describe("getting resource server", func() { Context("when resource pool is nil", func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) rs, e := f.GetResourceServer(nil) It("should fail", func() { Expect(e).To(HaveOccurred()) @@ -641,7 +641,7 @@ var _ = Describe("Factory", func() { }) }) Context("when resource pool uses overridden prefix", func() { - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) rp := mocks.ResourcePool{} rp.On("GetResourcePrefix").Return("overridden"). On("GetResourceName").Return("fake") diff --git a/pkg/netdevice/netDeviceProvider_test.go b/pkg/netdevice/netDeviceProvider_test.go index dbc3e0a79..974b180da 100644 --- a/pkg/netdevice/netDeviceProvider_test.go +++ b/pkg/netdevice/netDeviceProvider_test.go @@ -92,7 +92,7 @@ var _ = Describe("NetDeviceProvider", func() { defer fs.Use()() - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) p := netdevice.NewNetDeviceProvider(rf) config := &types.ResourceConfig{ DeviceType: types.NetDeviceType, @@ -139,7 +139,7 @@ var _ = Describe("NetDeviceProvider", func() { Describe("getting Filtered devices", func() { Context("using selectors", func() { It("should correctly filter devices", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := netdevice.NewNetDeviceProvider(rf) all := make([]types.HostDevice, 5) mocked := make([]mocks.PciNetDevice, 5) @@ -250,7 +250,7 @@ var _ = Describe("NetDeviceProvider", func() { Expect(actual).To(ConsistOf(matchingDevices)) }) It("should error if the selector index is out of bounds", func() { - rf := factory.NewResourceFactory("fake", "fake", false, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", false, false) p := netdevice.NewNetDeviceProvider(rf) devs := make([]types.HostDevice, 0) diff --git a/pkg/netdevice/netResourcePool_test.go b/pkg/netdevice/netResourcePool_test.go index e8d8119f7..f55c81829 100644 --- a/pkg/netdevice/netResourcePool_test.go +++ b/pkg/netdevice/netResourcePool_test.go @@ -34,7 +34,7 @@ import ( var _ = Describe("NetResourcePool", func() { Context("getting a new instance of the pool", func() { - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) nadutils := rf.GetNadUtils() rc := &types.ResourceConfig{ ResourceName: "fake", @@ -51,7 +51,7 @@ var _ = Describe("NetResourcePool", func() { }) Describe("getting DeviceSpecs", func() { Context("for multiple devices", func() { - rf := factory.NewResourceFactory("fake", "fake", true, false) + rf := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) nadutils := rf.GetNadUtils() rc := &types.ResourceConfig{ ResourceName: "fake", diff --git a/pkg/netdevice/pciNetDevice_test.go b/pkg/netdevice/pciNetDevice_test.go index fb28b1fc7..5298cafda 100644 --- a/pkg/netdevice/pciNetDevice_test.go +++ b/pkg/netdevice/pciNetDevice_test.go @@ -65,7 +65,7 @@ var _ = Describe("PciNetDevice", func() { defer fs.Use()() utils.SetDefaultMockNetlinkProvider() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn("0000:00:00.1") rc := &types.ResourceConfig{} @@ -266,7 +266,7 @@ var _ = Describe("PciNetDevice", func() { }, } - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn("0000:00:00.1") It("should add the vhost-net deviceSpec", func() { defer fs.Use()() @@ -352,7 +352,7 @@ var _ = Describe("PciNetDevice", func() { defer fs.Use()() utils.SetDefaultMockNetlinkProvider() - f := factory.NewResourceFactory("fake", "fake", true, false) + f := factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) in := newPciDeviceFn("0000:00:00.1") rc := &types.ResourceConfig{} diff --git a/pkg/resources/pool_stub_test.go b/pkg/resources/pool_stub_test.go index 2441530dc..e4ccceb7c 100644 --- a/pkg/resources/pool_stub_test.go +++ b/pkg/resources/pool_stub_test.go @@ -50,7 +50,7 @@ var _ = Describe("ResourcePool", func() { "sys/bus/pci/devices/0000:00:00.1/physfn": "../0000:01:00.0", }, } - f = factory.NewResourceFactory("fake", "fake", true, false) + f = factory.NewResourceFactory("fake", "fake", "/var/lib/kubelet", true, false) rc = &types.ResourceConfig{SelectorObjs: []interface{}{types.NetDeviceSelectors{}}} devs = []string{"0000:00:00.1", "0000:00:00.2"} }) diff --git a/pkg/resources/server.go b/pkg/resources/server.go index dc2864ce1..5eb1b042a 100644 --- a/pkg/resources/server.go +++ b/pkg/resources/server.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "os" + "path" "path/filepath" "strings" "time" @@ -36,6 +37,7 @@ import ( type resourceServer struct { resourcePool types.ResourcePool pluginWatch bool + kubeletRootDir string // Kubelet root dir endPoint string // Socket file sockPath string // Socket file path resourceNamePrefix string @@ -54,15 +56,24 @@ const ( ) // NewResourceServer returns an instance of ResourceServer -func NewResourceServer(prefix, suffix string, pluginWatch, useCdi bool, rp types.ResourcePool) types.ResourceServer { +func NewResourceServer(prefix, suffix, kubeletRootDir string, pluginWatch, useCdi bool, rp types.ResourcePool) types.ResourceServer { sockName := fmt.Sprintf("%s_%s.%s", prefix, rp.GetResourceName(), suffix) - sockPath := filepath.Join(types.SockDir, sockName) + sockPath := filepath.Join( + path.Join(kubeletRootDir, types.PluginRegistry), + sockName, + ) + if !pluginWatch { - sockPath = filepath.Join(types.DeprecatedSockDir, sockName) + sockPath = filepath.Join( + path.Join(kubeletRootDir, types.DeprecatedDevicePlugins), + sockName, + ) } + return &resourceServer{ resourcePool: rp, pluginWatch: pluginWatch, + kubeletRootDir: kubeletRootDir, endPoint: sockName, sockPath: sockPath, resourceNamePrefix: prefix, @@ -77,7 +88,11 @@ func NewResourceServer(prefix, suffix string, pluginWatch, useCdi bool, rp types } func (rs *resourceServer) register() error { - kubeletEndpoint := unix + ":" + filepath.Join(types.DeprecatedSockDir, types.KubeEndPoint) + kubeletEndpoint := unix + ":" + filepath.Join( + path.Join(rs.kubeletRootDir, types.DeprecatedDevicePlugins), + types.KubeEndPoint, + ) + conn, err := grpc.NewClient(kubeletEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { glog.Errorf("%s device plugin unable connect to Kubelet : %v", rs.resourcePool.GetResourceName(), err) @@ -101,10 +116,11 @@ func (rs *resourceServer) register() error { } func (rs *resourceServer) GetInfo(ctx context.Context, rqt *registerapi.InfoRequest) (*registerapi.PluginInfo, error) { + kubeletSockDir := path.Join(rs.kubeletRootDir, types.PluginRegistry) pluginInfoResponse := ®isterapi.PluginInfo{ Type: registerapi.DevicePlugin, Name: fmt.Sprintf("%s/%s", rs.resourceNamePrefix, rs.resourcePool.GetResourceName()), - Endpoint: filepath.Join(types.SockDir, rs.endPoint), + Endpoint: filepath.Join(kubeletSockDir, rs.endPoint), SupportedVersions: []string{"v1alpha1", "v1beta1"}, } return pluginInfoResponse, nil diff --git a/pkg/resources/server_test.go b/pkg/resources/server_test.go index db71c1322..45fd20790 100644 --- a/pkg/resources/server_test.go +++ b/pkg/resources/server_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "os" + "path" "path/filepath" "time" @@ -31,23 +32,23 @@ var _ = Describe("Server", func() { }) It("should have the properties correctly assigned when plugin watcher enabled", func() { // Create ResourceServer with plugin watch mode enabled - obj := NewResourceServer("fakeprefix", "fakesuffix", true, false, &rp) + obj := NewResourceServer("fakeprefix", "fakesuffix", "/var/lib/kubelet", true, false, &rp) rs = obj.(*resourceServer) Expect(rs.resourcePool.GetResourceName()).To(Equal("fakename")) Expect(rs.resourceNamePrefix).To(Equal("fakeprefix")) Expect(rs.endPoint).To(Equal("fakeprefix_fakename.fakesuffix")) Expect(rs.pluginWatch).To(Equal(true)) - Expect(rs.sockPath).To(Equal(filepath.Join(types.SockDir, "fakeprefix_fakename.fakesuffix"))) + Expect(rs.sockPath).To(Equal(filepath.Join(path.Join(rs.kubeletRootDir, types.PluginRegistry), "fakeprefix_fakename.fakesuffix"))) }) It("should have the properties correctly assigned when plugin watcher disabled", func() { // Create ResourceServer with plugin watch mode disabled - obj := NewResourceServer("fakeprefix", "fakesuffix", false, false, &rp) + obj := NewResourceServer("fakeprefix", "fakesuffix", "/var/lib/kubelet", false, false, &rp) rs = obj.(*resourceServer) Expect(rs.resourcePool.GetResourceName()).To(Equal("fakename")) Expect(rs.resourceNamePrefix).To(Equal("fakeprefix")) Expect(rs.endPoint).To(Equal("fakeprefix_fakename.fakesuffix")) Expect(rs.pluginWatch).To(Equal(false)) - Expect(rs.sockPath).To(Equal(filepath.Join(types.DeprecatedSockDir, + Expect(rs.sockPath).To(Equal(filepath.Join(path.Join(rs.kubeletRootDir, types.DeprecatedDevicePlugins), "fakeprefix_fakename.fakesuffix"))) }) }) @@ -63,11 +64,7 @@ var _ = Describe("Server", func() { rp.On("GetResourceName").Return("fakename") rp.On("CleanDeviceInfoFile", "fakeprefix").Return(nil) - // Use faked dir as socket dir - types.SockDir = fs.RootDir - types.DeprecatedSockDir = fs.RootDir - - obj := NewResourceServer("fakeprefix", "fakesuffix", shouldEnablePluginWatch, false, &rp) + obj := NewResourceServer("fakeprefix", "fakesuffix", "/var/lib/kubelet", shouldEnablePluginWatch, false, &rp) rs := obj.(*resourceServer) registrationServer := createFakeRegistrationServer(fs.RootDir, @@ -114,7 +111,7 @@ var _ = Describe("Server", func() { defer fs.Use()() rp := mocks.ResourcePool{} rp.On("GetResourceName").Return("fake.com") - rs := NewResourceServer("fakeprefix", "fakesuffix", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fakeprefix", "fakesuffix", "/var/lib/kubelet", true, false, &rp).(*resourceServer) err = rs.Init() }) It("should never fail", func() { @@ -143,8 +140,6 @@ var _ = Describe("Server", func() { Context("starting, restarting and stopping the resource server", func() { It("should not fail and messages should be received on the channels", func(done Done) { defer fs.Use()() - // Use faked dir as socket dir - types.DeprecatedSockDir = fs.RootDir rp := mocks.ResourcePool{} rp.On("GetConfig").Return(fakeConf). On("GetResourceName").Return("fake.com"). @@ -154,7 +149,7 @@ var _ = Describe("Server", func() { On("CleanDeviceInfoFile", "fake").Return(nil) // Create ResourceServer with plugin watch mode disabled - rs := NewResourceServer("fake", "fake", false, false, &rp).(*resourceServer) + rs := NewResourceServer("fake", "fake", "/var/lib/kubelet", false, false, &rp).(*resourceServer) registrationServer := createFakeRegistrationServer(fs.RootDir, "fake_fake.com.fake", false, false) @@ -183,9 +178,6 @@ var _ = Describe("Server", func() { }, 12.0) It("should not fail and messages should be received on the channels", func(done Done) { defer fs.Use()() - // Use faked dir as socket dir - types.SockDir = fs.RootDir - rp := mocks.ResourcePool{} rp.On("GetConfig").Return(fakeConf). On("GetResourceName").Return("fake.com"). @@ -194,7 +186,7 @@ var _ = Describe("Server", func() { On("Probe").Return(true). On("CleanDeviceInfoFile", "fake").Return(nil) // Create ResourceServer with plugin watch mode enabled - rs := NewResourceServer("fake", "fake", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fake", "fake", "/var/lib/kubelet", true, false, &rp).(*resourceServer) registrationServer := createFakeRegistrationServer(fs.RootDir, "fake_fake.com.fake", false, true) @@ -218,8 +210,6 @@ var _ = Describe("Server", func() { Context("starting, watching and stopping the resource server", func() { It("should not fail and messages should be received on the channels", func(done Done) { defer fs.Use()() - // Use faked dir as socket dir - types.DeprecatedSockDir = fs.RootDir rp := mocks.ResourcePool{} rp.On("GetConfig").Return(fakeConf). On("GetResourceName").Return("fake.com"). @@ -229,7 +219,7 @@ var _ = Describe("Server", func() { On("CleanDeviceInfoFile", "fake").Return(nil) // Create ResourceServer with plugin watch mode disabled - rs := NewResourceServer("fake", "fake", false, false, &rp).(*resourceServer) + rs := NewResourceServer("fake", "fake", "/var/lib/kubelet", false, false, &rp).(*resourceServer) registrationServer := createFakeRegistrationServer(fs.RootDir, "fake_fake.com.fake", false, false) @@ -273,7 +263,7 @@ var _ = Describe("Server", func() { On("StoreDeviceInfoFile", "fake.com", []string{"00:00.01"}). Return(nil) - rs := NewResourceServer("fake.com", "fake", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, false, &rp).(*resourceServer) resp, err := rs.Allocate(context.TODO(), req) @@ -319,7 +309,7 @@ var _ = Describe("Server", func() { On("StoreDeviceInfoFile", "fake.com", []string{"00:00.01"}). Return(nil) - rs := NewResourceServer("fake.com", "fake", true, true, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, true, &rp).(*resourceServer) cdi := &CDImocks.CDI{} cdi.On("CreateCDISpecForPool", "fake.com", &rp).Return(nil).Twice(). @@ -378,7 +368,7 @@ var _ = Describe("Server", func() { rp.On("GetResourceName").Return("fake.com"). On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.01": {ID: "00:00.01", Health: "Healthy"}}).Once() - rs := NewResourceServer("fake.com", "fake", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, false, &rp).(*resourceServer) rs.sockPath = fs.RootDir lwSrv := &fakeListAndWatchServer{ @@ -399,7 +389,7 @@ var _ = Describe("Server", func() { On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.01": {ID: "00:00.01", Health: "Healthy"}}).Once(). On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.02": {ID: "00:00.02", Health: "Healthy"}}).Once() - rs := NewResourceServer("fake.com", "fake", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, false, &rp).(*resourceServer) rs.sockPath = fs.RootDir lwSrv := &fakeListAndWatchServer{ @@ -434,7 +424,7 @@ var _ = Describe("Server", func() { On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.01": {ID: "00:00.01", Health: "Healthy"}}).Once(). On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.02": {ID: "00:00.02", Health: "Healthy"}}).Once() - rs := NewResourceServer("fake.com", "fake", true, false, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, false, &rp).(*resourceServer) rs.sockPath = fs.RootDir lwSrv := &fakeListAndWatchServer{ @@ -472,7 +462,7 @@ var _ = Describe("Server", func() { On("GetDevices").Return(map[string]*pluginapi.Device{"00:00.01": {ID: "00:00.01", Health: "Healthy"}}).Twice(). On("GetResourcePrefix").Return("fake.com").Twice() - rs := NewResourceServer("fake.com", "fake", true, true, &rp).(*resourceServer) + rs := NewResourceServer("fake.com", "fake", "/var/lib/kubelet", true, true, &rp).(*resourceServer) rs.sockPath = fs.RootDir cdi := &CDImocks.CDI{} diff --git a/pkg/types/types.go b/pkg/types/types.go index 48dadc6d2..42552c1dd 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -24,10 +24,11 @@ import ( ) var ( + // SockDir is the default Kubelet device plugin socket directory - SockDir = "/var/lib/kubelet/plugins_registry" + PluginRegistry = "plugins_registry" // DeprecatedSockDir is the deprecated Kubelet device plugin socket directory - DeprecatedSockDir = "/var/lib/kubelet/device-plugins" + DeprecatedDevicePlugins = "device-plugins" ) const (