Fork me on GitHub

longhron磁盘发现机制探索

发表于

longhorn 的crd nodes.longhorn.io 里是有存储disk信息的，这个node自定义资源的创建是在 longhorn-manager Daemon 启动时initDaemonNode

datastore/longhorn.go:CreateDefaultNode

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

// For newly added node, the customized default disks will be applied only if the setting is enabled.
if !requireLabel {
	// Note: this part wasn't moved to the controller is because
	// this will be done only once.
	// If user remove all the disks on the node, the default disk
	// will not be recreated automatically
	dataPath, err := s.GetSettingValueExisted(types.SettingNameDefaultDataPath)
	// dataPath : 默认是/var/lib/longhorn/,从settings crd中获取
	if err != nil {
		return nil, err
	}
	disks, err := types.CreateDefaultDisk(dataPath)
	if err != nil {
		return nil, err
	}
	node.Spec.Disks = disks
}

创建默认disk 函数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

func CreateDefaultDisk(dataPath string) (map[string]DiskSpec, error) {
	if err := util.CreateDiskPathReplicaSubdirectory(dataPath); err != nil {
		return nil, err
	}
	diskInfo, err := util.GetDiskInfo(dataPath)
	if err != nil {
		return nil, err
	}
	return map[string]DiskSpec{
		DefaultDiskPrefix + diskInfo.Fsid: {
			Path:              diskInfo.Path,
			AllowScheduling:   true,
			EvictionRequested: false,
			StorageReserved:   diskInfo.StorageMaximum * 30 / 100, // 磁盘预留空间，也即是我们在longhorn Dashboard Node.Size 看到的 Reserved 值
		},
	}, nil
}

CreateDiskPathReplicaSubdirectory 这个函数只是创建 replicas 子目录，而核心逻辑是GetDiskInfo获取disk信息，那么接下来我们就看下具体代码实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
func GetDiskInfo(directory string) (info *DiskInfo, err error) {
	defer func() {
		err = errors.Wrapf(err, "cannot get disk info of directory %v", directory)
	}()
	initiatorNSPath := iscsi_util.GetHostNamespacePath(HostProcPath)
	mountPath := fmt.Sprintf("--mount=%s/mnt", initiatorNSPath)
	// mountPath: /host/proc/pid/ns/
	// stat -f -c "{\"path\":\"%n\",\"fsid\":\"%i\",\"type\":\"%T\",\"freeBlock\":%f,\"totalBlock\":%b,\"blockSize\":%S}" /var/lib/longhorn/
	// nsenter --mount=/host/proc/pid/ns/mnt command(stat -fc ...)
	output, err := Execute([]string{}, "nsenter", mountPath, "stat", "-fc", "{\"path\":\"%n\",\"fsid\":\"%i\",\"type\":\"%T\",\"freeBlock\":%f,\"totalBlock\":%b,\"blockSize\":%S}", directory)
	if err != nil {
		return nil, err
	}
	output = strings.Replace(output, "\n", "", -1)

	diskInfo := &DiskInfo{}
	err = json.Unmarshal([]byte(output), diskInfo)
	if err != nil {
		return nil, err
	}

	diskInfo.StorageMaximum = diskInfo.TotalBlock * diskInfo.BlockSize
	diskInfo.StorageAvailable = diskInfo.FreeBlock * diskInfo.BlockSize

	return diskInfo, nil
}

可以看出，核心就是用了 linux 的nsenter命令，以及用 stat 命令获取 文件系统信息，最后反序列化转换成自定义disk结构体

附录：
neenter 命令：https://staight.github.io/2019/09/23/nsenter%E5%91%BD%E4%BB%A4%E7%AE%80%E4%BB%8B/