[PATCH] tuna: RFC: Use cpusets to isolate cpus

Use cpusets to isolate cpus This is a proof of concept, some more work would have to be done especially in tuna to deal with situations such as attempting to create a new isolated cpuset when one already exists, and so on. After running the following command on a machine with no cpusets su -c './tuna-cmd.py -c "5-7" -i' We have the base cpuset which by convention is mounted here /dev/cpuset and we have cpuset 'tuna0' which has all the cpus that were not to be isolated cpuset 'tuna1' which has all the cpus that were to be isolated and tuna1 is a cpu_exclusive set All the tasks that were migratable from the base cpuset are now in tuna0 and no tasks are in tuna1 ls -ld /dev/cpuset dr-xr-xr-x. 4 root root 0 Nov 8 15:20 /dev/cpuset [root@fionn ~]# cd /dev/cpuset [root@fionn cpuset]# cat tuna0/cpus 0-4,8-11 [root@fionn cpuset]# cat tuna1/cpus 5-7 [root@fionn cpuset]# cat tuna0/cpu_exclusive 0 [root@fionn cpuset]# cat tuna1/cpu_exclusive 1 Some tasks are not migrate-able, after the migration There are (as an example) 138 tasks that couldn't be migrated 971 that were migrated and 0 in the new cpuset with the isolated cpus [jkacur@fionn tuna]$ cat /dev/cpuset/tasks | wc -l 138 [jkacur@fionn tuna]$ cat /dev/cpuset/tuna0/tasks | wc -l 961 [jkacur@fionn tuna]$ cat /dev/cpuset/tuna1/tasks | wc -l 0 The programmer could now use affinity to run new tasks on the isolated cpus pushed to branch devel/cpusets so folks can try it out. Signed-off-by: John Kacur <jkacur(a)redhat.com&gt; --- tuna/cpuset.py | 225 +++++++++++++++++++++++++++++++++++++++++++++++++ tuna/tuna.py | 33 ++++++++ 2 files changed, 258 insertions(+) create mode 100755 tuna/cpuset.py diff --git a/tuna/cpuset.py b/tuna/cpuset.py new file mode 100755 index 000000000000..526ca2975d4d --- /dev/null +++ b/tuna/cpuset.py @@ -0,0 +1,225 @@ +#!/usr/bin/python3 +# Copyright 2021 John Kacur <jkacur(a)redhat.com&gt; +""" Module for creating and using cpusets """ + +import errno +import os +import subprocess +import sys +from glob import glob + +import procfs + + +class Cpuset: + """ Class for manipulating cpusets """ + + mpath = '/dev/cpuset' + + def __init__(self, name=None): + self.cpuset_name = None + self._cpuset_path = None + self.create_cpuset(name) + self.tasks = [] + + @property + def cpuset_path(self): + """ returns the cpuset_path, for example /dev/cpuset/tuna0 """ + return self._cpuset_path + + def __str__(self): + return (f'cpuset_name={self.cpuset_name}, cpuset_path={self.cpuset_path}') + + def create_cpuset(self, name=None): + """ creates a cpuset below /dev/cpuset """ + self.cpuset_name = name + path = os.path.join(Cpuset.mpath, self.cpuset_name) + if not os.path.exists(path): + os.mkdir(path) + self._cpuset_path = path + + def assign_cpus(self, cpu_str): + """ Assign cpus in list-format to a cpuset """ + path = os.path.join(self._cpuset_path, "cpus") + with open(path, 'w') as f: + f.write(cpu_str) + + def write_pid(self, pid): + """ place a pid in a cpuset """ + path = os.path.join(self._cpuset_path, "tasks") + pid_str = str(pid) + with open(path, 'wb', buffering=0) as f: + b = pid_str.encode('utf-8') + try: + f.write(b) + f.flush() + except OSError as err: + if err.args[0] == errno.EINVAL: + pass + if err.args[0] == errno.ESRCH: + pass + + def write_memnode(self, memnode): + """ place memnode in a cpuset """ + path = os.path.join(self._cpuset_path, "mems") + memnode_str = str(memnode) + with open(path, 'w') as f: + f.write(memnode_str) + + def write_cpu_exclusive(self, cpu_exclusive): + """ turn cpu_explusive on or off """ + path = os.path.join(self._cpuset_path, "cpu_exclusive") + cpu_exclusive_str = str(cpu_exclusive) + with open(path, 'w') as f: + f.write(cpu_exclusive_str) + + def set_cpu_exclusive(self): + """ turn cpu_exclusive on """ + self.write_cpu_exclusive('1') + + def unset_cpu_exclusive(self): + """ turn cpu_exlusive off """ + self.write_cpu_exclusive('0') + + def write_memory_migrate(self, memory_migrate): + """ turn memory migrate on or off """ + path = os.path.join(self._cpuset_path, "memory_migrate") + memory_migrate_str = str(memory_migrate) + with open(path, 'w') as f: + f.write(memory_migrate_str) + + def set_memory_migrate(self): + """ turn memory_migrate on """ + self.write_memory_migrate('1') + + def unset_memory_migrate(self): + """ turn memory migrate off """ + self.write_memory_migrate('0') + + def get_tasks(self): + """ get the tasks currently in a cpuset """ + path = os.path.join(self.cpuset_path, "tasks") + with open(path, 'r') as f: + task = f.readline().strip() + while task: + self.tasks.append(task) + task = f.readline().strip() + return self.tasks + + +class CpusetsInit(Cpuset): + """ + CpusetsInit inits cpusets if they are currently off + this class will create /dev/cpuset and mount it + It will also test whether the kernel supports cpusets + This class is a cpuset itself, it is the base cpuset /dev/cpuset + """ + + mpath = '/dev/cpuset' + + def __init__(self): + self._supported = self._cpuset_supported() + if self._supported: + self.create_cpuset_dir() + self.mount_cpuset() + self._numa_nodes = self._get_numa_nodes() + super().__init__('/dev/cpuset') + + @property + def supported(self): + """ return True if cpuset is supported """ + return self._supported + + @property + def numa_nodes(self): + """ return the number of numa nodes """ + return self._numa_nodes + + @staticmethod + def _cpuset_supported(): + """ + Return True if cpusets are supported + if the entry 'node cpuset' is in /proc/filesystems + then cpusets are supported + This function is called by the __init__ + Users of this class should use the supported method + """ + dpath = '/proc/filesystems' + with open(dpath) as fp: + line = fp.readline().strip() + while line: + elems = line.split() + if len(elems) == 2 and elems[1] == "cpuset": + return True + line = fp.readline().strip() + return False + + @staticmethod + def _get_numa_nodes(): + """ + return the number of numa nodes + used by init to set _numa_nodes + Users of this class should use the "numa_nodes" method + """ + return len(glob('/sys/devices/system/node/node*')) + + @staticmethod + def create_cpuset_dir(): + """ create /dev/cpuset if it doesn't already exist """ + path = Cpuset.mpath + if not os.path.exists(path): + os.mkdir(path) + + @staticmethod + def mount_cpuset(): + """ mount /dev/cpuset if it exists and isn't already mounted """ + path = Cpuset.mpath + if os.path.exists(path) and not os.path.ismount(path): + args = ['mount', '-t', 'cpuset', 'cpuset', path] + subprocess.run(args, check=True) + + +class TaskMigrate: + """ a class for migrating a set of pids from one cpuset to another """ + + def __init__(self, cpuset_from, cpuset_to): + self.cpuset_from = cpuset_from + self.cpuset_to = cpuset_to + self.cpuset_to.set_memory_migrate() + + def migrate(self): + """ do the migration, only attempt on pids where this is allowed """ + # ps = procfs.pidstats() + # ps.reload_threads() + for task in self.cpuset_from.get_tasks(): + self.cpuset_to.write_pid(task) + + +if __name__ == '__main__': + + cpusets_init = CpusetsInit() + # tasks = cpusets_init.get_tasks() + print(f'cpusets_init.supported = {cpusets_init.supported}') + if cpusets_init.supported: + print(f'cpusets_init.numa_nodes = {cpusets_init.numa_nodes}') + # print(f'tasks = {tasks}') + cpuset0 = Cpuset('tuna0') + cpuset1 = Cpuset('tuna1') + + cpuset0.assign_cpus('0-4,8-11') + cpuset1.assign_cpus('5-7') + cpuset0.write_memnode('0') + cpuset1.write_memnode('0') + cpuset1.set_cpu_exclusive() + cpuset0.set_memory_migrate() + + tm = TaskMigrate(cpusets_init, cpuset0) + tm.migrate() + + sys.exit(0) + for dirpath, dirnames, fnames in \ + os.walk(cpusets_init.mpath, topdown=True): + print(f'dirpath = {dirpath}') + print(f'dirnames = {dirnames}') + print(f'fnames = {fnames}') + print('\n') diff --git a/tuna/tuna.py b/tuna/tuna.py index 6b6edcc8ff72..d3758e0cb731 100755 --- a/tuna/tuna.py +++ b/tuna/tuna.py @@ -14,6 +14,7 @@ import procfs from procfs import utilist import help import tuna_sched +import cpuset try: fntable @@ -332,7 +333,39 @@ def parse_irq_affinity_filename(filename, nr_cpus): return utilist.bitmasklist(line, nr_cpus) +def _isolate_cpus(cpus, nr_cpus): + + base_cpuset = cpuset.CpusetsInit() + + if not base_cpuset.supported: + print('Warning: cpusets are not supported') + return + + # cpus not in the isolated list + cpus0 = set(range(nr_cpus)) - set(cpus) + cpus0 = list_to_cpustring(cpus0) + + # cpuset of cpus not in the isolated list + tuna0 = cpuset.Cpuset('tuna0') + tuna0.assign_cpus(cpus0) + tuna0.write_memnode('0') + + # cpuset to be isolated + tuna1 = cpuset.Cpuset('tuna1') + cpus1 = list_to_cpustring(cpus) + tuna1.assign_cpus(cpus1) + tuna1.write_memnode('0') + tuna1.set_cpu_exclusive() + + # Migrate all migratable tasks from the base_cpuset + # to the cpusset with the cpus not in the isolated list + tm = cpuset.TaskMigrate(base_cpuset, tuna0) + tm.migrate() + def isolate_cpus(cpus, nr_cpus): + _isolate_cpus(cpus, nr_cpus) + return + fname = sys._getframe().f_code.co_name # Function name ps = procfs.pidstats() ps.reload_threads() -- 2.31.1