[PATCH] tuna: Fix tuna displays incorrect cpu affinity when cpu is offlined
by John Kacur
If we create a process and query it's affinity, both taskset and tuna
display the same result, but if we then disable a processor, tuna
displays the wrong affinity, as below.
[jkacur@fionn tuna]$ sha1sum /dev/zero &
[1] 10640
[jkacur@fionn tuna]$ taskset -p 10640
pid 10640's current affinity mask: fff
[jkacur@fionn tuna]$ su -c './tuna-cmd.py -t sha1sum -P'
Password:
thread ctxt_switches
pid SCHED_ rtpri affinity voluntary nonvoluntary cmd
10640 OTHER 0 0xfff 0 423 sha1sum
[jkacur@fionn tuna]$ su -c 'chcpu -d 5'
Password:
CPU 5 disabled
[jkacur@fionn tuna]$ taskset -p 10640
pid 10640's current affinity mask: fdf
[jkacur@fionn tuna]$ su -c './tuna-cmd.py -t sha1sum -P'
Password:
thread ctxt_switches
pid SCHED_ rtpri affinity voluntary nonvoluntary cmd
10640 OTHER 0 0x7df 0 919 sha1sum
The reason for this is that after tuna gets the affinity for the
process, when it converts this to a hex value, it passes the number of
enabled processors to hexbitmask in procfs, when what we really need is
the number of processors including disabled ones.
Fix this by querrying SC_NPROCESSORS_CONF and passing that value to
hexbitmask.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
tuna-cmd.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tuna-cmd.py b/tuna-cmd.py
index d209a2e89493..7e33a128d676 100755
--- a/tuna-cmd.py
+++ b/tuna-cmd.py
@@ -199,7 +199,9 @@ def format_affinity(affinity):
if len(affinity) <= 4:
return ",".join(str(a) for a in affinity)
- return ",".join(str(hex(a)) for a in procfs.hexbitmask(affinity, get_nr_cpus()))
+ # We need the number of cpus on a system, including disabled ones
+ ncpus = os.sysconf('SC_NPROCESSORS_CONF')
+ return ",".join(str(hex(a)) for a in procfs.hexbitmask(affinity, ncpus))
def ps_show_thread(pid, affect_children, ps, has_ctxt_switch_info, sock_inodes,
--
2.31.1
2 years, 4 months
- 1
- 0
[PATCH] python-linux-procfs: Fix UnicodeDecodeError
by John Kacur
Commit 7570fc0d6082 meant to solve the UnicodeDecodeError
Instead it actually increased the problem by reading lines as bytes
and decoding them.
The original problem is hard to trigger and doesn't trigger consistently
with reproducers. In addition there seems to be a difference in how this
is handled between python-3.6 to python-3.9
For now, we should return the code to reading as utf-8 (the default)
since that handles more cases than the current code.
We can catch the UnicodeDecodeError and ignore it for now. It is not
ideal because we are not handling some pids that trigger the error.
This patch also includes a fix for FileNotFoundError which can occur if
a pid exits and disappears before we try to read it in the /proc file
system.
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
procfs/procfs.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/procfs/procfs.py b/procfs/procfs.py
index a78bac5376e3..de55dfc1aef4 100755
--- a/procfs/procfs.py
+++ b/procfs/procfs.py
@@ -44,7 +44,11 @@ def process_cmdline(pid_info):
if pid_info["cmdline"]:
return reduce(lambda a, b: a + " %s" % b, pid_info["cmdline"]).strip()
- return pid_info["stat"]["comm"]
+ try:
+ """ If a pid disappears before we query it, return None """
+ return pid_info["stat"]["comm"]
+ except:
+ return None
class pidstat:
@@ -374,9 +378,15 @@ class process:
return hasattr(self, attr)
def load_cmdline(self):
- with open("/proc/%d/cmdline" % self.pid, mode='rb') as f:
- cmdline = f.readline().decode(encoding='unicode_escape')
- self.cmdline = cmdline.strip().split('\0')[:-1]
+ try:
+ with open("/proc/%d/cmdline" % self.pid) as f:
+ self.cmdline = f.readline().strip().split('\0')[:-1]
+ except FileNotFoundError:
+ """ This can happen when a pid disappears """
+ self.cmdline = None
+ except UnicodeDecodeError:
+ """ TODO - this shouldn't happen, needs to be investigated """
+ self.cmdline = None
def load_threads(self):
self.threads = pidstats("/proc/%d/task/" % self.pid)
--
2.31.1
2 years, 4 months
- 1
- 0
[PATCH] python-linux-procfs: Various clean-ups
by John Kacur
- Replace f=open with 'with' (context managers), except in try-except blocks
- Reformat lines that are too long, especially in comments
- Use min() instead of if construct
- Use bool instead of complicated and True or False constructs
- Reorder imports
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
procfs/procfs.py | 292 +++++++++++++++++++++++------------------------
1 file changed, 142 insertions(+), 150 deletions(-)
diff --git a/procfs/procfs.py b/procfs/procfs.py
index a0e9977214fe..a78bac5376e3 100755
--- a/procfs/procfs.py
+++ b/procfs/procfs.py
@@ -19,10 +19,10 @@
#
import os
-import time
-from functools import reduce
import platform
import re
+import time
+from functools import reduce
from six.moves import range
from procfs.utilist import bitmasklist
@@ -37,8 +37,9 @@ def is_s390():
def process_cmdline(pid_info):
"""
- Returns the process command line, if available in the given `process' class, if
- not available, falls back to using the comm (short process name) in its pidstat key.
+ Returns the process command line, if available in the given `process' class,
+ if not available, falls back to using the comm (short process name) in its
+ pidstat key.
"""
if pid_info["cmdline"]:
return reduce(lambda a, b: a + " %s" % b, pid_info["cmdline"]).strip()
@@ -47,10 +48,12 @@ def process_cmdline(pid_info):
class pidstat:
- """Provides a dictionary to access the fields in the per process /proc/PID/stat
- files.
+ """
+ Provides a dictionary to access the fields in the
+ per process /proc/PID/stat files.
- One can obtain the available fields asking for the keys of the dictionary, e.g.:
+ One can obtain the available fields by asking for the keys of the
+ dictionary, e.g.:
>>> p = procfs.pidstat(1)
>>> print p.keys()
@@ -182,8 +185,7 @@ class pidstat:
Returns true if this process has a fixed smp affinity mask,
not allowing it to be moved to a different set of CPUs.
"""
- return self.fields["flags"] & self.PF_THREAD_BOUND and \
- True or False
+ return bool(self.fields["flags"] & self.PF_THREAD_BOUND)
def process_flags(self):
"""
@@ -193,33 +195,34 @@ class pidstat:
As of v4.2-rc7 these include (from include/linux/sched.h comments):
- PF_EXITING Getting shut down
- PF_EXITPIDONE Pi exit done on shut down
+ PF_EXITING Getting shut down
+ PF_EXITPIDONE Pi exit done on shut down
PF_VCPU I'm a virtual CPU
- PF_WQ_WORKER I'm a workqueue worker
- PF_FORKNOEXEC Forked but didn't exec
- PF_MCE_PROCESS Process policy on mce errors
- PF_SUPERPRIV Used super-user privileges
+ PF_WQ_WORKER I'm a workqueue worker
+ PF_FORKNOEXEC Forked but didn't exec
+ PF_MCE_PROCESS Process policy on mce errors
+ PF_SUPERPRIV Used super-user privileges
PF_DUMPCORE Dumped core
PF_SIGNALED Killed by a signal
PF_MEMALLOC Allocating memory
- PF_NPROC_EXCEEDED Set_user noticed that RLIMIT_NPROC was exceeded
- PF_USED_MATH If unset the fpu must be initialized before use
- PF_USED_ASYNC Used async_schedule*(), used by module init
+ PF_NPROC_EXCEEDED Set_user noticed that RLIMIT_NPROC was exceeded
+ PF_USED_MATH If unset the fpu must be initialized before use
+ PF_USED_ASYNC Used async_schedule*(), used by module init
PF_NOFREEZE This thread should not be frozen
- PF_FROZEN Frozen for system suspend
- PF_FSTRANS Inside a filesystem transaction
- PF_KSWAPD I am kswapd
+ PF_FROZEN Frozen for system suspend
+ PF_FSTRANS Inside a filesystem transaction
+ PF_KSWAPD I am kswapd
PF_MEMALLOC_NOIO Allocating memory without IO involved
PF_LESS_THROTTLE Throttle me less: I clean memory
- PF_KTHREAD I am a kernel thread
+ PF_KTHREAD I am a kernel thread
PF_RANDOMIZE Randomize virtual address space
PF_SWAPWRITE Allowed to write to swap
PF_NO_SETAFFINITY Userland is not allowed to meddle with cpus_allowed
PF_MCE_EARLY Early kill for mce process policy
- PF_MUTEX_TESTER Thread belongs to the rt mutex tester
- PF_FREEZER_SKIP Freezer should not count it as freezable
- PF_SUSPEND_TASK This thread called freeze_processes and should not be frozen
+ PF_MUTEX_TESTER Thread belongs to the rt mutex tester
+ PF_FREEZER_SKIP Freezer should not count it as freezable
+ PF_SUSPEND_TASK This thread called freeze_processes and
+ should not be frozen
"""
sflags = []
@@ -236,8 +239,8 @@ class pidstat:
def cannot_set_affinity(self, pid):
PF_NO_SETAFFINITY = 0x04000000
try:
- return int(self.processes[pid]["stat"]["flags"]) & \
- PF_NO_SETAFFINITY and True or False
+ return bool(int(self.processes[pid]["stat"]["flags"]) &
+ PF_NO_SETAFFINITY)
except:
return True
@@ -245,19 +248,21 @@ def cannot_set_affinity(self, pid):
def cannot_set_thread_affinity(self, pid, tid):
PF_NO_SETAFFINITY = 0x04000000
try:
- return int(self.processes[pid].threads[tid]["stat"]["flags"]) & \
- PF_NO_SETAFFINITY and True or False
+ return bool(int(self.processes[pid].threads[tid]["stat"]["flags"]) &
+ PF_NO_SETAFFINITY)
except:
return True
class pidstatus:
"""
- Provides a dictionary to access the fields in the per process /proc/PID/status
- files. This provides additional information about processes and threads to what
- can be obtained with the procfs.pidstat() class.
+ Provides a dictionary to access the fields
+ in the per process /proc/PID/status files.
+ This provides additional information about processes and threads to
+ what can be obtained with the procfs.pidstat() class.
- One can obtain the available fields asking for the keys of the dictionary, e.g.:
+ One can obtain the available fields by asking for the keys of the
+ dictionary, e.g.:
>>> import procfs
>>> p = procfs.pidstatus(1)
@@ -312,19 +317,18 @@ class pidstatus:
return fieldname in self.fields
def load(self, basedir="/proc"):
- f = open("%s/%d/status" % (basedir, self.pid))
self.fields = {}
- for line in f.readlines():
- fields = line.split(":")
- if len(fields) != 2:
- continue
- name = fields[0]
- value = fields[1].strip()
- try:
- self.fields[name] = int(value)
- except:
- self.fields[name] = value
- f.close()
+ with open("%s/%d/status" % (basedir, self.pid)) as f:
+ for line in f.readlines():
+ fields = line.split(":")
+ if len(fields) != 2:
+ continue
+ name = fields[0]
+ value = fields[1].strip()
+ try:
+ self.fields[name] = int(value)
+ except:
+ self.fields[name] = value
class process:
@@ -380,14 +384,13 @@ class process:
del self.threads[self.pid]
def load_cgroups(self):
- f = open("/proc/%d/cgroup" % self.pid)
self.cgroups = ""
- for line in reversed(f.readlines()):
- if len(self.cgroups) != 0:
- self.cgroups = self.cgroups + "," + line[:-1]
- else:
- self.cgroups = line[:-1]
- f.close()
+ with open("/proc/%d/cgroup" % self.pid) as f:
+ for line in reversed(f.readlines()):
+ if len(self.cgroups) != 0:
+ self.cgroups = self.cgroups + "," + line[:-1]
+ else:
+ self.cgroups = line[:-1]
def load_environ(self):
"""
@@ -416,12 +419,11 @@ class process:
>>>
"""
self.environ = {}
- f = open("/proc/%d/environ" % self.pid)
- for x in f.readline().split('\0'):
- if len(x) > 0:
- y = x.split('=')
- self.environ[y[0]] = y[1]
- f.close()
+ with open("/proc/%d/environ" % self.pid) as f:
+ for x in f.readline().split('\0'):
+ if len(x) > 0:
+ y = x.split('=')
+ self.environ[y[0]] = y[1]
class pidstats:
@@ -465,18 +467,18 @@ class pidstats:
def reload(self):
"""
- This operation will throw away the current dictionary contents, if any, and
- read all the pid files from /proc/, instantiating a 'process' instance for
- each of them.
+ This operation will throw away the current dictionary contents,
+ if any, and read all the pid files from /proc/, instantiating a
+ 'process' instance for each of them.
- This is a high overhead operation, and should be avoided if the perf python
- binding can be used to detect when new threads appear and existing ones
- terminate.
+ This is a high overhead operation, and should be avoided if the
+ perf python binding can be used to detect when new threads appear
+ and existing ones terminate.
In RHEL it is found in the python-perf rpm package.
- More information about the perf facilities can be found in the 'perf_event_open'
- man page.
+ More information about the perf facilities can be found in the
+ 'perf_event_open' man page.
"""
del self.processes
self.processes = {}
@@ -643,24 +645,21 @@ class interrupts:
def reload(self):
del self.interrupts
self.interrupts = {}
- f = open("/proc/interrupts")
-
- for line in f.readlines():
- line = line.strip()
- fields = line.split()
- if fields[0][:3] == "CPU":
- self.nr_cpus = len(fields)
- continue
- irq = fields[0].strip(":")
- self.interrupts[irq] = {}
- self.interrupts[irq] = self.parse_entry(fields[1:], line)
- try:
- nirq = int(irq)
- except:
- continue
- self.interrupts[irq]["affinity"] = self.parse_affinity(nirq)
-
- f.close()
+ with open("/proc/interrupts") as f:
+ for line in f.readlines():
+ line = line.strip()
+ fields = line.split()
+ if fields[0][:3] == "CPU":
+ self.nr_cpus = len(fields)
+ continue
+ irq = fields[0].strip(":")
+ self.interrupts[irq] = {}
+ self.interrupts[irq] = self.parse_entry(fields[1:], line)
+ try:
+ nirq = int(irq)
+ except:
+ continue
+ self.interrupts[irq]["affinity"] = self.parse_affinity(nirq)
def parse_entry(self, fields, line):
dict = {}
@@ -681,9 +680,8 @@ class interrupts:
def parse_affinity(self, irq):
try:
- f = open("/proc/irq/%s/smp_affinity" % irq)
- line = f.readline()
- f.close()
+ with open("/proc/irq/%s/smp_affinity" % irq) as f:
+ line = f.readline()
return bitmasklist(line, self.nr_cpus)
except IOError:
return [0, ]
@@ -741,11 +739,11 @@ class cmdline:
"""
Parses the kernel command line (/proc/cmdline), turning it into a dictionary."
- Useful to figure out if some kernel boolean knob has been turned on, as well
- as to find the value associated to other kernel knobs.
+ Useful to figure out if some kernel boolean knob has been turned on,
+ as well as to find the value associated to other kernel knobs.
- It can also be used to find out about parameters passed to the init process,
- such as 'BOOT_IMAGE', etc.
+ It can also be used to find out about parameters passed to the
+ init process, such as 'BOOT_IMAGE', etc.
E.g.:
>>> import procfs
@@ -762,15 +760,13 @@ class cmdline:
self.parse()
def parse(self):
- f = open("/proc/cmdline")
- for option in f.readline().strip().split():
- fields = option.split("=")
- if len(fields) == 1:
- self.options[fields[0]] = True
- else:
- self.options[fields[0]] = fields[1]
-
- f.close()
+ with open("/proc/cmdline") as f:
+ for option in f.readline().strip().split():
+ fields = option.split("=")
+ if len(fields) == 1:
+ self.options[fields[0]] = True
+ else:
+ self.options[fields[0]] = fields[1]
def __getitem__(self, key):
return self.options[key]
@@ -790,9 +786,9 @@ class cpuinfo:
Dictionary with information about CPUs in the system.
Please refer to 'man procfs(5)' for further information about the
- '/proc/cpuinfo' file, that is the source of the information provided by this
- class. The 'man lscpu(1)' also has information about a program that uses
- the '/proc/cpuinfo' file.
+ '/proc/cpuinfo' file, that is the source of the information provided
+ by this class. The 'man lscpu(1)' also has information about a program that
+ uses the '/proc/cpuinfo' file.
Using this class one can obtain the number of CPUs in a system:
@@ -801,14 +797,14 @@ class cpuinfo:
4
It is also possible to figure out aspects of the CPU topology, such as
- how many CPU physical sockets exists, i.e. groups of CPUs sharing components
- such as CPU memory caches:
+ how many CPU physical sockets exists, i.e. groups of CPUs sharing
+ components such as CPU memory caches:
>>> print len(cpus.sockets)
1
- Additionally dictionary with information common to all CPUs in the system is
- available:
+ Additionally dictionary with information common to all CPUs in the system
+ is available:
>>> print cpus["model name"]
Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz
@@ -836,28 +832,26 @@ class cpuinfo:
return self.tags
def parse(self, filename):
- f = open(filename)
- for line in f.readlines():
- line = line.strip()
- if not line:
- continue
- fields = line.split(":")
- tagname = fields[0].strip().lower()
- if tagname == "processor":
- self.nr_cpus += 1
- continue
- if is_s390() and tagname == "cpu number":
- self.nr_cpus += 1
- continue
- if tagname == "core id":
- continue
- self.tags[tagname] = fields[1].strip()
- if tagname == "physical id":
- socket_id = self.tags[tagname]
- if socket_id not in self.sockets:
- self.sockets.append(socket_id)
-
- f.close()
+ with open(filename) as f:
+ for line in f.readlines():
+ line = line.strip()
+ if not line:
+ continue
+ fields = line.split(":")
+ tagname = fields[0].strip().lower()
+ if tagname == "processor":
+ self.nr_cpus += 1
+ continue
+ if is_s390() and tagname == "cpu number":
+ self.nr_cpus += 1
+ continue
+ if tagname == "core id":
+ continue
+ self.tags[tagname] = fields[1].strip()
+ if tagname == "physical id":
+ socket_id = self.tags[tagname]
+ if socket_id not in self.sockets:
+ self.sockets.append(socket_id)
self.nr_sockets = self.sockets and len(self.sockets) or \
(self.nr_cpus /
("siblings" in self.tags and int(self.tags["siblings"]) or 1))
@@ -870,7 +864,8 @@ class smaps_lib:
Representation of an mmap in place for a process. Can be used to figure
out which processes have an library mapped, etc.
- The 'perm' member can be used to figure out executable mmaps, i.e. libraries.
+ The 'perm' member can be used to figure out executable mmaps,
+ i.e. libraries.
The 'vm_start' and 'vm_end' in turn can be used when trying to resolve
processor instruction pointer addresses to a symbol name in a library.
@@ -967,13 +962,12 @@ class smaps:
return self.entries[index]
def reload(self):
- f = open("/proc/%d/smaps" % self.pid)
line = None
- while True:
- line = self.parse_entry(f, line)
- if not line:
- break
- f.close()
+ with("/proc/%d/smaps" % self.pid) as f:
+ while True:
+ line = self.parse_entry(f, line)
+ if not line:
+ break
self.nr_entries = len(self.entries)
def find_by_name_fragment(self, fragment):
@@ -1055,18 +1049,17 @@ class cpusstats:
def reload(self):
last_entries = self.entries
self.entries = {}
- f = open(self.filename)
- for line in f.readlines():
- fields = line.strip().split()
- if fields[0][:3].lower() != "cpu":
- continue
- c = cpustat(fields)
- if c.name == "cpu":
- idx = 0
- else:
- idx = int(c.name[3:]) + 1
- self.entries[idx] = c
- f.close()
+ with open(self.filename) as f:
+ for line in f.readlines():
+ fields = line.strip().split()
+ if fields[0][:3].lower() != "cpu":
+ continue
+ c = cpustat(fields)
+ if c.name == "cpu":
+ idx = 0
+ else:
+ idx = int(c.name[3:]) + 1
+ self.entries[idx] = c
last_time = self.time
self.time = time.time()
if last_entries:
@@ -1082,8 +1075,7 @@ class cpusstats:
(curr.nice - prev.nice) + \
(curr.system - prev.system)
curr.usage = (delta / interval_hz) * 100
- if curr.usage > 100:
- curr.usage = 100
+ curr.usage = min(curr.usage, 100)
if __name__ == '__main__':
--
2.31.1
2 years, 4 months
- 1
- 0
[PATCH] python-linux-procfs: Remove procfs/sysctl.py
by John Kacur
python-linux-procfs makes no use of sysctl.py, nor does tuna
It appears to be code that was written to abstract some common
operations reading procfs, but was never actually put into use.
Just delete it
Signed-off-by: John Kacur <jkacur(a)redhat.com>
---
MANIFEST | 1 -
procfs/__init__.py | 1 -
procfs/sysctl.py | 68 ----------------------------------------------
3 files changed, 70 deletions(-)
delete mode 100755 procfs/sysctl.py
diff --git a/MANIFEST b/MANIFEST
index 5e86f6b3795b..ebbd4998d03a 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -4,7 +4,6 @@ Makefile
pflags-cmd.py
procfs/__init__.py
procfs/procfs.py
-procfs/sysctl.py
procfs/utilist.py
rpm/SPECS/python-linux-procfs.spec
setup.py
diff --git a/procfs/__init__.py b/procfs/__init__.py
index 86c34b46cf0c..071c54dacab8 100644
--- a/procfs/__init__.py
+++ b/procfs/__init__.py
@@ -26,5 +26,4 @@ __author__ = "Arnaldo Carvalho de Melo <acme(a)redhat.com>"
__license__ = "GPLv2 License"
from .procfs import *
-from .sysctl import *
from .utilist import *
diff --git a/procfs/sysctl.py b/procfs/sysctl.py
deleted file mode 100755
index 727b0bf60aad..000000000000
--- a/procfs/sysctl.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#! /usr/bin/python3
-# -*- python -*-
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2007 Red Hat, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-
-class sysctl:
- def __init__(self):
- self.cache = {}
-
- def __getitem__(self, key):
- if key not in self.cache:
- value = self.read(key)
- if value is None:
- return None
- self.cache[key] = value
-
- return self.cache[key]
-
- def __setitem__(self, key, value):
- oldvalue = self[key]
-
- if oldvalue is None:
- raise IOError
- elif oldvalue != value:
- self.write(key, value)
- self.cache[key] = value
-
- def keys(self):
- return list(self.cache.keys())
-
- def read(self, key):
- try:
- f = open("/proc/sys/%s" % key.replace(".", "/"))
- except:
- return None
- value = f.readline().strip()
- f.close()
- return value
-
- def write(self, key, value):
- try:
- f = open("/proc/sys/%s" % key.replace(".", "/"), "w")
- except:
- return
- f.write(value)
- f.close()
-
- def refresh(self):
- for key in list(self.cache.keys()):
- del self.cache[key]
- value = self.read(key)
- if value is not None:
- self.cache[key] = value
--
2.31.1
2 years, 4 months
- 1
- 0
- ← Newer
- 1
- Older →