From 34b0b8530f4f7b1082c2625bce68567ec1442ad5 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 14:49:22 +0000 Subject: [PATCH 01/10] util: read the kernel's CPU-list format sysfs lists sets of logical processors as "0-3,7": the online CPUs, the CPUs of a NUMA node, the CPUs sharing a cache, the SMT siblings of a core, the CPUs local to a PCI device. ParseCPUList reads that format once for all of them. An empty list, or "(null)" for a mask the kernel never set, is nil; anything malformed is an error rather than a guess. IDs of 65536 or more are an error too, so that a corrupt list, from a snapshot say, cannot make it allocate gigabytes. ReadCPUList reads such a file, and ReadInt a file holding one integer. Unlike SafeIntFromFile, ReadInt does not warn and tells a missing or malformed file from a value of -1, which the kernel itself uses for "unknown" in some topology files. Signed-off-by: Mikhail Malyshev --- pkg/util/util.go | 62 +++++++++++++++++++++++++++++++++++++++ pkg/util/util_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/pkg/util/util.go b/pkg/util/util.go index a9955b32..262b24b4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -83,3 +83,65 @@ func ParseBool(str string) (bool, error) { } } } + +// maxCPUID bounds the logical processor IDs ParseCPUList accepts, far above any +// kernel's CPU limit, so that a corrupt list, from a snapshot say, cannot make it +// allocate gigabytes. +const maxCPUID = 1 << 16 + +// ParseCPUList parses a list of logical processor IDs in the kernel's CPU-list +// format, such as "0-3,7", the format of sysfs files like +// /sys/devices/system/cpu/online. Surrounding whitespace is ignored, and an +// empty list or "(null)", which the kernel prints for a mask it never set, is +// nil. An ID of maxCPUID or more is an error. +func ParseCPUList(s string) ([]int, error) { + s = strings.TrimSpace(s) + if s == "" || s == "(null)" { + return nil, nil + } + var ids []int + for _, part := range strings.Split(s, ",") { + lo, hi, isRange := strings.Cut(part, "-") + first, err := strconv.ParseUint(lo, 10, 31) + if err != nil { + return nil, fmt.Errorf("invalid CPU list %q: %w", s, err) + } + last := first + if isRange { + if last, err = strconv.ParseUint(hi, 10, 31); err != nil { + return nil, fmt.Errorf("invalid CPU list %q: %w", s, err) + } + if last < first { + return nil, fmt.Errorf("invalid CPU list %q: range %s is reversed", s, part) + } + } + if last >= maxCPUID { + return nil, fmt.Errorf("invalid CPU list %q: CPU %d is out of range", s, last) + } + for id := first; id <= last; id++ { + ids = append(ids, int(id)) + } + } + return ids, nil +} + +// ReadCPUList reads a sysfs file holding a CPU list (see ParseCPUList). The +// error wraps os.ErrNotExist when the file does not exist. +func ReadCPUList(path string) ([]int, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseCPUList(string(data)) +} + +// ReadInt reads a sysfs file holding one integer. Unlike SafeIntFromFile it +// tells a missing or malformed file (an error, wrapping os.ErrNotExist when the +// file does not exist) from a value of -1, and it does not warn. +func ReadInt(path string) (int, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(string(data))) +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 20258e58..34bbceba 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -7,6 +7,10 @@ package util_test import ( + "errors" + "os" + "path/filepath" + "reflect" "testing" "github.com/zededa/ghw/pkg/util" @@ -109,3 +113,66 @@ func TestParseBool(t *testing.T) { }) } } + +func TestParseCPUList(t *testing.T) { + testCases := []struct { + in string + want []int + wantErr bool + }{ + {in: "0-3,7\n", want: []int{0, 1, 2, 3, 7}}, + {in: "5", want: []int{5}}, + {in: "0,2-3", want: []int{0, 2, 3}}, + {in: "", want: nil}, + {in: " \n", want: nil}, + {in: "(null)\n", want: nil}, + {in: "a", wantErr: true}, + {in: "3-1", wantErr: true}, + {in: "1-", wantErr: true}, + {in: "-1", wantErr: true}, + {in: "0,,1", wantErr: true}, + {in: "65535", want: []int{65535}}, + {in: "65536", wantErr: true}, + {in: "0-2147483647", wantErr: true}, + } + for _, tc := range testCases { + got, err := util.ParseCPUList(tc.in) + if (err != nil) != tc.wantErr { + t.Errorf("ParseCPUList(%q) error = %v, want error %v", tc.in, err, tc.wantErr) + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ParseCPUList(%q) = %v, want %v", tc.in, got, tc.want) + } + } +} + +func TestReadCPUListAndInt(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path + } + if got, err := util.ReadCPUList(write("list", "0-2,8\n")); err != nil || !reflect.DeepEqual(got, []int{0, 1, 2, 8}) { + t.Errorf("ReadCPUList = %v, %v; want [0 1 2 8]", got, err) + } + if _, err := util.ReadCPUList(filepath.Join(dir, "missing")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("ReadCPUList(missing) error = %v, want os.ErrNotExist", err) + } + if _, err := util.ReadCPUList(write("bad", "x\n")); err == nil { + t.Error("ReadCPUList(malformed) = no error") + } + if got, err := util.ReadInt(write("int", " -1\n")); err != nil || got != -1 { + t.Errorf("ReadInt = %d, %v; want -1", got, err) + } + if _, err := util.ReadInt(filepath.Join(dir, "missing")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("ReadInt(missing) error = %v, want os.ErrNotExist", err) + } + if _, err := util.ReadInt(write("nan", "abc")); err == nil { + t.Error("ReadInt(abc) = no error") + } +} From 94b72cd0d956a8aa42ccb03b347b58c6b561f28e Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 14:51:35 +0000 Subject: [PATCH 02/10] cpu: tell apart cores that share a core_id across clusters Cores were grouped by the kernel's core_id within a package. That ID is unique within a package on x86, but an ARM device tree numbers cores per cluster, so core_id repeats: on a part with two 4-core clusters, CPUs 0 and 4 were reported as two threads of one core, and the package as four dual-threaded cores instead of eight single-threaded ones. Where one core_id covers more than one set of SMT siblings in a package, as thread_siblings_list reports them, every core of that package is now numbered by the lowest logical processor among its siblings; a CPU of such a package whose siblings cannot be read gets core ID -1 rather than a core_id another core now uses. A core ID of -1 is unknown, so CPUs with it are never grouped into one core: each is a core of its own, which also stops CPUs whose core_id and siblings both cannot be read from being reported as one multi-threaded core. Where core_id is unique, it is kept, so x86 reports the same IDs as before. CoresForNode identifies cores the same way, and also by package: a NUMA node holding two packages no longer merges their cores that share a core_id. It decides whether a package is clustered from all its online CPUs, as Info does, not from the node's alone: where clusters are separate NUMA nodes, a node's core_ids need not repeat, yet its cores must carry the same IDs as in Info. The tests build their sysfs trees with a new internal sysfsfixture package, which later tests reuse. Signed-off-by: Mikhail Malyshev --- README.md | 3 +- internal/sysfsfixture/sysfsfixture.go | 154 ++++++++++++++ pkg/cpu/cpu.go | 3 +- pkg/cpu/cpu_linux.go | 145 +++++++++---- pkg/cpu/topology_linux_test.go | 292 ++++++++++++++++++++++++++ 5 files changed, 557 insertions(+), 40 deletions(-) create mode 100644 internal/sysfsfixture/sysfsfixture.go create mode 100644 pkg/cpu/topology_linux_test.go diff --git a/README.md b/README.md index 2f04c6f4..50dfb60d 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,8 @@ A `ghw.ProcessorCore` has the following fields: * `ghw.ProcessorCore.ID` is the `uint32` identifier that the host gave this core. Note that this does *not* necessarily equate to a zero-based index of the core within a physical package. For example, the core IDs for an Intel Core - i7 are 0, 1, 2, 8, 9, and 10 + i7 are 0, 1, 2, 8, 9, and 10. On Linux it is -1 where the core cannot be told, + and each such core holds a single logical processor * `ghw.ProcessorCore.TotalHardwareThreads` is the number of hardware threads associated with the core * `ghw.ProcessorCore.LogicalProcessors` is an array of ints representing the diff --git a/internal/sysfsfixture/sysfsfixture.go b/internal/sysfsfixture/sysfsfixture.go new file mode 100644 index 00000000..8b060e43 --- /dev/null +++ b/internal/sysfsfixture/sysfsfixture.go @@ -0,0 +1,154 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +// Package sysfsfixture writes small fake sysfs and procfs trees describing a +// CPU topology, for tests that point ghw at them with a chroot. +package sysfsfixture + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + + "github.com/zededa/ghw/pkg/util" +) + +// CPU describes one logical processor. An empty string field means the +// corresponding file does not exist. +type CPU struct { + ID int + Package string // topology/physical_package_id + Core string // topology/core_id + Siblings string // topology/thread_siblings_list + // Offline CPUs are present but not online: cpuN/online is 0 and they have + // no topology, caches or frequencies, as on a real kernel. + Offline bool + Caches []Cache + BaseKHz string // cpufreq/base_frequency + MaxKHz string // cpufreq/cpuinfo_max_freq + Capacity string // cpu_capacity +} + +// Cache describes one cache/indexN directory of a CPU. +type Cache struct { + Index int + Level string + Type string + ID string + Size string + Shared string // shared_cpu_list +} + +// Tree describes the whole fixture. +type Tree struct { + CPUs []CPU + // Nodes maps a NUMA node to its cpulist. Nil means no node directory at + // all, as on a kernel without NUMA. + Nodes map[int]string + // PMUCore and PMUAtom are the cpus files of the Intel hybrid core PMUs; + // empty means the PMU is absent. + PMUCore string + PMUAtom string + // CorePMU creates /sys/devices/cpu, the core PMU of a homogeneous part. + CorePMU bool +} + +// Write writes the tree under a new temporary directory and returns it, to be +// used as the chroot. +func Write(t testing.TB, tree Tree) string { + t.Helper() + root := t.TempDir() + write := func(path, content string) { + t.Helper() + full := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + optional := func(path, content string) { + if content != "" { + write(path, content) + } + } + + cpuDir := "sys/devices/system/cpu" + var online, present []int + var cpuinfo strings.Builder + for _, c := range tree.CPUs { + dir := fmt.Sprintf("%s/cpu%d", cpuDir, c.ID) + present = append(present, c.ID) + if c.Offline { + write(dir+"/online", "0") + continue + } + online = append(online, c.ID) + write(dir+"/online", "1") + optional(dir+"/topology/physical_package_id", c.Package) + optional(dir+"/topology/core_id", c.Core) + optional(dir+"/topology/thread_siblings_list", c.Siblings) + optional(dir+"/cpufreq/base_frequency", c.BaseKHz) + optional(dir+"/cpufreq/cpuinfo_max_freq", c.MaxKHz) + optional(dir+"/cpu_capacity", c.Capacity) + for _, cache := range c.Caches { + idx := fmt.Sprintf("%s/cache/index%d", dir, cache.Index) + optional(idx+"/level", cache.Level) + optional(idx+"/type", cache.Type) + optional(idx+"/id", cache.ID) + optional(idx+"/size", cache.Size) + optional(idx+"/shared_cpu_list", cache.Shared) + } + fmt.Fprintf(&cpuinfo, "processor\t: %d\nvendor_id\t: GenuineTest\nmodel name\t: Test CPU\nflags\t\t: fpu\n\n", c.ID) + } + write(cpuDir+"/online", List(online)) + write(cpuDir+"/present", List(present)) + write(cpuDir+"/possible", List(present)) + write("proc/cpuinfo", cpuinfo.String()) + + if tree.Nodes != nil { + nodes := make([]int, 0, len(tree.Nodes)) + for n := range tree.Nodes { + nodes = append(nodes, n) + } + sort.Ints(nodes) + for _, n := range nodes { + nodeDir := fmt.Sprintf("sys/devices/system/node/node%d", n) + write(nodeDir+"/cpulist", tree.Nodes[n]) + // Like the kernel, link each of the node's CPUs into its directory. + cpus, err := util.ParseCPUList(tree.Nodes[n]) + if err != nil { + t.Fatal(err) + } + for _, c := range cpus { + link := filepath.Join(root, nodeDir, fmt.Sprintf("cpu%d", c)) + if err := os.Symlink(fmt.Sprintf("../../cpu/cpu%d", c), link); err != nil { + t.Fatal(err) + } + } + } + } + optional("sys/devices/cpu_core/cpus", tree.PMUCore) + optional("sys/devices/cpu_atom/cpus", tree.PMUAtom) + if tree.CorePMU { + write("sys/devices/cpu/type", "4") + } + return root +} + +// List formats logical processor IDs as a comma-separated CPU list. +func List(ids []int) string { + s := make([]string, len(ids)) + for i, id := range ids { + s[i] = strconv.Itoa(id) + } + return strings.Join(s, ",") +} diff --git a/pkg/cpu/cpu.go b/pkg/cpu/cpu.go index 48d4da84..43729cd2 100644 --- a/pkg/cpu/cpu.go +++ b/pkg/cpu/cpu.go @@ -20,7 +20,8 @@ type ProcessorCore struct { // ID is the `uint32` identifier that the host gave this core. Note that // this does *not* necessarily equate to a zero-based index of the core // within a physical package. For example, the core IDs for an Intel Core - // i7 are 0, 1, 2, 8, 9, and 10 + // i7 are 0, 1, 2, 8, 9, and 10. It is -1 where the core cannot be told; + // each such core holds a single logical processor. ID int `json:"id"` // TotalHardwareThreads is the number of hardware threads associated with // the core diff --git a/pkg/cpu/cpu_linux.go b/pkg/cpu/cpu_linux.go index 6c4e8d59..d7fc913f 100644 --- a/pkg/cpu/cpu_linux.go +++ b/pkg/cpu/cpu_linux.go @@ -43,21 +43,19 @@ func (i *Info) load(ctx context.Context) error { return nil } -func processorsGet(ctx context.Context) []*Processor { +// onlineLogicalProcessorIDs lists the online logical processors, in directory +// order. +func onlineLogicalProcessorIDs(ctx context.Context) []int { paths := linuxpath.New(ctx) - - lps := logicalProcessorsFromProcCPUInfo(ctx) - // keyed by processor ID (physical_package_id) - procs := map[int]*Processor{} - // /sys/devices/system/cpu pseudodir contains N number of pseudodirs with // information about the logical processors on the host. These logical // processor pseudodirs are of the pattern /sys/devices/system/cpu/cpu{N} fnames, err := os.ReadDir(paths.SysDevicesSystemCPU) if err != nil { log.Warn(ctx, "failed to read /sys/devices/system/cpu: %s", err) - return []*Processor{} + return nil } + var onlineLPs []int for _, fname := range fnames { matches := regexForCpulCore.FindStringSubmatch(fname.Name()) if len(matches) < 2 { @@ -79,7 +77,20 @@ func processorsGet(ctx context.Context) []*Processor { // Assume the CPU is online if the online state file doesn't exist // (as is the case with older snapshots) } - procID := processorIDFromLogicalProcessorID(ctx, lpID) + onlineLPs = append(onlineLPs, lpID) + } + return onlineLPs +} + +func processorsGet(ctx context.Context) []*Processor { + lps := logicalProcessorsFromProcCPUInfo(ctx) + // keyed by processor ID (physical_package_id) + procs := map[int]*Processor{} + + onlineLPs := onlineLogicalProcessorIDs(ctx) + procIDs, coreIDs := topologyOf(ctx, onlineLPs) + for _, lpID := range onlineLPs { + procID := procIDs[lpID] proc, found := procs[procID] if !found { proc = &Processor{ID: procID} @@ -138,8 +149,12 @@ func processorsGet(ctx context.Context) []*Processor { procs[procID] = proc } - coreID := coreIDFromLogicalProcessorID(ctx, lpID) - core := proc.CoreByID(coreID) + coreID := coreIDs[lpID] + var core *ProcessorCore + // An unknown core (-1) is not known to be shared: a core of its own. + if coreID != -1 { + core = proc.CoreByID(coreID) + } if core == nil { core = &ProcessorCore{ ID: coreID, @@ -200,38 +215,74 @@ func coreIDFromLogicalProcessorID(ctx context.Context, lpID int) int { return util.SafeIntFromFile(ctx, path) } +// topologyOf returns the physical package and the core ID of each of the given +// logical processors, the core ID unique within its package. The kernel's +// core_id is that on x86, but an ARM device tree numbers cores per cluster, so +// core_id repeats within a package for cores that are not SMT siblings. Where +// one core_id covers more than one set of siblings (thread_siblings_list) in a +// package, every core of that package is numbered by the lowest logical +// processor among its siblings instead, and a logical processor whose siblings +// cannot be read gets -1. +func topologyOf(ctx context.Context, lpIDs []int) (procIDs, coreIDs map[int]int) { + paths := linuxpath.New(ctx) + type key struct{ proc, core int } + procIDs = make(map[int]int, len(lpIDs)) + coreIDs = make(map[int]int, len(lpIDs)) + firstSibling := map[int]int{} + siblingSets := map[key]map[int]bool{} + for _, lpID := range lpIDs { + procIDs[lpID] = processorIDFromLogicalProcessorID(ctx, lpID) + coreIDs[lpID] = coreIDFromLogicalProcessorID(ctx, lpID) + siblings, err := util.ReadCPUList(filepath.Join(paths.SysDevicesSystemCPU, + fmt.Sprintf("cpu%d", lpID), "topology", "thread_siblings_list")) + if err != nil || len(siblings) == 0 { + continue + } + first := siblings[0] + for _, s := range siblings { + first = min(first, s) + } + firstSibling[lpID] = first + k := key{procIDs[lpID], coreIDs[lpID]} + if siblingSets[k] == nil { + siblingSets[k] = map[int]bool{} + } + siblingSets[k][first] = true + } + clustered := map[int]bool{} + for k, sets := range siblingSets { + if len(sets) > 1 { + clustered[k.proc] = true + } + } + for _, lpID := range lpIDs { + if !clustered[procIDs[lpID]] { + continue + } + if first, ok := firstSibling[lpID]; ok { + coreIDs[lpID] = first + } else { + coreIDs[lpID] = -1 + } + } + return procIDs, coreIDs +} + func CoresForNode(ctx context.Context, nodeID int) ([]*ProcessorCore, error) { // The /sys/devices/system/node/nodeX directory contains a subdirectory - // called 'cpuX' for each logical processor assigned to the node. Each of - // those subdirectories contains a topology subdirectory which has a - // core_id file that indicates the 0-based identifier of the physical core - // the logical processor (hardware thread) is on. + // called 'cpuX' for each logical processor assigned to the node. Cores are + // identified as for Info: see topologyOf. paths := linuxpath.New(ctx) path := filepath.Join( paths.SysDevicesSystemNode, fmt.Sprintf("node%d", nodeID), ) - cores := make([]*ProcessorCore, 0) - - findCoreByID := func(coreID int) *ProcessorCore { - for _, c := range cores { - if c.ID == coreID { - return c - } - } - - c := &ProcessorCore{ - ID: coreID, - LogicalProcessors: []int{}, - } - cores = append(cores, c) - return c - } files, err := os.ReadDir(path) if err != nil { return nil, err } + var lpIDs []int for _, file := range files { filename := file.Name() if !strings.HasPrefix(filename, "cpu") { @@ -246,7 +297,7 @@ func CoresForNode(ctx context.Context, nodeID int) ([]*ProcessorCore, error) { // Grab the logical processor ID by cutting the integer from the // /sys/devices/system/node/nodeX/cpuX filename cpuPath := filepath.Join(path, filename) - procID, err := strconv.Atoi(filename[3:]) + lpID, err := strconv.Atoi(filename[3:]) if err != nil { log.Warn(ctx, "failed to determine procID from %s. Expected integer after 3rd char.", @@ -263,13 +314,31 @@ func CoresForNode(ctx context.Context, nodeID int) ([]*ProcessorCore, error) { // Assume the CPU is online if the online state file doesn't exist // (as is the case with older snapshots) } - coreIDPath := filepath.Join(cpuPath, "topology", "core_id") - coreID := util.SafeIntFromFile(ctx, coreIDPath) - core := findCoreByID(coreID) - core.LogicalProcessors = append( - core.LogicalProcessors, - procID, - ) + lpIDs = append(lpIDs, lpID) + } + + // A node can hold more than one package, so a core is a package and a + // core ID. Whether a package is clustered depends on all its CPUs, not + // only the node's, or the node's cores would be numbered unlike Info's. + procIDs, coreIDs := topologyOf(ctx, onlineLogicalProcessorIDs(ctx)) + type key struct{ proc, core int } + byKey := map[key]*ProcessorCore{} + cores := make([]*ProcessorCore, 0) + for _, lpID := range lpIDs { + k := key{procIDs[lpID], coreIDs[lpID]} + core, ok := byKey[k] + if !ok { + core = &ProcessorCore{ + ID: k.core, + LogicalProcessors: []int{}, + } + // As in Info, an unknown core is a core of its own. + if k.core != -1 { + byKey[k] = core + } + cores = append(cores, core) + } + core.LogicalProcessors = append(core.LogicalProcessors, lpID) } for _, c := range cores { diff --git a/pkg/cpu/topology_linux_test.go b/pkg/cpu/topology_linux_test.go new file mode 100644 index 00000000..f899585a --- /dev/null +++ b/pkg/cpu/topology_linux_test.go @@ -0,0 +1,292 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package cpu_test + +import ( + "fmt" + "path/filepath" + "reflect" + "slices" + "strconv" + "testing" + + "github.com/zededa/ghw/internal/config" + "github.com/zededa/ghw/internal/sysfsfixture" + "github.com/zededa/ghw/pkg/cpu" + "github.com/zededa/ghw/pkg/snapshot" + "github.com/zededa/ghw/testdata" +) + +// coresOf returns each core's logical processors, keyed by core ID, for the +// processor with the given ID. +func coresOf(t *testing.T, info *cpu.Info, procID int) map[int][]int { + t.Helper() + for _, p := range info.Processors { + if p.ID != procID { + continue + } + cores := map[int][]int{} + for _, c := range p.Cores { + if _, dup := cores[c.ID]; dup { + t.Fatalf("processor %d has two cores with ID %d", procID, c.ID) + } + cores[c.ID] = c.LogicalProcessors + } + return cores + } + t.Fatalf("no processor %d in %v", procID, info.Processors) + return nil +} + +// An ARM device tree numbers cores per cluster, so core_id repeats within one +// package; the sibling lists tell the cores apart. +func TestCoresAcrossClusters(t *testing.T) { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + cpus = append(cpus, sysfsfixture.CPU{ + ID: id, + Package: "0", + Core: strconv.Itoa(id % 4), + Siblings: strconv.Itoa(id), + }) + } + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus}) + + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + want := map[int][]int{0: {0}, 1: {1}, 2: {2}, 3: {3}, 4: {4}, 5: {5}, 6: {6}, 7: {7}} + if got := coresOf(t, info, 0); !reflect.DeepEqual(got, want) { + t.Errorf("cores = %v, want %v", got, want) + } + if info.TotalCores != 8 { + t.Errorf("TotalCores = %d, want 8", info.TotalCores) + } +} + +// Where core_id is unique within the package, as on x86, it is kept. +func TestCoreIDsKeptOnX86(t *testing.T) { + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: []sysfsfixture.CPU{ + {ID: 0, Package: "0", Core: "0", Siblings: "0,2"}, + {ID: 1, Package: "0", Core: "1", Siblings: "1,3"}, + {ID: 2, Package: "0", Core: "0", Siblings: "0,2"}, + {ID: 3, Package: "0", Core: "1", Siblings: "1,3"}, + {ID: 4, Package: "1", Core: "0", Siblings: "4"}, + }}) + + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + if got, want := coresOf(t, info, 0), map[int][]int{0: {0, 2}, 1: {1, 3}}; !reflect.DeepEqual(got, want) { + t.Errorf("package 0 cores = %v, want %v", got, want) + } + if got, want := coresOf(t, info, 1), map[int][]int{0: {4}}; !reflect.DeepEqual(got, want) { + t.Errorf("package 1 cores = %v, want %v", got, want) + } +} + +// In a renumbered package, a CPU whose siblings cannot be read has an unknown +// core rather than its kernel core_id, which another core now uses. Unknown +// cores are not known to be shared, so each is a core of its own. +func TestClusteredPackageCPUsWithoutSiblings(t *testing.T) { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + cpus = append(cpus, sysfsfixture.CPU{ + ID: id, Package: "0", Core: strconv.Itoa(id % 4), Siblings: strconv.Itoa(id), + }) + } + cpus[5].Siblings, cpus[6].Siblings = "", "" + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus, Nodes: map[int]string{0: "0-7"}}) + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + cores, err := cpu.CoresForNode(config.ContextFromArgs(config.WithChroot(root)), 0) + if err != nil { + t.Fatal(err) + } + want := []string{"0:[0]", "1:[1]", "2:[2]", "3:[3]", "4:[4]", "-1:[5]", "-1:[6]", "7:[7]"} + if got := coreList(info.Processors[0].Cores); !reflect.DeepEqual(got, want) { + t.Errorf("Info cores = %v, want %v", got, want) + } + if got := coreList(cores); !reflect.DeepEqual(got, want) { + t.Errorf("CoresForNode = %v, want %v", got, want) + } +} + +// CPUs whose core_id and siblings cannot be read are not merged into one core +// either. +func TestCPUsWithoutCoreID(t *testing.T) { + cpus := []sysfsfixture.CPU{ + {ID: 0, Package: "0", Core: "0", Siblings: "0"}, + {ID: 1, Package: "0"}, + {ID: 2, Package: "0"}, + } + info, err := cpu.New(config.WithChroot(sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus}))) + if err != nil { + t.Fatal(err) + } + want := []string{"0:[0]", "-1:[1]", "-1:[2]"} + if got := coreList(info.Processors[0].Cores); !reflect.DeepEqual(got, want) { + t.Errorf("cores = %v, want %v", got, want) + } +} + +// coreList lists cores as "ID:[logical processors]", by first logical processor. +func coreList(cores []*cpu.ProcessorCore) []string { + sorted := slices.Clone(cores) + slices.SortFunc(sorted, func(a, b *cpu.ProcessorCore) int { + return a.LogicalProcessors[0] - b.LogicalProcessors[0] + }) + var out []string + for _, c := range sorted { + out = append(out, fmt.Sprintf("%d:%v", c.ID, c.LogicalProcessors)) + } + return out +} + +// Renumbering is per package: a package whose core_id is unique keeps it. +func TestOneOfTwoPackagesClustered(t *testing.T) { + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: []sysfsfixture.CPU{ + {ID: 0, Package: "0", Core: "0", Siblings: "0"}, + {ID: 1, Package: "0", Core: "1", Siblings: "1"}, + {ID: 2, Package: "0", Core: "0", Siblings: "2"}, + {ID: 3, Package: "0", Core: "1", Siblings: "3"}, + {ID: 4, Package: "1", Core: "0", Siblings: "4-5"}, + {ID: 5, Package: "1", Core: "0", Siblings: "4-5"}, + {ID: 6, Package: "1", Core: "1", Siblings: "6-7"}, + {ID: 7, Package: "1", Core: "1", Siblings: "6-7"}, + }}) + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + if got, want := coresOf(t, info, 0), map[int][]int{0: {0}, 1: {1}, 2: {2}, 3: {3}}; !reflect.DeepEqual(got, want) { + t.Errorf("package 0 cores = %v, want %v", got, want) + } + if got, want := coresOf(t, info, 1), map[int][]int{0: {4, 5}, 1: {6, 7}}; !reflect.DeepEqual(got, want) { + t.Errorf("package 1 cores = %v, want %v", got, want) + } +} + +// CoresForNode identifies cores as Info does: clusters are told apart. +func TestCoresForNodeAcrossClusters(t *testing.T) { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + cpus = append(cpus, sysfsfixture.CPU{ + ID: id, Package: "0", Core: strconv.Itoa(id % 4), Siblings: strconv.Itoa(id), + }) + } + ctx := config.ContextFromArgs(config.WithChroot(sysfsfixture.Write(t, + sysfsfixture.Tree{CPUs: cpus, Nodes: map[int]string{0: "0-7"}}))) + cores, err := cpu.CoresForNode(ctx, 0) + if err != nil { + t.Fatal(err) + } + if len(cores) != 8 { + t.Errorf("CoresForNode = %v, want 8 single-thread cores", cores) + } +} + +// When a package's clusters are separate NUMA nodes, CoresForNode numbers each +// node's cores as Info does: the package is clustered whichever subset of its +// CPUs is asked for. +func TestCoresForNodeClustersAsNodes(t *testing.T) { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + cpus = append(cpus, sysfsfixture.CPU{ + ID: id, Package: "0", Core: strconv.Itoa(id % 4), Siblings: strconv.Itoa(id), + }) + } + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus, Nodes: map[int]string{0: "0-3", 1: "4-7"}}) + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + infoCores := map[int][]int{} + for _, p := range info.Processors { + for _, c := range p.Cores { + infoCores[c.ID] = c.LogicalProcessors + } + } + ctx := config.ContextFromArgs(config.WithChroot(root)) + for _, node := range []int{0, 1} { + cores, err := cpu.CoresForNode(ctx, node) + if err != nil { + t.Fatal(err) + } + for _, c := range cores { + if !reflect.DeepEqual(infoCores[c.ID], c.LogicalProcessors) { + t.Errorf("node %d: core %d has %v, Info's core %d has %v", + node, c.ID, c.LogicalProcessors, c.ID, infoCores[c.ID]) + } + } + } +} + +// Two packages in one NUMA node reuse core IDs; their cores stay apart. +func TestCoresForNodeTwoPackages(t *testing.T) { + tree := sysfsfixture.Tree{ + CPUs: []sysfsfixture.CPU{ + {ID: 0, Package: "0", Core: "0", Siblings: "0"}, + {ID: 1, Package: "1", Core: "0", Siblings: "1"}, + }, + Nodes: map[int]string{0: "0-1"}, + } + ctx := config.ContextFromArgs(config.WithChroot(sysfsfixture.Write(t, tree))) + cores, err := cpu.CoresForNode(ctx, 0) + if err != nil { + t.Fatal(err) + } + if len(cores) != 2 { + t.Errorf("CoresForNode = %v, want 2 cores", cores) + } +} + +// On the i7-1270P the E-cores 8-11 share an L2 but are four cores of their own. +func TestHybridSnapshotCores(t *testing.T) { + info, err := cpu.New(config.WithChroot(unpackSnapshot(t, "linux-amd64-intel-i7-1270P.tar.gz"))) + if err != nil { + t.Fatal(err) + } + cores := map[int][]int{} + for _, p := range info.Processors { + for _, c := range p.Cores { + cores[c.ID] = c.LogicalProcessors + } + } + for _, lp := range []int{8, 9, 10, 11} { + found := false + for _, lps := range cores { + if reflect.DeepEqual(lps, []int{lp}) { + found = true + } + } + if !found { + t.Errorf("E-core CPU %d is not a core of its own: %v", lp, cores) + } + } + if info.TotalCores != 12 || info.TotalHardwareThreads != 16 { + t.Errorf("TotalCores, TotalHardwareThreads = %d, %d; want 12, 16", + info.TotalCores, info.TotalHardwareThreads) + } +} + +func unpackSnapshot(t *testing.T, name string) string { + t.Helper() + dir, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := snapshot.UnpackInto(filepath.Join(dir, name), root); err != nil { + t.Fatal(err) + } + return root +} From f7646e3021c2d3c55a2e1aa39136d6b9a6e0a26d Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 14:52:28 +0000 Subject: [PATCH 03/10] memory: report every cache instance with its id A cache's identity was used to merge the per-CPU views of it, but never reported, so a consumer could not tell which L2 or L3 a CPU uses, or name a cache the way the kernel and resctrl do. Cache gains ID: the kernel's identifier of the instance, unique within its level and type, or -1 where the kernel does not number it. CachesForNode fills it in. Caches lists every cache instance of the online CPUs, read from /sys/devices/system/cpu rather than per NUMA node, with all the CPUs sharing each one. Instances without a kernel id are told apart by the CPUs sharing them, so two such caches are never merged, and one with an id is never merged with one without. A CPU whose online file says 0 is left out even if its cache directory is still there, and a cache index whose level does not fit Cache.Level is skipped with a warning rather than wrapped around. Signed-off-by: Mikhail Malyshev --- README.md | 3 + pkg/memory/memory_cache.go | 4 + pkg/memory/memory_cache_linux.go | 130 +++++++++++++++++- pkg/memory/memory_cache_linux_test.go | 190 ++++++++++++++++++++++++++ 4 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 pkg/memory/memory_cache_linux_test.go diff --git a/README.md b/README.md index 50dfb60d..ec6072c0 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,9 @@ Each `ghw.MemoryCache` struct contains the following fields: cache can contain * `ghw.MemoryCache.LogicalProcessors` is an array of integers representing the logical processors that use the cache +* `ghw.MemoryCache.ID` (Linux only) is the kernel's identifier of this cache + instance, unique among the caches of the same level and type, or -1 where the + kernel does not number it (older kernels did not number device-tree caches) ```go package main diff --git a/pkg/memory/memory_cache.go b/pkg/memory/memory_cache.go index fcfeda5c..6805323d 100644 --- a/pkg/memory/memory_cache.go +++ b/pkg/memory/memory_cache.go @@ -122,6 +122,10 @@ type Cache struct { // Type indicates what type of memory is stored in the cache. Can be // instruction (executable bytecodes), data or both. Type CacheType `json:"type"` + // ID is the kernel's identifier of this cache instance, unique among the + // caches of the same level and type, or -1 where the kernel does not + // number it (older kernels did not number device-tree caches). + ID int `json:"id"` // SizeBytes indicates the size of the cache in bytes. SizeBytes uint64 `json:"size_bytes"` // The set of logical processors (hardware threads) that have access to diff --git a/pkg/memory/memory_cache_linux.go b/pkg/memory/memory_cache_linux.go index 8896fd5b..f10286e5 100644 --- a/pkg/memory/memory_cache_linux.go +++ b/pkg/memory/memory_cache_linux.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "math" "os" "path/filepath" "sort" @@ -18,6 +19,7 @@ import ( "github.com/zededa/ghw/internal/log" "github.com/zededa/ghw/pkg/linuxpath" "github.com/zededa/ghw/pkg/unitutil" + "github.com/zededa/ghw/pkg/util" ) func CachesForNode(ctx context.Context, nodeID int) ([]*Cache, error) { @@ -103,6 +105,7 @@ func CachesForNode(ctx context.Context, nodeID int) ([]*Cache, error) { cache = &Cache{ Level: uint8(level), Type: cacheType, + ID: cacheID, SizeBytes: uint64(size) * uint64(unitutil.KB), LogicalProcessors: make([]uint32, 0), } @@ -218,7 +221,12 @@ func memoryCacheType( log.Warn(ctx, "%s", err) return CacheTypeUnified } - switch string(cacheTypeContents[:len(cacheTypeContents)-1]) { + return cacheTypeFromSysfs(string(cacheTypeContents)) +} + +// cacheTypeFromSysfs maps the content of a cache's sysfs "type" file. +func cacheTypeFromSysfs(s string) CacheType { + switch strings.TrimSpace(s) { case "Data": return CacheTypeData case "Instruction": @@ -246,3 +254,123 @@ func memoryCacheSharedCPUMap( } return string(sharedCpuMap[:len(sharedCpuMap)-1]) } + +// Caches returns every cache instance of the host's online logical processors, +// read from /sys/devices/system/cpu/cpuN/cache: one entry per instance with +// all the logical processors sharing it, sorted by level, type and first +// logical processor. Instances are told apart by their kernel ID, or, where +// the kernel does not number them, by the set of logical processors sharing +// them. The error is only for a CPU directory that cannot be listed. +func Caches(ctx context.Context) ([]*Cache, error) { + paths := linuxpath.New(ctx) + cpuDirs, err := os.ReadDir(paths.SysDevicesSystemCPU) + if err != nil { + return nil, err + } + caches := map[string]*Cache{} + for _, cpuDir := range cpuDirs { + name := cpuDir.Name() + id, err := strconv.ParseUint(strings.TrimPrefix(name, "cpu"), 10, 31) + if err != nil || !strings.HasPrefix(name, "cpu") { + continue + } + lpID := int(id) + if online, err := util.ReadInt(filepath.Join(paths.SysDevicesSystemCPU, name, "online")); err == nil && online == 0 { + continue + } + cacheDir := filepath.Join(paths.SysDevicesSystemCPU, name, "cache") + indexes, err := os.ReadDir(cacheDir) + if err != nil { + continue + } + for _, index := range indexes { + if !strings.HasPrefix(index.Name(), "index") { + continue + } + dir := filepath.Join(cacheDir, index.Name()) + c, err := readCacheIndex(dir) + if err != nil { + log.Warn(ctx, "cache %s: %s", dir, err) + continue + } + key := cacheKey(c, dir, lpID) + cache, ok := caches[key] + if !ok { + cache = c + caches[key] = cache + } + cache.LogicalProcessors = append(cache.LogicalProcessors, uint32(id)) + } + } + + result := make([]*Cache, 0, len(caches)) + for _, c := range caches { + sort.Sort(SortByLogicalProcessorId(c.LogicalProcessors)) + result = append(result, c) + } + sort.Sort(SortByCacheLevelTypeFirstProcessor(result)) + return result, nil +} + +// cacheKey tells cache instances apart: by the kernel's ID, or, where the kernel +// does not number them, by the logical processors sharing the cache, read from +// dir. A cache whose sharing list is blank is logical processor lpID's own. +func cacheKey(c *Cache, dir string, lpID int) string { + if c.ID >= 0 { + return fmt.Sprintf("%d-%d-id%d", c.Level, c.Type, c.ID) + } + shared, _ := os.ReadFile(filepath.Join(dir, "shared_cpu_list")) + cpus := strings.TrimSpace(string(shared)) + if cpus == "" { + cpus = fmt.Sprintf("own%d", lpID) + } + return fmt.Sprintf("%d-%d-cpus%s", c.Level, c.Type, cpus) +} + +// readCacheIndex reads the cache a cache/indexN directory describes: its level, +// type, kernel ID (-1 when the kernel does not number it) and size. The error +// is for a level that cannot be read or does not fit a Cache. +func readCacheIndex(dir string) (*Cache, error) { + level, err := util.ReadInt(filepath.Join(dir, "level")) + if err != nil { + return nil, err + } + if level < 0 || level > math.MaxUint8 { + return nil, fmt.Errorf("cache level %d is out of range", level) + } + typeContents, _ := os.ReadFile(filepath.Join(dir, "type")) + id, err := util.ReadInt(filepath.Join(dir, "id")) + if err != nil { + id = -1 + } + return &Cache{ + Level: uint8(level), + Type: cacheTypeFromSysfs(string(typeContents)), + ID: id, + SizeBytes: readCacheSize(dir), + }, nil +} + +// readCacheSize reads a cache's size, such as "32K", in bytes; 0 when it is +// not reported. +func readCacheSize(dir string) uint64 { + data, err := os.ReadFile(filepath.Join(dir, "size")) + if err != nil { + return 0 + } + s := strings.TrimSpace(string(data)) + multiplier := uint64(1) + switch { + case strings.HasSuffix(s, "K"): + multiplier = uint64(unitutil.KB) + case strings.HasSuffix(s, "M"): + multiplier = uint64(unitutil.MB) + case strings.HasSuffix(s, "G"): + multiplier = uint64(unitutil.GB) + } + n, err := strconv.ParseUint(strings.TrimRight(s, "KMG"), 10, 64) + if err != nil { + return 0 + } + return n * multiplier +} diff --git a/pkg/memory/memory_cache_linux_test.go b/pkg/memory/memory_cache_linux_test.go new file mode 100644 index 00000000..621fca22 --- /dev/null +++ b/pkg/memory/memory_cache_linux_test.go @@ -0,0 +1,190 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory_test + +import ( + "os" + "path/filepath" + "reflect" + "strconv" + "testing" + + "github.com/zededa/ghw/internal/config" + "github.com/zededa/ghw/internal/sysfsfixture" + "github.com/zededa/ghw/pkg/memory" + "github.com/zededa/ghw/pkg/snapshot" + "github.com/zededa/ghw/testdata" +) + +// cachesFixture builds four CPUs with private L1 caches, an L2 per pair of +// CPUs and one L3. withID decides, per cache level, whether the kernel numbers +// that level's instances. +func cachesFixture(t *testing.T, withID func(level, cpu int) bool) string { + t.Helper() + id := func(level, cpu int, value int) string { + if withID(level, cpu) { + return strconv.Itoa(value) + } + return "" + } + var cpus []sysfsfixture.CPU + for c := 0; c < 4; c++ { + pair := "0-1" + if c >= 2 { + pair = "2-3" + } + cpus = append(cpus, sysfsfixture.CPU{ID: c, Caches: []sysfsfixture.Cache{ + {Index: 0, Level: "1", Type: "Data", ID: id(1, c, c), Size: "48K", Shared: strconv.Itoa(c)}, + {Index: 1, Level: "1", Type: "Instruction", ID: id(1, c, c), Size: "32K", Shared: strconv.Itoa(c)}, + {Index: 2, Level: "2", Type: "Unified", ID: id(2, c, c/2), Size: "2048K", Shared: pair}, + {Index: 3, Level: "3", Type: "Unified", ID: id(3, c, 0), Size: "30720K", Shared: "0-3"}, + }}) + } + return sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus}) +} + +type cacheSummary struct { + Level uint8 + Type memory.CacheType + ID int + Size uint64 + CPUs []uint32 +} + +func summarize(caches []*memory.Cache) []cacheSummary { + out := make([]cacheSummary, len(caches)) + for i, c := range caches { + out[i] = cacheSummary{c.Level, c.Type, c.ID, c.SizeBytes, c.LogicalProcessors} + } + return out +} + +func wantCaches(id func(level, value int) int) []cacheSummary { + var want []cacheSummary + for c := 0; c < 4; c++ { + want = append(want, cacheSummary{1, memory.CacheTypeInstruction, id(1, c), 32 * 1024, []uint32{uint32(c)}}) + } + for c := 0; c < 4; c++ { + want = append(want, cacheSummary{1, memory.CacheTypeData, id(1, c), 48 * 1024, []uint32{uint32(c)}}) + } + want = append(want, + cacheSummary{2, memory.CacheTypeUnified, id(2, 0), 2048 * 1024, []uint32{0, 1}}, + cacheSummary{2, memory.CacheTypeUnified, id(2, 1), 2048 * 1024, []uint32{2, 3}}, + cacheSummary{3, memory.CacheTypeUnified, id(3, 0), 30720 * 1024, []uint32{0, 1, 2, 3}}, + ) + return want +} + +func caches(t *testing.T, root string) []cacheSummary { + t.Helper() + got, err := memory.Caches(config.ContextFromArgs(config.WithChroot(root))) + if err != nil { + t.Fatal(err) + } + return summarize(got) +} + +func TestCaches(t *testing.T) { + root := cachesFixture(t, func(int, int) bool { return true }) + want := wantCaches(func(_, value int) int { return value }) + if got := caches(t, root); !reflect.DeepEqual(got, want) { + t.Errorf("Caches() =\n%v\nwant\n%v", got, want) + } +} + +// ARM device trees give caches no id: instances are told apart by the CPUs +// sharing them, and their ID is -1. +func TestCachesWithoutIDs(t *testing.T) { + root := cachesFixture(t, func(int, int) bool { return false }) + want := wantCaches(func(int, int) int { return -1 }) + if got := caches(t, root); !reflect.DeepEqual(got, want) { + t.Errorf("Caches() =\n%v\nwant\n%v", got, want) + } +} + +// One L2 numbered by the kernel and one not must stay two instances. +func TestCachesMixedIDs(t *testing.T) { + root := cachesFixture(t, func(level, cpu int) bool { return level != 2 || cpu < 2 }) + var l2 []cacheSummary + for _, c := range caches(t, root) { + if c.Level == 2 { + l2 = append(l2, c) + } + } + want := []cacheSummary{ + {2, memory.CacheTypeUnified, 0, 2048 * 1024, []uint32{0, 1}}, + {2, memory.CacheTypeUnified, -1, 2048 * 1024, []uint32{2, 3}}, + } + if !reflect.DeepEqual(l2, want) { + t.Errorf("L2 caches = %v, want %v", l2, want) + } +} + +// A cache index whose level does not fit a Cache is skipped, not wrapped around. +func TestCachesLevelOutOfRange(t *testing.T) { + root := sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: []sysfsfixture.CPU{{ID: 0, Caches: []sysfsfixture.Cache{ + {Index: 0, Level: "1", Type: "Data", ID: "0", Size: "48K", Shared: "0"}, + {Index: 2, Level: "258", Type: "Unified", ID: "0", Size: "2048K", Shared: "0"}, + }}}}) + got := caches(t, root) + if len(got) != 1 || got[0].Level != 1 { + t.Errorf("caches = %+v, want only the L1", got) + } +} + +// An offline CPU is not counted even if its cache directory is still there. +func TestCachesSkipOfflineCPU(t *testing.T) { + root := cachesFixture(t, func(int, int) bool { return true }) + if err := os.WriteFile(filepath.Join(root, "sys/devices/system/cpu/cpu3/online"), []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + for _, c := range caches(t, root) { + for _, lp := range c.CPUs { + if lp == 3 { + t.Errorf("cache %+v lists offline CPU 3", c) + } + } + } +} + +// A blank shared_cpu_list must not merge private caches without ids. +func TestCachesBlankSharedList(t *testing.T) { + var cpus []sysfsfixture.CPU + for c := 0; c < 2; c++ { + cpus = append(cpus, sysfsfixture.CPU{ID: c, Caches: []sysfsfixture.Cache{ + {Index: 2, Level: "2", Type: "Unified", Size: "1024K", Shared: " "}, + }}) + } + got := caches(t, sysfsfixture.Write(t, sysfsfixture.Tree{CPUs: cpus})) + if len(got) != 2 { + t.Errorf("Caches() = %v, want two private L2 caches", got) + } +} + +// On the i7-1270P each 4-core E-module shares one L2. +func TestHybridSnapshotCaches(t *testing.T) { + dir, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := snapshot.UnpackInto(filepath.Join(dir, "linux-amd64-intel-i7-1270P.tar.gz"), root); err != nil { + t.Fatal(err) + } + found := false + for _, c := range caches(t, root) { + if c.Level == 2 && reflect.DeepEqual(c.CPUs, []uint32{8, 9, 10, 11}) { + found = true + if c.ID != 4 { + t.Errorf("E-module L2 ID = %d, want 4", c.ID) + } + } + } + if !found { + t.Error("no L2 shared by CPUs 8-11") + } +} From 28eab44ae8de004e830eca8896a7902d0be7dd96 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 14:53:15 +0000 Subject: [PATCH 04/10] memory: read CachesForNode's caches the way Caches does CachesForNode read a cache's size from the cache index numbered by the cache's level rather than from the cache itself: index 1, the L1 instruction cache, for every level-1 cache. On an i7-1270P the P-cores' 48K L1d was reported as 32K and the E-cores' 32K L1d as 64K. It now reads each cache index with readCacheIndex and tells instances apart with cacheKey, as Caches does, so the two cannot drift apart: an instance without a kernel id is keyed by its shared_cpu_list rather than its shared_cpu_map, the same CPUs either way. Its own level, ID, type, size and CPU map readers are gone. That also fixes a size the kernel does not report, which came out as a wrapped-around SizeBytes and is now 0, and an index whose level cannot be read, which is now skipped with a warning rather than reported as a level-255 cache. It also keyed caches with a kernel id and caches without one in the same namespace, so a cache with id 3 and one whose CPU map reads "3" were merged into one. The two keys now differ. Signed-off-by: Mikhail Malyshev --- pkg/memory/memory_cache_linux.go | 157 +++----------------------- pkg/memory/memory_cache_linux_test.go | 53 +++++++++ 2 files changed, 67 insertions(+), 143 deletions(-) diff --git a/pkg/memory/memory_cache_linux.go b/pkg/memory/memory_cache_linux.go index f10286e5..d829c76f 100644 --- a/pkg/memory/memory_cache_linux.go +++ b/pkg/memory/memory_cache_linux.go @@ -77,39 +77,23 @@ func CachesForNode(ctx context.Context, nodeID int) ([]*Cache, error) { if !strings.HasPrefix(cacheDirFileName, "index") { continue } - cacheIndex, _ := strconv.Atoi(cacheDirFileName[5:]) - - // The cache information is repeated for each node, so here, we - // just ensure that we only have a one Cache object for each - // unique combination of level, type and processor map - level := memoryCacheLevel(ctx, paths, nodeID, lpID, cacheIndex) - cacheType := memoryCacheType(ctx, paths, nodeID, lpID, cacheIndex) - cacheID := memoryCacheID(ctx, paths, nodeID, lpID, cacheIndex) - - // Generate cache key for deduplication. - // Use cache ID if available (modern kernels), otherwise fall back - // to shared_cpu_map for older kernels. This ensures that caches - // shared across NUMA nodes (e.g., L3 in Sub-NUMA Clustering) are - // correctly merged into a single cache object. - var cacheKey string - if cacheID != -1 { - cacheKey = fmt.Sprintf("%d-%d-%d", level, cacheType, cacheID) - } else { - sharedCpuMap := memoryCacheSharedCPUMap(ctx, paths, nodeID, lpID, cacheIndex) - cacheKey = fmt.Sprintf("%d-%d-%s", level, cacheType, sharedCpuMap) + dir := filepath.Join(cachePath, cacheDirFileName) + c, err := readCacheIndex(dir) + if err != nil { + log.Warn(ctx, "cache %s: %s", dir, err) + continue } - cache, exists := caches[cacheKey] + // The cache information is repeated for each logical processor + // sharing the cache, so keep one Cache object per instance. Keying + // by the kernel's cache ID also merges caches shared across NUMA + // nodes (e.g., L3 in Sub-NUMA Clustering) into one. + key := cacheKey(c, dir, lpID) + cache, exists := caches[key] if !exists { - size := memoryCacheSize(ctx, paths, nodeID, lpID, level) - cache = &Cache{ - Level: uint8(level), - Type: cacheType, - ID: cacheID, - SizeBytes: uint64(size) * uint64(unitutil.KB), - LogicalProcessors: make([]uint32, 0), - } - caches[cacheKey] = cache + c.LogicalProcessors = make([]uint32, 0) + cache = c + caches[key] = cache } cache.LogicalProcessors = append( cache.LogicalProcessors, @@ -130,100 +114,6 @@ func CachesForNode(ctx context.Context, nodeID int) ([]*Cache, error) { return cacheVals, nil } -func memoryCacheLevel( - ctx context.Context, - paths *linuxpath.Paths, - nodeID int, - lpID int, - cacheIndex int, -) int { - levelPath := filepath.Join( - paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), - "level", - ) - levelContents, err := os.ReadFile(levelPath) - if err != nil { - log.Warn(ctx, "%s", err) - return -1 - } - // levelContents is now a []byte with the last byte being a newline - // character. Trim that off and convert the contents to an integer. - level, err := strconv.Atoi(string(levelContents[:len(levelContents)-1])) - if err != nil { - log.Warn(ctx, "Unable to parse int from %s", levelContents) - return -1 - } - return level -} - -func memoryCacheID( - ctx context.Context, - paths *linuxpath.Paths, - nodeID int, - lpID int, - cacheIndex int, -) int { - idPath := filepath.Join( - paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), - "id", - ) - idContents, err := os.ReadFile(idPath) - if err != nil { - // The id file may not exist in early kernel versions, so we don't warn - return -1 - } - id, err := strconv.Atoi(string(idContents[:len(idContents)-1])) - if err != nil { - log.Warn(ctx, "Unable to parse int from %s", idContents) - return -1 - } - return id -} - -func memoryCacheSize( - ctx context.Context, - paths *linuxpath.Paths, - nodeID int, - lpID int, - cacheIndex int, -) int { - sizePath := filepath.Join( - paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), - "size", - ) - sizeContents, err := os.ReadFile(sizePath) - if err != nil { - log.Warn(ctx, "%s", err) - return -1 - } - // size comes as XK\n, so we trim off the K and the newline. - size, err := strconv.Atoi(string(sizeContents[:len(sizeContents)-2])) - if err != nil { - log.Warn(ctx, "Unable to parse int from %s", sizeContents) - return -1 - } - return size -} - -func memoryCacheType( - ctx context.Context, - paths *linuxpath.Paths, - nodeID int, - lpID int, - cacheIndex int, -) CacheType { - typePath := filepath.Join( - paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), - "type", - ) - cacheTypeContents, err := os.ReadFile(typePath) - if err != nil { - log.Warn(ctx, "%s", err) - return CacheTypeUnified - } - return cacheTypeFromSysfs(string(cacheTypeContents)) -} - // cacheTypeFromSysfs maps the content of a cache's sysfs "type" file. func cacheTypeFromSysfs(s string) CacheType { switch strings.TrimSpace(s) { @@ -236,25 +126,6 @@ func cacheTypeFromSysfs(s string) CacheType { } } -func memoryCacheSharedCPUMap( - ctx context.Context, - paths *linuxpath.Paths, - nodeID int, - lpID int, - cacheIndex int, -) string { - scpuPath := filepath.Join( - paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), - "shared_cpu_map", - ) - sharedCpuMap, err := os.ReadFile(scpuPath) - if err != nil { - log.Warn(ctx, "%s", err) - return "" - } - return string(sharedCpuMap[:len(sharedCpuMap)-1]) -} - // Caches returns every cache instance of the host's online logical processors, // read from /sys/devices/system/cpu/cpuN/cache: one entry per instance with // all the logical processors sharing it, sorted by level, type and first diff --git a/pkg/memory/memory_cache_linux_test.go b/pkg/memory/memory_cache_linux_test.go index 621fca22..f46e8848 100644 --- a/pkg/memory/memory_cache_linux_test.go +++ b/pkg/memory/memory_cache_linux_test.go @@ -188,3 +188,56 @@ func TestHybridSnapshotCaches(t *testing.T) { t.Error("no L2 shared by CPUs 8-11") } } + +func cachesForNode(t *testing.T, root string, node int) []cacheSummary { + t.Helper() + got, err := memory.CachesForNode(config.ContextFromArgs(config.WithChroot(root)), node) + if err != nil { + t.Fatal(err) + } + return summarize(got) +} + +// Each cache reports its own size: on the i7-1270P a P-core has a 48K L1d and +// a 32K L1i, an E-core a 32K L1d and a 64K L1i. +func TestCachesForNodeSizes(t *testing.T) { + dir, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := snapshot.UnpackInto(filepath.Join(dir, "linux-amd64-intel-i7-1270P.tar.gz"), root); err != nil { + t.Fatal(err) + } + want := map[[2]uint32]uint64{ // {CPU, type} -> size + {0, uint32(memory.CacheTypeData)}: 48 * 1024, + {0, uint32(memory.CacheTypeInstruction)}: 32 * 1024, + {8, uint32(memory.CacheTypeData)}: 32 * 1024, + {8, uint32(memory.CacheTypeInstruction)}: 64 * 1024, + } + for _, c := range cachesForNode(t, root, 0) { + if c.Level != 1 { + continue + } + for _, cpu := range c.CPUs { + if size, ok := want[[2]uint32{cpu, uint32(c.Type)}]; ok && c.Size != size { + t.Errorf("CPU %d L1 %v size = %d, want %d", cpu, c.Type, c.Size, size) + } + } + } +} + +// A cache with kernel id 3 and one without an id whose CPU list reads "3" are +// two caches. +func TestCachesForNodeIDAndCPUListApart(t *testing.T) { + root := sysfsfixture.Write(t, sysfsfixture.Tree{ + CPUs: []sysfsfixture.CPU{ + {ID: 0, Caches: []sysfsfixture.Cache{{Index: 2, Level: "2", Type: "Unified", ID: "3", Size: "1024K", Shared: "0"}}}, + {ID: 1, Caches: []sysfsfixture.Cache{{Index: 2, Level: "2", Type: "Unified", Size: "1024K", Shared: "3"}}}, + }, + Nodes: map[int]string{0: "0-1"}, + }) + if got := cachesForNode(t, root, 0); len(got) != 2 { + t.Errorf("CachesForNode = %v, want two L2 caches", got) + } +} From 6297a1e1937b712b1b879820ae028a0368f0397f Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Fri, 23 Oct 2026 22:38:18 +0000 Subject: [PATCH 05/10] cpu: report each logical processor's package, core, NUMA node and frequencies Info listed packages and the cores in them, but nothing about a logical processor on its own: not its NUMA node, its frequencies or the caches it uses, and nothing about CPUs that are present but offline. A consumer placing work on CPUs had to read sysfs again for all of that. Info gains LogicalProcessors, one entry per online logical processor with its package, core (the IDs of its Processor and ProcessorCore), NUMA node and base and maximum frequencies; OfflineLogicalProcessors; and Caches, every cache instance of the online CPUs (memory.Caches). An ID that cannot be read is -1, never a plausible 0: a CPU no NUMA node lists is on node -1, while on a kernel without NUMA every CPU is on node 0. A frequency the kernel does not report is 0. physical_package_id -1 from every CPU is the kernel saying it has no package information, which means one package: it is now reported as 0, for Processor.ID too, and clusters are looked for in that package, as on boards that report one. Next to CPUs with a real package, and for a package ID that cannot be read, it stays -1. Processors and their cores are otherwise unchanged, in content and order. Signed-off-by: Mikhail Malyshev # Conflicts: # pkg/cpu/cpu_linux.go --- README.md | 25 +++ pkg/cpu/cpu.go | 33 ++++ pkg/cpu/cpu_linux.go | 144 ++++++++++++---- pkg/cpu/topology_linux_test.go | 298 +++++++++++++++++++++++++++++++++ 4 files changed, 467 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index ec6072c0..c9adb222 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,15 @@ information about the CPUs on the host system. the host system contains * `ghw.CPUInfo.Processors` is an array of `ghw.Processor` structs, one for each physical processor package contained in the host +* `ghw.CPUInfo.LogicalProcessors` (Linux only) is an array of pointers to + `cpu.LogicalProcessor` structs, one for each online logical processor, + ascending by ID +* `ghw.CPUInfo.OfflineLogicalProcessors` (Linux only) is an array of the IDs of + the logical processors that are present but offline (for example with SMT + turned off), ascending +* `ghw.CPUInfo.Caches` (Linux only) is an array of pointers to + `ghw.MemoryCache` structs, one for each cache instance of the online logical + processors, with the logical processors sharing it Each `ghw.Processor` struct contains a number of fields: @@ -111,6 +120,22 @@ A `ghw.ProcessorCore` has the following fields: *zero-based* index of the processor on the host and are *not* related to the core ID. +A `cpu.LogicalProcessor` (Linux only) describes one online logical processor +(hardware thread) and where it sits in the host's topology. An ID that could +not be read is -1, and a frequency the kernel does not report is 0: + +* `cpu.LogicalProcessor.ID` is the logical processor ID the kernel and CPU + affinities use +* `cpu.LogicalProcessor.ProcessorID` is the physical package (socket) it + belongs to: the ID of its `ghw.Processor` +* `cpu.LogicalProcessor.CoreID` is the physical core it runs on: the ID of its + `ghw.ProcessorCore`, unique within the package. Logical processors with the + same known `ProcessorID` and `CoreID` are SMT siblings +* `cpu.LogicalProcessor.NodeID` is the NUMA node it belongs to; 0 on a kernel + without NUMA +* `cpu.LogicalProcessor.BaseFrequencyKHz` is its base (non-turbo) frequency +* `cpu.LogicalProcessor.MaxFrequencyKHz` is its maximum (turbo) frequency + ```go package main diff --git a/pkg/cpu/cpu.go b/pkg/cpu/cpu.go index 43729cd2..dad40ef9 100644 --- a/pkg/cpu/cpu.go +++ b/pkg/cpu/cpu.go @@ -11,6 +11,7 @@ import ( "github.com/zededa/ghw/internal/config" "github.com/zededa/ghw/pkg/marshal" + "github.com/zededa/ghw/pkg/memory" ) // ProcessorCore describes a physical host processor core. A processor core is @@ -135,6 +136,38 @@ type Info struct { // Processors is a slice of Processor struct pointers, one for each // physical processor package contained in the host Processors []*Processor `json:"processors"` + // LogicalProcessors describes each online logical processor, ascending by + // ID. + LogicalProcessors []*LogicalProcessor `json:"logical_processors,omitempty"` + // OfflineLogicalProcessors are the IDs of the logical processors that are + // present but offline (for example SMT turned off), ascending. + OfflineLogicalProcessors []int `json:"offline_logical_processors,omitempty"` + // Caches lists every cache instance of the online logical processors, + // with the logical processors sharing it. + Caches []*memory.Cache `json:"caches,omitempty"` +} + +// LogicalProcessor describes one online logical processor (hardware thread) +// and where it sits in the host's topology. An ID that could not be read is -1, +// and a frequency the kernel does not report is 0. +type LogicalProcessor struct { + // ID is the logical processor ID the kernel and CPU affinities use. + ID int `json:"id"` + // ProcessorID is the physical package (socket) it belongs to, the ID of + // its Processor in Processors (unless /proc/cpuinfo does not describe the + // logical processor, which leaves it out of Processors). + ProcessorID int `json:"processor_id"` + // CoreID is the physical core it runs on, the ID of its ProcessorCore, + // unique within the package. Logical processors with the same known + // ProcessorID and CoreID are SMT siblings; two with an ID of -1 are not + // known to be. + CoreID int `json:"core_id"` + // NodeID is the NUMA node it belongs to; 0 on a kernel without NUMA. + NodeID int `json:"node_id"` + // BaseFrequencyKHz is its base (non-turbo) frequency. + BaseFrequencyKHz uint64 `json:"base_frequency_khz,omitempty"` + // MaxFrequencyKHz is its maximum (turbo) frequency. + MaxFrequencyKHz uint64 `json:"max_frequency_khz,omitempty"` } // New returns a pointer to an Info struct that contains information about the diff --git a/pkg/cpu/cpu_linux.go b/pkg/cpu/cpu_linux.go index d7fc913f..2fda2fc9 100644 --- a/pkg/cpu/cpu_linux.go +++ b/pkg/cpu/cpu_linux.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -20,6 +21,7 @@ import ( "github.com/zededa/ghw/internal/log" "github.com/zededa/ghw/pkg/linuxdt" "github.com/zededa/ghw/pkg/linuxpath" + "github.com/zededa/ghw/pkg/memory" "github.com/zededa/ghw/pkg/util" ) @@ -29,7 +31,9 @@ var ( ) func (i *Info) load(ctx context.Context) error { - i.Processors = processorsGet(ctx) + online, offline := logicalProcessorIDs(ctx) + procIDs, coreIDs := topologyOf(ctx, online) + i.Processors = processorsGet(ctx, online, procIDs, coreIDs) var totCores uint32 var totThreads uint32 for _, p := range i.Processors { @@ -40,12 +44,21 @@ func (i *Info) load(ctx context.Context) error { i.TotalHardwareThreads = totThreads // TODO(jaypipes): Remove TotalThreads before v1.0 i.TotalThreads = totThreads + ascending := slices.Clone(online) + slices.Sort(ascending) + i.LogicalProcessors = logicalProcessorsGet(ctx, ascending, procIDs, coreIDs) + i.OfflineLogicalProcessors = offline + caches, err := memory.Caches(ctx) + if err != nil { + log.Warn(ctx, "failed to read CPU caches: %s", err) + } + i.Caches = caches return nil } -// onlineLogicalProcessorIDs lists the online logical processors, in directory -// order. -func onlineLogicalProcessorIDs(ctx context.Context) []int { +// logicalProcessorIDs returns the IDs of the online logical processors, in the +// order /sys/devices/system/cpu lists them, and of the offline ones, ascending. +func logicalProcessorIDs(ctx context.Context) (online, offline []int) { paths := linuxpath.New(ctx) // /sys/devices/system/cpu pseudodir contains N number of pseudodirs with // information about the logical processors on the host. These logical @@ -53,9 +66,8 @@ func onlineLogicalProcessorIDs(ctx context.Context) []int { fnames, err := os.ReadDir(paths.SysDevicesSystemCPU) if err != nil { log.Warn(ctx, "failed to read /sys/devices/system/cpu: %s", err) - return nil + return nil, nil } - var onlineLPs []int for _, fname := range fnames { matches := regexForCpulCore.FindStringSubmatch(fname.Name()) if len(matches) < 2 { @@ -71,24 +83,24 @@ func onlineLogicalProcessorIDs(ctx context.Context) []int { onlineFilePath := filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", lpID), onlineFile) if _, err := os.Stat(onlineFilePath); err == nil { if util.SafeIntFromFile(ctx, onlineFilePath) == 0 { + offline = append(offline, lpID) continue } } else if errors.Is(err, os.ErrNotExist) { // Assume the CPU is online if the online state file doesn't exist // (as is the case with older snapshots) } - onlineLPs = append(onlineLPs, lpID) + online = append(online, lpID) } - return onlineLPs + sort.Ints(offline) + return online, offline } -func processorsGet(ctx context.Context) []*Processor { +func processorsGet(ctx context.Context, onlineLPs []int, procIDs, coreIDs map[int]int) []*Processor { lps := logicalProcessorsFromProcCPUInfo(ctx) // keyed by processor ID (physical_package_id) procs := map[int]*Processor{} - onlineLPs := onlineLogicalProcessorIDs(ctx) - procIDs, coreIDs := topologyOf(ctx, onlineLPs) for _, lpID := range onlineLPs { procID := procIDs[lpID] proc, found := procs[procID] @@ -189,19 +201,6 @@ func processorsGet(ctx context.Context) []*Processor { return res } -// processorIDFromLogicalProcessorID returns the processor physical package ID -// for the supplied logical processor ID -func processorIDFromLogicalProcessorID(ctx context.Context, lpID int) int { - paths := linuxpath.New(ctx) - // Fetch CPU ID - path := filepath.Join( - paths.SysDevicesSystemCPU, - fmt.Sprintf("cpu%d", lpID), - "topology", "physical_package_id", - ) - return util.SafeIntFromFile(ctx, path) -} - // coreIDFromLogicalProcessorID returns the core ID for the supplied logical // processor ID func coreIDFromLogicalProcessorID(ctx context.Context, lpID int) int { @@ -229,25 +228,42 @@ func topologyOf(ctx context.Context, lpIDs []int) (procIDs, coreIDs map[int]int) procIDs = make(map[int]int, len(lpIDs)) coreIDs = make(map[int]int, len(lpIDs)) firstSibling := map[int]int{} - siblingSets := map[key]map[int]bool{} + allMinusOne := len(lpIDs) > 0 for _, lpID := range lpIDs { - procIDs[lpID] = processorIDFromLogicalProcessorID(ctx, lpID) + procIDs[lpID] = -1 + pkgPath := filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", lpID), + "topology", "physical_package_id") + if pkg, err := util.ReadInt(pkgPath); err != nil { + log.Warn(ctx, "failed to read the package of logical processor %d: %s", lpID, err) + allMinusOne = false + } else { + procIDs[lpID] = pkg + allMinusOne = allMinusOne && pkg == -1 + } coreIDs[lpID] = coreIDFromLogicalProcessorID(ctx, lpID) siblings, err := util.ReadCPUList(filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", lpID), "topology", "thread_siblings_list")) if err != nil || len(siblings) == 0 { continue } - first := siblings[0] - for _, s := range siblings { - first = min(first, s) + firstSibling[lpID] = slices.Min(siblings) + } + // physical_package_id -1 from every CPU is the kernel saying it has no + // package information: one package. Next to CPUs with a real package it + // is unknown. + if allMinusOne { + for _, lpID := range lpIDs { + procIDs[lpID] = 0 } - firstSibling[lpID] = first + } + // Clusters are found per package, so only once the package IDs are final. + siblingSets := map[key]map[int]struct{}{} + for lpID, first := range firstSibling { k := key{procIDs[lpID], coreIDs[lpID]} if siblingSets[k] == nil { - siblingSets[k] = map[int]bool{} + siblingSets[k] = map[int]struct{}{} } - siblingSets[k][first] = true + siblingSets[k][first] = struct{}{} } clustered := map[int]bool{} for k, sets := range siblingSets { @@ -320,7 +336,8 @@ func CoresForNode(ctx context.Context, nodeID int) ([]*ProcessorCore, error) { // A node can hold more than one package, so a core is a package and a // core ID. Whether a package is clustered depends on all its CPUs, not // only the node's, or the node's cores would be numbered unlike Info's. - procIDs, coreIDs := topologyOf(ctx, onlineLogicalProcessorIDs(ctx)) + online, _ := logicalProcessorIDs(ctx) + procIDs, coreIDs := topologyOf(ctx, online) type key struct{ proc, core int } byKey := map[key]*ProcessorCore{} cores := make([]*ProcessorCore, 0) @@ -488,3 +505,64 @@ func logicalProcessorsFromProcCPUInfo( } return lps } + +// logicalProcessorsGet describes each of the given online logical processors. +func logicalProcessorsGet(ctx context.Context, lpIDs []int, procIDs, coreIDs map[int]int) []*LogicalProcessor { + paths := linuxpath.New(ctx) + nodeOf := nodesOf(ctx) + lps := make([]*LogicalProcessor, 0, len(lpIDs)) + for _, id := range lpIDs { + dir := filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", id)) + lp := &LogicalProcessor{ + ID: id, + ProcessorID: procIDs[id], + CoreID: coreIDs[id], + NodeID: -1, + } + // A frequency the kernel does not report stays 0. + if khz, err := util.ReadInt(filepath.Join(dir, "cpufreq", "base_frequency")); err == nil && khz > 0 { + lp.BaseFrequencyKHz = uint64(khz) + } + if khz, err := util.ReadInt(filepath.Join(dir, "cpufreq", "cpuinfo_max_freq")); err == nil && khz > 0 { + lp.MaxFrequencyKHz = uint64(khz) + } + if nodeOf == nil { + lp.NodeID = 0 + } else if node, ok := nodeOf[id]; ok { + lp.NodeID = node + } + lps = append(lps, lp) + } + return lps +} + +// nodesOf maps each logical processor to its NUMA node, from the nodes' +// cpulist files. It is nil when the kernel has no NUMA support (no node +// directory); a logical processor no node lists is missing from the map. +func nodesOf(ctx context.Context) map[int]int { + paths := linuxpath.New(ctx) + entries, err := os.ReadDir(paths.SysDevicesSystemNode) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Warn(ctx, "failed to read %s: %s", paths.SysDevicesSystemNode, err) + return map[int]int{} + } + return nil + } + nodeOf := map[int]int{} + for _, entry := range entries { + node, err := strconv.Atoi(strings.TrimPrefix(entry.Name(), "node")) + if err != nil || !strings.HasPrefix(entry.Name(), "node") { + continue + } + cpus, err := util.ReadCPUList(filepath.Join(paths.SysDevicesSystemNode, entry.Name(), "cpulist")) + if err != nil { + log.Warn(ctx, "failed to read the CPUs of NUMA node %d: %s", node, err) + continue + } + for _, cpu := range cpus { + nodeOf[cpu] = node + } + } + return nodeOf +} diff --git a/pkg/cpu/topology_linux_test.go b/pkg/cpu/topology_linux_test.go index f899585a..28c800e8 100644 --- a/pkg/cpu/topology_linux_test.go +++ b/pkg/cpu/topology_linux_test.go @@ -7,11 +7,15 @@ package cpu_test import ( + "encoding/json" "fmt" + "os" "path/filepath" "reflect" "slices" + "sort" "strconv" + "strings" "testing" "github.com/zededa/ghw/internal/config" @@ -290,3 +294,297 @@ func unpackSnapshot(t *testing.T, name string) string { } return root } + +// twoSocketTree is 2 packages x 2 cores x 2 threads, one NUMA node per +// package, with frequencies on CPU 0 only. +func twoSocketTree() sysfsfixture.Tree { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + pkg, core := id/4, (id%4)/2 + first := pkg*4 + core*2 + c := sysfsfixture.CPU{ + ID: id, + Package: strconv.Itoa(pkg), + Core: strconv.Itoa(core), + Siblings: strconv.Itoa(first) + "-" + strconv.Itoa(first+1), + } + if id == 0 { + c.BaseKHz, c.MaxKHz = "2100000", "3500000" + } + cpus = append(cpus, c) + } + return sysfsfixture.Tree{CPUs: cpus, Nodes: map[int]string{0: "0-3", 1: "4-7"}} +} + +func logicalProcessors(t *testing.T, tree sysfsfixture.Tree) *cpu.Info { + t.Helper() + info, err := cpu.New(config.WithChroot(sysfsfixture.Write(t, tree))) + if err != nil { + t.Fatal(err) + } + return info +} + +func TestLogicalProcessors(t *testing.T) { + info := logicalProcessors(t, twoSocketTree()) + var want []*cpu.LogicalProcessor + for id := 0; id < 8; id++ { + lp := &cpu.LogicalProcessor{ID: id, ProcessorID: id / 4, CoreID: (id % 4) / 2, NodeID: id / 4} + if id == 0 { + lp.BaseFrequencyKHz, lp.MaxFrequencyKHz = 2100000, 3500000 + } + want = append(want, lp) + } + if !reflect.DeepEqual(info.LogicalProcessors, want) { + t.Errorf("LogicalProcessors =\n%v\nwant\n%v", info.LogicalProcessors, want) + } + if info.OfflineLogicalProcessors != nil { + t.Errorf("OfflineLogicalProcessors = %v, want none", info.OfflineLogicalProcessors) + } +} + +// A kernel without NUMA has no node directory: every CPU is on node 0. +func TestLogicalProcessorsNoNUMA(t *testing.T) { + tree := twoSocketTree() + tree.Nodes = nil + for _, lp := range logicalProcessors(t, tree).LogicalProcessors { + if lp.NodeID != 0 { + t.Errorf("CPU %d NodeID = %d, want 0", lp.ID, lp.NodeID) + } + } +} + +// A CPU no node lists is on an unknown node, not on node 0. +func TestLogicalProcessorsNUMAHole(t *testing.T) { + tree := twoSocketTree() + tree.Nodes = map[int]string{0: "0-3", 1: "4-6"} + for _, lp := range logicalProcessors(t, tree).LogicalProcessors { + want := lp.ID / 4 + if lp.ID == 7 { + want = -1 + } + if lp.NodeID != want { + t.Errorf("CPU %d NodeID = %d, want %d", lp.ID, lp.NodeID, want) + } + } +} + +// A CPU whose topology cannot be read has unknown package and core. +func TestLogicalProcessorsUnreadableTopology(t *testing.T) { + tree := twoSocketTree() + tree.CPUs[2].Package, tree.CPUs[2].Core, tree.CPUs[2].Siblings = "", "", "" + for _, lp := range logicalProcessors(t, tree).LogicalProcessors { + wantPkg, wantCore := lp.ID/4, (lp.ID%4)/2 + if lp.ID == 2 { + wantPkg, wantCore = -1, -1 + } + if lp.ProcessorID != wantPkg || lp.CoreID != wantCore { + t.Errorf("CPU %d = package %d core %d, want package %d core %d", + lp.ID, lp.ProcessorID, lp.CoreID, wantPkg, wantCore) + } + } +} + +// physical_package_id -1 is the kernel saying there is no package +// information: one package, 0. +func TestPackageIDMinusOne(t *testing.T) { + tree := twoSocketTree() + for i := range tree.CPUs { + tree.CPUs[i].Package = "-1" + } + tree.CPUs = tree.CPUs[:4] + info := logicalProcessors(t, tree) + for _, lp := range info.LogicalProcessors { + if lp.ProcessorID != 0 { + t.Errorf("CPU %d ProcessorID = %d, want 0", lp.ID, lp.ProcessorID) + } + } + if len(info.Processors) != 1 || info.Processors[0].ID != 0 { + t.Errorf("Processors = %v, want one with ID 0", info.Processors) + } +} + +// A device-tree board with no package information and core_id repeating per +// cluster is still renumbered: its one package, 0, is clustered. +func TestPackageIDMinusOneClusters(t *testing.T) { + var cpus []sysfsfixture.CPU + for id := 0; id < 8; id++ { + cpus = append(cpus, sysfsfixture.CPU{ + ID: id, Package: "-1", Core: strconv.Itoa(id % 4), Siblings: strconv.Itoa(id), + }) + } + info := logicalProcessors(t, sysfsfixture.Tree{CPUs: cpus}) + want := []string{"0:[0]", "1:[1]", "2:[2]", "3:[3]", "4:[4]", "5:[5]", "6:[6]", "7:[7]"} + if len(info.Processors) != 1 || info.Processors[0].ID != 0 { + t.Fatalf("Processors = %v, want one with ID 0", info.Processors) + } + if got := coreList(info.Processors[0].Cores); !reflect.DeepEqual(got, want) { + t.Errorf("cores = %v, want %v", got, want) + } +} + +func TestOfflineLogicalProcessors(t *testing.T) { + tree := twoSocketTree() + tree.CPUs = tree.CPUs[:4] + tree.CPUs[2].Offline, tree.CPUs[3].Offline = true, true + info := logicalProcessors(t, tree) + if want := []int{2, 3}; !reflect.DeepEqual(info.OfflineLogicalProcessors, want) { + t.Errorf("OfflineLogicalProcessors = %v, want %v", info.OfflineLogicalProcessors, want) + } + var ids []int + for _, lp := range info.LogicalProcessors { + ids = append(ids, lp.ID) + } + if want := []int{0, 1}; !reflect.DeepEqual(ids, want) { + t.Errorf("LogicalProcessors = %v, want %v", ids, want) + } +} + +func TestInfoCaches(t *testing.T) { + tree := twoSocketTree() + for i := range tree.CPUs { + tree.CPUs[i].Caches = []sysfsfixture.Cache{ + {Index: 3, Level: "3", Type: "Unified", ID: strconv.Itoa(i / 4), Size: "16384K"}, + } + } + info := logicalProcessors(t, tree) + if len(info.Caches) != 2 { + t.Fatalf("Caches = %v, want two L3 instances", info.Caches) + } + for p, c := range info.Caches { + want := []uint32{uint32(p * 4), uint32(p*4 + 1), uint32(p*4 + 2), uint32(p*4 + 3)} + if c.Level != 3 || c.ID != p || !reflect.DeepEqual(c.LogicalProcessors, want) { + t.Errorf("Caches[%d] = %+v, want L3 %d of %v", p, c, p, want) + } + } +} + +// The new fields come in addition to the existing ones, which keep their +// values. +func TestInfoJSONKeepsOldFields(t *testing.T) { + info := logicalProcessors(t, twoSocketTree()) + var got map[string]any + if err := json.Unmarshal([]byte(info.JSONString(false)), &got); err != nil { + t.Fatal(err) + } + fields, _ := got["cpu"].(map[string]any) + for key, want := range map[string]float64{"total_cores": 4, "total_hardware_threads": 8} { + if fields[key] != want { + t.Errorf("%s = %v, want %v", key, fields[key], want) + } + } + for _, key := range []string{"processors", "logical_processors"} { + if _, ok := fields[key]; !ok { + t.Errorf("JSON has no %q: %v", key, fields) + } + } +} + +// physical_package_id -1 means "no package information" only when every CPU +// says so; next to CPUs with a real package it is unknown. +func TestPackageIDMixedMinusOne(t *testing.T) { + tree := twoSocketTree() + tree.CPUs = tree.CPUs[:4] + tree.CPUs[0].Package, tree.CPUs[1].Package = "-1", "-1" + tree.CPUs[2].Package, tree.CPUs[3].Package = "1", "1" + for _, lp := range logicalProcessors(t, tree).LogicalProcessors { + want := 1 + if lp.ID < 2 { + want = -1 + } + if lp.ProcessorID != want { + t.Errorf("CPU %d ProcessorID = %d, want %d", lp.ID, lp.ProcessorID, want) + } + } +} + +// On the x86 snapshots, Processors and their Cores are what they were before +// this package learned about clusters: cores grouped by (package, core_id), +// in the order the CPU directories are listed. +func TestSnapshotProcessorsUnchanged(t *testing.T) { + dir, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatal(err) + } + names, err := filepath.Glob(filepath.Join(dir, "linux-amd64-*.tar.gz")) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + t.Run(filepath.Base(name), func(t *testing.T) { + root := unpackSnapshot(t, filepath.Base(name)) + cpuDir := filepath.Join(root, "sys", "devices", "system", "cpu") + want := legacyProcessors(t, cpuDir) + if want == nil { + t.Skip("no CPU topology in this snapshot") + } + info, err := cpu.New(config.WithChroot(root)) + if err != nil { + t.Fatal(err) + } + got := map[int][][2]any{} + for _, p := range info.Processors { + for _, c := range p.Cores { + got[p.ID] = append(got[p.ID], [2]any{c.ID, c.LogicalProcessors}) + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("processors =\n%v\nwant\n%v", got, want) + } + }) + } +} + +// legacyProcessors groups the online CPUs of a sysfs cpu directory by their +// raw (physical_package_id, core_id), cores in directory order, as ghw did +// before. +func legacyProcessors(t *testing.T, cpuDir string) map[int][][2]any { + t.Helper() + entries, err := os.ReadDir(cpuDir) + if err != nil { + return nil + } + type core struct { + id int + lps []int + } + cores := map[int][]*core{} + for _, e := range entries { + lp, err := strconv.Atoi(strings.TrimPrefix(e.Name(), "cpu")) + if err != nil || !strings.HasPrefix(e.Name(), "cpu") { + continue + } + if online, err := os.ReadFile(filepath.Join(cpuDir, e.Name(), "online")); err == nil && strings.TrimSpace(string(online)) == "0" { + continue + } + pkg, err1 := os.ReadFile(filepath.Join(cpuDir, e.Name(), "topology", "physical_package_id")) + id, err2 := os.ReadFile(filepath.Join(cpuDir, e.Name(), "topology", "core_id")) + if err1 != nil || err2 != nil { + return nil + } + p, _ := strconv.Atoi(strings.TrimSpace(string(pkg))) + c, _ := strconv.Atoi(strings.TrimSpace(string(id))) + var found *core + for _, existing := range cores[p] { + if existing.id == c { + found = existing + } + } + if found == nil { + found = &core{id: c} + cores[p] = append(cores[p], found) + } + found.lps = append(found.lps, lp) + } + if len(cores) == 0 { + return nil + } + out := map[int][][2]any{} + for p, list := range cores { + for _, c := range list { + sort.Ints(c.lps) + out[p] = append(out[p], [2]any{c.id, c.lps}) + } + } + return out +} From fcbd6e2a0ea1582557778092618321dbd619baf1 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 14:59:45 +0000 Subject: [PATCH 06/10] cpu: report each logical processor's core type Heterogeneous parts mix core classes: Intel P-cores and E-cores, ARM big.LITTLE. LogicalProcessor gains CoreType: performance, efficiency, uniform for a homogeneous part, or unknown. Intel hybrid parts list their cores under the cpu_core PMU (performance) and the cpu_atom and cpu_lowpower PMUs (efficiency). Elsewhere each CPU's cpu_capacity ranks it: the highest capacity is a performance core, any lower an efficiency core, and one capacity for all CPUs a homogeneous part. Before Linux 6.16 a homogeneous x86 part reports no cpu_capacity; its single core PMU, /sys/devices/cpu, tells it instead. When none of these tells, or only some CPUs report a capacity, the type is unknown rather than guessed. The ranking sees only online CPUs and ranks by capacity alone: on a part with three classes of core the middle one reads as efficiency. The doc comment says so. Signed-off-by: Mikhail Malyshev --- README.md | 9 ++++ internal/sysfsfixture/sysfsfixture.go | 10 ++-- pkg/cpu/cpu.go | 61 +++++++++++++++++++++ pkg/cpu/cpu_linux.go | 77 +++++++++++++++++++++++++++ pkg/cpu/topology_linux_test.go | 62 +++++++++++++++++++++ 5 files changed, 215 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c9adb222..2d1234a3 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,15 @@ not be read is -1, and a frequency the kernel does not report is 0: same known `ProcessorID` and `CoreID` are SMT siblings * `cpu.LogicalProcessor.NodeID` is the NUMA node it belongs to; 0 on a kernel without NUMA +* `cpu.LogicalProcessor.CoreType` is the class of core it runs on, a + `cpu.CoreType`: `cpu.CoreTypePerformance` or `cpu.CoreTypeEfficiency` on a + heterogeneous part (Intel P- and E-cores, ARM big.LITTLE), + `cpu.CoreTypeUniform` on a homogeneous one, or `cpu.CoreTypeUnknown` where the + host does not tell. Intel hybrid parts list their cores under the `cpu_core`, + `cpu_atom` and `cpu_lowpower` PMUs; elsewhere each CPU's `cpu_capacity` ranks + it, the highest capacity being a performance core. The ranking sees only + online CPUs and ranks by capacity alone, so on a part with three classes of + core the middle one reads as efficiency * `cpu.LogicalProcessor.BaseFrequencyKHz` is its base (non-turbo) frequency * `cpu.LogicalProcessor.MaxFrequencyKHz` is its maximum (turbo) frequency diff --git a/internal/sysfsfixture/sysfsfixture.go b/internal/sysfsfixture/sysfsfixture.go index 8b060e43..8fe4dd29 100644 --- a/internal/sysfsfixture/sysfsfixture.go +++ b/internal/sysfsfixture/sysfsfixture.go @@ -52,10 +52,11 @@ type Tree struct { // Nodes maps a NUMA node to its cpulist. Nil means no node directory at // all, as on a kernel without NUMA. Nodes map[int]string - // PMUCore and PMUAtom are the cpus files of the Intel hybrid core PMUs; - // empty means the PMU is absent. - PMUCore string - PMUAtom string + // PMUCore, PMUAtom and PMULowPower are the cpus files of the Intel hybrid + // core PMUs; empty means the PMU is absent. + PMUCore string + PMUAtom string + PMULowPower string // CorePMU creates /sys/devices/cpu, the core PMU of a homogeneous part. CorePMU bool } @@ -138,6 +139,7 @@ func Write(t testing.TB, tree Tree) string { } optional("sys/devices/cpu_core/cpus", tree.PMUCore) optional("sys/devices/cpu_atom/cpus", tree.PMUAtom) + optional("sys/devices/cpu_lowpower/cpus", tree.PMULowPower) if tree.CorePMU { write("sys/devices/cpu/type", "4") } diff --git a/pkg/cpu/cpu.go b/pkg/cpu/cpu.go index dad40ef9..916f6ed7 100644 --- a/pkg/cpu/cpu.go +++ b/pkg/cpu/cpu.go @@ -7,7 +7,10 @@ package cpu import ( + "encoding/json" "fmt" + "strconv" + "strings" "github.com/zededa/ghw/internal/config" "github.com/zededa/ghw/pkg/marshal" @@ -164,6 +167,8 @@ type LogicalProcessor struct { CoreID int `json:"core_id"` // NodeID is the NUMA node it belongs to; 0 on a kernel without NUMA. NodeID int `json:"node_id"` + // CoreType is the class of core it runs on. + CoreType CoreType `json:"core_type"` // BaseFrequencyKHz is its base (non-turbo) frequency. BaseFrequencyKHz uint64 `json:"base_frequency_khz,omitempty"` // MaxFrequencyKHz is its maximum (turbo) frequency. @@ -223,3 +228,59 @@ func (i *Info) YAMLString() string { func (i *Info) JSONString(indent bool) string { return marshal.SafeJSON(cpuPrinter{i}, indent) } + +// CoreType is the class of core a logical processor runs on. +type CoreType int + +const ( + // CoreTypeUnknown means the host does not tell the class of the core. + CoreTypeUnknown CoreType = iota + // CoreTypePerformance is a performance core of a heterogeneous part, such + // as an Intel P-core or an ARM "big" core. + CoreTypePerformance + // CoreTypeEfficiency is an efficiency core of a heterogeneous part, such + // as an Intel E-core or an ARM "LITTLE" core. + CoreTypeEfficiency + // CoreTypeUniform is a core of a homogeneous part, where every core is of + // the same class. + CoreTypeUniform +) + +var ( + coreTypeString = map[CoreType]string{ + CoreTypeUnknown: "unknown", + CoreTypePerformance: "performance", + CoreTypeEfficiency: "efficiency", + CoreTypeUniform: "uniform", + } + stringCoreType = map[string]CoreType{ + "unknown": CoreTypeUnknown, + "performance": CoreTypePerformance, + "efficiency": CoreTypeEfficiency, + "uniform": CoreTypeUniform, + } +) + +// String returns the core type's name, as it appears in JSON and YAML. +func (t CoreType) String() string { + return coreTypeString[t] +} + +// MarshalJSON writes the core type as its name. +func (t CoreType) MarshalJSON() ([]byte, error) { + return []byte(strconv.Quote(t.String())), nil +} + +// UnmarshalJSON reads a core type from its name. +func (t *CoreType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + val, ok := stringCoreType[strings.ToLower(s)] + if !ok { + return fmt.Errorf("unknown core type: %q", s) + } + *t = val + return nil +} diff --git a/pkg/cpu/cpu_linux.go b/pkg/cpu/cpu_linux.go index 2fda2fc9..57629d0e 100644 --- a/pkg/cpu/cpu_linux.go +++ b/pkg/cpu/cpu_linux.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "math" "os" "path/filepath" "regexp" @@ -510,6 +511,7 @@ func logicalProcessorsFromProcCPUInfo( func logicalProcessorsGet(ctx context.Context, lpIDs []int, procIDs, coreIDs map[int]int) []*LogicalProcessor { paths := linuxpath.New(ctx) nodeOf := nodesOf(ctx) + types := coreTypesOf(ctx, lpIDs) lps := make([]*LogicalProcessor, 0, len(lpIDs)) for _, id := range lpIDs { dir := filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", id)) @@ -518,6 +520,7 @@ func logicalProcessorsGet(ctx context.Context, lpIDs []int, procIDs, coreIDs map ProcessorID: procIDs[id], CoreID: coreIDs[id], NodeID: -1, + CoreType: types[id], } // A frequency the kernel does not report stays 0. if khz, err := util.ReadInt(filepath.Join(dir, "cpufreq", "base_frequency")); err == nil && khz > 0 { @@ -566,3 +569,77 @@ func nodesOf(ctx context.Context) map[int]int { } return nodeOf } + +// coreTypesOf tells each logical processor's core type. Intel hybrid parts list +// their performance cores under the cpu_core PMU and their efficiency cores +// under cpu_atom and, for the low-power ones, cpu_lowpower. Elsewhere, ARM +// big.LITTLE among them, cpu_capacity ranks the logical processors: the highest +// capacity is a performance core, any lower an efficiency core, and one +// capacity for all of them a homogeneous part (x86 reports 1024 for every CPU +// of a homogeneous part since Linux 6.16). Before that, a homogeneous x86 part +// is told by its single core PMU, /sys/devices/cpu. When none of these tells, +// or only some CPUs report a capacity, the type is unknown, never guessed. +// +// The ranking sees only the online CPUs: with every big core offline, the +// LITTLE ones read as performance or uniform. And it ranks by capacity alone, +// so on a part with three classes of core (prime, big, LITTLE) the middle one +// reads as efficiency. +func coreTypesOf(ctx context.Context, lpIDs []int) map[int]CoreType { + paths := linuxpath.New(ctx) + devices := filepath.Join(paths.SysRoot, "devices") + types := map[int]CoreType{} + + var hybrid bool + for pmu, coreType := range map[string]CoreType{ + "cpu_core": CoreTypePerformance, + "cpu_atom": CoreTypeEfficiency, + "cpu_lowpower": CoreTypeEfficiency, + } { + cpus, err := util.ReadCPUList(filepath.Join(devices, pmu, "cpus")) + if err != nil { + continue + } + hybrid = true + for _, id := range cpus { + types[id] = coreType + } + } + if hybrid { + return types + } + + capacities := map[int]int{} + for _, id := range lpIDs { + capacity, err := util.ReadInt(filepath.Join(paths.SysDevicesSystemCPU, fmt.Sprintf("cpu%d", id), "cpu_capacity")) + if err == nil { + capacities[id] = capacity + } + } + if len(capacities) > 0 && len(capacities) < len(lpIDs) { + return types + } + if len(capacities) > 0 { + lowest, highest := math.MaxInt, math.MinInt + for _, capacity := range capacities { + lowest, highest = min(lowest, capacity), max(highest, capacity) + } + for id, capacity := range capacities { + switch { + case lowest == highest: + types[id] = CoreTypeUniform + case capacity == highest: + types[id] = CoreTypePerformance + default: + types[id] = CoreTypeEfficiency + } + } + return types + } + + if fi, err := os.Stat(filepath.Join(devices, "cpu")); err == nil && fi.IsDir() { + for _, id := range lpIDs { + types[id] = CoreTypeUniform + } + } + return types +} diff --git a/pkg/cpu/topology_linux_test.go b/pkg/cpu/topology_linux_test.go index 28c800e8..7e7cf239 100644 --- a/pkg/cpu/topology_linux_test.go +++ b/pkg/cpu/topology_linux_test.go @@ -535,6 +535,57 @@ func TestSnapshotProcessorsUnchanged(t *testing.T) { } } +func coreTypes(t *testing.T, tree sysfsfixture.Tree) []cpu.CoreType { + t.Helper() + var types []cpu.CoreType + for _, lp := range logicalProcessors(t, tree).LogicalProcessors { + types = append(types, lp.CoreType) + } + return types +} + +func fourCPUs(capacities ...string) sysfsfixture.Tree { + tree := twoSocketTree() + tree.CPUs = tree.CPUs[:4] + for i, c := range capacities { + tree.CPUs[i].Capacity = c + } + return tree +} + +func TestCoreType(t *testing.T) { + P, E, U, X := cpu.CoreTypePerformance, cpu.CoreTypeEfficiency, cpu.CoreTypeUniform, cpu.CoreTypeUnknown + hybrid := fourCPUs() + hybrid.PMUCore, hybrid.PMUAtom = "0-1", "2" + pmuOnly := fourCPUs() + pmuOnly.CorePMU = true + partialPMU := fourCPUs("1024") + partialPMU.CorePMU = true + lowPower := fourCPUs() + lowPower.PMUCore, lowPower.PMULowPower = "0-1", "2-3" + for name, tc := range map[string]struct { + tree sysfsfixture.Tree + want []cpu.CoreType + }{ + // CPU 3 is in neither list: not guessed. + "Intel hybrid PMUs": {hybrid, []cpu.CoreType{P, P, E, X}}, + "Intel low-power": {lowPower, []cpu.CoreType{P, P, E, E}}, + "big.LITTLE capacity": {fourCPUs("1024", "1024", "446", "446"), []cpu.CoreType{P, P, E, E}}, + "uniform capacity": {fourCPUs("1024", "1024", "1024", "1024"), []cpu.CoreType{U, U, U, U}}, + "core PMU only": {pmuOnly, []cpu.CoreType{U, U, U, U}}, + "nothing to tell": {fourCPUs(), []cpu.CoreType{X, X, X, X}}, + "partial capacity": {fourCPUs("1024"), []cpu.CoreType{X, X, X, X}}, + // Some CPUs report a capacity: the core PMU does not make them uniform. + "partial capacity with core PMU": {partialPMU, []cpu.CoreType{X, X, X, X}}, + } { + t.Run(name, func(t *testing.T) { + if got := coreTypes(t, tc.tree); !reflect.DeepEqual(got, tc.want) { + t.Errorf("core types = %v, want %v", got, tc.want) + } + }) + } +} + // legacyProcessors groups the online CPUs of a sysfs cpu directory by their // raw (physical_package_id, core_id), cores in directory order, as ghw did // before. @@ -588,3 +639,14 @@ func legacyProcessors(t *testing.T, cpuDir string) map[int][][2]any { } return out } + +func TestCoreTypeJSON(t *testing.T) { + data, err := json.Marshal(cpu.CoreTypeEfficiency) + if err != nil || string(data) != `"efficiency"` { + t.Fatalf("json.Marshal(CoreTypeEfficiency) = %s, %v", data, err) + } + var back cpu.CoreType + if err := json.Unmarshal(data, &back); err != nil || back != cpu.CoreTypeEfficiency { + t.Errorf("json.Unmarshal(%s) = %v, %v", data, back, err) + } +} From 2c79e0912f96e470e76f007908cdb6873d00067d Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 15:00:53 +0000 Subject: [PATCH 07/10] pci: report the CPUs local to each device A device's DMA and interrupts are local to one NUMA node, and placing the work that drives it on that node's CPUs avoids crossing the interconnect. Device gains LocalCPUs, read from the device's local_cpulist: the CPUs of its NUMA node, or every CPU where the platform gives it no NUMA affinity. Unlike Node it does not depend on ghw having found a NUMA topology. Nil when the kernel does not report it; a list that cannot be read or parsed is also nil, with a warning. Signed-off-by: Mikhail Malyshev --- README.md | 4 ++ pkg/pci/local_cpus_linux_test.go | 72 ++++++++++++++++++++++++++++++++ pkg/pci/pci.go | 7 ++++ pkg/pci/pci_linux.go | 16 +++++++ pkg/pci/pci_linux_test.go | 28 +++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 pkg/pci/local_cpus_linux_test.go diff --git a/README.md b/README.md index 2d1234a3..3d0d0e82 100644 --- a/README.md +++ b/README.md @@ -738,6 +738,10 @@ The `ghw.PCIDevice` struct has the following fields: information is not available. If the information is not available, this does not mean the device is not functioning, but rather that `ghw` was not able to retrieve driver information. +* `ghw.PCIDevice.LocalCPUs` (Linux only) is an array of the logical processors + local to the device: those of the NUMA node its DMA and interrupts are local + to, or every CPU when the platform gives it no NUMA affinity. Nil when the + kernel does not report them. The `ghw.PCIAddress` (which is an alias for the `ghw.pci.address.Address` struct) contains the PCI address fields. It has a `ghw.PCIAddress.String()` diff --git a/pkg/pci/local_cpus_linux_test.go b/pkg/pci/local_cpus_linux_test.go new file mode 100644 index 00000000..3a15bb87 --- /dev/null +++ b/pkg/pci/local_cpus_linux_test.go @@ -0,0 +1,72 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/jaypipes/pcidb" + + "github.com/zededa/ghw/internal/config" + pciaddr "github.com/zededa/ghw/pkg/pci/address" +) + +func TestDeviceLocalCPUs(t *testing.T) { + root := t.TempDir() + devices := filepath.Join(root, "sys", "bus", "pci", "devices") + if err := os.MkdirAll(filepath.Join(devices, "0000:00:1f.0"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(devices, "0000:00:1f.0", "local_cpulist"), []byte("0-3\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(devices, "0000:00:1f.1"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(devices, "0000:00:1f.2"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(devices, "0000:00:1f.2", "local_cpulist"), []byte("0-x\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx := config.ContextFromArgs(config.WithChroot(root)) + + if got, want := getDeviceLocalCPUs(ctx, pciaddr.FromString("0000:00:1f.0")), []int{0, 1, 2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("local CPUs = %v, want %v", got, want) + } + if got := getDeviceLocalCPUs(ctx, pciaddr.FromString("0000:00:1f.1")); got != nil { + t.Errorf("local CPUs without local_cpulist = %v, want nil", got) + } + if got := getDeviceLocalCPUs(ctx, pciaddr.FromString("0000:00:1f.2")); got != nil { + t.Errorf("local CPUs from a malformed local_cpulist = %v, want nil", got) + } +} + +func TestDeviceLocalCPUsJSON(t *testing.T) { + dev := &Device{ + Address: "0000:00:1f.0", + Vendor: &pcidb.Vendor{}, + Product: &pcidb.Product{}, + Subsystem: &pcidb.Product{}, + Class: &pcidb.Class{}, + Subclass: &pcidb.Subclass{}, + ProgrammingInterface: &pcidb.ProgrammingInterface{}, + LocalCPUs: []int{0, 1}, + } + data, err := json.Marshal(dev) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"local_cpus":[0,1]`) { + t.Errorf("JSON %s has no local_cpus", data) + } +} diff --git a/pkg/pci/pci.go b/pkg/pci/pci.go index 28bdd682..a863e1a9 100644 --- a/pkg/pci/pci.go +++ b/pkg/pci/pci.go @@ -45,6 +45,11 @@ type Device struct { // Useful for callers that want to do custom vendor/device/class matching // beyond what the pcidb lookup provides. Modalias string `json:"modalias,omitempty"` + // LocalCPUs are the logical processors local to the device: those of the + // NUMA node its DMA and interrupts are local to, or every CPU when the + // platform gives it no NUMA affinity. Nil when the kernel does not report + // them. + LocalCPUs []int `json:"local_cpus,omitempty"` // Parent is the resolved parent Device pointer for this device (the // PCIe upstream port or root complex device). It is nil for root @@ -75,6 +80,7 @@ type devMarshallable struct { Interface devIdent `json:"programming_interface"` IOMMUGroup string `json:"iommu_group"` Modalias string `json:"modalias,omitempty"` + LocalCPUs []int `json:"local_cpus,omitempty"` } // NOTE(jaypipes) Device has a custom JSON marshaller because we don't want @@ -113,6 +119,7 @@ func (d *Device) MarshalJSON() ([]byte, error) { }, IOMMUGroup: d.IOMMUGroup, Modalias: d.Modalias, + LocalCPUs: d.LocalCPUs, } return json.Marshal(dm) } diff --git a/pkg/pci/pci_linux.go b/pkg/pci/pci_linux.go index 52052b06..e9487040 100644 --- a/pkg/pci/pci_linux.go +++ b/pkg/pci/pci_linux.go @@ -7,6 +7,7 @@ package pci import ( "context" + "errors" "os" "path/filepath" "regexp" @@ -95,6 +96,20 @@ func getDeviceIommuGroup(paths *linuxpath.Paths, pciAddr *pciaddr.Address) strin return filepath.Base(dest) } +// getDeviceLocalCPUs reads the logical processors local to the device; nil +// when the kernel does not report them. +func getDeviceLocalCPUs(ctx context.Context, pciAddr *pciaddr.Address) []int { + paths := linuxpath.New(ctx) + cpus, err := util.ReadCPUList(filepath.Join(paths.SysBusPciDevices, pciAddr.String(), "local_cpulist")) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Warn(ctx, "failed to read the local CPUs of device %q: %v", pciAddr, err) + } + return nil + } + return cpus +} + func getDeviceModalias(paths *linuxpath.Paths, pciAddr *pciaddr.Address) string { fp := getDeviceModaliasPath(paths, pciAddr) data, err := os.ReadFile(fp) @@ -412,6 +427,7 @@ func (info *Info) getDevices(ctx context.Context) []*Device { device.ParentAddress = getDeviceParentAddress(paths, pciAddr) device.IOMMUGroup = getDeviceIommuGroup(paths, pciAddr) device.Modalias = modalias + device.LocalCPUs = getDeviceLocalCPUs(ctx, pciAddr) devs = append(devs, device) } linkDeviceTree(devs) diff --git a/pkg/pci/pci_linux_test.go b/pkg/pci/pci_linux_test.go index 0327a8fa..ed12c156 100644 --- a/pkg/pci/pci_linux_test.go +++ b/pkg/pci/pci_linux_test.go @@ -12,6 +12,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "testing" "github.com/jaypipes/pcidb" @@ -298,6 +299,33 @@ func TestPCIMalformedModalias(t *testing.T) { } } +// The dual-socket Xeon's devices behind 0000:05:00.0 and .1 are local to one +// socket each; the others list every CPU. +func TestPCIDeviceLocalCPUsXeon(t *testing.T) { + info := pciTestSetupXeon(t) + var socket0, socket1 []int + for cpu := 0; cpu < 24; cpu += 2 { + socket0, socket1 = append(socket0, cpu), append(socket1, cpu+1) + } + all := make([]int, 24) + for cpu := range all { + all[cpu] = cpu + } + for address, want := range map[string][]int{ + "0000:05:00.0": socket0, + "0000:05:00.1": socket1, + "0000:00:1a.1": all, + } { + dev := info.GetDevice(address) + if dev == nil { + t.Fatalf("no device %s", address) + } + if !reflect.DeepEqual(dev.LocalCPUs, want) { + t.Errorf("%s LocalCPUs = %v, want %v", address, dev.LocalCPUs, want) + } + } +} + func pciTestSetupXeon(t *testing.T) *pci.Info { const snapshotFilename = "linux-amd64-intel-xeon-L5640.tar.gz" return pciTestSetup(t, snapshotFilename) From d446797d64abe7375c2ee230619c9c46f44fe47e Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Wed, 21 Oct 2026 15:02:09 +0000 Subject: [PATCH 08/10] snapshot: capture the files CPU topology reads A snapshot lacked files the CPU discovery now reads, so a snapshot taken on a hybrid or big.LITTLE host reproduced neither its offline CPUs, its frequencies nor its core types. Capture each CPU's online state, its cpufreq base and maximum frequencies and its cpu_capacity, the cpus files of the cpu_core, cpu_atom and cpu_lowpower PMUs, and the core PMU /sys/devices/cpu. The NUMA nodes' cpulist and the PCI devices' local_cpulist were already captured. The test snapshots three fixture hosts (Intel hybrid with an offline CPU, big.LITTLE, homogeneous x86) and checks that ghw reads the same CPU topology from each snapshot as from the host. Signed-off-by: Mikhail Malyshev --- pkg/snapshot/clonetree_linux.go | 8 ++++ pkg/snapshot/clonetree_linux_test.go | 64 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/pkg/snapshot/clonetree_linux.go b/pkg/snapshot/clonetree_linux.go index 88bff08f..59c8b35e 100644 --- a/pkg/snapshot/clonetree_linux.go +++ b/pkg/snapshot/clonetree_linux.go @@ -37,8 +37,16 @@ func ExpectedCloneStaticContent() []string { "/proc/cpuinfo", "/proc/meminfo", "/proc/self/mounts", + "/sys/devices/system/cpu/cpu*/online", "/sys/devices/system/cpu/cpu*/cache/index*/*", "/sys/devices/system/cpu/cpu*/topology/*", + "/sys/devices/system/cpu/cpu*/cpufreq/base_frequency", + "/sys/devices/system/cpu/cpu*/cpufreq/cpuinfo_max_freq", + "/sys/devices/system/cpu/cpu*/cpu_capacity", + "/sys/devices/cpu_core/cpus", + "/sys/devices/cpu_atom/cpus", + "/sys/devices/cpu_lowpower/cpus", + "/sys/devices/cpu/type", "/sys/devices/system/memory/block_size_bytes", "/sys/devices/system/memory/memory*/online", "/sys/devices/system/memory/memory*/state", diff --git a/pkg/snapshot/clonetree_linux_test.go b/pkg/snapshot/clonetree_linux_test.go index 3cc6e38d..72ed2b70 100644 --- a/pkg/snapshot/clonetree_linux_test.go +++ b/pkg/snapshot/clonetree_linux_test.go @@ -12,10 +12,13 @@ import ( "path/filepath" "reflect" "sort" + "strconv" "strings" "testing" "github.com/zededa/ghw/internal/config" + "github.com/zededa/ghw/internal/sysfsfixture" + ghwcpu "github.com/zededa/ghw/pkg/cpu" "github.com/zededa/ghw/pkg/linuxdt" "github.com/zededa/ghw/pkg/snapshot" "github.com/zededa/ghw/pkg/util" @@ -207,3 +210,64 @@ func TestCloneDeviceTree(t *testing.T) { t.Errorf("snapshot carries the serial number %q", serial) } } + +// A snapshot reproduces the CPU topology ghw reads from the host: online and +// offline CPUs, packages, cores, NUMA nodes, frequencies, caches and core types. +func TestCloneStaticContentCPUTopology(t *testing.T) { + cpu := func(id int, core string, siblings string) sysfsfixture.CPU { + return sysfsfixture.CPU{ + ID: id, Package: "0", Core: core, Siblings: siblings, + BaseKHz: "2100000", MaxKHz: "4800000", + Caches: []sysfsfixture.Cache{ + {Index: 0, Level: "1", Type: "Data", ID: core, Size: "48K", Shared: siblings}, + {Index: 2, Level: "2", Type: "Unified", ID: "0", Size: "2048K", Shared: "0-3"}, + }, + } + } + hybrid := sysfsfixture.Tree{ + CPUs: []sysfsfixture.CPU{ + cpu(0, "0", "0-1"), cpu(1, "0", "0-1"), cpu(2, "8", "2"), cpu(3, "9", "3"), + {ID: 4, Offline: true}, + }, + Nodes: map[int]string{0: "0-3"}, + PMUCore: "0-1", PMUAtom: "2", PMULowPower: "3", + } + bigLittle := sysfsfixture.Tree{Nodes: map[int]string{0: "0-3"}} + for id, capacity := range []string{"1024", "1024", "446", "446"} { + c := cpu(id, strconv.Itoa(id), strconv.Itoa(id)) + c.Capacity = capacity + bigLittle.CPUs = append(bigLittle.CPUs, c) + } + homogeneous := sysfsfixture.Tree{ + CPUs: []sysfsfixture.CPU{cpu(0, "0", "0-1"), cpu(1, "0", "0-1")}, + CorePMU: true, + } + for name, tree := range map[string]sysfsfixture.Tree{ + "Intel hybrid": hybrid, + "big.LITTLE": bigLittle, + "homogeneous x86": homogeneous, + } { + t.Run(name, func(t *testing.T) { + host := sysfsfixture.Write(t, tree) + var specs []string + for _, spec := range snapshot.ExpectedCloneStaticContent() { + specs = append(specs, filepath.Join(host, spec)) + } + clone := t.TempDir() + if err := snapshot.CopyFilesInto(context.Background(), specs, clone, nil); err != nil { + t.Fatal(err) + } + want, err := ghwcpu.New(config.WithChroot(host)) + if err != nil { + t.Fatal(err) + } + got, err := ghwcpu.New(config.WithChroot(filepath.Join(clone, host))) + if err != nil { + t.Fatal(err) + } + if got.JSONString(true) != want.JSONString(true) { + t.Errorf("from the snapshot:\n%s\nfrom the host:\n%s", got.JSONString(true), want.JSONString(true)) + } + }) + } +} From 6856e44b263908bf86195a33493627d94de07f33 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Tue, 27 Oct 2026 08:25:35 +0000 Subject: [PATCH 09/10] testdata: add a snapshot of an Intel hybrid with its PMU files No snapshot showed the files the CPU topology reads on a hybrid part: the i7-1270P one was taken before snapshots captured the cpu_core and cpu_atom PMUs and the cpufreq files, so core types and frequencies were tested on fixture trees only. linux-amd64-intel-i5-1335UE is a 13th-gen Core i5-1335UE: two P-cores with two threads each (CPUs 0-3, cpu_core) and eight E-cores (CPUs 4-11, cpu_atom), each module of four E-cores sharing an L2. It is a full capture, taken with this branch's ghwc from / in EVE's debug container (which shares the host's /sys, /dev and PID and network namespaces; only /proc/self/mounts is the container's), so other packages can test against it too, including its two NICs. Snapshots hold no DMI files or MAC addresses; the only identifying values left, the NVMe namespace's eui and wwid and an app instance UUID in /proc/self/mounts, are replaced by fake ones. ghwc reads the same cpu, memory, topology, block, pci and net information from it as from the unscrubbed capture, and the same CPU topology as from the machine. Turbo is off on that machine, so each CPU's maximum frequency equals its base frequency. TestHybridPMUSnapshot checks the cores, core types, frequencies and the L1d and L2 caches on it; the golden test of Processors covers it too. Signed-off-by: Mikhail Malyshev --- pkg/cpu/topology_linux_test.go | 47 ++++++++++++++++++ .../linux-amd64-intel-i5-1335UE.tar.gz | Bin 0 -> 39214 bytes 2 files changed, 47 insertions(+) create mode 100644 testdata/snapshots/linux-amd64-intel-i5-1335UE.tar.gz diff --git a/pkg/cpu/topology_linux_test.go b/pkg/cpu/topology_linux_test.go index 7e7cf239..4618f814 100644 --- a/pkg/cpu/topology_linux_test.go +++ b/pkg/cpu/topology_linux_test.go @@ -21,6 +21,7 @@ import ( "github.com/zededa/ghw/internal/config" "github.com/zededa/ghw/internal/sysfsfixture" "github.com/zededa/ghw/pkg/cpu" + "github.com/zededa/ghw/pkg/memory" "github.com/zededa/ghw/pkg/snapshot" "github.com/zededa/ghw/testdata" ) @@ -253,6 +254,52 @@ func TestCoresForNodeTwoPackages(t *testing.T) { } } +// A 13th-gen Core i5-1335UE: two P-cores with two threads each (CPUs 0-3, +// under the cpu_core PMU) and eight E-cores (CPUs 4-11, under cpu_atom), each +// module of four E-cores sharing an L2. Turbo is off on that machine, so each +// CPU's maximum frequency is its base frequency. +func TestHybridPMUSnapshot(t *testing.T) { + info, err := cpu.New(config.WithChroot(unpackSnapshot(t, "linux-amd64-intel-i5-1335UE.tar.gz"))) + if err != nil { + t.Fatal(err) + } + if info.TotalCores != 10 || info.TotalHardwareThreads != 12 { + t.Errorf("TotalCores, TotalHardwareThreads = %d, %d; want 10, 12", + info.TotalCores, info.TotalHardwareThreads) + } + var lps []string + for _, lp := range info.LogicalProcessors { + lps = append(lps, fmt.Sprintf("%d:%d:%s:%d", lp.ID, lp.CoreID, lp.CoreType, lp.MaxFrequencyKHz)) + } + wantLPs := []string{ + "0:0:performance:800000", "1:0:performance:800000", + "2:4:performance:800000", "3:4:performance:800000", + "4:8:efficiency:600000", "5:9:efficiency:600000", + "6:10:efficiency:600000", "7:11:efficiency:600000", + "8:12:efficiency:600000", "9:13:efficiency:600000", + "10:14:efficiency:600000", "11:15:efficiency:600000", + } + if !reflect.DeepEqual(lps, wantLPs) { + t.Errorf("logical processors (ID:core:type:max kHz) =\n%v\nwant\n%v", lps, wantLPs) + } + var caches []string + for _, c := range info.Caches { + if c.Level == 2 || c.Type == memory.CacheTypeData { + caches = append(caches, fmt.Sprintf("L%d %s %dK %v", c.Level, c.Type, c.SizeBytes/1024, c.LogicalProcessors)) + } + } + wantCaches := []string{ + "L1 Data 48K [0 1]", "L1 Data 48K [2 3]", + "L1 Data 32K [4]", "L1 Data 32K [5]", "L1 Data 32K [6]", "L1 Data 32K [7]", + "L1 Data 32K [8]", "L1 Data 32K [9]", "L1 Data 32K [10]", "L1 Data 32K [11]", + "L2 Unified 1280K [0 1]", "L2 Unified 1280K [2 3]", + "L2 Unified 2048K [4 5 6 7]", "L2 Unified 2048K [8 9 10 11]", + } + if !reflect.DeepEqual(caches, wantCaches) { + t.Errorf("L1d and L2 caches =\n%v\nwant\n%v", caches, wantCaches) + } +} + // On the i7-1270P the E-cores 8-11 share an L2 but are four cores of their own. func TestHybridSnapshotCores(t *testing.T) { info, err := cpu.New(config.WithChroot(unpackSnapshot(t, "linux-amd64-intel-i7-1270P.tar.gz"))) diff --git a/testdata/snapshots/linux-amd64-intel-i5-1335UE.tar.gz b/testdata/snapshots/linux-amd64-intel-i5-1335UE.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f700ad85f93beb14ee65f8d0ea6029fe0b19eb2c GIT binary patch literal 39214 zcmce-c|26_`#)|ey`+_*U7M7M5?a)>P$4gsBuP`rR+ctPg`-6(N-IK5D}_v@Qpiye zC5^I{H1?6Pjlpc^oa=X=QLoqg^Zk5&kH`1>$M@^6W9B~ha^2VUyszi;x<@aRiT zHe9w=I=a9$X=O^vr!*pE0e{wL-Al$)I`i`JS*I_BDBTEk=Z;-%qN`zVmLY4gsKBp8 z-R%45X=}gq(!FUGwKEEaVY)kx)lYA;QBkfg@0&mjcbae4?YPHQk!a2xqo>i(HOl&K zlOJoQ#B22`Gi!ZLcq(>r)V9GA$2fIR-FD{LV`UB&F-AD&%=QKto~u_@8$RZ{;x4mi zhYQYGJH6_{V5M}ib&&`oq$iuk8=gxVk5N1gTdPUO>eYMp*&XMFjmSIc_1I*rE$1F7 z<=}B)5A~yOT{2(%B4fIGANBFngSNwZWKWrW&e7zXuCh%@PY8?GuSN!Y#qd5Te$dyk z*=_#qyuNMdQ&-NM%Q>Ys%XU7^F7!2MFtItNdo|>FQRTD)ZI<<&Q}{;dO;?6*RcyPp z{B?n>J$HCdz0yakgs^q&9RBuZS(gi zpQ+9t&v_*^iPbP$$IA(uJ_PKx8+zi=vudTeL%r+qq1ZhKz5B<m7aGCMCBUTYHorulTED*?NPd3peg{bDf;#TWPKwV?F=6z1N6?>gvwLvk%@kIv3)V zu;xJ1+Ty^Mry4dUUi<8`)5GnX_LkLrlc=Ao_FcNj-Zbgxrgrh0@6Z-HxI@*%U-e#) z*VT3No=>XFIi9jXY&YAUo#_Iih?->8`Pr20F0e{-gMB zqx4lwqVHC#maQKdTRrdDpxyU(tWf_jYxDO_$5zigru^Q1{SfcV)zfx*vNbiwy}vo+ za?6SS1FJM=`foWxriH)mw?CVI=;ES49lx6!#P8Jtk@LxM^XUim@pcmlf`X zC!{r;wNWj4?Bc#)>AdfjgK{Q=&j`t_cTW2UPuRLsmX@n;B5PPzqhrOJsDs~AKObWF zoG3an*!6zC)%E@RG*(%exg6i)IU#Mhv{e6=W!lf({oJ8<%bjyPm_EACf971Qh;dmJ z`+0d|)%}8Qrh&PahRV%nBXYK$V^(dlXmp(Z`rD47NA5K%S@?`wkg(d=YPsL@?{b!I8^jaE}h6-!saLeH)Lad1J2-_ihY&awyhhtytv{w!x- z=*Q zmzw(+^A^oo5u@rJGR2(rEa_~4zIQ=?{N-fHlE5_!RZeN_R-|O?z5Nv@MhtfpimFx2 zR&m>dRCi|tOn#@me~Ow`;@hE*hpt#R%>1s~po{Jo$~TvP8I#EWytaeLc*vu^E8U3M zwc_nU+$(!tIm`Xq)&oAXN|t$U51)v|^m>JQH0u2N-p^`i9ezu-y>y5AdtOto z0>?d>HR?(E!!NG+n?BEoIWbyohUOamsIB9kbS~Zf5PdtSSBllgO z-o}N`=jXb{9Eg~2qh(wi?{daztlG(;>BDx*zfyCpyiuqx9O0_zJbv?@NBcf)nl}mG zF!%3aDM5Qiozzq@7`AQPvrcnQ+fE}-x5$%wc{5I)J2=hr;nO25uM5p&_sx2S`qo+YRQ+gVpGr3C!^y6KtP%G8EQr_b?>j68Rh zzC|7JKc9>_7;X3_ZhiR|i#%yvxUh3?fz`0FcDr|$zkLwCQSEN!>$Q)vgGaA%dq3*i z$h=-}y?AQM$2mRg6~mmqb3SFf)QLIg?P>4-@oR3FY>7?m%5hE6BgjTzRH3wciUA)BDM*cNZ3S%NT$8 z?eSg89&3)B(gPLWS1YVGdYn$b?nH(@ZxnB}bx82EelIK@npiNOQE>6Ren%|#Y-7!i z{SR_dim!NGFnD&gad-S)&sQ$F+x0}cgKG8tnMYG5E`EVuoH>(k$i0Z;1sqmCOCde&6{MD6_EZE1KF9!jtxuE`9#c{cr0J*jME(S?NUw z=XCHYmIN{YZbqoVC~)EKdvNQxlc|$ z`r*iv)!&aiG52A*2(wp$jPcF(vr|?x-<7>l$jhzHZga)$!w~?y|28 zGnaWQ4Y?xbxatnIuTk5zb;u)~;$iJaN8r;d$# z=fC7sQW=-qi#xNn_dX~Us_XeSol8g|CMCZy%??|+ZRu%V+WIn!EyJCzEn!$q9+#Xv z592MLGwfZ+3(tz@#Dn81OGR(3bx*CK-18@WI5O(Hf3i1uU>ifl+4AZUA76p@3Hz%P zJhZ-P-yAXM=Hb!zw$y!b9pNhdyYN+VZE`Fw?EdNGR%qfB5j5E0HVhv2Z zUS955y4~R9qJ1yygFo#g;td$lk((c0J`h%B)dc$ezDbZ+O>1R=H`vwZ0XhJ&d$!0#Ri3uSI#Y9PBNU?&v4DNo-%ft z+Q^)0uQ`UQKif}5P4iFW-TJy*H?|_y>e+=y2Y2fAez-E>NZpUJ$%{>OcVsB5Udt)W z&cAVQ--3Wwf!p~`rge@+8IyS@E;#8i%Jh$^WI&sxiMMd!-$_SW^#2|ecM`Aa{-JYl zL2dKT#dA%aQg%7tJJ@6J)Z>QHv?jyLTP`R`R~r{rORx5Zw1G&1Us51E;mVw-2C@v%l=RLH=RA&_0Lv~ z(eHZGRlfgJ>eWHMXMf#(5wK#p)-Rs&>&|^wgQKQ|KJ3Yff8?k?GVMa;mh_0otZQxi zHAXGo-0*GJN2Q@h7hWqet++m6!!hZUC3*cPOgmyX8g_f;#2-EGc*U|p%{!j`=%(?$ z-29{03Y8z(h89`hFj&-^+4->;FMjW!Jl@CbWqjl)tv%bUq+uIdQzj2O^|0mX5{nU` z)09@J;g5ZvL@@sz`R3R5ZGK<)t`;Z!GAeKEF6>a+!<-nGqi5c~Mi(;(_c>!a>e~S; zmW@TS2{unP#Czx&jZd$qM^D$@x<`%o)2s2-*^BcgmK!uZt1j}LZdrcXO!j`9=|USX zgEJnZXLI%ZUaZYEwoa-$^VK0^ujc*Y8m@VBK=zTfH9^mv1E!1G2KjtF`~8N-D}yS3 zMtc34?IV}&f9?JH?2khlncrp|jkRsTwJyDOYi@nbUiS7Kj~1P9idRVUy8P(IVy~}zw{5;Ey6d*8J;46ui6K$t z34gD;FuuLT`n-pW(ab$#A7r@n-7e!Bn>Nh5ZGR;*AiBIew)5%f57WXuZXe_%^)2YV zsm!rH?6mJ@LF-+jWC~9*&P3y@#gs2fRoO-AD&u{#=Y4c~7Aat#ty{T>_hrKCoJAuK zA9Fjhf9W3I%4yn}rnNP1lq;l1Yd8_5hS?(OIsdrrk;BKsoHa+*>3-U#eC%z+kb2?K z?iDT@Q=D+J+C;PZ@$%Ddi=+?wF+|dS&^?SM8#>3IC zTtaGXPBoQ8#~670^&>oc;d}n7pzpzN?)zVvH?Di(ugf3JZL<t(L}(q8=2l(>(W2gDTWr_n{(^AonH5fqZ<;36mRNupDa1K*>;7-kqsK={VSM6 zJ}-+?Ju10RX^7vqc^!h3VK=wnu3wj^bbJ&(Yd)E4?(^zu_VwhW?GsczM@2gvx)2)o zer55FB8R6}J{a26wH=SLJoag;*S!3I_=EKhO+MAaDNkqJsK>5OVET3X1+=C8c+l*; zM`9B=e{sZ^DV`$lZFe(yc_()|?VBp7Ig({EP1S zI_kLz^<0j6p0)+`tfBpH&(7*QLPSO3ot3wOwC)Pe)UOofC7YZWqG@h4WzOf5%?FOV z-w-sG|GnP%$l7!$`qZ1-dnS}xz3M=^hX7{om?F$@)4BjW?b>#dil`XiR#yHubJq7^?UEN*NkSj z;VQcgr`~C7sLI%(N42jTU$Vx!e#pFKL`c#WfrV(9USN=qbeU%$pS!k`l?BeE!_p!-d(b zF&r` zkNtD4@sGOA{`vP2{V#4xY22UNz-0gxo(BswD{b$+{WI>4rK%e6 zfyTOIk!LtO7aLm3JN9BM*jA4jP?2B93K-RgE9n*Se>Z-cBoU-c*|YFH)8Sf~fY~0T z&((FB(zTp1r}M$f4fPY|Z5`Wlw7W`0z^!LhznOdkN)4shaWwr#Nfy^QY&=!+8S~N@ z&%uVT98A?QkL^=FFL2yrSv3l#J&3~U_gNNuUSJGTP!?#PNa;3I+3LMsXVn^^m@(hM zBpM{R+TFZ#>*-XVIMpnl+f;zxy?h2u^jzpt!sLbq&|k%NYdtOgse-w*6yjIVx67#a z!(|LMbfwH}z*Mnm#mC8hxFGK%A(Ib;IB>JNfL(|MtR`NO*jk;@U?b*!f{8y!dFRWh z&m~gW^ER6*bFj+Cr(mMf;zJx*s&`gg)oAB?h7m|~Js9XYxqV_I`|adTio20+2?gh+ zczkU`KHU%K5aH%-UsSOBiqws(7KCBF;U2ZzvDA-7`CQPXd#+l}C0;ekXE1@dR&xV3 zjJ>5Ign1KYP$uHk%mVxafk(bS1B#Ge^H{6bO zIJNL(i$MxlX=2 zm&QEsP^uAQ$|OwP9M0INFnYT0plWv_xlYJBxj~20SY5j<|M_6cFjI9DE7BA# zEmm87b>`a(@T?79QcH&+I~87NUOac#p)gHxs$M4yU*#_@LrzjpagoKa6)^$3kEL7s zOq-FnMW9`X#x$XBj{VU~B+qK3zzEGwZz(03-ZuW0CdoAtNVqG}3KmridPPNj&&LvB z!nI$)sPEKp zg>7{sHCE(ff2HJWXEAK6gwFlgc9?v6Z#S#vGauP=5jFAD-frCkSN}RT=+=3tqPM)F zPTRik1s@Yrne;F-?>isJ#yg~pbQrPPL2PeIjWjho4=U+siOj;0QFdn_UR~L=4Un;(3YRtD`3jZZa}?+v+IE- zoY!riY~<5j$iq&fM!$TxaE{(iUSvMReoW?3Y9`HLt73cS%UWY=-+eqIK zN{fRmdrf@IO7H35KhCBOE8_T^2PNW>A}j*=6*nzt_2i=&6uVJ^AAc|beAaF1c+mxU zayQ3sOnC3Z1C^UF4mkz)a<@)@$KlaB$Ioosd59Q3z7V6`C(sC3brhO^9iH@difHNG z7IE=VWWUvJgv(RGr0JcnwFjFNGUvOQqgBl(CUZ6WQmU-?M&ahyi(aCSCU-~7VcPlJ z3Wuus)X3F^V*K)99ktj?^^EJ?jqHyLmcsoDYh1;Sr*y{Md78%Bb0kMb+|ys#p^9+oDg(Er(Vd6N5xa~^RTsjORcj$?o>#~3RT4VyvgX-;6$vucOv%yQIYJtFYK_*l3p-d>c7SJ zS&kY+3$UpvdANA`C(=|KQnP#$&%?JT_nvpCZ5bnAzhhQgJYGzl{um0e`zUtwYep!g z9ZBv!k~(23_4<)atNaI?9Fj+}Rz8TrHP{AA&z~|>MsvDmVh;0ZOkqA)F@9KGv5?U? z22_{ANg+;IVH?LJDurer{tn*^aqdYd6v}OWQihSAAVUbPAk&ptt`@;?`&SSY=p^rg@%d&ijZ12PMFjFA#XEB$4QkLhZ1 z$NXQgf1bD4W73!@Jn$}NlDG^evo<*Ax}?SP3u=jgUbDPcltCXeU0a*Q=K5B0>`pG9 zkxNU!<}$`-wj)3L_Kdu4DRT-R z{~*Au#4#0I6HU^uALrgsnIs)AATftPiJ^S?3vMc0K=UGZ`3{&HSqTX z*9)q66H{i1RTfp@_bDra=~e_LU)olX@A??oz!wu9P7b}G-gBZp+5`2`fcluL2(fv~6&~x4r<`B##+%pfF6CP8M3n{}+xVrNpJ#UJs zDx=t))_FtgG~^SEgLR9oREnVOrJn{wn!=YGBle_}{uj=Gm(}cOqlkY4;!jzA3s+W83BfJHj zH=8hm8C#qS`vjPz;Ew#CHxMqZ-Tf8+0oSyS2;zZ%;7LyCB!Pu=C~Xb2|EWVIc#2o9 zB8>tWGCU<0A}Qvo9h>zj6ldB{)qDbq%{0kI$l8vq;igM44JfwFJ9+N|D~v`HeFM#C z=L7>&)A$Mw)4}mtBWyxWgZ%Fu;ix3}?PIBRl$PoZ;J;WyqZq@Ia2@UXo$8Ag?SxZ43ke&zD3Ueu|ds?eg5HmaQQloGfjoaNTQ zKDA0rRTA}_Yy3s3y;EfvAA-OS>}p@DKZY4QH;~x2_MevdC-STC4_PwiLy(N$2$I^` zmZQ0~^SJ5hf!eIUyHrofpuv;MV4hqAy2qdYFcV0l8N)_Zf`DLu*hb)J4l;TUBa~0| zahF2Pi!?w0?^K|XXNYFIf85kGxCzO6ALw1=BH!c-8L3ec{4^=$?io8HgpGV+I%K$! zv8Bz}&riru>!q_NEF=nV5N!d4H z2Pvj9rA^}ln!z-%QJMPu0^TlxEma48HpW?!U^&DcSRK4nPZ1lzGMnc(Qld}YnC%O%ib zUhLYat>AJ?lz1+Y4m@2fk}g zsKkyC6szJB%p7AfQh~GnHWUbpM9@m}Y>;WY_K4J1=Zh;h+un%k&dlK$Ec zWpyh_t!2QxltI~uTQt6zv9Pzoys_xb15XmTC4oYILUeY4+MzsgA3haq2OXuY!t{v^ zWQV*Dqwusiu-ZxR60CY5C&Xm<^anr$QLJmjDQ#6r+M{Bp8*au^22Htc0j$vACI0F; zGSS`7SQi%}PYAg~lO92_?wdx0`+FC`oZ}(}cYP|v+MNjR!-A=~euoesWC@u$%2K!8 zBp#&horgj!2aQLHN}frvBf5(oz)#BWO9PbVQEb1Y3MdQCaPGXo9vY2yBxg6;!`C?d zu{-(IBcIP;@9W2O9{-fNZzKI;LGt$R{To1{#sm8JlQ=lIkF`@wUY`c9W2C0tC?C}3 zFmWaWWEc?w?8%|+)N7pL zi?~E%V?bt*#XB5JD#zyEhm3r9xbwTzt`(XjhR4?wo0YX;OTgm|$yQ$ix}wE0WK=sI z?4jl!@8i;6t^!>lQ+3#m5aUObZZ858o8RU;McENd^{rj>7ZFkhvl}ylD;xGWc2LJS zu6c7b_*fidbiurr=&Vi4`8c6dVSjOxl=Trl5X@C(!c<{E|8r$wn_hxRLA`e59etH? zi#!~^#bGz*-;46!6j}JG+(7SLm~Nu z7oTw(t`OR)3%BK7-vSy^#s|1oDq+~%RP?>R70M)xZ*VP7+LeI%MGYnFK`LyzA^Kt; zp{?FZ{$5-I#N-|Qrxj}u!OBIiE0e5cq?Q3nUJD$)jX4sjSU%?pAJdc1joXxQr&^2W zIHX12k597w@lAXDH!=$*EE2xZ`|jP1e;&8#!!s$XlwS1PCy>#OCKU-8#CZ@JJ`mw* z_HB6lS8!^C%|<>XGH(y1`+HN6-FcYKVam=I>nYBcvOdEHian?jMQJLik}Q;38U^(s zhev&%Md$i}OCP2|ZNa4sXvQd#iGy~l2p@tLbOUXjyO=R{y)!-0UQFHdS@3}KEI{WrGWo!+ zT#+yi5nT1HKpQ_}ldDdEMGT?tAShw>v9o*XS@GTn_b5Q)rLo-oH z@DZ7&lM3vIneBbttNzES8w*}hx;@xsxKp1HY`U-JTa|=mIls8L4ewzHou=%o`&5(l zAma@zab2f_x_BYUEklP?hN~hFQf-nO`(jICCqHhPqRffpvQ52yhePC5+LmS}YzpIik)zeF}_aRfF z;Q5SM|6Wxa=*y^)Er`uIZS-)sd2ON6JhWN%%~;HM%23sR%a&$`51L{2v#MY!@YMK> zE>O??P@Lv?&%;mCo8_O)w8U4=KGDBhRiNTP+tE~FX1Hs{#lIL0Jw zO)xkOxB$E@xrSLIL{OWq;kO3K{TNy8~sUD;;28m!T5$nh@42laN* z=gb|x&`~7~oU9NvL+yr=c)OIic8fYjaiaQ|7R&eX4{q`J7B%Gf+F4&9GM&v-{e_O) zkJdfRg~7(fsqR~vaVh>W^@UGlPG{(^lt6JAoElY*?IKwS2|{lo%f3!WEdg zOOQh(w`cBDJ!J8WGLtcKmi-+RbmJD4F@xm!*EGP@iy-X+V?u~08N&&87L-d=i?9xT*L#g)<#IwbeVE`adl)Z~Jo5@ti?WI2GK&4b z(s)G0Qin@$_diLy2%Fpmb+GA@vFyNre9;cHd?A+;kcgsFuLD&q55!(O`U7I>A#0|b zu{EHWA5ckX3NL4AD8)z5JUQL8-3o5%;CQM250Nb(9N*?bb^$bN$!CkSiTXOoS}CWp z;gFgS_wzQQmAphTP zN)K$e6%OB@j@gq)@0e4BSr0nT`YppCelkE-ke(tnL?+@?1hRiXY$Lp)?Q0`e}=BbLqso?I0JvD#ZAhxHiQ4A0qyeM`8oRr~&@3Hbn#uN2|S- zEMz^M0987>zM$pnqNCv-Ss)Q&|FKp*#L&;X$SN-+2U3QE<)aADi7x|M6dO=%hJa)E zA48o)i+D(>_OQCa^Ois;R~;Ci0Cg`Ict#|2M=xyt0WE{+y~K(zL_Z4={nQn};{lBd z*{mMvNo-oLdMow&D#r8(2BP{%GOQ=2;)WM+JRHhxLjGByej*T&5&`Ep% z6b!%7-+z{kZUVw4bXp-gz{~8?)F2E{BmW}oF2&Q`jtG*kU|0D&G)aGlr<9FR6$FsJ zPn^(?DY3~%; z;-^Wm#KPvVVWpVUW6Gu)KXw}{tnSl(QXh!UBcTcz`M{lV3oDH4^J8>=qjhzmwyhO) z<#bm|_TrHWcw8Uk1mRnUTVYRIp&BX{nn+`Jrh}m{y2e-Jkz1SY7b~poArFea^pCHX ziU_7pF%^9(r|wtR_`B4hSD=M5(?eth#YCA-HHgkT!Co+w7^0w+zE8uRK;|^CRLX|j z`Fv&vxS_D1=mn*cI67BQKIK}8e+Sg#AOg=Fe+N4Yv3&>#nB3|AQ(dPY@f z)MmtqNQ@iT$L(SDz-M`)Q~Z zXs(#Jn_eR7?ag+~pE9KUR2Qp^sv6hFOd)ZH)8Kgg6@@H@7!j@d&`l#@ty0JDfv+P5 zEaD)8u`1AD{oqWXd@?CfFtkofzIGHcFA>aCf)k0q7BWmI`Dq-jP&n)ho)5(cYAI@p zWa;~KI3&fK3G8f9a0mNFxVG~llmutL>&G!e}re932_TXAyH3&p4|XiIBB<%lFHcX>N*6KR_%naCpu3$X+X z24`oFPuR?h(3IdwuFb|zH#4@hgY{JSVzyDL9i+05c4MZ`1+Yaue zT@y2?`x1`raj;VsGPrTQun;G`sDSRJd0`T!!(${qq7lDH8Ih1_2B~{LV^;K(Y=M=* zVle({*r9Sq1|cHazk5-r{K7=O8FAd<0SXpKmyNlL4M)CVYynT$3IZudXu2APqVQ@# z;r-@6WVOs>VoB+kn!|kDXt`y2M%gUF_ri@3I_(C4zWbwKfz%`B@7Dq0~>^$6548cgd#!nG4;DS zt(?p4r4ELCbgtvMPx#DsumtswO2GC&OGpN@E4$f|3Y-}7KK}Zh~n@9Z~_7ynMX!YOWnPzsu<hx?%QzdP9@>3`N z5_IGsyhMX(0e&iHjz^A?xPZoL^smYf>^C6(l&~fM>vVff;@U7*gnad2fEfA@E_Vif zzF>cYUnCfgKC$0gR(!X=BSr^+82KxwViwa8W~{Io=VKA%bxy!u-ATSY7vN zt!8)>KLOZ-Y)HSBngV}O2JbUWgzIy?95SI9H26Let+ zW$cyjYICc5@t^vqO~LPUFW4eCR6&uE{|j3o1osWVYEFch70@*?n~E5+qCf<;XAu-2 z=WLg9=lM;ENzGLLsi^V^5+;!dQj+}|`m|xhs7lNbZqq0;72cT5BDn^IQE>Vbpi1RJ zWW3k*{f;wd{(l8r{Z(}#XE z;O9pRaw_eJ9U8CO;cyhAB7ZSVF+=}W6OzyBR zSo_=+Ig~J`gcj!U+#{Cot+ft^JN7}_snt%0C4NneM3y5T<|qX8dZ6lyUbYPBEt2`= zFiOF^W!ESsWOep%(I<$A(OJ|lr4@`?g>eV8*S!ROGsDpY#r%YMa2NqHs+l`Xkujd0 zkVK!FL}sHHh$xU1yt~Cg_{)Qoxk!nOqr~Xr3Snc(?lKp}443vh{b2~i)6lIan*mg3 zuAdr#E*ePOGk+LV&3)%#!3N8XTTG;mG%RPM`j$d)56S(BqSR=>E^}K*zVe3GoE6dRyM`b z>e5P8{AH*~YRmBW7YIn&`%rRopr(WRlv&f)aWL08m)YqZyK-b4xTDJ#af0r^^A7N8 z0N-9yy8)%)sx-c(qpj0`3J=&#OGVBXZYlZUM zXaQZ5sxmv-%kBo0vgQFl;V*I@iMKLW+1Eo~lnZ$DLQ4qAYh}I_rFQrVgKKOL28~#h zPmicY+B~j8*q}q_W{7FE;iF_&9U}`U=FR_rq+=9=z^A_?KcE9MlEj-ZtJ$Ua zzIGZzMd32m2Z(tLTa1+sq5;vWP!=JFTS%m6lh?r)Bbh00bY0v^siPR9@uLy^ZBJ30 zR`8KhC~vlD{RsyYWCe2gL)Az<9x74&i*@o`=q#6x>duf;vmq=pp~ zcha9Z_s|5Y4>{E~@H>k#sIvdG6Mfnbq-!{$1x>b#8^`7u2GHgbgUL>W%JRv}F!col zo<9B_b{xz+$k_cD7EMgnxDCetM{c0eXa{`H*Vo}-v2$^(8Y=y(vP@+ z9Y>TuV(5)z2R#8UICUn5C;Ab)>N_WdQ4>f3WV|Z>)3p2kVLku#Qb*-6I<7{K{yo zyYnAd7xNqI*sxW^GeQLd0woOy>&oDw{-?-egcz@Y zFY-Lght{=@zKeGgW;;p03BU2~Z#YTg9rp9ZcvSwQ$w3vq5MJj&@^Jy~=PkwJ zxzs{zY47Bnc?A1&5k>=ULJ)n1`hSqv$up-mIMgrW4Ov-Uyy7r0n`Dj(z3ecQdYO`4 zKYcWX*|$f8wNiLBLlG&A$Wh1NgCfT;BxH$j&35pjgUGR$6c*|CF%P>WGL_Icwl1`ec_NRAAJK_OeTYx1AGm)vF2Gcaa8`P>__Z z=$>6|>7$5_5hsTVWIy>i0u73bb0|iMS$4DUs~zK)&p1jNo_k6czEE1$E`>I#Yu%96 zus-|)D%0hWzY$|kNJ4x$a~m>9a&g3s_P#m_tyE0Q!hs4Yy+ z_yvp5VbqQ76Z_^S2o<^`U4RoQigXAT7b9&e8cF5F61(C3Qdj^lI|8tlnht5eVy*$# zFz`k8ts$SVjCfS4F`k01mb77?c7^?2|AgWyA#(T!QD|}~k4}28Xs$!U9i7g-pf~#e zG2CsCVTTBOXnP+5)gyBBM4^0N#At-E8>eGca)iAMI=5HA!2WSjKIdRdWu784<55D_ zF0GOx&2Xb~KZDMCQ7+_%eijX$aUqe))y5}#4ZEQQ<#n9`$U^4|0%Fq;(B`3%L6 zIP7H0&Gv6vTL3eT|DrtrEd|lVJb4m=-beV#{Fg0eQwY})Pm4oru?nw5dA8l^(@u`i5pP3MU&-xL=(|bhhF?>KD$K6RNrz;ftfYZA zx{|9LVU{x+{3>F^m`+?9X3U+Jp3PboNO4;^XE5NR@(6R)Au*Mq5`=ERkkQ2zq*Qu< zs_!0mhKTYcwPd;q=r-LgjGL1JeJMg|6czOt`5o?1VFa;xEr>r>{KFr61{+Z%`}5pR z?9$FuB%V_>&#Di%M9=1lbl-BIQKns!<%Wu#gc%prf-A|zpTNloF0G(aX=2zuENCk< z3#818@Zfe`ckI3>Dbr6=IRVU0N+F6c8;tiwI;sTyZl%Bq9srTC`&O6*dP0 z4wob>c_1Tm1soAR-v0VLHe_5WWFp%Q`ix24!>GGlI@_yJM_1l@w1ZvCp!Q(C*bmjT z7dA1xXa<=`4Usd*bnrmRr#ymVTW~EgEPO2Ep_Gl)lF2CHjtCht&InSCdswtmG;0T_ zgP+Smq<|qrh?1gTiBOW2qxu$%szulQatE(~p{*#E%Am8>eLg<1P2o%@x<;g-K+3a& zmD1?o!8EajeQzk13R+NNB{tu*zf0}L+r}R4k9TTd2GkQdTfk1hM$8_8-9`^2*uA78 z#9Z%B%#&him4g`|!K`pQ)bwQ+Bt-&&K=zZ*RAnjt)D70IdMEL#itA(ipyubn#!QEk z)Yz#sdxXb9c+j~1@d7E2NFMs0ffxeTM+9Y5AJ)qA*2;c4`IeYs%$-PZ)#>S1f|gp* zA0@1~?_db~v&v%6(986@A9{(M;)lTQ2YYLtp0- zxXvGjO{aWGUC6SCQpUN_Feg`~5X_-7Q>VVLq9^p#%SP3@F&PL`xTjIhgwzOvi44~h zh0?hwsbMpHV6c_`sEGp#;&a4GqX&Iw4A={0y?LE-6#^-3zhsD_+Y}`nd_Ok~iEqgI zC{r8|v}oe|8Ol*O?cuhVD@{V;iChdgC6gU%qCJBKNCD_fH$3-knDS$1`P(1pA_%SE zrpndHPkpgcEV486As!}2x|H8YH~(LxD@RDT^*@nLOe38iAFb|f`#@4>`CpSd8J*NU zHRlnrbcM637vJH(gN5XpNqE*F3eSUk3)siJJL{neIFiD zjm2svd?rTo2x>qnb9LsSTLTxNMXr=FBAW3ns5oDQ^&ydJ$W6s)4E7x&SQB0nZ%`7% zHj6~Jz!IjBSlMMrqBW@lQteU#A72h0i+{_}=}@n@0bPVfETU{h|H=QxV3q#Q87x)> zhtP#AxqO6B&TKG}DJPNL=dHj_Si@@Z8-gnjivx$jC;>+ZfS7oO7(0d-Z3~f1M@n@s zES8bNlT$n0re=ZRQ<`tlNb&VI(6!5fPJb;-7>FNNreZ@Mi|2Ek8cVAzUSxDukId}w z(CA`pc4XQ#SovmOA6$hclyL)}Io3p?t^T%;(FE?U)C9z5(^I0PbAML=e)TXIE(+rg zO-9^x>1XUGPN7SjrTZ1~P25QvQrq8oKDzcshd8`zARBg7qnk$<%s@22n?maMZ`xX` zfM{#O3ps7|o-mM)x|5h7Y|;*3wItLY%-qh_XN`VMX%$e~NOAj(awPZ32&DDPBy3dM zqVGJNM%T2cF@58edzqTFXoTu9>2T&>&T1l~lesiHnG?#BxjBM=lQ}1OGFK7S9gE5+ zG-*98WKH<=Kakd_z?Bhvt`O_MtjV!WgY7WRxIR`fB1-rlJ?FRjr6t4m^>B6M2#-{; z%P@;BM#%#Sr6B`Yu^ar5@I9bhprAsSNvGJ;Xf9~;55%ElgfjUr5Ql|RI>Gq&PE$^1 z?d%9%*y>Moax?M&Pv|%3{vU+?gOKqX>X7(O;e#$<^o}6e>zIfU`va107wN58aRF&n z9dI9FX`MtaiKF7yz!YQUbajAI^5hAeAxhwy3iRJ8wj=GG(*_+9_=xJFs*Lm)QE|a! zJB8#wkdCy?cyK<9+meHu#z%)D3o<{FID+-TEmC9#d9Cla?2hivJwsB?pW;X@eJ;=l zTqc4BR9G-;j&h`HG|0dRfN91#F!T+0C;+db+>#@M7X5Q#h(w|y?)(9!+WDoQ;+lI(0oh*s>Oz(WS12CjIA3u!oLzClFvZL;e}B8jburOoiOvH zJP}Mefo}C8Wgp!FaU$^7=(5e)KfDzC916tLGsni$$y(YY#7pVipcg-u>GY{fWvK!x z3`{Tnf~(MM^$t`!h6RzlBhGiQ!6-jCv<)g+f8d+}XcPGSc4tcg2cM7sRY+z^cFnYNMC*nxVc;gpN-Q014tpc%uz z)@*#I3hvC_MwfR1xjLlEaM#JVP$dKXpl!ldimq%RLGHnTAg4zIgVP1&a^nDUt+#mc z=ts~=`Slv!)(>3)C|)s<;8!rJxMH5x=6z0;L+l26DH_RH7M`(9yk}q;afs$xWGoneqlQ+Z!|4H7OaFp-- zR?U$bAy3|D_>jo}@qtr0x0Ji1BtAu^WE zm?ADZg5;2DvqEmvG$gqFk2ME*fp4IYU?RaLr1&|@^W}k3>#BfgRA2o^97DCt($~=V zzf*faAVuqO)ofM75#$tgBi#Q1&h?0*rXzys?DoIF+h0+0_qHC#vRPuXBo;DyM{xR$SP=YW*3gSSe2;H*5==}Ci#g(pw z{AQ~EXZhP7rh4bUGF6ghs!luhqYJa>&iFnllf8t^5hK@yYH)TKkMP2qlC_wSU6pqH zh`yR=SEfQE7gajz(Qi@ysdRc!XAXs8(+PCR1)e-Q)PkdYO^EK%->qf6>tW87GE|#z zDU$z{Ibjif%qeVGEQv-xmlB|YA-`m|Cz9aS0ry!G?p1V9igEG>iV#H-j(#J_RQAAW zIi*|SfhWnc4M$aoSe#B2I(k}my*j$Ijuy}GIS6JDQALDO(50}8q)T-1sANP_)vy0w zbzdG>)3yDd5Mn6$sHVo$)Kt(=(oouCEUiZnX{h0*RoYTwe2FP1hEhUfj`;?ykf=+2 zi6ItpEF zCi1QY&?l1$J1OC2uHM;Uid$utZa9cDz?AN4#XtAGV9RY^M;5UAvWu9^Oz>WX&ej^Q z3QFcomJCN)0*ml8CcWvG7;3$3oBee7GDE2E?dwQ1k8H-naV_&x zjnoeqr1uynX~>NSlY(x(^RFo*y2CL3^iTuQM>80NfbO*oFk&xGIx=Ff&$@)~M^o_j zuS;@~WWWn&1f9+1B4LT`x*|*{G2uXR)c&0l-9D$|8P)HWd8+>8A)*LK18fM^;^`mC z&Y4`NR*GLO;Pa<7wF`v&oS7fEzu~J&0pK`e$miN*#cRP4MYzCCI(AzRT+NA5m;B?4 z;D$pU@|lP{DcT7Q`>2`v1o zxvJ@KRdZzLe+Rjt;!9**ACI~_l}c**LGxFU!F;a2x|7Ld4e+w9H?d4Ovs<$#_ur}csiAahvF!iN{s$MFr(DYQT&SnmBJR+P zORPfA50()BKh|l(lnH%+jgWkpL3Ou5-aaKh)y%mduxde8sub4ofHti|Sk5{+*fD6# zJq_8BCVjOoUomX^NJfN~53m~wG{nx+6rNH;O(gzhahGeQRVHT}L(^14UYOERtpY#G-SH{eOR$GkhViu?L(*b=FSZpj<`qOxzL&xx8N{N5mUe$)2! zC~%WlQ_Z;!rX#=GFeS(f#+Y=Kt+tcdiUu&nqB=*vXrRgqDt=2KPi4WPXx~a{4OE#^ zSmwXNJkqq2TfQylimX8lL!cpXLbKpsP7`_438hV;TZ%R5Ux3g*`lw4c$HuB*gE*e* zb*-Z2^GZdYdx=j7d8|dQu3GiLoxHt5z}7%i-oz^vriNfF9k=RHmFZW#&!-)bZ&i&? zE#In%m#qYq?b*vbF}~i(4#4H8bBsd{ViwdtCI-AmD8%;BBEIoSrbZx<@RL1pv??!k z{7hlw@A(dsmzIn=O@@cdQV_S@2J`k^98ZoKlgPI&qW6kzD42IOTBt^7G2gPi8pVXa z|KDKVtt(4J{2}uWF!#T1-c9%e=AG-a=3P8X^*A@Rt8XL^??pEErt(SSS#v*F;pLgm|e}A4Mye>o;HA@njrS4X*X9jI%p`|mPNl2&@=@(Z$XzImCk)L(K z*fy}~82N+A0cU~>=+gnq$!GpgiihdZy`$Ef(=ur}oTDN-Gzkt#r^UJ4gdlXzFw)_X zw+#m8Oy}Am^KK*{JGL^tiY@PoFdECdk}dDLwxccYT87(wtmR!Bvb<{q%ex*^^RAuC z@~-cHrR80FXLG;FUE0#YowSA?s>Sa0};o4+M5N|aI$i(6uKKqmpdW&u!8Qwz06%V<}BYUgLk)h zHUy-k-cR&-s!(j^jTE|>R%oG{pDeipPD#4T$M|PJbu9t#M6$IMROem+sP5C}f$FYo zvFocsbrGUvNr`0Y<_9z;J5|?2{qjqtFq63c3}%ANtED9Kx|*6b8>zxfGE|sJLJVs-VlZCdNHZYWT)if{! z=6kVLLIuM!;X9KaOay}7q}3}@Xhtc_xk6u-WoSyE!)iJP5kwX!>|3dbvu1e9+y_R# z*SV1WO4H8KDSa$_ zi(d^xwAdJNo~7+gMi0XSJ}U4tZjUjrv9;10r1*SQt4Qbv+}`8QY1C)mzHjQg*<`Jk zl1ce;$iyujX>E696J&$)Bc*W$?b*NrElFC1Lbp9s5FjU<;}Zu>E{=D9I`!HtoctLE zq!Zx8o9V*4cgTIRW%ZMm-CcyApH5K#8C?n=hWlykgd)C|%DIOnAOmD5cKRxYdAGS| zN)%R6Tb2&KA|@B}krnu`f7qs@@zZmRU%mfbY&y55&*+sDcK4vjpXZwZ=501C-6++m zo+3II@CQ2ZwUCMR@$Mx9Vb326jJh%1qSwD;x9mvmmcLVzOfC~_=4FGIlsD;COtU}y zqzijzXuB85IEd*#A>-N<=Bi~@T%Wlbnr=7{w{Pe1N1jZ;YM|lk3^Z&}LBnf`1@}1~ z#)~uUfUl=aV4KHxFGbUx0h%u3IcU1GDw^(CyJlH#cEn&l|E5jH$_M`gVLKR*1-h2| ztW5gWEH5!oWu-;mLhsNr@~QH;%(-STI?dr!$s{*ss-gO@xNvujg9wN**XF1hXy-hSqp9mb9y$)Qup?}w6_D~lS#)Re?24MkUy zzCNkOMj@4-MA02!aNoM9lDGg%L5=T`XHN#B>&0N$pEAf_Me(}wGVUQwgh}?&94yS` z=axtO*?8So6~*fgz93$As(ic-y+6hw1NdybuGgs)x8xg7RY)wFuNd8J0F$Z8G-?w~ z{SWsniPy~`BLmf=IE0xBQSqA=r)ND1Vxe3v5ncwZv^8f!uw ziCW-&1+EyYXk)J*yHRlqCoj~DFI7UUIyN>d8%7#Kn-nYLCZEgH`l&Il(n@NF<|b+- zwJ5AoJXxi;$ib?+Pppu76&p@n*B*KcLtJ_85UsT#qP`J1fF{wv??*~S`6tFj`Sb&d z<5zX3Otc#OiakKAcVh()7_JubY0s>b&9WsMt%sbsB!|LiwJ$pGC4|;g3E@JPsoPX4 ze3OH&9>W*>q@;1WcT_U!rOf#6)aAJxr3EN&UMH@mIY(6T@CAA|C+x{Wv;MwPN1eB) zVoL8xnLu8j<4=qP1<>$syOl$B?fEanWbi_Ge}DH2U$DZr(P$b2>R%62P%GdLpH7TUGUJ z?uzV;ko&&G5_+4JpUK= z>_5sR8yQz#e*ss%XoVc>M*6@eys*zW@CEvon(%VPR;z;Rzn=|W#OO>kSG2;4^jY=H?z?pG4r-&iQ6&9iNDUlmcqA*?&J(1v{9e>X=C`*Ryoi94` zaxBUPkiS|ahL>_w55m+C$kdjXYM?hjvNX7KPtFI;PmyxBhoG*W#Y<-AGa{EmYkhY? zwi)he>Z7WRQEjqqh2qj#>DrbyD4x*R-xTOsOrQNYn;Wcjlx?fmw<@43^f6v_QE}7$ z_(q}rEcIp8;Hn1sou-_7BP&66c6tSCcty-qLTWBjzY0G}%h5mM|IU|CT9{C$iYFv- zApA6ntA00d2F3E8<7C2dO*U*IT)F-cZ?0pQU)@(2D;nQt?w z8@|Hq3?`MA_h1G!6Qk9$xFpq}ZqE#ARryq91~ubm_>V(1gZhuZaE5BZ|H&7=XoY%* zUh=g<9&icuyiq6F+2YAOM0nxo?jKXUGq=q>tFV217iyC-R5o6WN=8qp)pI^V6T-2~ z3Ub*2;CL^S8=LeCxulrU&*lTqyjMR=2)mTs+#z=C~u zxw6^WaIBd*w}1=D;NOmV^+gXrDtB!0eITB}`@ZFY<0&xVPiv1a{U+pv-9 zu*a3sNA!1F%^4)mmW>;B+9#4(^ho8#^pj0@MDCPRL0$HjDEtLaWRrs~uKyrP&gRgE zV?ma21Zh(fMw4w+bLpr&ZfR)2?P!yiZ57P>iAz8?Js{VnImC77!Lr|HSs&#j;Dggq z>Z8>q7_w=SP@LcswPPR+FJ!3JDi&SCsjv95dimU~#&@=6_!S8r869iq@`Dw_(dN`+ z!9hhA_;})0?*m6C28zL!+}_jWB2RjEhf?EgB2CIae5c*ssCb%lyhk+y_00tK^5-r{ z{on90g$tJ}2aH{&po3A2In>x$W|*}t^!9mI7sN6i34Ukq^9T3u;#Vl9%~p z$1_cZdC|2#bhu~M)$f90B&4G6nd!mu4&&bJ?5i9g#|*AU)nubsP&95B`Anm5hplv? z!5%(4-U!gQ8~T|nW>sCM#HbU}@dp2+MAb)49|jqnOsV%6t}aAc7Ni9K^3Wo51c*tR ztIPEJ#ZNgDAcdGGDS5GeAFfRHnYi-cMGsC+uOlcc`P+hNu2OII`8{#LH~lCs zP;OQ7fN3+l=gNJR{CMesE+Z=Ugy_}erVpQ%5OplwYWUE_%`nb;n)e)Y-xIovymfKG zrFYkOAE)!=uKU>s{nTO$wx4j_yYXgCTC1Z$-Wg*@6jGa__{A|Pb*Ah7dFS<{{ObRsZD>gUUOc@K5$NNsm5AJW8S~%A->B!Bw-IU74k#k4wAio2@ zx0{5rRXIP6PuCUc(uEn>=1gu#TGF@gS;Y`(qEc9R%WbUiX0l+3&&!nkA7;}hqemu9 z>|h=6lkt(hsUW{!aMN2h^XZG#YeQmlvfYYZX+VQxbu1w{jf&E<<((HI?`69cIS1dm zqjxm~n))h=0hb<4IVcaP^1}%?emG~CGNWPbojS9VlSOC;=s|}ArlJe;jse=GtLJeat*Zht?e#~ z35CQ@FK$7P1N;j^?-d({5_gqGBwItIjF34Nw~@TL<#|!CCiS}iol%ZG(M~Qnwi{hE zYgO9|xOVLR?7YvFM$;6-CK`td0XgHtjpt@p%FM`mR|vowFn)CU+mi_mb;fpn+!IOIMk_JUM{|UgAHpO}%;s zA1(?jb^YsP`R(H8Ta4$-dW&(*jV!&d;y+dXO7UoQNQ}?q!!x-S!n&sHZ4M@3bv9=b zTJKNdPD#0GVuIxN$+u?D_SE%=(vddR^^$|}jF;Kjaj^TeOjL8I*z(_p$0bEK7V4n>4ry+z1eMGIt`E>dorG~p8WtU zCkji9%5Lypq6UJ0}$mS=muvPD!gvteK~ zc}_%WIas{$ky;V7+sxm~cg8mIX>k(=$(}z`y%1sy-J5s zaR_73L+MQi(Y-{AarZ4Ad`JD_(_fWC!?QSZl{|Lu$jWu`n9UXTwjc4f@b$FyK7}9R zhk%*AF7Td(7B&%mWP<0JMmdTpVtM`gA^_R0v(Wc;{85*H&C^(Ia|yJyUv9q5BA z6e5FtAPVYXG8gGAJ50DL;@_UYa@)_@T7ISryh{vP!C1@WbC^*yZo}*YP6N2qp3== zk^xHJs;x5-huMD!t~tRf58^Oyd9A-3`{7i_M4|HxicbuAdN$9j-$kUC&PLmW7t-LV z*|evL12sAgq5Z2MeBbRln0BW##)i!Jz?~XdnW#yzaY}62epEOgUoeHD31n?kwdHCA zer8WFpHos%qU84sao35upK!W&(G4ci>Li0(a!Vw=I&jx3RI*eihcEJF+`=<<-=__&KVZV>wX$Y=jFH5fH-Q^c&Flo-Ek zedvHqf@NIRU}JD+1OIq&qR#%uv^9$U`Yi^yU*&?%VUgQW=~4-1kikrWM{a(Q;H=Ed zO6ZKR;>27kh`#unhw-0*$(j8-CsThdLDnh7IKNm79lj35iQ(D8(IfFYE&bbvDFX%% z=BvyT6P(xi`Gf=~<;J`Z&00PUy{J|wp|@^XGIGTD*(X;dpmB%OaMr3K5cwBe z=#g(atp3fa>q(XNqSsX=Q5S7ASZ=LkRr>i-)FoHkE`VD6t$y6zNsg@0a?|!7;T}hp zUfR*5@a<~J0$PG?p%788_nX$xE@5FIeeQpJgW=8(1@pc$>zWv7#t+oL=U|U+c8%*@ zMt$k++h4dBbi8Mz$V2Ju<_-i<3x$!lxwgg=jWVp3e-5HPXzIpt%qJ|pA79MW=XIOh zr@LBwG`p*Fvi+|ZaX2paxQmXlXuKpv2aH}eW1fGF5EJh+F_adNRa&1V3`fU{^jYrw z8-)#USKNDTMr=jXkfJ%5!TpXGC1MC9c`jKK%MS0GbwND12iymuQ zfZErEg~HamxH?6b?hg1lHaZB|TWaiV->ExWlNSo(2K1w2mcz%5KdexK-_6Z-Cd0xz zN^WkNOSg5M913Yp`yg1tcSU1kJ?%Y!RvX2)^ym*`$>k%?bJ8QLkjt-Q;QtDy^YI%7{i!n~WMv4jbGWev2KkG>c;H^^)B;A z)wb0tD>$X|?y}@233Ve1LWXs3T<6LxQ<0aF1mrD ztG-xrYg%a%0^;-S_OVW0tT(}rxhxUjh{ucx)SO22x-_x&UNl`_YPSzvv@E(V7NOy{ zW@jh6*LAe>OVTaauk5IF62QkpdsquYMlUnh!)_R^(~}JA002tkE#R`&cqWsd@sYBa zH8#}73q$Jzo%box%|7)YmacDZhdW_(Yr+4F()|dNoxzwpc{ti*I*gEGs|3YxbSwI< zgk1OEoH~sz2a{D^9rR=cr_8Sm*l@h|sJd z1P|2u8AQ*{5xcwRrr!1-u1>g-Wk{fVQb^S?ve}|T9ZGj46w~)y3x4RYu1y^hApdBv z^N;McAXNL&WuG;oVG1=QV+|P9D<}rH!?2@oI|#s$rL#CvI-f$T0Hn}9Um;dstW6j< z&*AE%9&CPDD8{Z7==_Z?jsK~+`Z{1cI$xrQU?ZGAw@iM@_=iKlgMF|vUW}*z%2Rl_aQI6|1;~e(1SiM z5I%#LeytU*+ET@4u!*m$ax`iQPYvVeDo*IsQJ$OGVNr!=44NiFv1-3U)8exe%03%x zY>qoDN6b^1>@ay(lDFD^i5)9pG((Of(A7}0I11-kqkL3oX{6TVwC!A$%4xg~@*GF{ zG%j^?w+>KzW`Gu6V;El;uBG$!nc_-fBEhD(v~5#EN0|0nT5HlyNdBvnOW7i*aPAl+ zTO~I}iu!+)6u8)dHC2;XW|*B!^fb@^4WF@|)AaMaWTxezkRngf&nMv|T6DjYD^pq8 z(9u@aRoNpJcQl*??AeuWq)Nl>;L4z}9shGL-8J}OOagqZxj+Fc=d6rpW zY!xSI)Is+mu7CR^ex#KACV6D?oi69H3t3y=6BecM?ya!vn%t^~HO{2SY3v3rlqy^$ z2h8s=Ux0Q}OhIczI8W!%!CMu|B{Q5?-X_2#d;(;@-xN6{jqBGL7D9v}RahE&P3YY5 zUtOXj=-tm1pLoSEzq<_H+Yj6$?hDgQ5v2vwdSQg~iN;n#)13{S?eBxAkX`#V<@CAi za-J!to!W?qV=i9VT!eyOKXfmde3A2F;EoXPNVhx2EANN(d6Ih~0V4OgKWm`JWs^@1r(g zvGqIqC^!btlqgc01_m1sBaa_UnEBOg|Vq?tL;%8;T=GN}e( z2kiuEwAW5x$>)=>PCzIKz!dsaqc1uWg&9pC<+%dE^)aswV>@c9x>i5i)b)L41#9YB zrBgW_RY^=v;}6OvXLPv(b7Wnb&)2v6bW1h6)XO)1%8O947iz*8M1h{N31@XYO*m^a zBp(Rpq&&zY)vRHsvT!aSY8xfPW>`csV=ABQXnV3R8gI8&H8x^@Z^Be28@FQ}dHlz0 zU$rNJPSw~0o1s~KDh2SF$s?7xk!K-~5YZ@=Ma|l2o30)h?|7^vFA$-Ocm2El^W|&yZ}Q^BVh$X zV|=O5C=x!=4#@+yl9i>3pION`QcrRPj$@UI4v8ue3=4NihMcy=J{H|0h>dPLLQhAE z`c@Yx;#t1HVC|kcAu<2xE~qRQtW}FvX8<85-1Jtk&N1k#()~hgD2)fyWnU84qttjQ zJ==LDp5jXn2>9_BHK8I4qDqjUe~1`N=D+laWM$xp+yp6VFCSIIkVURgB^O_&O27s8 z3xshM%g#MA6tm55FB6^2>aaOF>b8|QI+`rvt3P`_w6nq-AXSBRQT(bPFdIh^M>IpO zl-{MLaq0Phv5TT`P050FP*7Lq_2{Y#A^+o6APo73s%H=~Xq?Gk9H-0VM{Us#$mGvZ ziZo{3J23JT7Z@+lV+a{^-(i#UIQgm=LRvF9-@+9LL(}-Q>UTG&&dGk-VQ)qRf_mG0 z9o!kJGJ6WG!x2Hy0sYKwWWmCB?Fbd8849}#ginBcVeG9gst*o5bDZzPi1fh)Tg+wF zR$E?p!m`tZCzOz?S}`G?T!}v;a}610MHLxZyMv0g>ss=@M{Dz}{jANYO?`WIiSNdv snKT~FR6Q*^3m?Oe)twfOW(kdzm?a(Tp7|G)C{drV%Wv7;Xkz#O0JF=G4FCWD literal 0 HcmV?d00001 From 58599f789930052327f77e7fffa843efd7c6fab0 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Tue, 27 Oct 2026 10:30:42 +0000 Subject: [PATCH 10/10] testdata: add a snapshot of an NVIDIA Jetson AGX Orin No snapshot had several cache instances at one level without kernel IDs, or a cpu_capacity file: the s390x one has caches without IDs, but its only shared level spans every CPU, so merging distinct instances would go unnoticed, and the x86 ones come from kernels that report no capacity there. linux-arm64-nvidia-jetson-agx-orin is a Jetson AGX Orin on NVIDIA's 5.15 kernel. It reports each cluster of four Cortex-A78AE cores as a package, so core IDs repeat across packages but not within one and are kept; every CPU has capacity 1024, a homogeneous part; and no cache has a kernel ID, so the three L3s, one per cluster, are told apart by the CPUs sharing them. It is a full capture, taken from / in EVE's debug container with this branch's ghwc plus the snapshot fixes of jaypipes/ghw#480 and #481, so it holds the board's network interfaces and the DeviceTree properties ghw reads (the CPU model comes from compatible). The NVMe namespace's IDs and two app UUIDs in /proc/self/mounts are replaced by fake ones; ghwc reads the same information from it as from the unscrubbed capture, and the same CPU information as from the board. TestARMClusterPackagesSnapshot checks the packages, CPU model, cores, core type, frequency and caches on it. Signed-off-by: Mikhail Malyshev --- pkg/cpu/topology_linux_test.go | 48 ++++++++++++++++++ .../linux-arm64-nvidia-jetson-agx-orin.tar.gz | Bin 0 -> 34761 bytes 2 files changed, 48 insertions(+) create mode 100644 testdata/snapshots/linux-arm64-nvidia-jetson-agx-orin.tar.gz diff --git a/pkg/cpu/topology_linux_test.go b/pkg/cpu/topology_linux_test.go index 4618f814..f72fbc2f 100644 --- a/pkg/cpu/topology_linux_test.go +++ b/pkg/cpu/topology_linux_test.go @@ -300,6 +300,54 @@ func TestHybridPMUSnapshot(t *testing.T) { } } +// An NVIDIA Jetson AGX Orin: its kernel reports each cluster of four cores as +// a package, so core IDs repeat across packages but not within one and are +// kept. Every CPU has capacity 1024: a homogeneous part. The kernel numbers no +// cache, so cache instances are told apart by the CPUs sharing them. The CPU +// model comes from the DeviceTree. +func TestARMClusterPackagesSnapshot(t *testing.T) { + info, err := cpu.New(config.WithChroot(unpackSnapshot(t, "linux-arm64-nvidia-jetson-agx-orin.tar.gz"))) + if err != nil { + t.Fatal(err) + } + if len(info.Processors) != 3 { + t.Fatalf("Processors = %v, want 3", info.Processors) + } + for _, p := range info.Processors { + if p.Model != "nvidia,tegra234" { + t.Errorf("package %d model = %q, want the SoC from the DeviceTree", p.ID, p.Model) + } + first := p.ID * 4 + want := []string{ + fmt.Sprintf("0:[%d]", first), fmt.Sprintf("1:[%d]", first+1), + fmt.Sprintf("2:[%d]", first+2), fmt.Sprintf("3:[%d]", first+3), + } + if got := coreList(p.Cores); !reflect.DeepEqual(got, want) { + t.Errorf("package %d cores = %v, want %v", p.ID, got, want) + } + } + for _, lp := range info.LogicalProcessors { + if lp.ProcessorID != lp.ID/4 || lp.CoreID != lp.ID%4 || lp.CoreType != cpu.CoreTypeUniform || lp.MaxFrequencyKHz != 2201600 { + t.Errorf("logical processor %+v", lp) + } + } + var l3 []string + for _, c := range info.Caches { + if c.ID != -1 { + t.Errorf("cache %+v has a kernel ID", c) + } + if c.Level == 3 { + l3 = append(l3, fmt.Sprintf("%dK %v", c.SizeBytes/1024, c.LogicalProcessors)) + } + } + if want := []string{"2048K [0 1 2 3]", "2048K [4 5 6 7]", "2048K [8 9 10 11]"}; !reflect.DeepEqual(l3, want) { + t.Errorf("L3 caches = %v, want %v", l3, want) + } + if len(info.Caches) != 39 { + t.Errorf("%d caches, want 39: L1i, L1d and L2 per CPU and an L3 per cluster", len(info.Caches)) + } +} + // On the i7-1270P the E-cores 8-11 share an L2 but are four cores of their own. func TestHybridSnapshotCores(t *testing.T) { info, err := cpu.New(config.WithChroot(unpackSnapshot(t, "linux-amd64-intel-i7-1270P.tar.gz"))) diff --git a/testdata/snapshots/linux-arm64-nvidia-jetson-agx-orin.tar.gz b/testdata/snapshots/linux-arm64-nvidia-jetson-agx-orin.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a19ee7f170337656820c946c1ed34b3c15b89159 GIT binary patch literal 34761 zcmb@udt8iN+drIbxoonz$bM|gwy{r?GrH)qNp_WxK}d-~Qc0Z%AsTcMMS~D+5$$W* zs3bL#nz|~ZrkZN1X_{s~=Q`ds-1l=o_w(M*=Y2n)-_IY+oO7OQ9c!&)^F6-DaiRqq z=>7cDuQO}6=Y*o<_*Xo6;=VsPr)}u#A2^w98xc}1f9s!IwoBgQl znYcLAXl-PRh%danp8xRRBqDLCS&4nxa`V-DVpyLel!ijH%H9K2l}p3*gl_6>T2JTu zLJ4oq!uHZ@?d^jsFH2}4VI?(_Y7Uvh)Dz`le$Hxc`QNh8g z^=WYlHYNgYY_t1G|7{^*CQ?yEannIx`}P)^=b}ABMtN>aILxa&;lyulN1J857VBl{ z8n=b%s~*g|B5f!>vYcHH#YX~8B~82+8`|FWB6z*8um2)?&><M_%MM~r0vG|C+`-ulrFdX)-ce*JX}=>-XSf;mRg+kpyJQ6^%;%!8G@qVE;)%Tx1vVI9!M3ru^bbzTwZJcfAU= z9HVP$YTELT@%>F>*ZZz=g4M@&l=n|e>(oR92TKm5r7`MT*)d7I z@6tNO&E4)6XEWQHW|X|OaOR6sIG=nq@=)0^{!qtCp(K=l)Op4(Vd*hibhC7~Nc3pT zdgQ}_*oS2=K#||+s_sg&s1r-7EX?D@>ZSmr%7f>;IqF@-PDmRNvAZG;olj(HTy~kA z%CF3??de#mwRkX;IF^+fR2$Ai^M>zEu^jv%v^L=7m{@u(%V%30LjxHvC$vdVDcy=( z0vtI_jFJ+G%U`GB%gR2heeX$C(rCBd2iX~&DY5Bqe2~vN6K}DMK$ep?e`;4t3L<8# zoZr>uNN$Vd&o{4BNHoXz6uNKPb=*HSW%J_D7b4M3qY6n=N@b|yOZyi*#nHXB;?jPr zRAXDzr~NsM0QIr%qSI-Uo@H>mIu5U>3y2moYcts#1~)~lP}Io0-m6{hAt`(I!t_5Y zDc1o1T6c@_=M=T-b=yS^JL1_Tq;puP+U3n4e`;_0lH#(@P6?59sOZ$F;q^=I+KDxa z*oe01PN7tky1Ctq%dk0nR&fp`?YV*G7zsGsF~Je4_J-SL>|n@Dn@9vKYfs4-gN)@| zW?dcAzc!H<70)eGC$rrjyc5fPc599+#BE+_Gq0`aY1zz{9ov29$k&g~9cH}o78x05 zPvs|RR%JEtOSn;w%(Q2c?lbEbFps}Pep0KJRo_dG#TKjNK}V?Y11XByI4PX6NNYVY z;v-2#e-sfywc+_9C4)Dg?_N`|RIY&QPd_GW$n@h)HDJ9!W3hMBC%TE)ZgTc+tt_R$ zXInyyNWwQx;uHpNeXo>LCzd(it$f#2Rmr}$w!UB-ozE&1nuUpZq%b)2W5t5)5yu}d5}3f%+aVMw4g(x3NZ?>Hfw{2`?q_->AWd@VDdAwaTpx)enWukHDDTy4Jtv0-q6~A(E zbK}`$gA5YM&+aj0juE7Ivz-!c++sfp`Xze_TPhg*B<|3Bo@CCMqaK!PuUSmN^fJaiczk@x#yde{RR-4CRsE zZ-~6lPRi4W#hh;}vq$rpG3ESlKQaL(@k4^R_Nkfa*%?Qrr7dnd#SRdVo$iOk5zH%8P?5}omCt@IQja!>ZyJ z)33y+i>F_@rYY_>(O!_vI_h{a&c@@)=ZK=+@ER4(eCO4Zow|2uLTt0t#)qSQdyWjN zGfN?;@6B)68~2}F?4|kGSeJ_RZr)y+!_1^+ zDL7fsIODZyt)}=J$HEOU2rEyntM@p@4w>+zW^ZL<`h#gCgRe$i^m`@|oYTMYn-82> zOtUEUFysomjA;Yvq7z(c(2zOo+h7)jP@!gSO~OgwDOl$4W$^8@eZH$)0kvn#39 zJTiNjv}yZNzlQzdDA;JSvcaK9KR_vMxezU=+_&7fZ5&b8uFNqcv+pT=PA+_DlTZ=1%FqtwNE+C6 zf`(UIBYMDsWoCXBDjDC5Ymq$*aru=tW)eV6YL@whrmR`cT9 z8N&Tqp=YgO3lV?R|AylNy4pVLe$^i~-kw zhb{jfiWcNv+l8F-RYy?=HgwPI4!V|nF@joT748p34mhMz`AdiL2IjY{DD7@hbe zo%6wkUY5G2C#P~k^u~$rG6knEn^ut_-us0GMtgfkGZsDGPaLtW6;2C^^Vi>FvhQY{ za8A*#jm+$$lfOuh`7X*5EK2lDnb@8(>@2*9WBw5m?iHZr<|>M|O_j{iM7 zb&k*H=A7yA{U#sn)BSNSsc)*9lwKuF`^d{}-)mBGwrCG`<@Yt;-#B)Mq@VWgSehZA zS4|@CCWJPpl~pyT@4U6LXG2~2y&oZ|@-f-1Enhdku{{v+VcE4~BV%mS#o6MAlfUnu zbZ*hhG$+$4&kOmL(~i7UoPgqvQwvS=uRJ$oZDp-Gr|-a}{Rq$sE$_swd{sKwr!C7b zcb@O1apjWMx5tS>@^YumDUKpi*xEhC=D{)k9uuEDf2o0~XO|u0UdcO6634rCuq>T% zyTqn(Y1ZrMFC}lgQf}luYxUt-a?9ifHX|D9*Ivv&>+KzO|M=dU8?9(iZR?XLw%;j0 zzwP}zqW>A5Ccw`!!eK5qJ~Nl~MyOgreB1gi#q0f#t4zy~jjgrxZ_SHe3-%sYYgaH{ zY4qE?I6qukwNoz8tZSyMr%gP;{?qSDH6uAuUb+0-$i1b(*He4yOX8o{90_?L;IxfYB-G2T50Bz$+$XLYl-RO%l6AoQhx`zx zFsZC1S7hKX86>{+AfJ(PAyQNz!8)#=5V7RLH;^z8nkEdoaKk}cOZ zE9w;Y#N>Iwtk-GleEy8^TX|WEo-e=L)4-fGS`oAN!u&daa;nXiZ=F8w@y2g$94347 zrSqI8H_z(pm$~J{5gILTW;S>4gN?V?{9N19iC}N;zih86ID>yujXZr9J01Hp++yOW z-ZMpnVYC5ax7W9%9~b9Ij~?r?ILj!VOmpu&Za!aF5?kZGbZ@36j$cn4P7+Aam1e(3 z&s3V=<{-hUgJ-lsn^%qNz7Bppz%J8!A3EM>lJ@m6x|i-OsEL-|>2Kx4iPi zs)_ZZclpj2B}VW#vm>YVHgCMCxxm(}_d@Eu(*0~1PjzQzfg*={I_bIEKD#S_6)#b7 zXk)PS0$VI~SO2`H^Pl(h_Sk2)=CAse*t<0NdiRL*X^A~vwC|&RAJKRJ`H-J!wY{+I zd2PsAX~JkHc?%j^htknrS7Z8hi&6~elb-r`-tnS7j6Z%a_SDH!7gKl8+#1PM7%_V( zXLqy6+eYCMo9?k75;drrLfe0g7b$rc&xCM_w>`=-Unu@=xnub`BC)nAxTaA}sY-i@ zZ<=Q+>*wDq?b3Z41C5WG@n6aaT6@GhLCFCvo3qt+^NOCJoRw((BfZZj1<+XHqOFzi zin7*i(N=UdLPzr;7Gx;I-=T z5v;w|wuA5Hu_~@mEF#;*6}y};K&gT(KjmtI%00IwI)c`du{(V6%&uYge3tK>CLe5a zbaE|pHIDoOCEt)7d}x~+)Z`6#n%VVK3dU2=ijs;(UoGJnTxn}=jyw-(Y5a~TA)W%ceE2PR@Sv!Pa7w4-+a^ZTo|rtzyKTn`EH6_3Eq zj9zcTGq}!)2k$uHCne!)SFt%^#7NSm6l~5W+<;KUyNJ;#kCnTna659Y0F;quLQ#Gp znXIIV(iR~W9>0y4hmsNh&A)zbWQt(YKYq5k5^+l@{MYX-^?RnzLdjb4=XBZ8Yt3}i z&xd~u*XH&=zB}RX*U0@PP=v)HU0$`GHq2Rp0+9IGmR@qY&s8|L(@Q?CJA~t~!geg# z-%bEuswQj3BMH-34}Z^SOpNqt4LLbftYvNs{jLBpTKQqX5#LicV6cQYLco4*h#+6U z^flLAFvAywzGf08XIUXMDe_W+0mbzdm4$ZlgE#W7gXb$}1ICLf!( zKBT0i6C?-5d_fWV;3}kTZd1Ez=qpkEus34zV>MFz6;=y^f_X0~;%_ypIe)e@c)+0| zPVMYuSNrn^I1XQ1nz52B9P9?t1Pw%cz=Ucvm_?>+?@$oGdw}1ke~xdmo=3r*PO@Ju z*&q?0xz>q-wkLyS^RLI3h2c{1+0CJ z+Qh_|6d9LEFREBEXow-2cd3<#MpHYkL$9=1+j4cj)p->BkRlB-1f*q%CF0gV5S}$D zwqZq|iK4|=|7VoX{y9)YO>b^%dge-dN%ztIP4Jh17(y=~SM8oSz4vI}UapEZJfo4b zyi(;>hZr%xMzMLQ-aYCu8DIT#OxyTU_cNK3_gQHN7szROWl9Erb5U&5k)c*rTAX?; z#4;VTL=*N5wOLq14iJ)4Ek3|u-#T-MRu(bb%HV*{c$i#CmW~#|n(EOc<>3Ft=A9@( zAZ~`ZBGMGW3|{FY{!+Va#0awREF^8TEWhFRNo_x!-9s3Z6#nBwyr}Zu%0+9DB&~DQ{fJZf> z$%;S~4e!RN9xTZui8xow+DPFKLM@9%|3D<8!J`&HQpnG-c1xJs+7>2`cuGdAxwR^y zsdyoli5cR1^RZGt4*FZV6#{i)n`wqcAD>AiM@sU9aDqzln=Bw=4po*e%@icsf1-%7 zkwxTA({)AkiqYS|Zx8stQf^v)d-kpKJYbd*hc8>oR;DwZl1d0 zz)l$5r=vUNy;V`{T!#ytqq((kyiveMuD%E2C=i@m{#?ZFac6aTmS@CjCptAovxLlUu0qp-a1o$3dgdCkeL;tHg zth{RXr$+o5`8_m@K0y&Ih*tMIQJ2LSiX>)k$DN_rj;A>1NHj~H`u6OnpO^5jgT%4k zx{>W9BugnZb1n+NS2-+TG`ssO4R@Z*@BcQ0!=S=$AY#bMM(8drcevQMhm;?LqK#@t zQUC7a_RcJ$25zK89M@IR52+Iu>xszKSVM2-gq#}+$_aPhqUo0&NjOLMvCiP{CL1bX z=zcqG%FwD@9*(z%mxy3>_f@ic;O+BDqXD&~L8Qb`Jp}6^cTpttnK}iDhAvyh?h8sD zS*nKN7X_rZN%YY;f!GXOuM0J#5zjfR&kas(-;zGOZXOwK@{Q;(R1!}6q*?2%P-0@K za=O7a&e`;w_7gYHYuVje`VjphhNbg9zW-1$7Vpp=xNxFQ8LJ;ykYpH0YQ1iIBK5vQd5K~>n**sLn zvZ@c?e(YtytJAMQcJd;#Zwri!V)LM0Eq>lF*800%gIZjGSnqt~PCLn6Z_P}gqg7>L z?W0}!ja-_Mojq=k>0vWzb>f}O{?pLvPR{oVM7j^*hep$0H*3E52Y(}7B@C^v_fS6| zj;Ua92u!{Imo-`3`xL_3&Vbz;2zs0^Cl^h)7n*w_VDYPatF0G*wY;jVm6BB}AXG|@ z{kA*4rq8X7L26Fq%}?v?O*<*D;8&4V55bo5wwqLC$mTtIExtKrw|nM~v6?rjjX$viS`gD<58R#v!(@q( zk!Y00>rTB@qqrVLtT_oLmjd2_ImNzh^&TQh_dRob{5Q#=NEse#^_CQO{~L2$^$vbz zX*n)nQ>JC<-37}}Ye!7U`UvqVIIAsEkp}G!uIuwU=%ZW68g)=J1eN}H=9tq$X35Ez zX|*ZfX*1}2dyhrO(BTQN_Z|5DeH?HcGuYBe1o!C;4M>I+>%UT+P`*LWfJf5*1`itz z`!!{wcNA#XSKr!O&Qft4C|0Hu2(=8w*t;it>=xkJRl~qg2vZ*S5WyGlC{56*6WUx# z^sRzhwnPtvP_%FrtD=^$~jEL*Kh#7W8xRT03CQ`xbvdM$7G;21&?n$rcsq zId(|fXnbcJ7!CHS9ZS~qyR{)m&Hw9MN;}6MR&MPDmzNqy+u+s~_UpQw3)fmyoUz}I zd44ymq~A4_!ImDc_!Eh0=DG<<(Wv%}gxfNx%r*FN`p7&PKUl>+U8FUn*uz(RBQ^$P z_Z==Vbcaw&d`j2J>?dgD4X`e@@dG>={SZYAF@E0X&qZ0SAkhksoUTSmP5ilfBS=5Q zm%|KmAuM=}*dAC*Uo=J7L%f=?{zWPn=Dk2+=zFVIeBxot9|_1YKS|9{oqD;g3e@4P zTOeXSL$1Y)RdgBNk2OCKtw!ex?y=4z@}7G;?LxY}N1y%`PmtL?W(L}g*e_-b|L=NF z_$r=#0evMjDe0+*TaU5*RKf_x}9Epu!Wf?@l zsJ<~2-(%qPtiF{<(+L6VtV6ekwx3t{TZ+Llb> zF!^c$lvEM(fbbY-lGS$^h~C5Y(KXo=d*ueb9bv-#%=0HG#Z!t~aYnM2S6+E>@hC2~C zYamTCAk?*YIxu;V$Z^G(Fb?Pucz@O8feW7?B7e%CiP$ir!E8*s!y7eg-`U&X1SaEO zYB}aZuJm62c@n{w(~sTYboBZ2xi!VykiA=m4gAqJ@5gAwHwyMVf7iS5=MSqiLii(k z4|d$gBc$1{euj>qJBddx`&E*Q|L%x|G-qv+??q|Nw3i8&@ChA2CiM%!8au{eGVUFA zqTV0cz`x2^-rWci*-%Xl6w)qH)QPbwQoPMwErEB}E9w z!0wJGGVCXU;uk*!bkYfU_n_?7dA0uQSMZ!;cGg)G>tvW33zgk*$T53j%wIB2wlOx8 z!zE#sz!bwPDS>16zbpS&&|h6YCt2?z4hR!UQk-Evu-w{-txz9zT$Z+f@RWNYz2q)< zomTi$`+L<|qKXtvPJoXgB#ug&sho_Na?y;83m^Cyl-cDf4;axKMe$D}8aDSWop5;Z z*tXFx<$NaaohjE4$cTOb+oP`=TRp+&m4s~OVgHjf?yuhKdz)aAj97xP$d;7CrZ_$0 zG5_xX;9ma_q>YF>X{O!_c4Ge7Fr9Oe6Lz(o1ed&g-W{BVJZAU4qwK!=Yk=j{*D6)# zW?fhQ4VOBJzB@J8OJ2ms{Tbf3lfPfrkW>G6|367*0Y?4*Hw$q2trWZdA~j(QZ3&1~ z9NOICSW+r33)(;rYJd0CNjdi^O~rpF}?{u;O4-2KxR))GWY}bQTo^p)@rbqUh=q;lp1dABl|5G-s+agsV+xJU^sP+hJ(Cg z9_Xv}{$VuC-VixkMoj&~14`C=nP*M5a}2C0C^)sr@yHPM8|1PR=8v=}S-%X2Irk#9 z&J9m%f&J!mZtVK^>X91T;hTsaf}*|ov!>5*TUrmtG`XGACNpXw^+rfZ|C;K=NPJW& z3UXE?{FZyrl~z6A)<%3=`M4suef0svHox9;V_<~W`D*dyKd|pG2ioA4fEw&pcnA1p zU@jt7UyT4_e$l+aRY;7Syxr$x$Ol3vS&sr!cD_@01Ib%y%i8+ zF5g;%C)wJmwli0dtMNPpD|+sjOE)&sb-|diVKGc^U8_m7&!Ai>eS`8^Se=F`^TL1$ zAM)gkWTYua6)-AT*$L;CJs6S@CYT!3cSNg=hAZRje2ro~unVXR(Q;*k!9po?y1$+k z9X4+e?KYM#62=b>ZSK%BctKOKMdX{S_aQN+{D{R(YM?0aP@OWil6Ikr5i{%zDeGHbj zo@Z`%@rD7H{3~9EfezzkNobewCyPOg#!rOeDuU;Cs3KUK=p6@NVdxVh729O>+9)i% zkSKU+4%a+$PPMH?W^4U09x%G3TXg@?!T6sLaLJW$bekg!QIS}*$N15exleI);!=$A zABbLqbwNB9JSo~5d=Qy?;$rNVx-1;K94psEET$*uHb$tTy=h&G7@rSth920D!s5vg z#i0s}`l8TLjQa4Pq!VQpoKu7)V~BOj8j$C-?e`75`KFpa)9?UxJ7MWayk0MZR3+^) z-h#>mcyS7E^}6TLo2XN|DU`$*|6Qr>UR$Mm8flwgDnG`$Lcl8_okbR1u_QCTC6&smLnw^6-U_)V|RWAEzr7b|g;3 znbH1NM9!f+EqyoO$GrWEq7AW8a0G9@gMYZFlj1)p3(mlQ|Hg+$v7t%nbG_Z)=DF2+ zzt|Q!_Sohd>%%pNI_{p<`5XmVFzo;mCWt403DPEBmn#_)@DbN)IF)iFF6|O;eXs33 zE%5Ge-5jzS-(Cc7up3~;Z0!i6N8)gC+J?eT%D`}{PlkZ)=krIM9`>gecA*gZSElj2x% z0sCxrQAN4O2u{+E0XLvgz6Sfr` z%UOd%skfn~a5cnh+24q5VDK&8yz)wW*oH7QV+Xsa^cFL|`)PF)#8bqz^7Zv#CjOqq zS1~AUxX_clEK>+n^EwPp>T*x5UAhYj(hyTHT6I6;@4q;8&*$SUZx}Ou2kCGGnH5dJ zlD(UXerBA0qm%=;z@q|EBCKE9LBVN1M0cU+LyVhy*ay#+L&+H>xf3R=3F?G{%jTH` zHfRrTo1a#i5g@uWpd0 zkCJDVN3}#8w%n&uOFLD zESc0!CN)og4D9D|I>2#?WFN%9q_F20k1PR~Rw#Jr{^?tc3j3}#zY(6`{ z1M!YRw8Q$ht{@B_WZ(s%U;Gp!^3fw?V@foP+c2`q8`0M*mOFH}-ka0UK&lQZU0U^ZdKS9}?!ITr!8%F=T>a+e!^#L#U z-&NlS(SP%b-88Cincl}|IP%MgpSe)!V|+5xKpX?Bu{wIyK2#L_N3O;F?m<5}*)^-r z7U#&IENp-O8E8)cVYVEO%+$Li;uv+3$x)PDHJX0@Js1|~lCB~xBNo}L2KvCLMVrx} z7Z~I58=yf;!(@o-!dn44CP<*A;e79ZRs7Gp;4{=Gy@HB~{oW2Udoer}Qs;ZX_k&qck9fJuC^up^_*fCFe04 zb)fqvkjISB8A?Z3srmo+couUmT-u@l&%0m0((ewFab~MJ*f7fF{x42|8N-<4+zwBg zL_agJO)i7ZrTw2UEhWoQ&_2wMgl?j%NeeI^KPegJsT99iec(qz3Pr5XRS`qtz$D_Q zB!???naiE#K{D39?XZz#_NG@kCQUx!2(b-LR3hdZXNyRG+n=lo9aA;mVY?6>s%>&~ zP+7;n(E3nN_Zpe>_lm9VYoNTG%v8WP<1kI3G%)l@njmC_G6(o9u z#9}j1j6*7h(I8CGeSx_V(?yWmqq*@uFH6gqb4(6ha&l6IRu_i$MJv&eywT})q zY9u#F`4b8uRl@iEl!Q}Y4ZLeWj3LJ6E{ZBc z`^y)II}D0$mlDhmBKVE6I$Kdt3OMi>Y?kSZ$dqQ`I0q_4OYAC#;D&OD3Q$VQz@-dr2G;MxsyO+% zU6>lfT-+7BHNEh$WgoS2Vn+ubcwYUibfoZ1?5lJ`2Re7EnThZQnGE!btvvF1jMW^! zUgpTGdzihfj7yLE9<9}BZIN-s86QI8yB{j?*lo-7o|?xX zk^bLaLotacGW6)%bZQUfSh7ED?9gR`^BG6`_rj>PVA9{6v8<>HqMK9S z(<~kX^EX|6;Nj5o2SZ{&B3vQVe+bp#V7pXAxU^0|fu~f5R-8M9OEQ=rN3=_(jRf1) zI8fB=ZWV;%$NN+W{f3aTM=&MQ%3x3N-w1INP!HP$SW)@g2<7A=jA~sG; zj9mv$dC=7hspCE*K&P7R^yDRJP>NU|UJtspfhYtwKP~$N&M0veMR9n_*SVNk9*1Ga zl);Rgydjq~cwM5t(T7P+u zc1QGCpNKsmhobLS{Q z>aOWhK7q`%yJf2u|FAv}v%0-5RFH<_QvMzpCE2IN9IiMuQgio?^OQz0f#we3LT(O< z^hmD&m#O=7v^CQOn@nFtz$ffpS&6Rf66v)PTqA^|1-NQ|A18KL+(S8(jD1MP-VS=H zXl?{XOU=GGQNYmuqA4a(JOw`vr^grvy1D2-vEat4gG{1me6pGWF1c}R1ws`5hTbl_3{`KR$LY5$ms;EKNUx@94Wdo!G6qN>1LkPuA8|U z8?ltsS%RSj1gV35DG*t$BFrgrcq-DV6Zu0$e8vSHuhlwn@$EQuSoeh`!$2xz?Z4)f zF!t;C)P1DgKW@`$2N!PJ2I*FeZ$6)+rQ`6f8{7idDtII&jZ{P=jJj#6Q}GHCo6`d* zNyWcuFtM{4^dm8qt=~Tq2eMy}VJW)GVC(vHe`F`fC5kuQj!8<&^k2@+W^NXB)wE}6TWH5ZfFuv06r-9j1Wt6^+v`<#f7UV?5E z8&jj;@5v_-OT?MATpng?qbBRLGUFoZmX3ad-(gS6-$i=Wvz6bb^!zbqEZ@jB1U8mh(5B4y!1K?teH%>Pjpe$o7z_$ z*DL0zWEmu&js7Y4t(^b@_2Sny=p~iP@V;HdqgMXJ{a}+z(G1iL(1=(&Pd~%Co6k&~ z-m{_R4FqC%tkZWJHE|B~x#X2E9hh*YrS(Pu7t_D3F_aLoczvd!DlO+I=u3(IUAXp% z@Ad(kJSltXiAsCjE_740UefAEHRB&&9%Ar*j*j+#H%b23-1j z7g2KZT@}22s1Dk4a6l#fepjzJ38E^&ZnOr5Z!X($c(b(Wq-JeGyVdW;W+o=S;SZK@ zwaLt9#`eB{{+uAw$7i$hYVMr4n;nA&vg?Pd|kw zEZeS=pnowy$w|dj>7+KchWLm2oaulcTEqS!Ne3%#BSSvi-U##kM&sb6&f^z&LE1AP7i~`cvPZ)i(d^szI+9_0IYu?!y4>+5-Hj-%5@wUE*i)pI>1%^Ig?? z%S+pzD1u(=D^euGK_-osuqt%;COm;mob~}6To%i7mjq%1CTaE26cnd9N{Pk0lrkBT4 zH0&a<4NO1Q34c1v$!9DLflwkNK&>QZ@p-vhWgsk79@b`i-E7%vp}@U_kP1>Fy1fH7X6q&+1?Z0zloh#)S{mL{I1?Mb8 zGFYLs{y~G!zsG?=t_f1ds~_`ci@6stPP5+v!`C#z8Z+@)4~jEmV@e}qMjfyko_IUz z`)sjw6cnsApGZpcIRxUd0Lj~1v=Jn=WT&^jh&YlBZ7dRbQc^1D8IqB8Qb=yAT7F}KBG<>Z{N&!5R|NSyVKj*Nx4|4q8Qnge63ncQr$&-5*%1PQbmjZizH^1 zU0X-i*+h;?S%M*sCR&UF6iMn9A-=JmAOaM-rI%Knt0J2C+=$tiZpVnt#>XU;pL+T2 zNWy7**s_$^A*1csUhtsHS=mMmYVU{J{cfmefjBh)zDnM|Uk}GkOWYt8mkLNK(Ub9o zK$o}PATC)8IBknN(yqiq^kK0;Ud8?yutxM@-!nb4THu)U_Qv&HmgjT3sWUQhKmPqk)tw?lIZY%rxnr zl@JoX^>1p6t$jW_ASTFNMo87jZ<-&!O<;OYj|N-Xz6O) zVzzL1>6N5w%~%b2wqdy1Yeki`OrhY6XV=#O246UvouE%&89l;^;?Kk6YXa}Jh>v@H zUW~(qw=aKjj^y<+_)#vDRLdN4y1K-86G#g@PA@+F{=Ab8W=99tVT!9S4-SrcK@$2e zZQ*_0ZWSR^(()+8t_Y%t;=^D^O+Au_vYooNx4-%AVA3`p#WmlZ#vUGzl1Jx>LD?CG z8wQiJ5BXn^rF8gxFtLK~etO|Hl3sX@%hQOi_e7#7)=QT2TkiqkN|_CD04RLKc+sAf zFyT!6`yc``c9h;dzKJHITDjs$*=mJnK;3>exT%O8c`AlM`;E7un;xS{k7(ZgTZy@t z51D1Ecg=VUgp{#XuIc5fnPaOv@7+v6Sr2L6)iVArun)&x?>;&th$RVO(WI$r$2u5r z!Hey1@or6Q?hBZN3=d&?8d*V9xFCN+J&F^T;W$QIJPp8Q-5W)Fs51beiYQ z_!(z@ox)Zb4v<)-NZoGlqzn_l>d%*Cxw4&>3~U?*u=i@YEaQK0rM!z4mh=E}%*ZZD8tu-GR64TrX{#CL=e73_T_ybWNh2B%%bxWEp<1nZtb%Ti~&1 z9KCwR$6Wl}QF7=q%A{DXW<-wD#oE2FogN2nC2Bq=V0@g|3n)8MD`kt=Vq&8#5d=v3 ztxl^TDXqwxLuwJ581^ltf?N4=4kV9fm~IGt#$<|Vf?^ug6CV(_VAUrsKSCihD0g5y z^q;V@OKrCLJ!b;8kvKSpmltTKE#2LmrAWyoknHYK0c*+(luhX!MciE)a#RAiyzYVl zCh{iV19r)BBc@T`KVl#X^ zl!?eMV3?bNY%Hs)K3NjPH`p@Wp3hRHdE--YfIvhu=!H+%qUw3@LrwHQo_E{!Frf7tAi@>Z8Tj#%vOt^q98Xm#N(7{e=S6p)(q2s?1mJv zdS7?Vs$SmvIm((9Z-ec2dWsm4WV5@2hd756n@KH9l=Lr=QYH zyr4VGFKU{uTWhjYjk1-t3;&?&nwUC)MtM8@LRMZB`UsO~f#UGy6$Ou^%ygF@< zLs&&_d&#*>7+)pf%eAb97BISh7domKX4GS@2PlzR=`kL?1V{q*|-_Hp;Bp}Yr~B;eh&RETB6F+OR_CrF2pUo_l$9> z@=!oLbiQdMXzMQ7x6QUF@Q{ozzreSYq@OO&pPA6!_=$Y8r(fW`($4{(8{z-Z8#rE3)kYFDCRoK`a}qBehxddShL$SENQ=e zLJYUEJ-BwMR!E-C#gQRZJ9mN&rQf0UdA2n*1%hyzUna59kk!aqN064O^)=+F1l$(Y z!SQ$~W67agOJHO8h{Znwh7=6PRIcq+G!@Y>G}c{rhJ9ct`^gP27jZTM>$r$`f~KB( zXzCao(GU}ETQkhy!_S9hhPmbl4}yD>a+(=6%iH;w4Yz1aLQV%OZjzAWT1>*cqbyqR>9?cZyyevl8fco4t7T= zq5(H$qWca;l|CPgt!9;}b?d~=U?TYrf@JI|58pN$-Hbw`-hoN8ijP$QFZVSSWCg5W zOFyACocct&RuBi9DZ=R}mAH?h8?1XivF~qqHy)|UT_PB#0GBOH+#$N@=bj{h^_KRF zYNH!Jg=*@Q;f(XbmtJtwyA%9V{w}!uU&Ft~KFA#X9ET%Ixk2x%QU^5@KV>%n&vR; zmFE$~*>orlH#N;h`85aXmJ?bJ_Jod3Qlt&Ks38`Wc9)Q>DzQw1KpFmp^wY?^D52~7s}D#A%xzHK#AG&V)vVmvsltUAh3^E z!r8huPv>w^7@pbDG5DnUZ)f!SH<7l~&-l8gSG=-6PIh)lwcbN;+p4b$XmtQ=|8>zzJHV9LxZhNyNR{#Cwo9%Lh&GZwj9Z zw}SqIJ`*ncX>j59;xC2oBy_bJL7YzcMLO!tWP&Rdr7JiMxPXg2%SE_p;0Z2#<6fCH zy@nFNxx3CIbm=?-p5vDfa8{G2cf%l=VT0gw+OE1Gj+Gvt&R9+PX<9!8!6t@1d0!kG z1TV=zIXt_LX#M})!{A#|k^Y4wz+RL18}c!+doXC5`8`86EQOz}Zb1b)4y;jj6eBNJt8>}SINlNcD& z-r9nmo~$CH5j*ZZa@6%WMr|9`XTUF;kC(v>4J>?_f_q$y{{REPlpXHJTTKndZVzuo zk6evg&#jKAVToQbJ`L>LaDDqChUf0=CEtt%->WjFZPW<@7~!t?O@R~FNa}s~-p>Ph&7F@`m&Hpn zi(s8wDPHJy4o~J+_WP8-{MBsR39;e^-fIOUQ>0r=D@G8wd7>s7Dm_+3d^Y7w{JT%D z7!J+Bd9cA|39W35pitG|s;S$@#O=+*?{#qPf{bxc5KS^g!tr9nb}}6FiqykV7caGP zR_}Nu;8--MX}UT{mzv=q40S&{Zo$BAF&L%2K)9ASe*#HqSbZ{O%pH;CxCp4>9K@!& z)$3y~^SajP(gKu#|_3uzbpwE2d5Neb8{<8MOnG{qr@t`loFg=ED!p)0;can+l@oweZ0)?Eug& zw)A~{#aS33e9S3Tb_G_8&v`lf&4xIQLN~11ZAeXT=xC3;Y?s%yFTTeMpc%zSc$JU-tjM zOTd5f@AjXn0~)#jU)c-RN|#;EI|L&l^q!=OXdSR0Z>^seud9l@c;eGN#*zfJ-4{f& z{ZC@xvc;dX$nsyqRqcY3oiKO}<=|;BYpgfr-3D%L`2V>fu2iIi&(s?*>KS<+drlZ@ z+6p7^LE)|V_W+-P@T|W83SOlPMa;&GZk<5qy!EF2ZL+=j$8P*F26l7os+A5=_CN|8}17Z3*}mrNZH1(kF_HgO3E5doETfH}-L z@9%!k8F0y^`hBnOb$$P^EAep7JMVd)=eeKzxu0iFtB6DIH#uTE7yx>vo5RA5WmR9M z87@?J{-cnM`XW(a0?)+6t~zS+%>uQsC?mPluL3eR%~Cjx`*0fhe`>sIOW6UtweQ8J zI+%Yc;~Ur?L5_6XFv}1ef9doJf8X4ZgKpl2)Jp!wC0=DSgNzv%$<>|kKYFYTttW0B zYglIweo(IO15M`^bCXy@ICTC=38m`em1*SoP}P)-j|pv1)d-QC3ytu|P)C@>m1$0p zm~n%lAfO)L06aiKjnYZbR&SC(_UZr+{czQKJkHRwe1#*2{An ztP#A6iz?jLzBSa$;s=uY1L^A{Cxe>XrO`-77k` zMxAsHQRXp5ny^5^CYAra#FM3P!n3w80x!mDO_WypF_Hep&sb1 zV#S%+y_W^Wd_Hi&Fk36pezBFJO}|wb8jz|vz9LIAs)2QcivKUqN9NQ#n+*p_KSoV5 zdE1hcvh9)+Li_?C5zWe_zXUe09cd&!3WN3SOziDH#9~1sob)&#q8>x)dus{Gv6BO)Z8f2>@_^FNets`?Px{)qFx5CXc=Pcc`s*RI9u*2?w)ABZ|KPx9V1vebQ z;xw}+3*wV+Um}qOhoVGE!9b>wZA6*K}Giydp;%?6F7(C zm6;l~Zq*iAmtP*emvm`%R|T_X2fT4Mh}5g~5V!k2kf8j}#Kv|nQT8lF2+2F% zGEGxwlR(p7_1gso2NBEf29`Ggbk{T5{zt)4z&Z^za!0n>wqBje9(EpS28GUzg+eH3k>dJb zIq7(VeEbHeyL`P8u$8SxF}XnlzA&Lcg=7%|_vx2W=BPF(EZ*}pnQ;Yqi=kc}pi`b0 zNq*$Y(I9jc01|E7EsD@bV7bq~oKgaFNMZ`R=^^<^7l>o!IfIxtbAj{B)Qm;U8^p@J(3@*`?eklLLKu4mzi=+7I?T1*khhdgs87IOEC&)UA2FNpog02b8tqZp*| zWbZtWEf>zc1Y=XEUxsC719!Mr7}i+)xUiM&TjHJQGPnip_p^b-?q6X2n-|X~ZL)GULwg^`>d| z~lgsD|HwJrqDcE0Z4EERhBiNq?dG%DVuM>m)FAIr$ z^xdrNS77~WGCy9??SMac#@({z z5FBN0ISs`F-Z%#z;Gpo2|J~0Ki;|@3GWD8MLodT8u>upx{si4ATLhmm&n>77nef)a2o1AZj(Ypkg%LL z-%Rs+2!55_Y4BghX}IG@f{Nx>q*>LsF{2q!N8o@_`PVJ!qgS7>36JtHcXAS-9X6w3 z{v&eUSq5F<04)o*!sebmLN;zStQ{*w`AO)a>VDIj{KhGQl}U_n_zF0Bu!3cr@(q6> ze4e2ltZ5x$E!t*_9n_n6$TfJ*~0hWs0Q4D z(!^RyY>hx#G@;p46iVqTgnb`kZPt|GgM~oGE+(yt3&_2YQv*emk$pm6>C6J&rZ45u z{~j2cfx)Q#XHSjqb=ze9V2Ys&^jaH<4HF!Ubb{Gq(zk}#`fQ~cyD^mzvv;(N9U<>a zeg&-X=mWXU#=U|pMArOeCUh2*6f}&UJW zbWGW%@x9>a8~M5&FXrol{sv!{_#9t%hG;oUs~`9U;7uw@7?x;`1{0djI|h7{>`E}I z_3+u&-6(kfIS4?E@^hyI(`1=RHM=AkPKxFQgLvh-Pp2iV?B~1K!eC}flbrwVXuy#x z4)%_y(VUsNIdxK;sjV&XDwUfS6B;-2x(BdA&Mq|LQ1zKZ88gB+kC*~b)w1vEECL!d zRu4nS9R6UEx$&^ANEwz-(`NV}S5^0rNg;Lxd%u{N zn0#El8JXgTm1@njjF8;?GiSRxYAVhJ)z#pA7+rx2ddl_hAZ1Xl5usU-}mvt2h9)GTP_M&QD~@$U$-po6p2N;-U#9|G^YUK z*AQ){^Jtj2Pn81PoOV@>SPeUrsW@_ozlieI(ZURrvG$?q0zgbc??OT?2 z2sMN*&Fa`Iu+ym0Z6%s*dd-pidU>;AGQaR=5?7)LsATUF+Eq-=76|nqG4N#QAetHO zG_y)a^qt$5`93^RXVnrO>G{z1F-bpr4z~ zepzvzm`nJ5^Rzo)gO!HzQl_gvMBoSCE5zmly_d1Q>h;Q?N*A!+5A{mdMC56{LO8AH zz}TTvh#=p;#=f^r;`hU_?in0V*Unuu@teN+Z(E^ zpOo8>ql^Cc`5gUI5FU!FFALyH{4f3Ag6Q!h-wfi+UjWeynZH!r{(S_eaTw{Qg6JE_ z>Sd5H6!f-}y|&~=OA6FCyrAT^+8mt*|HQ=opO|=Qr&mVs4?Yt?x}Pw8G9{TJ$-5MU zBEUigR1ErAs<5d@A{T1&NpiEe?KICrQ7xP8&uRKh+I$BxZW`F==gy$3JZ}4l*&=F214-i6H*)p^%rDPRo&~fv7Vm7v6D{{ z+V#{_&wgTT;6fT>nlsiNt$bY`PIf%c!HMX&F%PHBT1^JtPcrbo@&L~I5DAr|`Yv&B zG$BXKe{s3(B?i9Z)gb*T1JC%2fx{dD8v_HYtf~9ek@-tONV22)ykBHBv5$z^Ax+hu zCUR%)r0 zeFK24FSnt3i{%nimLJPTRc*!)vO6#UJ3chp`2ZPxpB%@aIngM?g`@+dSu$BoR64mW zO{JSEZg&=gWSZ64>9rNNR~Y!1tEj2p^`i>QPz@XH=>-y^(v}L%z>;i1G*h+Y4u-GvLfcaVzOXoanM^ortlB}<=cyr z?4pq7XrnhvF;nz+6*IbR+X@+rFOcOtDX15rpiZSQuOr!t z|2D~%`&32!CH8$I@c9&+@`SHS!I9^|wWyDFDqPtClYlOhU7X0N1wP1IW09{SC+?%) z^JW9J@26Anv39S-=pc}hZ$iNWpAU9pJHkR@F8FMbe2cczvktVuMJc7*RmD?3QJy@;L8@@>UhE*yFiF``$?s{#dw*36EW?QH^P zOKAW#Eh@md6-oMDQ`@kzSFmhG`+8{>j$r=(pM^`$M;7k>&$Do>>~Ck`$6u0#!&A&4 zErFZBMUdu?$dwK#=Dk`FmkVuh^pk?QX!9tT7d|PNH?rtr!TcX7;!ftUZW-^+H<&Em z$y#5V%WIdLC9Ry>cF*_cyOisddtRZJDSnk+rm0~5dP53k@d!CV)eT`a-!W>bG^7it~kNaO#n}Xogga1eI`)WgU=$ zx1&;B&(p|l909Lxkg!uZsP&qU&L9U(#zB}2uW}k{neEQMsNv`+fJ^LO3AU+me~(~O3lmowg4gTvfZxdH9lmNq zDwUBdQCChuW%Cu_+ynD8$n?lng7tF@+kMOt4tPcn_rGc0*V;NkBlA3mPD!>eAFvo! zW%)-n*@22hgXWW&Hju+qOg5;mF6e)E)ebM;Q)KAW8g`yXImJTkkjSmpSw0ipVx~?# zx-cZc>!waUtAS}BNm{J03P-s@iN5=gjqhvjw%bPHH^U7;Sqp4c;=N+dgtp7g3_4Z3 zU~K_Sp+kvA@6vTir^~Eh=ek?3yiUH*3iK-gT^0xCRH{u&yOBE`boJV^es$+DW_HB> zDDQ;&@=(o41HQSz;bb8jXm`9-a^9OHX|=T#aa@w3*=-h3dFmqZxx48ilNDRPCPUUo zXeeCXRx{Sg&_3(Gq+R6Y4{ZovslZPr8p0mc%L~4JxS=vnPHHe1|B@z)Fc;eAwkSlS zpNmbPIsHM|;(-D50z-vY;6>%Bs-}B5i1(PKCl-qN)|-h?uj=NH5${i3!3?7dU0y_b z4F7HpU5I>IRq0T8A+27C)KnR?lWDb-cS&p+Dch4K0R#hRa?9I~2SsL?-;75k1= z=hq0I_@-!<@pv!h1>5SC_Q{2b_k+UKbYw~c37c~8cVdC|uvOm(2U}U|%uAGB7*t04 z>a2K&<(;apT)T10qP~n7unXoGaHnXdd8XYDAYo(5VJd&K;X2rY`pZ2Iq9_%(leIAcmKJxuVL zQM3G$^jqX`Kk&Vp%oQUi*)gGNRuy~oN^sb)Do0nviOF`V^W=V|!av$&oh7?sdrXG? zun&H(X7rvzx-Av%z2mCgJn*1GxQDMs)aRksBtsLu_N#s6K{2J(^;fRVwEi|*AI9hh z_g0bbrY*X*_n59HnLcEX8z~QFhGmqLY;{>{h#H=CCH|lv=}EsJv971CdffG@J)y_- zQI77~ym02gmJ8F0^!G|qzpCBOM*be8I{C+rSVq5Z=8^N>B+<39W}Tk3&iedoaPjMW zD<}C!tBTZa$^P|W_gj=?EYkiQp;a**qqN5tnbaTkPl+6H)pf3>e@TXm{!#R@5s{2# zSi}gwHKZ?{o|iXG5g9k_M}IDRfVKYI+QM$?DaumEbyap6L(T2{(2|u6kJfRs&#Uf! z5?q+=cks(|t2Iik(su`I@EabnBTHAf>67hyMuyZZ534fh7i@RmvrsEsESN;UWKmdL zee$PIYwgFn$%OdFUym@`cYb{DE01Ku$?iwfe-XlzYL{8<@`SSw*=;(z&&xf%A2Yr; zE+49mRon{idPNcaU$fR7*OsLubCTPlb@3}X~*C%IB zTOBksc>ugi=On1iaCQ_r@@qGbJ3GR9?BVLhK?A@k&OmgT#jYL& zZ?Q6m6n4^Fd5J77puQMGoJPf6)I>}P&nT<>V!#h`X4|KH_I(1Rl&h;1(Dv7o`JU+{ zE+V`*>!+Ek{RBTuMVjklFz>#=)V6boP`10RCWvaRi0f!v&9BEW8*fg%Gihbt&fdu; z-E%A+kSd>6 zaOHudd@p>%z@WDpSSs=|&b_h8>`<>I9b0P2#fqI>W$8C-*xx$aUo9ybHL?%UYwSP9 zP&72ivWRaNx#?{`ES#;l zqu8hsGJs7Www^w4pfrNTw9dr*cQPLs7!BoJb_PTeCLWl=kPbxTS#i7Fu}30rr1+jF7QLG3D`bm)O?D$F9ThuP+wY09eY$4Z!) zJW$YR4`^lPknc*xGVVmO|JVa&E-#1ZXz(RwEDwdC?deO*Xz5a3o_n)K9*ZVbCB~(> z4D7_G#E`~Lmi@$6C#tf1drQx{@_>8<2GK@`l2AZG?Oa^$5<|kM$UZ}J91EDns{$MM z6O$x>_|O0?{VsG8&C1ZFb$~3^QE4ZCs5XgXj>UWDf5*H?xM6Jmo1zW#Uf6H^z zU3$rWd0fztMS8U^BTjLbh)s(CbdSBUhoYEaGx0$BlZSLXVK0bgFoK)j-m`=3co}m$ zlDMuO%lAl=J7P-K3>_QT%*li&A3+V%?2Gd;A}n`6eB*kV=> z`hzc^zxH{45p?$Gb3~X>a9B96ouJY+{cU3|VvlGv3zccOpsntvd3? zwQ5zLxO3rK#9z9nCck*_maQY8xtU5_ddfQVAz_dk(4H9a*?R*F%V$76h3(xfgiexO zK8mr4Dy+p+TuT;d_l7LD26KJU5w~BSP(2a-4QI{}c}evAUVQ9V<$whpCnHDcN5*sODRZV)GYI=r@hW;Y>mw z^e5!_iZnKYW1zT7voHRz^KJnLpa4@GXBm2A;v`sZ@HABFL=;ZnQ#8RlK_f~6#~2UJ zh6asD3J;`biV6vmy`-}VrqQn1sd|?~5FRnjPu9`vrIVgtqsDH8LU=44x){N;>ClPK zZ#;D3)1EqX;tkMl5Qn{GVn~$o5J?)*L)oT5xBO3-6rIh`iJntbKB?7t(`TTm+xp1~ zMmDiQNJx*EFFhZMa@y6tmkidi=;hhl9uuu3d)NU~vaB+6B0~woIPky6bIkosc&q)Zvf`A)^(@Eqdd(TY84QBx z@PXxjuDMFjr|7CgIj}7k>P~|3nRj_<_Dd^(=aC!^JkQ`{xsn9qHzgyYYeL^Ab!Nr| zIT#HGC$3MU#o&mA_2QUz(p``VOJ;&ge0b|-2iXOlSu7TWG885Y&(O(Z1WTk~fQLrc z#)z&>JgUyT^8=R?kX(Pe$&DN(UrO>N(jKGxqg13cg{Yyy;tv~)1_oz)AZ^RNr^vX;^F z<4Rk<;d~`Sf$d370-pxWF&HL>-S~-`Xb2Te8g19;6{Kwn^qmF7^BZqj=;`u8nN@+Cs_9ORJK7U_}0mS-A{r7$+h){cZHOm zr2Q{K%wYJ}uDvO!*j&YQZ(z&H5q_mu96Cv`LuGMS8d*Pz?c*(@%KCO!Nnj1xLQ92k zkjF4H62B!lBTk0Hu%jhrtB!pgt&p=>J!sVHyuLP#piF(yuj`=TK3ysWh-un!S zalFd@GdPkbk31xxcIioVE%pdudet_L0q#n4Dm-EQfs8Dgc@G1+HnWO56fh~aAD8M%}6{Vj(0q+HIqIG49;S4Q5MO~!b9%fi=Cm*(;@ zg&xtwy{FN@>?zQ(1Nel2X3H_R!ooGkQWO!o0g*vqn5yQPqs0^JX z23aZEHkl331Qjf2Ljo~|REmCSJ@_sO6(h}doNN3Pj=ahu;!pWL0f56?#2M1d)pVD# z2704Lv-qx**j#E%6vSgdcbFsO%-PaMeQ$w;^c4G(Rmc! zt6YwduvNVE`ga8^Dh=uf(XWVGE?Kp z;C)#6QIgcP2i*YWM@NV|0{Ck-0HuYd4UkaZ)A>!o4$iviwW4dOFGe+jm+dXIlfZ$L z5{OXqLX%n1T!_jKI#HzQ{`3Ywg+en)+$9a5{?R$(enHbfV{GHgr7>27XyCZ_3e4_w9~OOc{Yq9jd&BQ*PjCs z(Mu_ZXxbEO6*R6M<4(5ZNNxc$M-|N_e*%q%ce&9+4#8WLTmeO-kV^csgRrrb%zs2- zDvbx%BeThn+jB_pu(9Xg>&z>sgy8!`E+v+obvX98z%Q0WPq9OQtgsi>eCu(-MqU?{ zFK5Hvg_sl!xHh#YePnacL^pO6Gox;HAq5Y=ntL?bG@M1(?L0rhE)~5&RtHoulcoww zBRjZOp8WO_ac5(`s9suoFF7{vNOE|{jdzwjT%>4=hF$fds1-jHm0=Y1!P|uzd|Rxt zwZ@Q>A|L6$2E)~3ioCz7x=ZTG1mVP?19nVip>V}}J literal 0 HcmV?d00001