From 9ec6db34888a266f561441c9c9e865c071ee15a9 Mon Sep 17 00:00:00 2001 From: m96-chan Date: Sun, 6 Sep 2026 00:25:51 +0900 Subject: [PATCH 1/5] feat: Decode nested structure properties Struct-typed properties were skipped outright, so their contents were unreachable from Python. On this machine 1512 properties across 904 providers are declared as structs, and a single Kernel-Processor-Power core-parking event carries ten of them, all previously absent from the property dict. TDH addresses a struct member with a chain of PROPERTY_DATA_DESCRIPTORs -- the struct with the index of the instance, then the member -- so `property_bytes` grows into `property_bytes_at`, taking the whole path. Struct values are read recursively into `EventValue::Struct`, and a struct that is itself an array into an `Array` of them. Recursion is capped by MAX_STRUCT_DEPTH: a schema whose members pointed back at an enclosing struct would otherwise not terminate. Two supporting changes fall out of this. `read_layout` now walks all `PropertyCount` properties rather than only the top-level ones, because struct members live past `TopLevelPropertyCount` and `structType` addresses them by their index into that same array. Only the top-level entries become keys of the returned map; the rest are reached through the struct that owns them. Keeping that vector positionally aligned with `EventPropertyInfoArray` also fixes #81. A property with no name used to be dropped from the vector, which shifted every later index, so `PropertyParamCount` resolved its count from the wrong sibling. Unnamed properties now occupy their slot as a placeholder that is skipped when values are read. The struct work needs this alignment to be correct at all, so it is not separable. Verified against tracerpt on live Kernel-Processor-Power events. For the same core-parking event tracerpt reports Group {Number 0, Affinity 0xFFFF} and OldPark/NewPark/NewSoftPark {Number 0, Affinity 0x50C0}, and those are exactly the values that now come through. For an array of structs -- HardwareTable, 64 elements of {LogicalProcessorIndex, WorkLoadClassIndex, PerformanceClass, EfficiencyClass} -- every element matches too, and unlike tracerpt, which flattens all 64 into one repeated list because XML cannot express the grouping, the elements stay grouped. Closes #81 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --- crates/pyetwkit-core/src/tdh.rs | 373 +++++++++++++++++++++++++++----- 1 file changed, 317 insertions(+), 56 deletions(-) diff --git a/crates/pyetwkit-core/src/tdh.rs b/crates/pyetwkit-core/src/tdh.rs index 613a068..f078710 100644 --- a/crates/pyetwkit-core/src/tdh.rs +++ b/crates/pyetwkit-core/src/tdh.rs @@ -113,12 +113,44 @@ struct PropertyDesc { out_type: u16, is_struct: bool, count: PropertyCount, + /// For a struct, the index of its first member in [`EventLayout::properties`]. + struct_start: u16, + /// For a struct, how many members follow `struct_start`. + struct_members: u16, +} + +impl PropertyDesc { + /// A stand-in for a property TDH gave no name for. + /// + /// It occupies the slot so that later properties keep the index the schema + /// refers to them by, and is skipped when values are read: an unnamed + /// property has no key to store a value under. + fn placeholder() -> Self { + Self { + name: String::new(), + name_utf16: Vec::new(), + in_type: IN_NULL, + out_type: 0, + is_struct: false, + count: PropertyCount::Fixed(1), + struct_start: 0, + struct_members: 0, + } + } + + fn is_placeholder(&self) -> bool { + self.name.is_empty() + } } /// The parsed, owned form of a `TRACE_EVENT_INFO` for one event layout. #[derive(Debug, Clone, Default)] struct EventLayout { + /// Every property TDH reports, positionally aligned with + /// `EventPropertyInfoArray` so an index from the schema means the same here. properties: Vec, + /// How many of those are top level; the rest are members of some struct. + top_level_count: usize, } type LayoutKey = (u128, u16, u8); @@ -203,25 +235,33 @@ unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { unsafe { std::slice::from_raw_parts(buf.as_ptr() as *const u8, size as usize) }; let info = unsafe { &*info_ptr }; - let count = info.TopLevelPropertyCount as usize; - let mut properties = Vec::with_capacity(count); + // Walk every property, not just the top-level ones: members of a nested + // structure live past `TopLevelPropertyCount`, and `structType` addresses + // them by their index into this same array. The vector is therefore kept + // positionally aligned with `EventPropertyInfoArray`, with a placeholder for + // anything unnamed, so an index from the schema means the same thing here. + let total = info.PropertyCount as usize; + let top_level_count = (info.TopLevelPropertyCount as usize).min(total); + let mut properties = Vec::with_capacity(total); let array_ptr = std::ptr::addr_of!(info.EventPropertyInfoArray) as *const EVENT_PROPERTY_INFO; - for i in 0..count { + for i in 0..total { let prop = unsafe { &*array_ptr.add(i) }; let Some(name) = wide_string_at(bytes, prop.NameOffset) else { + properties.push(PropertyDesc::placeholder()); continue; }; let is_struct = prop.Flags.0 & PROPERTY_STRUCT != 0; - // `nonStructType` and `structType` overlay each other; only the former - // carries in/out types, and reading it for a struct would be nonsense. - let (in_type, out_type) = if is_struct { - (IN_NULL, 0) + // `nonStructType` and `structType` overlay each other; only one of them + // is meaningful, and reading the wrong one would be nonsense. + let (in_type, out_type, struct_start, struct_members) = if is_struct { + let s = unsafe { prop.Anonymous1.structType }; + (IN_NULL, 0, s.StructStartIndex, s.NumOfStructMembers) } else { let non_struct = unsafe { prop.Anonymous1.nonStructType }; - (non_struct.InType, non_struct.OutType) + (non_struct.InType, non_struct.OutType, 0, 0) }; // `count` and `countPropertyIndex` overlay each other; the flag says @@ -242,10 +282,15 @@ unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { out_type, is_struct, count, + struct_start, + struct_members, }); } - Some(EventLayout { properties }) + Some(EventLayout { + properties, + top_level_count, + }) } /// Fetch the layout for this event, consulting and populating the cache. @@ -269,24 +314,37 @@ fn layout_for(record: &EventRecord, raw: *const EVENT_RECORD) -> Option = (&'a [u16], u32); + +/// How deep a nesting of structures we are willing to follow. +/// +/// Guards against a malformed or hostile schema whose struct members point back +/// at an enclosing struct, which would otherwise recurse without end. +const MAX_STRUCT_DEPTH: usize = 8; + +/// Retrieve the raw bytes of the value `path` leads to. /// /// # Safety /// -/// `record` must point to a valid `EVENT_RECORD` that stays alive for the call. -unsafe fn property_bytes( - record: *const EVENT_RECORD, - name_utf16: &[u16], - array_index: u32, -) -> Option> { - let descriptor = PROPERTY_DATA_DESCRIPTOR { - PropertyName: name_utf16.as_ptr() as u64, - ArrayIndex: array_index, - Reserved: 0, - }; - let descriptors = [descriptor]; +/// `record` must point to a valid `EVENT_RECORD` that stays alive for the call, +/// and every name in `path` must stay alive for it too. +unsafe fn property_bytes_at(record: *const EVENT_RECORD, path: &[PathLink]) -> Option> { + if path.is_empty() { + return None; + } + let descriptors: Vec = path + .iter() + .map(|(name_utf16, array_index)| PROPERTY_DATA_DESCRIPTOR { + PropertyName: name_utf16.as_ptr() as u64, + ArrayIndex: *array_index, + Reserved: 0, + }) + .collect(); let mut size: u32 = 0; let status = unsafe { TdhGetPropertySize(record, None, &descriptors, &mut size) }; @@ -305,6 +363,21 @@ unsafe fn property_bytes( Some(buf) } +/// Retrieve one top-level property's raw bytes by name. +/// +/// `array_index` selects a single element, or [`WHOLE_PROPERTY`] for all of it. +/// +/// # Safety +/// +/// As [`property_bytes_at`]. +unsafe fn property_bytes( + record: *const EVENT_RECORD, + name_utf16: &[u16], + array_index: u32, +) -> Option> { + unsafe { property_bytes_at(record, &[(name_utf16, array_index)]) } +} + /// Decode a UTF-16 blob, trimming a trailing NUL if the provider included one. fn decode_utf16(bytes: &[u8]) -> String { let (pairs, _) = bytes.as_chunks::<2>(); @@ -518,25 +591,159 @@ fn read_unsigned(bytes: &[u8]) -> Option { } /// Resolve how many elements `prop` holds in this particular event. -fn element_count(record: *const EVENT_RECORD, layout: &EventLayout, prop: &PropertyDesc) -> u32 { +fn element_count<'a>( + record: *const EVENT_RECORD, + layout: &'a EventLayout, + prop: &PropertyDesc, + parent: &[PathLink<'a>], +) -> u32 { let raw_count = match prop.count { PropertyCount::Fixed(n) => u32::from(n), - PropertyCount::FromProperty(index) => layout - .properties - .get(index as usize) - .and_then(|counter| unsafe { - property_bytes(record, &counter.name_utf16, WHOLE_PROPERTY) - }) - .as_deref() - .and_then(read_unsigned) - .and_then(|n| u32::try_from(n).ok()) - // A count we cannot read is treated as a plain scalar, which is what - // this code did before arrays were handled at all. - .unwrap_or(1), + PropertyCount::FromProperty(index) => { + // `countPropertyIndex` indexes the schema's own property array, and + // `layout.properties` is kept aligned with it, so this is the + // property the schema meant. An index past the end (or an unnamed + // slot) leaves the count at 1. + match layout.properties.get(index as usize) { + Some(counter) if !counter.is_placeholder() => { + // The counter is a sibling of `prop`, so it is reached + // through the same enclosing struct. Fall back to the top + // level for schemas that point outside the struct. + let mut path = parent.to_vec(); + path.push((&counter.name_utf16, leaf_index(parent, 0))); + unsafe { property_bytes_at(record, &path) } + .or_else(|| unsafe { + property_bytes(record, &counter.name_utf16, WHOLE_PROPERTY) + }) + .as_deref() + .and_then(read_unsigned) + .and_then(|n| u32::try_from(n).ok()) + // A count we cannot read is treated as a plain scalar, + // which is what this code did before arrays existed. + .unwrap_or(1) + } + _ => 1, + } + } }; raw_count.min(MAX_ARRAY_ELEMENTS) } +/// `ArrayIndex` to use for the last link of a path. +/// +/// At the top level TDH accepts [`WHOLE_PROPERTY`], which is how a whole array +/// is fetched in one call. Inside a struct the element has to be named +/// concretely, so `index` is used instead. +fn leaf_index(parent: &[PathLink], index: u32) -> u32 { + if parent.is_empty() { + WHOLE_PROPERTY + } else { + index + } +} + +/// Read one property's value, following `parent` to the enclosing struct +/// element; `parent` is empty for a top-level property. +fn read_value<'a>( + record: *const EVENT_RECORD, + layout: &'a EventLayout, + prop: &'a PropertyDesc, + parent: &[PathLink<'a>], + pointer_size: usize, + depth: usize, +) -> Option { + let count = element_count(record, layout, prop, parent); + + if prop.is_struct { + // A schema whose members loop back to an enclosing struct would recurse + // forever; stop rather than trust it. + if depth >= MAX_STRUCT_DEPTH { + return None; + } + if count <= 1 { + return read_struct(record, layout, prop, parent, 0, pointer_size, depth); + } + let mut items = Vec::with_capacity(count as usize); + for index in 0..count { + let Some(value) = read_struct(record, layout, prop, parent, index, pointer_size, depth) + else { + break; + }; + items.push(value); + } + return Some(EventValue::Array(items)); + } + + if parent.is_empty() { + // Top level: fetch the whole property in one call, then split it if it + // turns out to be an array. Without the split an array collapsed to its + // first element -- a 176-byte array of UInt8 came back as just 228. + let bytes = unsafe { property_bytes(record, &prop.name_utf16, WHOLE_PROPERTY) }?; + return if count > 1 && !spans_whole_blob(prop.in_type) { + to_event_array(record, prop, &bytes, count, pointer_size) + } else { + to_event_value(&bytes, prop.in_type, prop.out_type, pointer_size) + }; + } + + // Inside a struct every element is addressed explicitly, so there is no + // whole-blob shortcut to take. + if count > 1 && !spans_whole_blob(prop.in_type) { + let mut items = Vec::with_capacity(count as usize); + for index in 0..count { + let mut path = parent.to_vec(); + path.push((&prop.name_utf16, index)); + let Some(bytes) = (unsafe { property_bytes_at(record, &path) }) else { + break; + }; + let Some(value) = to_event_value(&bytes, prop.in_type, prop.out_type, pointer_size) + else { + break; + }; + items.push(value); + } + return Some(EventValue::Array(items)); + } + + let mut path = parent.to_vec(); + path.push((&prop.name_utf16, 0)); + let bytes = unsafe { property_bytes_at(record, &path) }?; + to_event_value(&bytes, prop.in_type, prop.out_type, pointer_size) +} + +/// Read one element of a struct property into a map of member name to value. +fn read_struct<'a>( + record: *const EVENT_RECORD, + layout: &'a EventLayout, + prop: &'a PropertyDesc, + parent: &[PathLink<'a>], + index: u32, + pointer_size: usize, + depth: usize, +) -> Option { + let mut path = parent.to_vec(); + path.push((&prop.name_utf16, index)); + + let start = prop.struct_start as usize; + let end = start + .saturating_add(prop.struct_members as usize) + .min(layout.properties.len()); + if start >= end { + return None; + } + + let mut members = HashMap::with_capacity(end - start); + for member in &layout.properties[start..end] { + if member.is_placeholder() { + continue; + } + if let Some(value) = read_value(record, layout, member, &path, pointer_size, depth + 1) { + members.insert(member.name.clone(), value); + } + } + Some(EventValue::Struct(members)) +} + /// Decode a property that carries more than one element. /// /// Prefers slicing the single blob we already fetched, falling back to asking @@ -605,28 +812,15 @@ pub fn parse_properties(record: &EventRecord) -> HashMap { 8 }; - let mut out = HashMap::with_capacity(layout.properties.len()); - for prop in &layout.properties { - // Nested structures are left to a follow-up; skipping them keeps the - // scalar properties around them intact. - if prop.is_struct { + // Only the top-level properties become keys of the returned map. The rest of + // the vector is the members of those properties' structs, reached through + // them rather than reported alongside them. + let mut out = HashMap::with_capacity(layout.top_level_count); + for prop in layout.properties.iter().take(layout.top_level_count) { + if prop.is_placeholder() { continue; } - let Some(bytes) = (unsafe { property_bytes(raw, &prop.name_utf16, WHOLE_PROPERTY) }) else { - continue; - }; - - // Without this, an array collapsed to its first element: the whole blob - // was handed to a scalar decoder, so e.g. a 176-byte array of UInt8 came - // back as the single number 228. - let count = element_count(raw, &layout, prop); - let value = if count > 1 && !spans_whole_blob(prop.in_type) { - to_event_array(raw, prop, &bytes, count, pointer_size) - } else { - to_event_value(&bytes, prop.in_type, prop.out_type, pointer_size) - }; - - if let Some(value) = value { + if let Some(value) = read_value(raw, &layout, prop, &[], pointer_size, 0) { out.insert(prop.name.clone(), value); } } @@ -740,6 +934,8 @@ mod tests { out_type: 0, is_struct: false, count: PropertyCount::Fixed(count), + struct_start: 0, + struct_members: 0, } } @@ -797,6 +993,71 @@ mod tests { assert!(!spans_whole_blob(IN_UINT32)); } + #[test] + fn test_leaf_index_depends_on_nesting() { + // At the top level a whole array can be fetched in one call; inside a + // struct TDH needs the element named explicitly. + assert_eq!(leaf_index(&[], 3), WHOLE_PROPERTY); + let parent: Vec = vec![(&[0u16], 0)]; + assert_eq!(leaf_index(&parent, 3), 3); + } + + #[test] + fn test_placeholder_holds_its_slot_without_being_reported() { + let p = PropertyDesc::placeholder(); + assert!(p.is_placeholder()); + assert!(!array_desc(IN_UINT32, 1).is_placeholder()); + } + + #[test] + fn test_count_from_an_unnamed_property_falls_back_to_one() { + // The counter resolves to a placeholder, so there is no name to fetch by + // and the property is treated as a scalar. Reaching TDH here would + // dereference the null record, so passing proves it short-circuits. + let layout = EventLayout { + properties: vec![PropertyDesc::placeholder()], + top_level_count: 1, + }; + let mut prop = array_desc(IN_UINT32, 0); + prop.count = PropertyCount::FromProperty(0); + assert_eq!(element_count(std::ptr::null(), &layout, &prop, &[]), 1); + } + + #[test] + fn test_count_from_an_out_of_range_index_falls_back_to_one() { + // `countPropertyIndex` can point past the top-level properties into + // struct members that are not in the vector at all. + let layout = EventLayout { + properties: vec![PropertyDesc::placeholder()], + top_level_count: 1, + }; + let mut prop = array_desc(IN_UINT32, 0); + prop.count = PropertyCount::FromProperty(99); + assert_eq!(element_count(std::ptr::null(), &layout, &prop, &[]), 1); + } + + #[test] + fn test_struct_recursion_is_bounded() { + // A schema whose struct members point back at the struct would other- + // wise recurse forever; the depth guard returns before touching TDH. + let mut prop = array_desc(IN_NULL, 1); + prop.is_struct = true; + prop.struct_members = 1; + let layout = EventLayout { + properties: vec![prop.clone()], + top_level_count: 1, + }; + let value = read_value( + std::ptr::null(), + &layout, + &layout.properties[0], + &[], + 8, + MAX_STRUCT_DEPTH, + ); + assert!(value.is_none()); + } + #[test] fn test_read_unsigned_widths() { assert_eq!(read_unsigned(&[7]), Some(7)); From 2c2663424d2a9d1d812ef3a6cc78187a57fdcb5a Mon Sep 17 00:00:00 2001 From: m96-chan Date: Sun, 6 Sep 2026 00:26:02 +0900 Subject: [PATCH 2/5] test: Cover nested structure decoding with a fixture that has some sample.etl cannot exercise the struct path: Microsoft-Windows-Kernel-Process declares no struct-typed property, so the feature would have had no automated coverage at all and would only ever have been checked by hand on a live session. Add nested_struct.etl, 40 KB of Kernel-Processor-Power core-parking events carrying seventeen distinct struct properties, plus the script that produces it. The Algorithm keyword (0x20) is what selects those events; wider masks add hundreds of KB of unrelated telemetry without adding a struct, which is what keeps the fixture the same order of size as sample.etl. The tests assert the members are reachable, that they do not also leak in beside their struct at the top level, that structs survive into to_dict(), and that nested values are Python-native recursively. Arrays of structs are not in the fixture. The events carrying them fire rarely enough that capturing one reliably would have meant a multi-megabyte file; that case is covered by the bounds tests in tdh.rs and was verified by hand against tracerpt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --- tests/fixtures/create_nested_struct_etl.ps1 | 30 +++++++ tests/fixtures/nested_struct.etl | Bin 0 -> 40960 bytes tests/test_etl_reader.py | 95 ++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 tests/fixtures/create_nested_struct_etl.ps1 create mode 100644 tests/fixtures/nested_struct.etl diff --git a/tests/fixtures/create_nested_struct_etl.ps1 b/tests/fixtures/create_nested_struct_etl.ps1 new file mode 100644 index 0000000..40b3bb0 --- /dev/null +++ b/tests/fixtures/create_nested_struct_etl.ps1 @@ -0,0 +1,30 @@ +# Create a sample ETL file containing events with nested structure properties. +# Run as Administrator. +# +# Microsoft-Windows-Kernel-Process, which sample.etl uses, has no struct-typed +# properties, so it cannot cover the TDH struct path. Kernel-Processor-Power +# emits core-parking events carrying several structs (Group, OldPark, NewPark, +# ... each { Number, Affinity }) and fires continuously on an idle machine, so a +# fraction of a second is enough. +# +# The Algorithm keyword (0x20) is what selects those events; the wider keyword +# masks add hundreds of KB of unrelated telemetry without adding a struct. + +$OutputPath = "$PSScriptRoot\nested_struct.etl" +$SessionName = "PyETWkitNestedStructSession" + +logman stop $SessionName -ets 2>$null | Out-Null +Remove-Item $OutputPath -ErrorAction SilentlyContinue + +# Small buffers keep the fixture near the size of sample.etl; ETL files are +# written a whole buffer at a time, so this is what bounds the file. +logman create trace $SessionName -o $OutputPath -ets ` + -p "Microsoft-Windows-Kernel-Processor-Power" 0x20 0xff ` + -bs 4 -nb 2 2 + +Start-Sleep -Milliseconds 120 + +logman stop $SessionName -ets + +Write-Host "Created nested-struct ETL file: $OutputPath" +Write-Host "File size: $((Get-Item $OutputPath).Length) bytes" diff --git a/tests/fixtures/nested_struct.etl b/tests/fixtures/nested_struct.etl new file mode 100644 index 0000000000000000000000000000000000000000..b216f5814c22cedcf843f7c170f2d8ada7e15329 GIT binary patch literal 40960 zcmeHQf2 zF%oL3K9gFcL0MBxDrkZzq<_i#!$=_h2r(qBf$;ulLc$O8(u%dH?>TdJ=gczq?%nfp z_Uxq1PSf4l*}3!0cV>6?d}sFF2a5n6o|XQGzUd;EZO+@eOQF?Oa~8}RpBw-1I|Z0z9mpLum)`0IEU+nZ;$rU^I;j{vruYd@1F z(dkc5ug%M5+L)>Thw15Q`?2xm ze>wH;>Q@Ik`+tnBJhSdzv_**TR?KE3gf=8eTUbt5LJP)tp@mYhWYXWO&|Y z`0g_KyWmR0b0utsJK$?@w<)>DoY?_)ni`wU=X=!%F*cfd)^?-V;ueFu+vM-fEV|LO znx3_N*o5_VnzOd`Rp!j7u{MAm@Aanlh2w3>%^(?bHL0;cV}Zs3jRhJDG!|$q&{&|c zKx2W%0*wU}Enszdve0O5v$tt{f2!M^K21vj=0mFgZQUVPnA^Vyi^p?^@aDd>oyU*e zzVODW$8Y%E*p5$J_otDMA2rHx5T2tpF4=H6gu-TQVXB}lb{aFFiid0!7bO5ViAwvu zW%ejcu07v!zJ@<*OG7?OepEdlihLv%(M31_P7t5&I7^Dvs!_VjSpc; z?3Rw;IPswo#A@lrv+b4h75nqKQGpLxKe4@_(Sk|h#j6OC6t@+7NC>;`8=t>hi2%+b|P`1To@rf9eW~36DI1Lrd*Cb z;m(C~csO5=EATnnfhU;peBN>=a+3rV0rBxUZvkJe|Mx^O+IK z_R7)8{(Lql@S%rg2RqTGuJ}x^vR-F<)O#riGFbY3?H{r-7DzlTk;EwfTNlr>+}@9ebgZ`$WpS`y+M662tj zD8>1;7zfQ%iu1G>2lY}Hj-vF(rbGSzn>qL4y1iF#>G&<2q5glNm9|s%{5W~(vpA61 zbwfK_=J?4_;XoEU_E%{%g_y z=M7drK6K3>A3CZ<|F2NslTBUO_|Rz8qW?duz$d2u-=M%JrvBfdz=vi4ynkTK&I&fw zq5tHnXJad0@g>{f^VKWuLNhqIgI z)DsV&{wM8|4XD%qw(d|?mfwZrdt7V}>_L}3LY$Lg98^W6IR6pjpjs)#nG)lmiYUby znA=SfVjT2>x^P1Oc;uJqJXz*X{`)kEZ|+mzLlLvYJ}D@=QA>RDumT^7Xd)jvswKX8 zM1fDs4vS5R-Oy;&65l+bz$a#Wb3x`eRXHETjBhSe;6pP2UT^irH`@w)V#YUz75K!A zZ|+p!6EnW~T?IZd?e`5N_%N6*<^pCGm;6oEYdjE7FOS#o=Ds$(|euDCpoB8|ywdms=z1a`v0>Ed}6Nu|5brc%=Q2O7Vrtz|2g`n&h`Hlnf?oPTevP* z)bDIA?OHzx*KI!~#zEJv6laYXhqwNJwHOEWQWu}9bbe7gR#@lP|9|-X@1XuCUo;y~ zr~kR@|1XK{f$FQv9^rL#RE)!0|DO=!@Yerd7vtc_mD=rXF%ED2|AZKaxBh=hjKf?1 zpBCfr*8k^h@2y|>^DuAyf4&%pxBh>g7>Bq1e}Nc>xBmac-gKS}pOfM5o7{7n#5ez` zz$a#WbIJFrA0LWnCibC7N43N^uTtQHPgqMOu^Sq#TGs!+s=y~^eDgsCJ~88)k16mW zpAxUPde{GdufQi}eDgI0J~88)Qwn@y#y6K_I;u+lAZC2?3I#qf=T5%|61N}I-tOZ6hHEzqgwR;Qwn@y>i_2z_{7xzuPE?|ssASx z_|Oc1_fNh0e@^DtRykkC)c+q;;1g5-uTtO>Q~$43;1g5-4=eDAssDE=@Ht)e|D?I` zCoi-<8o<>r??wGj&(gFQ-1q-&-Jz^3zYE{Dy6(y9zaJa(eXDH>d}6+D^^gLenD1LX zUBD-N--`2n>^k4KdcC0EVVta}-`PD_{=SD8hxfkKaWM|6#!^0~#5kzmN^z#dIH&@1@Zy+D#5lY-=7+^Nyg24EF%B<| zxm=9Hi({@7u|Q*iGt&Zg{Xeh&KbPr$iUno@!~Oj~ zuKs^XA-)n*|G%lgC#L>~;p*2fW9t8p6z~c4KW9C*I{jb%_g2;w^gGmjMg8XL|Mg-V zp8mf{jKkCa!(tqs{@*0V;pzX)VjQ0Sze9|})BoGVI6VD-w-|?~|Gy!|;pzWTF%D1v z-zUc5>Hqu1I6VFTpcsdz|2twFp8o%~7^l(yXQs#Auxc#OSfH^$H5Lf<|GG^7)4NTX zz;NIHxB7o+R+is|>;E76LG|CijH&h%9W{_m}z zfDV7ZV!WW=q5d!GH&_4vQH;aW|9=+a@bv%R#5g?t|93GCPyfFr#^LGzV`3bh{(nP^ z!_)thVjQ0Se@BeN)Bo>^ad`S4#(F=W`ST7>|F^_AJpDgUjKkCa3&l7*{eQj~r_uk_ Sc*spFjRhJDG!{5BE%1N-S@7}z literal 0 HcmV?d00001 diff --git a/tests/test_etl_reader.py b/tests/test_etl_reader.py index 142990b..d181383 100644 --- a/tests/test_etl_reader.py +++ b/tests/test_etl_reader.py @@ -231,3 +231,98 @@ def test_property_values_are_python_native_types(self) -> None: for event in events: for name, value in event.properties.items(): assert isinstance(value, allowed), f"{name} produced {type(value)!r}" + + +NESTED_STRUCT_ETL_PATH = Path(__file__).parent / "fixtures" / "nested_struct.etl" + + +class TestNestedStructProperties: + """Regression tests for properties whose type is a nested structure. + + ``sample.etl`` cannot cover these: Microsoft-Windows-Kernel-Process declares + no struct-typed property. ``nested_struct.etl`` is a short capture of + Kernel-Processor-Power core-parking events, which carry several + (see ``fixtures/create_nested_struct_etl.ps1``). + """ + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_struct_properties_are_dicts_of_their_members(self) -> None: + """A struct property must expose its members, not vanish.""" + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NESTED_STRUCT_ETL_PATH)).read_all() + assert events, "fixture produced no events" + + structs = { + name: value + for event in events + for name, value in event.properties.items() + if isinstance(value, dict) + } + assert structs, ( + "no struct-valued property was decoded; before nested structures " + "were supported these were skipped outright" + ) + + # Every core-parking affinity struct is a { Number, Affinity } pair. + assert "NewPark" in structs, sorted(structs) + assert set(structs["NewPark"]) == {"Number", "Affinity"} + assert all(isinstance(v, int) for v in structs["NewPark"].values()) + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_struct_members_do_not_leak_into_the_top_level(self) -> None: + """Members belong to their struct, not beside it. + + The layout vector holds struct members too, so that the schema's own + indices stay usable; only the top-level properties may become keys. + """ + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NESTED_STRUCT_ETL_PATH)).read_all() + assert events, "fixture produced no events" + + for event in events: + props = event.properties + if "NewPark" not in props: + continue + assert "Affinity" not in props + assert "Number" not in props + return + pytest.fail("no event carried a NewPark struct") + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_struct_values_survive_into_to_dict(self) -> None: + """Structs must round-trip like every other value.""" + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NESTED_STRUCT_ETL_PATH)).read_all() + assert events, "fixture produced no events" + + for event in events: + assert event.properties == event.to_dict()["properties"] + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_nested_values_are_python_native_types(self) -> None: + """Members must be usable from Python, recursively.""" + from pyetwkit import _core as pyetwkit_core + + allowed = (bool, int, float, str, bytes, list, dict, type(None)) + events = pyetwkit_core.EtlReader(str(NESTED_STRUCT_ETL_PATH)).read_all() + assert events, "fixture produced no events" + + for event in events: + for name, value in event.properties.items(): + assert isinstance(value, allowed), f"{name} produced {type(value)!r}" + if isinstance(value, dict): + for member, inner in value.items(): + assert isinstance( + inner, allowed + ), f"{name}.{member} produced {type(inner)!r}" From 8f3cb3c8727945375ecf96e391c302a02d66503b Mon Sep 17 00:00:00 2001 From: m96-chan Date: Sun, 6 Sep 2026 11:33:14 +0900 Subject: [PATCH 3/5] test: Make the nested-struct fixture decode on machines other than mine CI failed on all five Python versions: the fixture produced its events, but not one struct-valued property, so both struct assertions fired. The cause is the fixture, not the parser. It was a capture of Kernel-Processor-Power, a manifest provider, and decoding a manifest event needs that manifest installed *and of a matching version*. It decoded on the machine that recorded it and yielded nothing on the CI runner's different Windows build. A fixture that only decodes where it was made is worse than no fixture, because it looks like coverage. Record from a TraceLogging provider defined in the generator script instead. Such events embed their own schema, so the ETL decodes identically anywhere, which is a property of the format rather than something to hope for. Windows ships the C# compiler the script uses, so this still needs nothing beyond a stock machine, and the fixture drops from 40 KB to 16 KB. Each event now carries two { Number, Affinity } structs with known values, so the tests can assert the contents outright rather than just their shape. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --- tests/fixtures/create_nested_struct_etl.ps1 | 66 ++++++++++++++++---- tests/fixtures/nested_struct.etl | Bin 40960 -> 16384 bytes tests/test_etl_reader.py | 42 ++++++++++--- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/tests/fixtures/create_nested_struct_etl.ps1 b/tests/fixtures/create_nested_struct_etl.ps1 index 40b3bb0..29bee02 100644 --- a/tests/fixtures/create_nested_struct_etl.ps1 +++ b/tests/fixtures/create_nested_struct_etl.ps1 @@ -1,28 +1,68 @@ # Create a sample ETL file containing events with nested structure properties. # Run as Administrator. # -# Microsoft-Windows-Kernel-Process, which sample.etl uses, has no struct-typed -# properties, so it cannot cover the TDH struct path. Kernel-Processor-Power -# emits core-parking events carrying several structs (Group, OldPark, NewPark, -# ... each { Number, Affinity }) and fires continuously on an idle machine, so a -# fraction of a second is enough. +# This deliberately uses a TraceLogging provider defined right here rather than +# a Windows one. A TraceLogging event carries its own schema inside the event, +# so the resulting ETL decodes identically on any machine. A manifest provider +# does not: decoding needs that manifest installed and of a matching version, so +# a fixture captured from, say, Kernel-Processor-Power decodes on the machine +# that recorded it and yields nothing in CI on a different Windows build. That +# is not hypothetical -- it is why this script exists in this form. # -# The Algorithm keyword (0x20) is what selects those events; the wider keyword -# masks add hundreds of KB of unrelated telemetry without adding a struct. +# Windows ships the C# compiler used below (.NET Framework 4.x), so this needs +# no toolchain beyond a stock machine. $OutputPath = "$PSScriptRoot\nested_struct.etl" $SessionName = "PyETWkitNestedStructSession" +$source = @' +using System; +using System.Diagnostics.Tracing; + +public static class NestedStructEmitter +{ + public static Guid ProviderGuid() + { + using (EventSource es = new EventSource("PyETWkit-NestedStruct")) + { + return es.Guid; + } + } + + public static void Emit() + { + using (EventSource es = new EventSource("PyETWkit-NestedStruct")) + { + for (int i = 0; i < 3; i++) + { + es.Write("CoreState", new + { + Label = "core-state", + Sequence = i, + Park = new { Number = 7, Affinity = 0x50C0 }, + Unpark = new { Number = 9, Affinity = 0x000F }, + }); + System.Threading.Thread.Sleep(40); + } + } + } +} +'@ + +Add-Type -TypeDefinition $source -Language CSharp + +$guid = [NestedStructEmitter]::ProviderGuid() +Write-Host "Provider GUID: $guid" + logman stop $SessionName -ets 2>$null | Out-Null Remove-Item $OutputPath -ErrorAction SilentlyContinue -# Small buffers keep the fixture near the size of sample.etl; ETL files are -# written a whole buffer at a time, so this is what bounds the file. -logman create trace $SessionName -o $OutputPath -ets ` - -p "Microsoft-Windows-Kernel-Processor-Power" 0x20 0xff ` - -bs 4 -nb 2 2 +# Small buffers keep the fixture tiny; ETL files are written a whole buffer at a +# time, so this is what bounds the file size. +logman create trace $SessionName -o $OutputPath -ets -p "{$guid}" 0xffffffffffffffff 0xff -bs 4 -nb 2 2 -Start-Sleep -Milliseconds 120 +[NestedStructEmitter]::Emit() +Start-Sleep -Milliseconds 200 logman stop $SessionName -ets diff --git a/tests/fixtures/nested_struct.etl b/tests/fixtures/nested_struct.etl index b216f5814c22cedcf843f7c170f2d8ada7e15329..bef0f6531a8d6f850d1b241c8d3b11c3c4a47654 100644 GIT binary patch literal 16384 zcmeI1&1(};6va=Hme^MOKoJqa7B^j(f-Tf471UCxC}A)y6$#Nv@`7pFq)ujv5f?68 zxfW@~l?Z;-MgM~=#GQ5LqR@p@aN%EIMB;rj6Bwc`hBljW?l5_8a?iVO<~MJ1-rxZ6 zI_#a;GL9o6d9Ce3$6v}Y!qslcJy-y;Dd1iI(#o>PLJg(N(~VH)jT|6d{5ZC@^6LD0 z=w@05`Jt<7{T}~1w@;7pBI>u_e%I?4VBeWy^0C3Awi5AR}$U z{l6@l5{go34Cj!+Up0S3!eYi+TLKo@7Hjssy*$?OLEQtqoZ%!Iwyv| zUs*Qa8t+pZg~2BqkHzixbOy$2&#!j?mcFKHwUG!OEu8$N{cSyI)Q(w($x$lmtkC1W14c zNPq-LfCNZ@1W14c+D@R^|F8P~UwiprL|XL!mph*ALjSLc(Eqo6QE~zjAOR8}0TLhq z5+DH*AOR8}fgLB%?EgRe{$GFT4n`>8|NDXozu)}*|IE;fUFiSyt_c1AjxRzMBmoj2 V0TLhq5+DH*AOR8}0TO6Cfj=b+Z6^Q# literal 40960 zcmeHQf2 zF%oL3K9gFcL0MBxDrkZzq<_i#!$=_h2r(qBf$;ulLc$O8(u%dH?>TdJ=gczq?%nfp z_Uxq1PSf4l*}3!0cV>6?d}sFF2a5n6o|XQGzUd;EZO+@eOQF?Oa~8}RpBw-1I|Z0z9mpLum)`0IEU+nZ;$rU^I;j{vruYd@1F z(dkc5ug%M5+L)>Thw15Q`?2xm ze>wH;>Q@Ik`+tnBJhSdzv_**TR?KE3gf=8eTUbt5LJP)tp@mYhWYXWO&|Y z`0g_KyWmR0b0utsJK$?@w<)>DoY?_)ni`wU=X=!%F*cfd)^?-V;ueFu+vM-fEV|LO znx3_N*o5_VnzOd`Rp!j7u{MAm@Aanlh2w3>%^(?bHL0;cV}Zs3jRhJDG!|$q&{&|c zKx2W%0*wU}Enszdve0O5v$tt{f2!M^K21vj=0mFgZQUVPnA^Vyi^p?^@aDd>oyU*e zzVODW$8Y%E*p5$J_otDMA2rHx5T2tpF4=H6gu-TQVXB}lb{aFFiid0!7bO5ViAwvu zW%ejcu07v!zJ@<*OG7?OepEdlihLv%(M31_P7t5&I7^Dvs!_VjSpc; z?3Rw;IPswo#A@lrv+b4h75nqKQGpLxKe4@_(Sk|h#j6OC6t@+7NC>;`8=t>hi2%+b|P`1To@rf9eW~36DI1Lrd*Cb z;m(C~csO5=EATnnfhU;peBN>=a+3rV0rBxUZvkJe|Mx^O+IK z_R7)8{(Lql@S%rg2RqTGuJ}x^vR-F<)O#riGFbY3?H{r-7DzlTk;EwfTNlr>+}@9ebgZ`$WpS`y+M662tj zD8>1;7zfQ%iu1G>2lY}Hj-vF(rbGSzn>qL4y1iF#>G&<2q5glNm9|s%{5W~(vpA61 zbwfK_=J?4_;XoEU_E%{%g_y z=M7drK6K3>A3CZ<|F2NslTBUO_|Rz8qW?duz$d2u-=M%JrvBfdz=vi4ynkTK&I&fw zq5tHnXJad0@g>{f^VKWuLNhqIgI z)DsV&{wM8|4XD%qw(d|?mfwZrdt7V}>_L}3LY$Lg98^W6IR6pjpjs)#nG)lmiYUby znA=SfVjT2>x^P1Oc;uJqJXz*X{`)kEZ|+mzLlLvYJ}D@=QA>RDumT^7Xd)jvswKX8 zM1fDs4vS5R-Oy;&65l+bz$a#Wb3x`eRXHETjBhSe;6pP2UT^irH`@w)V#YUz75K!A zZ|+p!6EnW~T?IZd?e`5N_%N6*<^pCGm;6oEYdjE7FOS#o=Ds$(|euDCpoB8|ywdms=z1a`v0>Ed}6Nu|5brc%=Q2O7Vrtz|2g`n&h`Hlnf?oPTevP* z)bDIA?OHzx*KI!~#zEJv6laYXhqwNJwHOEWQWu}9bbe7gR#@lP|9|-X@1XuCUo;y~ zr~kR@|1XK{f$FQv9^rL#RE)!0|DO=!@Yerd7vtc_mD=rXF%ED2|AZKaxBh=hjKf?1 zpBCfr*8k^h@2y|>^DuAyf4&%pxBh>g7>Bq1e}Nc>xBmac-gKS}pOfM5o7{7n#5ez` zz$a#WbIJFrA0LWnCibC7N43N^uTtQHPgqMOu^Sq#TGs!+s=y~^eDgsCJ~88)k16mW zpAxUPde{GdufQi}eDgI0J~88)Qwn@y#y6K_I;u+lAZC2?3I#qf=T5%|61N}I-tOZ6hHEzqgwR;Qwn@y>i_2z_{7xzuPE?|ssASx z_|Oc1_fNh0e@^DtRykkC)c+q;;1g5-uTtO>Q~$43;1g5-4=eDAssDE=@Ht)e|D?I` zCoi-<8o<>r??wGj&(gFQ-1q-&-Jz^3zYE{Dy6(y9zaJa(eXDH>d}6+D^^gLenD1LX zUBD-N--`2n>^k4KdcC0EVVta}-`PD_{=SD8hxfkKaWM|6#!^0~#5kzmN^z#dIH&@1@Zy+D#5lY-=7+^Nyg24EF%B<| zxm=9Hi({@7u|Q*iGt&Zg{Xeh&KbPr$iUno@!~Oj~ zuKs^XA-)n*|G%lgC#L>~;p*2fW9t8p6z~c4KW9C*I{jb%_g2;w^gGmjMg8XL|Mg-V zp8mf{jKkCa!(tqs{@*0V;pzX)VjQ0Sze9|})BoGVI6VD-w-|?~|Gy!|;pzWTF%D1v z-zUc5>Hqu1I6VFTpcsdz|2twFp8o%~7^l(yXQs#Auxc#OSfH^$H5Lf<|GG^7)4NTX zz;NIHxB7o+R+is|>;E76LG|CijH&h%9W{_m}z zfDV7ZV!WW=q5d!GH&_4vQH;aW|9=+a@bv%R#5g?t|93GCPyfFr#^LGzV`3bh{(nP^ z!_)thVjQ0Se@BeN)Bo>^ad`S4#(F=W`ST7>|F^_AJpDgUjKkCa3&l7*{eQj~r_uk_ Sc*spFjRhJDG!{5BE%1N-S@7}z diff --git a/tests/test_etl_reader.py b/tests/test_etl_reader.py index d181383..aaeec97 100644 --- a/tests/test_etl_reader.py +++ b/tests/test_etl_reader.py @@ -240,9 +240,16 @@ class TestNestedStructProperties: """Regression tests for properties whose type is a nested structure. ``sample.etl`` cannot cover these: Microsoft-Windows-Kernel-Process declares - no struct-typed property. ``nested_struct.etl`` is a short capture of - Kernel-Processor-Power core-parking events, which carry several - (see ``fixtures/create_nested_struct_etl.ps1``). + no struct-typed property. ``nested_struct.etl`` holds events from a + TraceLogging provider defined by ``fixtures/create_nested_struct_etl.ps1``, + each carrying two ``{ Number, Affinity }`` structs. + + TraceLogging matters here rather than being an implementation detail: such + events embed their schema, so the fixture decodes identically anywhere. A + capture from a manifest provider does not -- decoding needs that manifest + installed and of a matching version, so an earlier version of this fixture, + taken from Kernel-Processor-Power, decoded on the machine that recorded it + and produced nothing at all in CI on a different Windows build. """ @pytest.mark.skipif( @@ -266,10 +273,27 @@ def test_struct_properties_are_dicts_of_their_members(self) -> None: "were supported these were skipped outright" ) - # Every core-parking affinity struct is a { Number, Affinity } pair. - assert "NewPark" in structs, sorted(structs) - assert set(structs["NewPark"]) == {"Number", "Affinity"} - assert all(isinstance(v, int) for v in structs["NewPark"].values()) + # Both structs in the fixture are a { Number, Affinity } pair. + assert "Park" in structs, sorted(structs) + assert set(structs["Park"]) == {"Number", "Affinity"} + assert structs["Park"] == {"Number": 7, "Affinity": 0x50C0} + assert structs["Unpark"] == {"Number": 9, "Affinity": 0x000F} + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_counted_string_has_no_length_prefix_in_its_value(self) -> None: + """TDH returns the count as part of the value; it is not content. + + The fixture's ``Label`` is a counted string, which decoded as + ``'\\x14core-state'`` -- the ``\\x14`` being its own 20-byte length. + """ + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NESTED_STRUCT_ETL_PATH)).read_all() + labels = {e.properties["Label"] for e in events if "Label" in e.properties} + assert labels, "fixture no longer carries a Label property" + assert labels == {"core-state"} @pytest.mark.skipif( not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" @@ -287,12 +311,12 @@ def test_struct_members_do_not_leak_into_the_top_level(self) -> None: for event in events: props = event.properties - if "NewPark" not in props: + if "Park" not in props: continue assert "Affinity" not in props assert "Number" not in props return - pytest.fail("no event carried a NewPark struct") + pytest.fail("no event carried a Park struct") @pytest.mark.skipif( not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" From 036723e94232a80a6204390d453c86011f685fdd Mon Sep 17 00:00:00 2001 From: m96-chan Date: Sun, 6 Sep 2026 11:33:14 +0900 Subject: [PATCH 4/5] fix: Drop the length prefix from counted strings and blobs TDH returns the two-byte count as part of the value for the counted in-types, so it was being decoded as content. The new fixture's `Label` holds "core-state" and came back as "\u{14}core-state", the `\u{14}` being its own 20-byte length. The same was visible earlier in a TraceLogging capture, where "hello-from-32bit" read as " hello-from-32bit" -- the leading space was the 32-byte count, quiet enough to miss. Affects win:CountedString, win:CountedAnsiString, win:CountedBinary and their MOF equivalents. The prefix is only removed when it actually describes the rest of the buffer. Stripping unconditionally would eat the first character of any value that TDH had already stripped, and this way a provider or TDH version that behaves differently loses nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --- crates/pyetwkit-core/src/tdh.rs | 75 ++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/crates/pyetwkit-core/src/tdh.rs b/crates/pyetwkit-core/src/tdh.rs index f078710..3b23bf0 100644 --- a/crates/pyetwkit-core/src/tdh.rs +++ b/crates/pyetwkit-core/src/tdh.rs @@ -378,6 +378,24 @@ unsafe fn property_bytes( unsafe { property_bytes_at(record, &[(name_utf16, array_index)]) } } +/// Drop the two-byte length prefix a counted string or blob carries. +/// +/// TDH returns that prefix as part of the value for the counted in-types, so +/// without this it is decoded as content: a TraceLogging counted string holding +/// "core-state" came back as "\u{14}core-state", the `\u{14}` being the 20-byte +/// length. The prefix is only removed when it actually describes the rest of the +/// buffer, so a provider or TDH version that hands the value over already +/// stripped is left alone rather than losing its first character. +fn strip_counted_prefix(bytes: &[u8]) -> &[u8] { + if bytes.len() >= 2 { + let declared = u16::from_le_bytes([bytes[0], bytes[1]]) as usize; + if declared == bytes.len() - 2 { + return &bytes[2..]; + } + } + bytes +} + /// Decode a UTF-16 blob, trimming a trailing NUL if the provider included one. fn decode_utf16(bytes: &[u8]) -> String { let (pairs, _) = bytes.as_chunks::<2>(); @@ -493,17 +511,21 @@ fn to_event_value( let value = match in_type { IN_NULL => EventValue::Null, - IN_UNICODESTRING - | IN_COUNTEDSTRING - | IN_NONNULLTERMINATEDSTRING - | IN_MANIFEST_COUNTEDSTRING - | IN_UNICODECHAR => EventValue::String(decode_utf16(bytes)), + IN_UNICODESTRING | IN_NONNULLTERMINATEDSTRING | IN_UNICODECHAR => { + EventValue::String(decode_utf16(bytes)) + } - IN_ANSISTRING - | IN_COUNTEDANSISTRING - | IN_NONNULLTERMINATEDANSISTRING - | IN_MANIFEST_COUNTEDANSISTRING - | IN_ANSICHAR => EventValue::String(decode_ansi(bytes)), + IN_COUNTEDSTRING | IN_MANIFEST_COUNTEDSTRING => { + EventValue::String(decode_utf16(strip_counted_prefix(bytes))) + } + + IN_ANSISTRING | IN_NONNULLTERMINATEDANSISTRING | IN_ANSICHAR => { + EventValue::String(decode_ansi(bytes)) + } + + IN_COUNTEDANSISTRING | IN_MANIFEST_COUNTEDANSISTRING => { + EventValue::String(decode_ansi(strip_counted_prefix(bytes))) + } IN_INT8 => EventValue::I8(bytes[0] as i8), IN_UINT8 => EventValue::U8(bytes[0]), @@ -519,7 +541,9 @@ fn to_event_value( // A Win32 BOOL is a 32-bit value, not a byte. IN_BOOLEAN => EventValue::Bool(fixed!(bytes, u32, 4) != 0), - IN_BINARY | IN_HEXDUMP | IN_MANIFEST_COUNTEDBINARY => EventValue::Binary(bytes.to_vec()), + IN_BINARY | IN_HEXDUMP => EventValue::Binary(bytes.to_vec()), + + IN_MANIFEST_COUNTEDBINARY => EventValue::Binary(strip_counted_prefix(bytes).to_vec()), IN_GUID => EventValue::Guid(decode_guid(bytes)?), @@ -831,6 +855,35 @@ pub fn parse_properties(record: &EventRecord) -> HashMap { mod tests { use super::*; + #[test] + fn test_counted_string_drops_its_length_prefix() { + // TDH hands the two-byte count back as part of the value, so without + // stripping it "core-state" decoded as "\u{14}core-state". + let mut bytes = 20u16.to_le_bytes().to_vec(); + bytes.extend("core-state".encode_utf16().flat_map(u16::to_le_bytes)); + let v = to_event_value(&bytes, IN_COUNTEDSTRING, 0, 8); + assert!(matches!(v, Some(EventValue::String(s)) if s == "core-state")); + } + + #[test] + fn test_plain_string_keeps_every_character() { + // A non-counted type must never lose its first character to the strip. + let bytes: Vec = "core-state" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + let v = to_event_value(&bytes, IN_UNICODESTRING, 0, 8); + assert!(matches!(v, Some(EventValue::String(s)) if s == "core-state")); + } + + #[test] + fn test_counted_prefix_is_left_alone_when_it_does_not_fit() { + // Only a prefix that actually describes the rest of the buffer is a + // length, so a value TDH already stripped survives intact. + let bytes: Vec = "ab".encode_utf16().flat_map(u16::to_le_bytes).collect(); + assert_eq!(strip_counted_prefix(&bytes), bytes.as_slice()); + } + #[test] fn test_decode_utf16_trims_trailing_nul() { let bytes: Vec = "abc\0".encode_utf16().flat_map(u16::to_le_bytes).collect(); From c6c5f353b5fd70348df789d4542d1a97e2763c44 Mon Sep 17 00:00:00 2001 From: Yusuke Harada <49523406+m96-chan@users.noreply.github.com> Date: Sun, 6 Sep 2026 11:41:16 +0900 Subject: [PATCH 5/5] feat: Recover schema-less payloads and offer TDH's display strings (#85) * feat: Recover schema-less payloads and offer TDH's display strings The last two gaps #72 listed for the TDH parser. **Events TDH cannot describe.** `TdhGetEventInformation` has no schema for WPP events without a matching TMF, nor for manifest providers whose manifest is not installed on this machine. Those events came back with an empty property dict and nothing else: their payload was read and then dropped. There are no names to report it under, so it cannot become properties -- keep the bytes instead. `EtwEvent.raw_data` has existed all along but was never filled in, the comment in session.rs explaining that ferrisetw keeps its `user_buffer` private. The record handed to TDH carries the same bytes and this module already holds it, so read `UserData` from there. Only for events with no schema: carrying a second copy of every decoded event's data would cost memory for nothing. `set_wpp_tmf_search_path` passes TDH a TDH_CONTEXT_WPP_TMFSEARCHPATH so a caller holding .tmf files can have WPP events decoded properly rather than falling back to bytes. Unset by default, in which case TDH is passed no context at all, exactly as before. **TDH's display strings.** #72 asked whether typed values or `TdhFormatProperty`'s strings should be the default and whether the other should be available. Typed values stay the default -- the Parquet/Arrow exporters need them -- and the strings become an opt-in second map, `formatted_properties`, behind `set_property_formatting`. Off by default, so the normal path pays nothing. They earn their place by expressing what typed values cannot, because TDH applies the manifest's value maps and output types: DynamicAddress typed=1 formatted='dynamic ' Address typed=b'\x02\x00\x00\x00\xc0\xa82\x01...' formatted='192.168.50.1' QueryBlob typed=2525199111760 formatted='0x24BF1989650' `TdhFormatProperty` is given one property's bytes, already isolated by `TdhGetProperty`, rather than the event payload with running offsets. That walk is what the rest of this module avoids on purpose, and skipping it costs nothing because TDH has already located the value. Arrays and structs are left out of the string view: `TdhFormatProperty` renders a single value, so handing it an array would silently describe only the first element -- a 176-byte TimeZoneInformation rendered as "228". `properties` reports those in full. The TMF search path is not verified end to end. This machine has no .tmf files and none of the WPP tooling that produces them. What is checked is that the context reaches TDH, that setting a path leaves all other decoding byte-identical, and that a path matching nothing is harmless. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn * test: Cover schema-less payloads, display strings, and the TMF path The existing fixtures cannot exercise the schema-less path: everything in sample.etl and nested_struct.etl decodes cleanly, which is the whole point of them. Add no_schema.etl -- the capture that used to be sample.etl before #76 replaced it. It was never a good property-parsing fixture, holding unrelated telemetry and no Kernel-Process events at all, but that is exactly what makes it the right one here: 73 of its 149 events have no schema TDH can find. Tests assert the payload survives for those, that decoded events do not also carry a redundant copy, that display strings appear only when switched on and leave the typed values untouched, and that a TMF search path matching nothing leaves all other decoding byte-identical. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --------- Co-authored-by: Claude Opus 5 (1M context) --- crates/pyetwkit-core/src/event.rs | 36 +++ crates/pyetwkit-core/src/lib.rs | 20 ++ crates/pyetwkit-core/src/session.rs | 12 +- crates/pyetwkit-core/src/tdh.rs | 396 +++++++++++++++++++++++++++- tests/fixtures/no_schema.etl | Bin 0 -> 21077 bytes tests/test_etl_reader.py | 118 +++++++++ 6 files changed, 573 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/no_schema.etl diff --git a/crates/pyetwkit-core/src/event.rs b/crates/pyetwkit-core/src/event.rs index fb06ff3..aca9189 100644 --- a/crates/pyetwkit-core/src/event.rs +++ b/crates/pyetwkit-core/src/event.rs @@ -42,6 +42,8 @@ pub struct EtwEvent { pub properties: HashMap, /// Raw event data (if schema parsing failed) pub raw_data: Option>, + /// TDH's own display strings, populated only when property formatting is on + pub formatted_properties: HashMap, /// Stack trace (if enabled) pub stack_trace: Option>, } @@ -93,6 +95,7 @@ impl EtwEvent { channel: 0, properties: HashMap::new(), raw_data: None, + formatted_properties: HashMap::new(), stack_trace: None, } } @@ -290,6 +293,39 @@ impl PyEtwEvent { self.inner.channel } + /// Undecoded payload, present only when TDH could not describe the event. + /// + /// WPP events with no matching TMF, and manifest providers whose manifest is + /// not installed on this machine, have no property names to report a value + /// under. `properties` is empty for those, and this carries the bytes so the + /// payload is still recoverable. `None` whenever the event decoded normally. + #[getter] + fn raw_data(&self, py: Python<'_>) -> PyResult>> { + use pyo3::IntoPyObject; + match &self.inner.raw_data { + Some(bytes) => Ok(Some( + bytes.as_slice().into_pyobject(py)?.into_any().unbind(), + )), + None => Ok(None), + } + } + + /// TDH's own display strings for each property, name to text. + /// + /// Empty unless property formatting has been switched on, and empty for + /// events TDH has no schema for. These differ from `properties` in that TDH + /// resolves the manifest's value maps, so a bare number becomes the name the + /// manifest gives it. Struct-typed properties are absent: their members are + /// values in their own right and `properties` already reports them. + #[getter] + fn formatted_properties(&self, py: Python<'_>) -> PyResult> { + let dict = PyDict::new(py); + for (key, value) in &self.inner.formatted_properties { + dict.set_item(key, value)?; + } + Ok(dict.into()) + } + /// Get event properties as a dictionary #[getter] fn properties(&self, py: Python<'_>) -> PyResult> { diff --git a/crates/pyetwkit-core/src/lib.rs b/crates/pyetwkit-core/src/lib.rs index cfce7a3..2c167de 100644 --- a/crates/pyetwkit-core/src/lib.rs +++ b/crates/pyetwkit-core/src/lib.rs @@ -71,6 +71,26 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { wrap_pyfunction!(discovery::py_get_provider_info, m)?, )?; + // Property formatting toggle + m.add( + "set_property_formatting", + wrap_pyfunction!(tdh::py_set_property_formatting, m)?, + )?; + m.add( + "property_formatting", + wrap_pyfunction!(tdh::py_property_formatting, m)?, + )?; + + // WPP decoding needs a directory of .tmf files to consult + m.add( + "set_wpp_tmf_search_path", + wrap_pyfunction!(tdh::py_set_wpp_tmf_search_path, m)?, + )?; + m.add( + "wpp_tmf_search_path", + wrap_pyfunction!(tdh::py_wpp_tmf_search_path, m)?, + )?; + // Register ETL reader m.add_class::()?; diff --git a/crates/pyetwkit-core/src/session.rs b/crates/pyetwkit-core/src/session.rs index ded6b3d..6294335 100644 --- a/crates/pyetwkit-core/src/session.rs +++ b/crates/pyetwkit-core/src/session.rs @@ -364,7 +364,17 @@ pub fn parse_event_record(record: &EventRecord, schema: Option<&Schema>) -> EtwE if let Some(schema) = schema { event.provider_name = Some(schema.provider_name().to_string()); } - // Note: raw_data extraction removed as user_buffer is private in ferrisetw 1.2 + + // When TDH has no schema -- WPP with no matching TMF, or a manifest that is + // not installed here -- there are no names to report a payload under, and + // the payload used to be dropped entirely. Keep the bytes instead, so the + // event is at least recoverable. Only in that case: carrying a second copy + // of every decoded event's data would be a memory cost for nothing. + event.formatted_properties = crate::tdh::format_properties(record); + + if event.properties.is_empty() && !crate::tdh::has_schema(record) { + event.raw_data = crate::tdh::raw_user_data(record); + } event } diff --git a/crates/pyetwkit-core/src/tdh.rs b/crates/pyetwkit-core/src/tdh.rs index 3b23bf0..7b71f74 100644 --- a/crates/pyetwkit-core/src/tdh.rs +++ b/crates/pyetwkit-core/src/tdh.rs @@ -26,12 +26,16 @@ use crate::event::EventValue; use chrono::{TimeZone, Utc}; use ferrisetw::EventRecord; use parking_lot::Mutex; +use pyo3::prelude::*; use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use uuid::Uuid; +use windows::core::{PCWSTR, PWSTR}; use windows::Win32::System::Diagnostics::Etw::{ - TdhGetEventInformation, TdhGetProperty, TdhGetPropertySize, EVENT_PROPERTY_INFO, EVENT_RECORD, - PROPERTY_DATA_DESCRIPTOR, TRACE_EVENT_INFO, + TdhFormatProperty, TdhGetEventInformation, TdhGetEventMapInformation, TdhGetProperty, + TdhGetPropertySize, EVENT_MAP_INFO, EVENT_PROPERTY_INFO, EVENT_RECORD, + PROPERTY_DATA_DESCRIPTOR, TDH_CONTEXT, TDH_CONTEXT_WPP_TMFSEARCHPATH, TRACE_EVENT_INFO, }; /// `EVENT_HEADER_FLAG_32_BIT_HEADER`, used to pick the pointer width of the @@ -48,6 +52,9 @@ const PROPERTY_STRUCT: i32 = 1; /// rather than in this one's `count` field. const PROPERTY_PARAM_COUNT: i32 = 4; +/// `PROPERTY_FLAGS` bit meaning the byte length lives in another property. +const PROPERTY_PARAM_LENGTH: i32 = 2; + /// Ceiling on how many elements we will materialise for one array property. /// /// A fixed `count` cannot exceed `u16::MAX` anyway; the cap only matters for a @@ -117,6 +124,11 @@ struct PropertyDesc { struct_start: u16, /// For a struct, how many members follow `struct_start`. struct_members: u16, + /// Declared byte length, which `TdhFormatProperty` wants for fixed types. + length: u16, + /// Name of the value map for this property, if the schema declares one, + /// NUL-terminated and ready for `TdhGetEventMapInformation`. + map_name: Option>, } impl PropertyDesc { @@ -135,6 +147,8 @@ impl PropertyDesc { count: PropertyCount::Fixed(1), struct_start: 0, struct_members: 0, + length: 0, + map_name: None, } } @@ -155,6 +169,54 @@ struct EventLayout { type LayoutKey = (u128, u16, u8); +/// Directory TDH should search for WPP `.tmf` files, NUL-terminated UTF-16. +/// +/// WPP events carry no schema of their own: the format strings live in a `.tmf` +/// generated from the emitting binary's PDB. Without one TDH can say nothing +/// about the event, which is why such events come back with no properties and +/// only `raw_data`. Given a search path it can decode them. +fn tmf_search_path() -> &'static Mutex>> { + static PATH: OnceLock>>> = OnceLock::new(); + PATH.get_or_init(|| Mutex::new(None)) +} + +/// Point TDH at a directory of WPP `.tmf` files, or pass `None` to stop. +/// +/// Clears the layout cache, since whether an event has a schema is exactly what +/// this changes. +pub fn set_wpp_tmf_search_path(path: Option<&str>) { + let encoded = path.map(|p| { + let mut utf16: Vec = p.encode_utf16().collect(); + utf16.push(0); + utf16 + }); + *tmf_search_path().lock() = encoded; + clear_layout_cache(); +} + +/// The directory currently searched for `.tmf` files, if any. +pub fn wpp_tmf_search_path() -> Option { + tmf_search_path().lock().as_ref().map(|utf16| { + let end = utf16.iter().position(|&u| u == 0).unwrap_or(utf16.len()); + String::from_utf16_lossy(&utf16[..end]) + }) +} + +/// Build the `TDH_CONTEXT` array for a decoding call. +/// +/// Empty unless a TMF search path has been set, so the common path passes TDH +/// no context at all, exactly as before. +fn tdh_contexts(path: &Option>) -> Vec { + match path { + Some(utf16) => vec![TDH_CONTEXT { + ParameterValue: utf16.as_ptr() as u64, + ParameterType: TDH_CONTEXT_WPP_TMFSEARCHPATH, + ParameterSize: 0, + }], + None => Vec::new(), + } +} + /// Cache of event layouts keyed by (provider GUID, event id, version). /// /// A `None` entry records that TDH has no schema for this event, so we do not @@ -213,10 +275,15 @@ fn wide_string_at(buf: &[u8], offset: u32) -> Option { /// /// `record` must point to a valid `EVENT_RECORD` that stays alive for the call. unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { + // Held across both calls so the path it points at stays alive for TDH. + let path = tmf_search_path().lock(); + let contexts = tdh_contexts(&path); + let contexts = (!contexts.is_empty()).then_some(contexts.as_slice()); + let mut size: u32 = 0; - let status = unsafe { TdhGetEventInformation(record, None, None, &mut size) }; + let status = unsafe { TdhGetEventInformation(record, contexts, None, &mut size) }; if status != ERROR_INSUFFICIENT_BUFFER || size == 0 { - // Anything else means TDH has no manifest/MOF schema for this event. + // Anything else means TDH has no manifest/MOF/WPP schema for this event. return None; } @@ -226,7 +293,7 @@ unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { let mut buf: Vec = vec![0u64; size.div_ceil(8) as usize]; let info_ptr = buf.as_mut_ptr() as *mut TRACE_EVENT_INFO; - let status = unsafe { TdhGetEventInformation(record, None, Some(info_ptr), &mut size) }; + let status = unsafe { TdhGetEventInformation(record, contexts, Some(info_ptr), &mut size) }; if status != ERROR_SUCCESS { return None; } @@ -256,12 +323,27 @@ unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { let is_struct = prop.Flags.0 & PROPERTY_STRUCT != 0; // `nonStructType` and `structType` overlay each other; only one of them // is meaningful, and reading the wrong one would be nonsense. - let (in_type, out_type, struct_start, struct_members) = if is_struct { + let (in_type, out_type, struct_start, struct_members, map_name) = if is_struct { let s = unsafe { prop.Anonymous1.structType }; - (IN_NULL, 0, s.StructStartIndex, s.NumOfStructMembers) + (IN_NULL, 0, s.StructStartIndex, s.NumOfStructMembers, None) } else { let non_struct = unsafe { prop.Anonymous1.nonStructType }; - (non_struct.InType, non_struct.OutType, 0, 0) + // A map turns a bare number into the name the manifest gives it. + let map_name = wide_string_at(bytes, non_struct.MapNameOffset).map(|name| { + let mut utf16: Vec = name.encode_utf16().collect(); + utf16.push(0); + utf16 + }); + (non_struct.InType, non_struct.OutType, 0, 0, map_name) + }; + + // `length` and `lengthPropertyIndex` overlay each other. Only the plain + // length is of use here, and only as a hint to `TdhFormatProperty`; the + // value bytes themselves always come from TDH. + let length = if prop.Flags.0 & PROPERTY_PARAM_LENGTH != 0 { + 0 + } else { + unsafe { prop.Anonymous3.length } }; // `count` and `countPropertyIndex` overlay each other; the flag says @@ -284,6 +366,8 @@ unsafe fn read_layout(record: *const EVENT_RECORD) -> Option { count, struct_start, struct_members, + length, + map_name, }); } @@ -819,6 +903,240 @@ fn to_event_array( Some(EventValue::Array(items)) } +/// Whether events should also carry TDH's own display strings. +/// +/// Off by default: typed values are what the exporters want, and rendering every +/// property twice costs a `TdhGetEventInformation` plus a `TdhFormatProperty` +/// per property on top of the parse. +static FORMAT_PROPERTIES: AtomicBool = AtomicBool::new(false); + +/// Turn TDH's display strings on or off for events parsed from now on. +pub fn set_property_formatting(enabled: bool) { + FORMAT_PROPERTIES.store(enabled, Ordering::Relaxed); +} + +/// Whether TDH display strings are currently being produced. +pub fn property_formatting() -> bool { + FORMAT_PROPERTIES.load(Ordering::Relaxed) +} + +/// Look up the value map named by a property, if it has one. +/// +/// A map turns a bare number into the name the manifest gives it, which is the +/// part of `TdhFormatProperty`'s output that typed values cannot express. +/// +/// # Safety +/// +/// `record` must point to a valid `EVENT_RECORD` that stays alive for the call. +unsafe fn map_info(record: *const EVENT_RECORD, map_name: &[u16]) -> Option> { + let name = PCWSTR(map_name.as_ptr()); + let mut size: u32 = 0; + let status = unsafe { TdhGetEventMapInformation(record, name, None, &mut size) }; + if status != ERROR_INSUFFICIENT_BUFFER || size == 0 { + return None; + } + // Over-aligned for the same reason as TRACE_EVENT_INFO. + let mut buf: Vec = vec![0u64; size.div_ceil(8) as usize]; + let ptr = buf.as_mut_ptr() as *mut EVENT_MAP_INFO; + let status = unsafe { TdhGetEventMapInformation(record, name, Some(ptr), &mut size) }; + if status != ERROR_SUCCESS { + return None; + } + Some(buf) +} + +/// Render one already-isolated property value the way TDH would display it. +/// +/// The bytes come from `TdhGetProperty`, so this hands `TdhFormatProperty` a +/// single value rather than walking the event payload with running offsets. That +/// walk is the part the rest of this module avoids on purpose, and skipping it +/// costs nothing here because TDH has already found the value for us. +/// +/// # Safety +/// +/// `info` must point to the `TRACE_EVENT_INFO` describing this event. +unsafe fn format_one( + info: *const TRACE_EVENT_INFO, + map: Option<*const EVENT_MAP_INFO>, + pointer_size: usize, + prop: &PropertyDesc, + bytes: &[u8], +) -> Option { + if bytes.is_empty() { + return None; + } + let length = if prop.length != 0 { + prop.length + } else { + u16::try_from(bytes.len()).ok()? + }; + + let mut size: u32 = 0; + let mut consumed: u16 = 0; + let status = unsafe { + TdhFormatProperty( + info, + map, + pointer_size as u32, + prop.in_type, + prop.out_type, + length, + bytes, + &mut size, + None, + &mut consumed, + ) + }; + if status != ERROR_INSUFFICIENT_BUFFER || size == 0 { + return None; + } + + let mut out: Vec = vec![0u16; (size as usize).div_ceil(2)]; + let status = unsafe { + TdhFormatProperty( + info, + map, + pointer_size as u32, + prop.in_type, + prop.out_type, + length, + bytes, + &mut size, + Some(PWSTR(out.as_mut_ptr())), + &mut consumed, + ) + }; + if status != ERROR_SUCCESS { + return None; + } + + let end = out.iter().position(|&u| u == 0).unwrap_or(out.len()); + Some(String::from_utf16_lossy(&out[..end])) +} + +/// The event's raw user data, copied out of the `EVENT_RECORD`. +/// +/// ferrisetw keeps its own `user_buffer` crate-private, which is why +/// `EtwEvent.raw_data` went unpopulated; the record TDH is handed carries the +/// same bytes, so read them from there. +pub fn raw_user_data(record: &EventRecord) -> Option> { + let raw = as_raw_record(record); + // Safety: `record` is a live `EVENT_RECORD` for the duration of the call, + // and `UserDataLength` is the length ETW itself recorded for `UserData`. + unsafe { + let len = (*raw).UserDataLength as usize; + let ptr = (*raw).UserData as *const u8; + if len == 0 || ptr.is_null() { + return None; + } + Some(std::slice::from_raw_parts(ptr, len).to_vec()) + } +} + +/// Render each top-level property the way TDH itself would display it. +/// +/// This is the string view #72 asks about, offered alongside the typed values +/// rather than instead of them: typed values are what the Parquet/Arrow +/// exporters need, while these are what a human reading a trace wants, because +/// TDH resolves value maps here -- a bare `1` becomes the name the manifest +/// gives it. +/// +/// Empty unless [`set_property_formatting`] has been called, and empty for +/// events TDH has no schema for. Struct-typed properties are skipped: their +/// members are values in their own right, and a single string for the whole +/// struct would be less useful than the typed map already returned. +pub fn format_properties(record: &EventRecord) -> HashMap { + if !property_formatting() { + return HashMap::new(); + } + let raw = as_raw_record(record); + let Some(layout) = layout_for(record, raw) else { + return HashMap::new(); + }; + + // `TdhFormatProperty` wants the TRACE_EVENT_INFO itself, which the cache + // deliberately does not retain -- keeping every event's raw schema buffer + // alive would cost far more than re-reading it on the opt-in path. + let Some(mut info_buf) = (unsafe { read_event_information(raw) }) else { + return HashMap::new(); + }; + let info = info_buf.as_mut_ptr() as *const TRACE_EVENT_INFO; + + let pointer_size = pointer_size_of(record); + + let mut out = HashMap::with_capacity(layout.top_level_count); + for prop in layout.properties.iter().take(layout.top_level_count) { + if prop.is_placeholder() || prop.is_struct { + continue; + } + // `TdhFormatProperty` renders one value, so handing it a whole array + // would silently describe only the first element. Arrays and structs are + // both left to `properties`, which reports every element. + let count = element_count(raw, &layout, prop, &[]); + if count > 1 && !spans_whole_blob(prop.in_type) { + continue; + } + let Some(bytes) = (unsafe { property_bytes(raw, &prop.name_utf16, WHOLE_PROPERTY) }) else { + continue; + }; + let map_buf = prop + .map_name + .as_ref() + .and_then(|name| unsafe { map_info(raw, name) }); + let map = map_buf + .as_ref() + .map(|b| b.as_ptr() as *const EVENT_MAP_INFO); + + if let Some(text) = unsafe { format_one(info, map, pointer_size, prop, &bytes) } { + out.insert(prop.name.clone(), text); + } + } + out +} + +/// Pointer width of the process that emitted the event, not of this one. +fn pointer_size_of(record: &EventRecord) -> usize { + if record.event_flags() & EVENT_HEADER_FLAG_32_BIT_HEADER != 0 { + 4 + } else { + 8 + } +} + +/// Fetch the raw `TRACE_EVENT_INFO` for an event, over-aligned. +/// +/// # Safety +/// +/// `record` must point to a valid `EVENT_RECORD` that stays alive for the call. +unsafe fn read_event_information(record: *const EVENT_RECORD) -> Option> { + let path = tmf_search_path().lock(); + let contexts = tdh_contexts(&path); + let contexts = (!contexts.is_empty()).then_some(contexts.as_slice()); + + let mut size: u32 = 0; + let status = unsafe { TdhGetEventInformation(record, contexts, None, &mut size) }; + if status != ERROR_INSUFFICIENT_BUFFER || size == 0 { + return None; + } + let mut buf: Vec = vec![0u64; size.div_ceil(8) as usize]; + let ptr = buf.as_mut_ptr() as *mut TRACE_EVENT_INFO; + let status = unsafe { TdhGetEventInformation(record, contexts, Some(ptr), &mut size) }; + if status != ERROR_SUCCESS { + return None; + } + Some(buf) +} + +/// Whether TDH can describe this event at all. +/// +/// False for WPP without a matching TMF, and for manifest providers whose +/// manifest is not installed on this machine. Those events have no property +/// names to report, so their payload is only reachable as raw bytes. +pub fn has_schema(record: &EventRecord) -> bool { + let raw = as_raw_record(record); + layout_for(record, raw).is_some() +} + /// Parse every top-level property of an event. /// /// Returns an empty map when TDH has no schema for the event, which is normal @@ -851,6 +1169,37 @@ pub fn parse_properties(record: &EventRecord) -> HashMap { out } +/// Turn TDH's display strings on or off for events parsed from now on. +/// +/// Off by default. Typed values stay in `properties` either way; this only adds +/// `formatted_properties`, and costs an extra TDH round trip per property. +#[pyfunction] +pub fn py_set_property_formatting(enabled: bool) { + set_property_formatting(enabled); +} + +/// Whether TDH display strings are currently being produced. +#[pyfunction] +pub fn py_property_formatting() -> bool { + property_formatting() +} + +/// Point TDH at a directory of WPP `.tmf` files, or pass `None` to stop. +/// +/// WPP events carry no schema of their own, so without a matching `.tmf` they +/// arrive with no properties and only `raw_data`. Clears the schema cache. +#[pyfunction] +#[pyo3(signature = (path=None))] +pub fn py_set_wpp_tmf_search_path(path: Option<&str>) { + set_wpp_tmf_search_path(path); +} + +/// The directory currently searched for `.tmf` files, if any. +#[pyfunction] +pub fn py_wpp_tmf_search_path() -> Option { + wpp_tmf_search_path() +} + #[cfg(test)] mod tests { use super::*; @@ -884,6 +1233,35 @@ mod tests { assert_eq!(strip_counted_prefix(&bytes), bytes.as_slice()); } + #[test] + fn test_property_formatting_is_off_by_default_and_togglable() { + assert!(!property_formatting()); + set_property_formatting(true); + assert!(property_formatting()); + set_property_formatting(false); + assert!(!property_formatting()); + } + + #[test] + fn test_tmf_search_path_round_trips() { + assert_eq!(wpp_tmf_search_path(), None); + set_wpp_tmf_search_path(Some(r"C:\tmf")); + assert_eq!(wpp_tmf_search_path().as_deref(), Some(r"C:\tmf")); + set_wpp_tmf_search_path(None); + assert_eq!(wpp_tmf_search_path(), None); + } + + #[test] + fn test_no_tmf_path_means_no_tdh_context() { + // The common path must hand TDH nothing, exactly as before the WPP + // search path existed. + assert!(tdh_contexts(&None).is_empty()); + let path: Vec = "x\0".encode_utf16().collect(); + let contexts = tdh_contexts(&Some(path)); + assert_eq!(contexts.len(), 1); + assert_eq!(contexts[0].ParameterType, TDH_CONTEXT_WPP_TMFSEARCHPATH); + } + #[test] fn test_decode_utf16_trims_trailing_nul() { let bytes: Vec = "abc\0".encode_utf16().flat_map(u16::to_le_bytes).collect(); @@ -989,6 +1367,8 @@ mod tests { count: PropertyCount::Fixed(count), struct_start: 0, struct_members: 0, + length: 0, + map_name: None, } } diff --git a/tests/fixtures/no_schema.etl b/tests/fixtures/no_schema.etl new file mode 100644 index 0000000000000000000000000000000000000000..e980af4747c5f4108e7026f39fcc9f98ee32c378 GIT binary patch literal 21077 zcmeIadt6gj)<3+?NkT3pB;kHh6F@K|0tq*RqDi=;BH1+}e+RkW=YwKizmDRrig+SV!B3utGiYNw;NrJ`2PyAImccIKJ)_xtz#yhkA< zS!bVp_Fij!*IIjj4=@2v6Yzg@Fi=WL0uX>F{ox>i99o0z@Sg)1(E&_9_AfyK%KX+e z0X!g~vKZUI&j;%A-&^$b=fsjyY!D9u2qtIp6@Y%Fk7vdAULXAZw>uQ21v|7tJ(!^Z zqM#hA@!w2{gan9%m5 z|I4`K;ayaN4Jz@U8Se=o1!luCC;}_g;=S2m72f4C(1Quz+rcs^wG2}6tb)mNqwu#K z-`zXtdC;Tu|2_}*1^*9U@HYPc&oRpIHr!gA6moWiro_8ipiAdUBmkrD4p&4u!tLaJW%?4QM8aWiJs) z6fx~b0^Xzu%0G32YlS8YuO9NjP8PG?oZ!|MLxe|iGul@wmJ=1sB7 zL*kP!@z3jG+D3S}dzW2_5a#&bb@y1I;RKp539R7or?|0K1h*MgI0+6$NX;WLwMXZ4I^xdsLCNw%oR=A<9r!ZnB&6%nj!HCUbTO>XNC} zRB5gUFCb>KzWN}FpE`fjj=Ye6MtyXBCAksYmGC&NZ}ji1TC3e^sbAj?d9bH_{C{_+*PNy3`hnR^7o&)yU~ z8?kQR`qd{FItQQLJHRUVfD*`^a4Ls<<6BaBOArpz^&kp|{0VlQzS?B3u+`TTn5!X2 zV3{p=2p$5bbOTY5)->S`8NeZ+-qu*R$W&tnBHe1PF3*QTQ*~n}j&6h9R9j}wN`)Pk z-K9i^$AI8EKz?zKu{mi`CKQ~Uvk)GJbWYMuFlPogLY&?loO$L< zO@lhiHWDKAm{mR^jE7nNLUpI86e4A4s0p1A*({1H7RGl$B3wCbFvMpHV=}=!0&&IS zSsWrp4T)+n`Ge3)KZnn7d9-PDV6F$pXYMuPi)Q!^IAI`aN1z1=U-G>gJ||or ze)=OGsc5GeB0pW3-P&Zex8&PR$KHBtUcJ3Hpu@u^$nfx8>y~O;0MUL%Mlc)lf|5Bd z4QD{Zp%|ZV;X0w=MAyycR9HMI--$Gi#*gc%@zePB7Wzz01;bU}7FZl?lJUyGGT~P{ z9FlsZtX39k>+~y^7jg-EbAfe!3BYAGzWy3Wls2q{zPfTxBqecf_IyW08 zJYxHTPld$h2#l<+-48kh{)9t7Q9(pbS4`@ZM_zcy=b}3=0-}S80M@z$6}QP4W(sTtR@!IAH$DW$@wB)=?6cRpxJZ|OPSroN=DQTd&v zroMx`*8(>$8U}M;QUKoUM@>KEv@}-?8)I7^if^9BHJ14kra|LPqjDI^ZW+~3+JAnq z{{@)USCX&Qf?s__&~R&$Uaf5Cs>|B* z^^VUjg(QMWy|{XgyBtmlj76U?p4S9*`u%{TBwDWtvQSBsp9aSinob_w!#^izP*5m|p1r1T4lp(ID>UmCsxQgZ z9C<>sJl-E7FNsQ5C=%rZr}q|?7e}boa?^Z8#CmntTJ;O-R2KO;If9r*p7(H2mm}z@ z^>YNS1j_yGL63J_9u`Lsl}ouNY7!{YFfI_QRIoZo`ZqZ8%9me6Omp$wb6-ne{k)e@ z4vrMc;0W3sfByO`8I+A|NjUj(1t%~u6N;+$$m1Ylis?vN;;r<^`BO~AY4Odw%!6yr z1(i30>y@8gdL-zWhu1&%xjgs*TCxBw4KEwJ_Tabk*Y{PuWR=PBe=X!kZP>leL&hr~ zq&`XXrSPvFR1Hi2<5z&wXw zNy9*E;FI4b9|({8M*P+j&wTsm%DWdl-0yyY{@7a!&=P2H1PVch|I(Yxsb;&m%#I$+ zTw}J^w*coc^HKfFb^H zEcL)%i1@Lnv8EI*a*p^Gv5TwvzZ|5vuO-PPn%E8Qv3=x01GBVv7uh+355%Pr^w?&5 zBQ>H~l@DosO$Psw?7opjk`pb{v*puUrrTQoQakd-bL*mNQcwNmde)OBnKE%j`P!L~ zm-${=COA}y#xNu6h)b{{BhDqD`1tFkoaQxY@U5Ug{!NAjj@KAoq5koG`mm}V5}T6! zJ#5Wk`Lf5Xm)7KDhMO(K&2Q{7qT2&xA~c7c3!7QqL(}sCR4^omJAyxkZGMDK4{qQi z1Ie_|zwvHe73*05ufE`{wbfe7URM6BJgEFg`L1$U898w)eZp7T$u%9B-GsqhQGS|_ z6tcTUQq-`}zqI9Ky%ZK&IF*)j_AbKw5XOm%d07qSVZX>FKP$7-S21lsQI8Yy{@(tZ zPE)hpKI~B?6K2SSHOu^@nD#{#&(9bPOB;doFEANuu|UeY8TL$}?V(kt&)(YbqOizc zq`w4jS(94dLd+bh`5t`g z6cJ0#y%E?3vDdjLFi`Nnia5WO+%uaPUdthg(ZBYq#}{gdA6<&TI;I==DoLqJk;BZ* z49xEe(7!$25-S~yE#oM<2OB2>zYSErBtJDd0tX0}Viw*2H9+`1a`1hZf+{99#=3#u zepxWSVJ=To_sO#|Vr@m#R|NDuY?jRlGCPW>SUhJJo=c%ThLCP>eMyK*LGG?Mc2Lw+ z;$Y!&y{@^!;E&^83>g!jrCj>VTJhmCs);!nw<=ROTO%{~1Uzf@e|8n`W@4nS`F~7Z zFS+ha9ZY&u-eK6OS(F(6ZYcui#BGD~TA+Yd}f(bZ|RFx~w|6huk< zvY0r{bAvEqs`Q@Tz50U4cJ7Pqb+RLn*TiX9b!prV=gwg)h9Q6%Dmr@_18?j;Pk}Z< zU7rapef4J9Pmxe~%b?%mZ!f$=)DJf_%C|cR>9Dx{yQAo*Zlb$=E)3m;sq|g{&As3SVlI5Z zpxh-yo~FYQ$WH8-k+|NaNSpAr6f5exWaeU+eX5NbRZ#9*(Jr>Nuj1ubw-qL|>d6h#dxIzy>_KYoV> zyvOHlO#8l7N}*4AMY%)yO4$pm<$Ze-GWV?xw6){es9i)6=wMN&FuSY2)*r5ndU@A@ zZRD7}6Nf(w^p?4ikRzhLCM05754f6+TZ}h}?!8~`O9W~kJVML@%Ax59bU&uWh?Ol(_a6A}*tc^Rc1BzXE^>w`LIAEl=o4w}+|vvnDZed-50t0-;I(FmIONV15;kg( za7)W2vat+Q2KAgmOwKAM3Z<|>T~t3@h&3ZwQK|(>p^>hr<;>K<*gr`u7nz{0k1+d_ zvv9ZuiO&RbHwh`m_?&dWIam$6U%eSRXwZuI=%g{9zFq_!la8Lv5+iO%W;Mg_9sOAX z%=;xW-EMNCut>Y>$D%71$GNAG;sT)5{}Vc%Uz6#S<2U4aSDGiVqZ zAv7Hf)N8O+Qq-T>X9P;1_pn0L?6&b5`=eXi zwq`t^-^IV`cHs)D>CMGA*pD7!Qq)513L6!86#et00;8AY?ZX_cGwdJQiJUWeTaxz& zpGcubR~qm%_0@1ZV-=5Tb14|MeLrQNY`2cI+bsKXiYYFtfAn1zGnBq6H(>wJ?W~Wy zpG@DK`|hWA4xIg(+J{(qLrS%sVp4^K4l5gzF2C)mCE8;3zDO)SViAD@DVEJykf8>B zADVAu;DB-FPr6g*sdtE*`|f%989@}mg8f$~9d$Uh&u2f`eLi5)*;6089adNm_~smm zZTRY6^Y_f!ZSCLvbx3maaAW7mr>?LH?3w8{b-G0sbT2TtIy^AYI$-0YV;DehA{3=| zike2wV-SqyAsgsE*RZ1K15Mq!fG?z=Yc4X&K9_=K=8uwM>HwFj_$gRU>-d5%Z!>5eZ+Y$aDt`I&w;%`hVa0qIIJ=3b zJNB=?1I6-za#%={xdx9!D0~LRX5{EQ=-??fX0?9*NfbL#g~AY<;6tZ5vZ3ic7cr= zMNL>MJf>I-uJaWCK7t$)H=**={ykjnR{L2$mh4{%-o5So{2j-oWAgVUZ#Rvsw0A1Z z6KSqAe3{{kj1h(k>V%54%(Sb1*eqIIyj zZO=DuKU4xb+^d1;#Qebf8u?G=l}jW%H8H(1biw}3F^{}skza6~k1t|U=!>Z#0{h9( zb2{{pGm?~3Nu43SnHnAY*VEyCYe;?+XpbcUBMH%P6MFiqALER<_p&pgTR-QKaNSc& z-@f>I?w|kUQm8JSe3OePKz%{v!3FBM;=TD`ZEF}Vs%_qk_{bm+Nv;OC6j@~Vl4Bh> z&39&QU<7O+Fv%$j0`P~#rSA7#Tbe!G*F!U>TCQ8N-d{HK%R zBslgcwGV4G+D{SJ?SLJwaVe&rmXV?i2)f4NOMZz(-2aXR{N|JCaxCPh57u+XP3kFF zOoYCWb^Q^F?B>iAi(=v6losAhjCz}i%D(+}DC$IKU%3RN6NGie7iPeJKsVFVk3@&vj1KCwTLw(YbgN7*U3I#pD1O52VRFyXoFIMC zO3gHkk8*xG2Ww|KSiVN|Y)Tj6thGWXu4466=WWhJ& zqRvA5NZ_2J0R>{>z>dNsEFCH^bfbHSf6&3(!3>x-s1Rs6$lS~x?w|F`2F#WYDdcDV z*_ z?xVh>0pIKX)r@#G77Qw+gNm&##dqHB7|mc*^T&v$S&^tY8I7*phV_!{;b+P^kJ*6X zDqh~uTv-uVIp8X>zGJ!=EtjZsTWbVJuw7r%Q+>Q zZiBM}cP~zcYv=Et=4C*zme)Y-8%Kvd8t|iI;NDHd_lVJN+#3*fOHs(6Vw(6!G+PV@OLF{o9(T&_|rI_FzR>%hxnIXTZp{Voe?RA=t zyXe$~1JvK>OT|hdl%lR+CEX>a_7TSvn~XUY?C;He*jom*?=J7KLZRioMeUiF zs)ntTpR3`;$yX+D$jR6P@XX2JqwWvE$K}&P0&v#oRmTjN>4dD`?r$>WpNvu`=TnPP z4@9YmAb96!@42{^kZ(c2HlYKTKcr(-(P5qi%a~v(#;~aos?|`RNhq6%0O(<0JVd=r z$6n1I^_fCC^pf?-ckqNoUD#H-44X}rLA^{^b`lC#fPGkK8WAEmLHUTNj?NwGo&DYG zMns{LV~m6BmO0haBHd(sekw-rLiQIdZYV`?^+4Lea~WVlc|R^|?{5XnH19dD!%#rS z#cR+;`MWuheQxt{2E2<#g7lU^%<(3*y%@3w)`j#J2hmA%b5Kjaf=;5RRD$JF(68>~ zp0ncKa#RNE;i^RPlmp{7rkBxC2sc+nvx^0E; z8;_n!zqTRt+y=JryGg&(rB-XdoAzn(Y@3L7Jx|V=^gESY>bq#p+lLibR@rX2P5K>F zT!v3lk)!B5nuKtIome|-&w)qwZ;OV+>=#-un{Mp*n-i5X(T~P_iuwyGaqMSk*l!T4 zi`sH10g>22>mu5U2!ZGVMrbYT*Zz2eN%jcQ-7U=MYIOeeceV6!m*VEWXg-}$P#@k` z?VJ0qim1J))u_H0mtF)1bbyD`Uo1z^jPcwYv4y#){o0kNoDgqI0dKGQ(y*dw8j>b=rcjtCyAN<%X=@(9F=OJIM2sDQH~~~C<(^ILOQp*g@eExROmBivDg2@ zBWts==d;H^f0y&o%C{0MavLsf$u)u0IF17LjKX}(fI5t$N8JLaxT( z_rH8`Yk1Ahz$yQuYeB_aq`MLc34+!q&_ee$K!UHRcN%&FNr< z!a#WeqyaQB5&vQU?`SGH%!1)XRlzvB-{Xy)=D@TNK7W*PKJ~ddpA;;}fAJ9(6d!dd zV!e3LIKE@p3}a!Z0`-atCEs)@?l2B3663m`U${#4=EbbReI+Byzu6XxmyO$?Kxq*1 z&wB~!OX~+Q%*JyD@%PCO2wZ>M86dysWinx6wBS4Ldp|97b+9ggw2k+VK<;}!1fd04 z@A)SWKoS&*w$;MmRy{8D%>yzCGFxy#5uDCT=Q;FZWO3Q0ZDc*noQLJ2D3B#!g0m*t z_sJyqYc{$eAj64iEgr42$R?j;C#Eb=2aVqGp~4^{S`P+X*}{zujWAYPyuA_#!5qfk z+@KjrQ0R1)J6X23A$bsl9C0=>s3z8=~Qo`bebk2+Ym&J3vyP;P@--XfV)i&sTN2(iMn z2I#EgKteJ^gKv8XJc_6(74k}=aH|P@%mR`KOe%ue5(DvY8hDj~u*C-=>@!>9ph^je z*baOPEaG7`5V61@9-eC0^Tgwu^I~4rshr6oc(~2c(E(cYOlx$zqMz&7l^knuYU7=d zcm~|!uC~uNBx?^!_Rmg?BIH{i)Mfwaa6P>SzE3Mv<39lcHt%K1hRoLCck42w+vT3JSdSGqCd$f$i{i=^U zg==5q{JbAS&dhP}_c-_teuz?;kjIU4;64Wcs-C}+I1{t!x!5h&pF23Uho@=bp9$9l z#RfHWB_4>lGUd2*lNZO=dnimZRp?vX1aR?9_0)X0F4s@JE>D4LxY$(DI~6LQnVJJT z==B$XW%U@yJz^!R*A8Qz(kIZa*- z<(%*%d9bAkd|26&Y84R8e0X0zzlQK(+vY)y62!I9JTD@cwR)bLgpK(g&TuYg?L4)F zNN^IWo$X$0`AML6`q`W;OCyvyA1!cBj#k@a^XecD)a|~{37?v>$lzR5)6OqyPbqf_ zR*gHYQfvs?jnXx~8)Q!ebv!0s%MOO6&a-9NDx48PmD;0)hPR$8smX?z}#3&t2re#8OiSK%zVx1D^4H3uP z`lVW>dC0x&#Lm#L#VhcSXK1Wp2@#@DI1)N1o9jt&vJ_QqsW%0Tv`x*?x_!w?i`ISYoOQfL|Nvi|HLi9yRu(}qch=Cv-; zvZ99D&iPg8RUUdnFOQbVD&eG+xEJ?oXYDPSnO35^K<1Q?MZJu>=_T-uz&0tf%WQ7G zTp8?rr!h2H>Ta5EB239UA8HIe7`(i`WL{Ya-OjY&&1UI~1HrMu(@gB`<}4{uCZE0` z(tRMOMsUHTq(vaMnlr1)?ZVbdYwAVO4=qZ#k|^aUrB6j)tl3^W!~H5PbAz+J?wimH zNoBnQTm@2`yti<@GWmBtWL`OA; zdXDMjL-?~FDrQ0ddNqegkL-sye zyyK{8z%+g#+AO=wl@7$gHxX|LH{n7y0{BK&`ZKJbRx16(oU;9#mA32bAoy(a90|jV zk+hN3NXu+w`LN$<_-np{I7k`YNz>_q+-6$F z7;F83;UYc}G!S_r_uAEYj@plV5*f#cGu5gcwO3!KWilGHRGMG1R79*S&Nn<-eLDEM z@{OojjiZbkY58~RXVEfsB8cZ~4ssBi8^z+Es=HWzMAClw^{7qGT=_m#T_i1|B9{_L z%9`faS(MoQj87sncAA7-?Di$GoPG6^SR+Ye_EL^YQuBSFSZwq=yP2sLK}?rmf8cW= z14hQtzyn?mf^I13^oy#PkkKAsC z+aY)u11Nyy%lQkHVu#?YXRMwTf9EtSZcA>DAhFOT;DJjZ;RMiEtYWRKEy%Hoq>oMC1QwiqBfRN!vUd9sq)%cO;JleUY&dF#!H8; zkJ1g1iREXquWouz^`Qhycd>8jKOwKrAJ28G{5U}Aot;Q9r5LhocX1dl&Wr9-UdL(^ zMm#VQPAA3mzEbHtGR=Y!E*||=P^=8(#KLRQoF#o2EJAw|xK%@-n~I#x4FM+(B&IG_ zCbW_#>0P(8>IS81GK}pQVSSmzzcC<}-LU5oOCvb8 zIJQtOxzR`CW%QAdngyB#Mj!OW(_M}da3t%&*I`m>7ElMk#c^>o3mk$To&=~K9wn7r zB&C@NTyROb_`x)n^g1C)a7i^@5+6$9gZ+_XiG2MSUZaG7kTE1a z7Z^q#F05oyeau#-!^@HEsAEz_-$zVFUmKI7y<9~#Fl|gJ{_d$|;>rwjHe&^o-N6WZ zy-R4lCNcvrAL?46Zk|-AjRP&-eG$lmMa>7X@jy{SZKmpruT%%9*4#2*|c)2 z$rS;~Se7UNk2-P-E&;&B6i6=PcnOI_cF0lA zR7d`Cl#$=aa=aUHwj*|Ey{`S;Dysbs)t>4Y(aaFd5-};q7MI)|uJVSRSy)w}Tr;nn zxGn;*PP5UI(o{aK8J_Bhj3bk{FeH& zt-f;?B)krFoGHhHIng+R8>=^Fg8hq-(iDt=F}EBNG9MAx^#!;Sd`cM-)}3b0;?e;?{&c#V;mxYiwmsZbxO) zN+)557LrH?r$$uRFJ{=Crwb=dGvp*NaO6Bn-VR5B}l>9a&e_ZMca~1cRcTS-lgL}=Je5$*;8E75BQ5j@AD}MIa*T6 z_xfhZ$qVoDdBi3dEg3DjE^3iRhXp&loZyg$NmlKVQZvGKOQ}wugzo8kePp;{HZ07_ z)G!&|jJJMLiJhPkOMOtS37xNIrOY)?tP)d)!+IiCn~24};jec}4v5onJxE+x7joL- zufy6gMwXq?ye=Xsj|pZn)voi@q8HU)5gR~$U3^7sUQYuk`H)~q3iuLUv-7e6cPbG- zDMdtV2FIOfVMo&bRHq_!(DW3!bRKgTSG+0SMz)gXXT&g=fY!G@9yXP(kKZItdK0#Y zJ-L}&C!w})5yQHT;<|!(s;xMF)Bg3{RG(u|yr4i6Pilp&ns^hpM||<>#SQ_uLnpozZ7@XtBQ$#atS}!`#Ok3Bqs$I3`NId zF$6cM#Z}Ng4=f91Q<{JuSPgM1(#}oQipF|5LQ$Q#6$0nsqT!JAKsovY1ed)%9#7Do zT#!e;e}|#7Ia8kSX%oKa@x-RJ+xB$8uD+NT`rPBA;@hR1)8rniKV^}DNDFaKBhs#1 zja$C>ud#n&WH0_U?zjh)vhVA-@s8zk2RI!tLbcy^=o)mXhec(gzSI=>GBqacl{k}U zl}kRcMt_l2wPXG2$aIP{VW18*k<>HS1-N;<-&NHkr4D4Zi5v!(-z|4caTG=^di%Rc z46U38qcTb^Asq&zmXe30t(p!JpV`x&5G4{C)h>sjL^_*sS284SgE8-C0Cyw<$LCN$ z!3CdKqfe_mR^)Rle!pj2jKbmL4tI=Rs)8U5*@FnBMO<)sb@cdrMl2d`sksKX7M+H2 z+%4fFSux>bU^?b>~A`oWIni68uVl-*4LPzH-mmeENdC`=_-S+YJlA{QV}K#vw=y>%!l&HI)?P6 ztl$|7aL4_^SQe1+fN@zIC=0MJ>CvM3N${u%8ArI`Hn0&Yz@rT8KHymo$@OGV-J_}S zP^AtDG>oO*bkDIK7f}Vcw!dmGu$BWsghdCY=)r3R?m%AzK6FXM2_N|^O%X%df+7FC z8Bm-Hhq58YsgHMZl*^s4q`egJjsW#@@^oi*{Zmd(`kK@>x4PHc1D0+Uv^mXAUYlEs zGp&WbS~b);*=;>>Yic%mB>6nfN=0A0=CAWWHSR2mcakx5me)RYv$&1F7Iz(eO|5i? z+bjRA!oT`8wq8m!kGWTmU*xE0 z87ai~dYRgd#IA$@81)(TF*+Cy z#;A{m@LiCx5ll7veJpvxSVA|M>IqbY{ix?smAGL)amG(YSo4?%DW9bxufmN!XMA+m zaI7Kl>5MwYC(F)wVjM!HuMNxKVDi!&e!ZiVx0s$BBqIrQaqpCJ&C#z#RKAB;4iPcKec(suU08dRl^JpJByZ zi;qQ`foh{dOnX1Lu2AQ(Rq)m9f|(jteu7F;Ua((96qaA(Y!=(TScPhYLE2^&wUH$* z^?LR!UxT>k^@xl7DZhU30MvXzKIbl>Wf&61opDyaKK{&n2iZcZo)ib&N)+)E?Y*D4 z;gTSOzF33XUmLF}u7h1k#>RB#&b!-iEi6C$tS&ceGwTAe-irYufR&!uzeLE{NW7Dr z_sknHXPK0?CCNMN%j?kmBY$($X8P8mD?Z+GMslTTKigQxG`=Zp<6q4WJ;Za6tH`>a zSw_a>STk;qvKq-4m&cnTiloS*L#th6y^`l5T`Y<`%34Twd#)%T_4Pe(FmAw^>P+}5 zS40>hZ-9!sNC~+M43cbGX6Z&EWk0d(bU>P+jx@erm%1Oq^PsRuM2NN5)50eeglEum z^AC!e^6QBB0^{}B2}<&<@M|Sr4Bh#ix7m0G9^8G|8%gn-o2qJ+_8dLj&Cjd59z09d zLvAH0W-e)@eeI#SE+Uqzso%uJQe2hh$61FG8;Q@>8=fe-d**;X!PFZH{!T~*jO}nM zz$LX)iM6VE7?5{5fm^i*`WkUjAyo+r+nG#=_9)=sY5-KwBtY5Q?Zh-J7{N8^BAq3~ zZ7D3@(|3X{>Pw?NPS5u&hw%Pt&sq8Kd<|4JLaeM3tzMW9Q>8SsgQHE9hOL)^9{Tc- zlcW?yCv(gOcSAl5w1PlNW>~!^X45yBI~OjLLQ4&}n?XQ)r*g{LASTU8#5tqYkbrI( z50%YHw#gm~ZkthrVcL|GTxSj@k4|QqbEe7Ql%cCeSIH`OdKNer8f@tFrf&D^P@&^% z^QzhN_<^-2q|V8OhVu4h%bajN%1P**VQm3cr+ocbzv|gB)j!8l-m8urqTAuR#SB?= zEA+VP8@qaFTDq1=Ra4bBtM6E2?tE&EL4(s0R>yfwjSKUu=wNs&VGofq7ElBIbvs!0 zG1At{&X*E&OZ}!2!0NVi$JB-rcR{eAu7bIJim0?FZ{R}E&X?zMX_@T;-54}n4^rQpQ7^vxNyRxL1{3K7 ze+)3A=vrcb-#Xmb0?kuT$JIdjAbwP#6(cRC_L9iB1gd+*CTNgB)F2VlOdziyVQfn~ z4>L8@icgdxMp5$LCds3jxS{OXa+#gc+2otrYX!1jmlCW$-*e3?YNN)vjt$mUT3 z?u~~q4*a_RUxx&(dOOVz3vAFkwM_UO4Pb&80(Tj_f&ZM4LKIo;7F~TMc1Y9!rXxh* zeM%UozR3g}yS2uQB&V7>tREoH2CEj8=6s}+6~H7PiY!AafF1V&+fCJ(Ry*$Mx7AxQ z_l8Pqt;vq$fMnb{&;n#>BeDzZQbuD9vdmP#c><=#fRlmtmu)L=tTw0Es;iN~V7<=B zsJ4}wsy9F$=Q3;SMiRt%k>pIkAO&Z;%vMusYA{EYS62fu#QQ?a?SYu;GF!R5)@(KK zKEW76OWbxxD%d2=`i($$)~qr3BNdE*v9*-D6YrM~xxxH21m)ZA;c$;z@E{$G;D?~! z{v*W>z;es|7miqAT9dgLcd&qnLvzF~@*X6JAzkbNh8UrzYgG?$@U97X4{+_Uf{9MZ z2%S(W$^x1^hDC^l9GX6co5xuBlgo7X7-V27#__OOI6M;FWepS#<4Xf>$bi~Rg5J{+ z6-e6(I?@)1&BSqwTh}k8i-0mHDkf?n*+SCsWVwY<;jWG)gz7w@os36^vQ_4LTI>6OuLDhi}c%9-7Jih0PKEuzqxX-%$DYrx?xsAfK!#~=Q zGw7c-?4Je&9Eb=&S_$RX^GV{6CT- zg$JjLej`Y#<`96Cm+_)wF2Uc^hxwPo?9J0{oNzlT93>ETDBRFh;Xc8VAlfPY=Gy2) zoj#=S!u`ES_gvegEPk%OzlTW;TPDA!UE3%3UE41fnD_NAF}k+jEKpivBJUA0e{pS2 z46wBjO7mb%_dHoRXOiVphu44y$(iWR{@oVQ@UT4Veu?VV|CviIa0E7(k3~~|6r33*dwPmSoYs+@&S7!<^Apd zn@z4D%@#&%Gaxe*eTbhwMfxFNMR5iaus6ny^0+(?uNP z8~A-Lk8z7+wvW%J5rSDR58m4)qGQZRQUljq9tN(H7&xh5={rFQWfkgm-UV!gW$ z7Wsqu65nTUe;Zwbrb&i3eY4!vo%{rg0MRs3MwH5l$V~SS`RZoA2@)-W(v$pIGCpdN z!Q91H4f9Kj0XHd^y~|Z&3;(8}swjgYQX6xh!mENK8o{djG{j+dnuaJtOrK(tO|kn= ztDOy|kwD}ievEuknulmZ9^wEc_~*RYL7$%bJ++#O*pe9ZjwUD*Zbs1SlXOM*zY?k$ z7-b@<`aYq0KzZ(mX8 zBLeB)pmkPp^!{lsnR~z`!k|mEV_NT$Q9arIGnxlvoCx-ASv*Hp-Z`UuIHCd)mKzE> z24^6<2TEnK-LeFAgsF@6-YOas%8yJ{4#wdh@w|xgVWCEJZ9v9|RT8=01%{Iv!FYv6 z@SaYVq8kohIGmb1oSHVw&q5nm;1BvP+^v)L&$RRvX<1O)oKU}{xy$LnlBWe(u&x0dy`O6N$_=iJ|xGxL*1@*H7d*Ue8kEa831WUK<$u~TQ z!wG-V2)@(}Ys)OVtb>7O*$R87uKrd4w`ZAg*=sKT%s;#M)Mm#F6%{a1-x*lT5m*NK zH{&iA78#7g0&}0tJO~XNW|X{z*HJqJ|4Oqobq9X?c9sPob9=oko_o0AonwhD1IuXg z=<6=Q^ZjT0d{}W3$7aes?vhurW3vS*yM8$nC)2tfn}j9npAnXfNB*3C^0Ci{^O_&e zx4dn&yq!Zy+zu8$Zk7?;6#K~!D@WM(!Kp{Elr|=BjB1sbIV)ob`t7e#@MJL!-%IaD zyYzcH3Z`XC@b#DNtiMFVG*rC!2dG&8)RQcv*8Bz)BISR83cVIg%*PNbN`8kGcnx?k za-&1vf5!?AUKX)p?7v|JHYEC$;crF~>u(hQ-@rnI4c&i;!mnTfh?n`DtVy)E7raS3 zou$77a2#e155uT>EZSNF>6OJUjLurYJOM})#vQa+i=zz_y*TaVO&Gpm;05LA-OdZ0Zr&eY{CExgRiOB!tdPddO3^u-IEd$U*zpZ#h$U5;&1ju z{a0pL_zi>Ty{i##=fh%wNG#G9SOhgZ0n+kuija~o#u`#RqrxV@A{Ua9;~#(ese7zs z%s=4)W^$|}P>Y5jbo(P8xgF8{9w&KiAm$>!%R~BF12NbG=UO6#p11p_llWhLa;T|?6%~$;v^4)jgzmNCG9XpbMbHcYf)ZH%&wO>r11i#+_U+Flcl>3L%j|k}sqn{9R@97|Q z5xlS=xs;4wjY>Kf2I@5I>BIjK$p)e+dm9tp&@9 zvM%uvKoC3{Cc$GX#V@D9C9hLHZfZ}!g) zw5vPv9oj|Bf|_k{KT11t1jeO^CEI4;R*HLA^6c*AX8)(Gf%lMP*&Y{veE!k?{@Kb6 zRKPQi85X7@&OI~K`5QS*sm#==T01TF$*_OCPxw(d8J&*m6-w%uVErOdl z)~-F3>gDzRf+n6mtrJXB9xvx`5K1g$nlNI?0c`JZ*%Ft4|Cynvpur`Gekq63lvccX z5}B|g2u&L{S9Rn~c;q*4w`R8I6bm>5C)@2y)>Li~*vh}H)oeB>-jpw0=MqqKMpM7; zvE(k9t?wa9@YqYPpz$W5ljn$VpXPLRJ0kD_4Hpb**_gU7HKDl&gIB8oW6$C#rRR@D zHjkD2gSz>GY2C$AbhVuq)EGEpa&_baom{;eP`n2m7mN&*YBsU^X@3L81zkAz&w6TF zlv$1oly?XmTOzOtijdUr5RA7cw>bizaePJ@PX#WfgQ!ik)#LjG8wIVOJ-mGRCuqI8 zh+YhZ#QR)9oAE&vs?0~W=QQUzK4e7W<25j53C8xn%5r?jImI^?HKbywBB!;@@gY|t zur1c;7tq&tJ?^(<-{qZaoAZa8Cu;<*=In5X!QUvuqPzZEs6Y{bRK&|^|r@GFi@fKyHf8nqZ`;0HdZ z1d9htbYB!RK9@6gx^)O=99k^v)~G2sf7#gzRdWJz9fvr z!>UG8qq(-S%vOKDo`oOZ8$tX7pQ%PKKAgo5CMJLGee+Az5i|DR9rQ9CJ+`#0318Ll z(*3IAz0{o^+krpCTlMijaWUTdErX)tt$PosdLT9T?j_bw##{RzM3j2^>i_K_Rdhau z6(@YU)PIV^tR!B6en=H&S%}2H%d{R$$R{6CMOPy+^@9KUkSaR+(qdEp{UKE}1VJ20 z-g`*ZUMyPwRxDfbYpo1Bgmp4t=K2uyQwEcPMxy^vEW@W%(KWeDVzN-7jSa-XUy`X> d5nCRoYBsE_%TPA9L@DVz!~?&>ITg}>{y%#u!!iH> literal 0 HcmV?d00001 diff --git a/tests/test_etl_reader.py b/tests/test_etl_reader.py index aaeec97..8531482 100644 --- a/tests/test_etl_reader.py +++ b/tests/test_etl_reader.py @@ -350,3 +350,121 @@ def test_nested_values_are_python_native_types(self) -> None: assert isinstance( inner, allowed ), f"{name}.{member} produced {type(inner)!r}" + + +NO_SCHEMA_ETL_PATH = Path(__file__).parent / "fixtures" / "no_schema.etl" + +requires_no_schema_etl = pytest.mark.skipif( + not NO_SCHEMA_ETL_PATH.exists(), reason="Requires schema-less ETL file" +) + + +class TestEventsWithoutASchema: + """Events TDH cannot describe must not lose their payload. + + WPP without a matching TMF, and manifest providers whose manifest is not + installed here, have no property names to report a value under. Before the + fallback the payload was simply dropped. ``no_schema.etl`` is a capture where + roughly half the events are of that kind. + """ + + @requires_no_schema_etl + def test_undecodable_events_keep_their_payload(self) -> None: + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NO_SCHEMA_ETL_PATH)).read_all() + assert events, "fixture produced no events" + + undecoded = [e for e in events if not e.properties] + assert undecoded, "fixture no longer contains schema-less events" + + with_raw = [e for e in undecoded if e.raw_data is not None] + assert with_raw, "schema-less events came back with no payload at all" + assert all(isinstance(e.raw_data, bytes) for e in with_raw) + assert all(len(e.raw_data) > 0 for e in with_raw) + + @requires_no_schema_etl + def test_decoded_events_do_not_also_carry_raw_data(self) -> None: + """The fallback is a fallback, not a second copy of every event.""" + from pyetwkit import _core as pyetwkit_core + + events = pyetwkit_core.EtlReader(str(NO_SCHEMA_ETL_PATH)).read_all() + decoded = [e for e in events if e.properties] + assert decoded, "fixture no longer contains decodable events" + assert all(e.raw_data is None for e in decoded) + + +class TestPropertyFormatting: + """TDH's own display strings, offered alongside the typed values.""" + + def test_formatting_is_off_by_default(self) -> None: + from pyetwkit import _core as pyetwkit_core + + assert pyetwkit_core.property_formatting() is False + + @pytest.mark.skipif( + not NESTED_STRUCT_ETL_PATH.exists(), reason="Requires nested-struct ETL file" + ) + def test_formatted_properties_appear_only_when_enabled(self) -> None: + from pyetwkit import _core as pyetwkit_core + + path = str(NESTED_STRUCT_ETL_PATH) + + off = pyetwkit_core.EtlReader(path).read_all() + assert off, "fixture produced no events" + assert all(not e.formatted_properties for e in off) + + pyetwkit_core.set_property_formatting(True) + try: + on = pyetwkit_core.EtlReader(path).read_all() + assert any(e.formatted_properties for e in on) + for event in on: + for name, text in event.formatted_properties.items(): + assert isinstance(name, str) + assert isinstance(text, str) + # Typed values must be unaffected by the string view. + assert [e.properties for e in on] == [e.properties for e in off] + finally: + pyetwkit_core.set_property_formatting(False) + + assert pyetwkit_core.property_formatting() is False + + +class TestWppTmfSearchPath: + """WPP needs a .tmf to decode; this is how one is offered to TDH.""" + + def test_search_path_is_unset_by_default(self) -> None: + from pyetwkit import _core as pyetwkit_core + + assert pyetwkit_core.wpp_tmf_search_path() is None + + def test_search_path_round_trips(self) -> None: + from pyetwkit import _core as pyetwkit_core + + try: + pyetwkit_core.set_wpp_tmf_search_path(r"C:\tmf") + assert pyetwkit_core.wpp_tmf_search_path() == r"C:\tmf" + finally: + pyetwkit_core.set_wpp_tmf_search_path(None) + assert pyetwkit_core.wpp_tmf_search_path() is None + + @requires_no_schema_etl + def test_setting_a_search_path_does_not_disturb_other_decoding(self) -> None: + """A path that matches nothing must leave every other event alone.""" + from pyetwkit import _core as pyetwkit_core + + path = str(NO_SCHEMA_ETL_PATH) + + def snapshot() -> list[tuple[int, tuple[str, ...]]]: + return [ + (e.event_id, tuple(sorted(e.properties))) + for e in pyetwkit_core.EtlReader(path).read_all() + ] + + before = snapshot() + try: + pyetwkit_core.set_wpp_tmf_search_path(r"Z:\no\such\directory") + assert snapshot() == before + finally: + pyetwkit_core.set_wpp_tmf_search_path(None) + assert snapshot() == before