diff --git a/pyroscope/pprof-bin/Cargo.toml b/pyroscope/pprof-bin/Cargo.toml index daf87289..0f481fb5 100644 --- a/pyroscope/pprof-bin/Cargo.toml +++ b/pyroscope/pprof-bin/Cargo.toml @@ -18,12 +18,14 @@ bytes = "1.5.0" prost = "0.12.3" json = "0.12.4" lazy_static = "1.4.0" +bytemuck = "1.16.1" # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. console_error_panic_hook = { version = "0.1.7", optional = true } +base64 = "0.22.1" [dev-dependencies] wasm-bindgen-test = "0.3.34" diff --git a/pyroscope/pprof-bin/pkg/pprof_bin.d.ts b/pyroscope/pprof-bin/pkg/pprof_bin.d.ts index 87b92058..f4204d23 100644 --- a/pyroscope/pprof-bin/pkg/pprof_bin.d.ts +++ b/pyroscope/pprof-bin/pkg/pprof_bin.d.ts @@ -9,13 +9,20 @@ export function merge_prof(id: number, bytes: Uint8Array, sample_type: string): /** * @param {number} id * @param {Uint8Array} bytes +* @param {string} sample_type */ -export function merge_tree(id: number, bytes: Uint8Array): void; +export function merge_tree(id: number, bytes: Uint8Array, sample_type: string): void; /** * @param {number} id +* @param {string} sample_type +* @returns {Uint8Array} +*/ +export function export_tree(id: number, sample_type: string): Uint8Array; +/** +* @param {Uint8Array} payload * @returns {Uint8Array} */ -export function export_tree(id: number): Uint8Array; +export function export_trees_pprof(payload: Uint8Array): Uint8Array; /** * @param {number} id */ diff --git a/pyroscope/pprof-bin/pkg/pprof_bin.js b/pyroscope/pprof-bin/pkg/pprof_bin.js index 6d0e9559..913b1d40 100644 --- a/pyroscope/pprof-bin/pkg/pprof_bin.js +++ b/pyroscope/pprof-bin/pkg/pprof_bin.js @@ -110,11 +110,14 @@ module.exports.merge_prof = function(id, bytes, sample_type) { /** * @param {number} id * @param {Uint8Array} bytes +* @param {string} sample_type */ -module.exports.merge_tree = function(id, bytes) { +module.exports.merge_tree = function(id, bytes, sample_type) { const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc); const len0 = WASM_VECTOR_LEN; - wasm.merge_tree(id, ptr0, len0); + const ptr1 = passStringToWasm0(sample_type, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len1 = WASM_VECTOR_LEN; + wasm.merge_tree(id, ptr0, len0, ptr1, len1); }; let cachedInt32Memory0 = null; @@ -132,17 +135,40 @@ function getArrayU8FromWasm0(ptr, len) { } /** * @param {number} id +* @param {string} sample_type +* @returns {Uint8Array} +*/ +module.exports.export_tree = function(id, sample_type) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(sample_type, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + wasm.export_tree(retptr, id, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var v2 = getArrayU8FromWasm0(r0, r1).slice(); + wasm.__wbindgen_free(r0, r1 * 1, 1); + return v2; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +}; + +/** +* @param {Uint8Array} payload * @returns {Uint8Array} */ -module.exports.export_tree = function(id) { +module.exports.export_trees_pprof = function(payload) { try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.export_tree(retptr, id); + const ptr0 = passArray8ToWasm0(payload, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + wasm.export_trees_pprof(retptr, ptr0, len0); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; - var v1 = getArrayU8FromWasm0(r0, r1).slice(); + var v2 = getArrayU8FromWasm0(r0, r1).slice(); wasm.__wbindgen_free(r0, r1 * 1, 1); - return v1; + return v2; } finally { wasm.__wbindgen_add_to_stack_pointer(16); } diff --git a/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm b/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm index b7f92a70..12123cb7 100644 Binary files a/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm and b/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm differ diff --git a/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm.d.ts b/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm.d.ts index 10694398..c1259bcb 100644 --- a/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm.d.ts +++ b/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm.d.ts @@ -2,8 +2,9 @@ /* eslint-disable */ export const memory: WebAssembly.Memory; export function merge_prof(a: number, b: number, c: number, d: number, e: number): void; -export function merge_tree(a: number, b: number, c: number): void; -export function export_tree(a: number, b: number): void; +export function merge_tree(a: number, b: number, c: number, d: number, e: number): void; +export function export_tree(a: number, b: number, c: number, d: number): void; +export function export_trees_pprof(a: number, b: number, c: number): void; export function drop_tree(a: number): void; export function init_panic_hook(): void; export function __wbindgen_malloc(a: number, b: number): number; diff --git a/pyroscope/pprof-bin/src/ch64.rs b/pyroscope/pprof-bin/src/ch64.rs index eb11e98c..dfb419bd 100644 --- a/pyroscope/pprof-bin/src/ch64.rs +++ b/pyroscope/pprof-bin/src/ch64.rs @@ -1,5 +1,4 @@ - -pub fn read_uint64_le(bytes: &[u8]) -> (u64) { +pub fn read_uint64_le(bytes: &[u8]) -> u64 { let mut res: u64 = 0; for i in 0..8 { res |= (bytes[i] as u64) << (i * 8); @@ -7,34 +6,34 @@ pub fn read_uint64_le(bytes: &[u8]) -> (u64) { res } -const kMul: u64 = 0x9ddfea08eb382d69; +const K_MUL: u64 = 0x9ddfea08eb382d69; pub fn hash_128_to_64(l: u64, h: u64) -> u64 { - let mut a = (l ^ h).wrapping_mul(kMul); - a ^= (a >> 47); - let mut b = (h ^ a).wrapping_mul(kMul); - b ^= (b >> 47); - b = b.wrapping_mul(kMul); + let mut a = (l ^ h).wrapping_mul(K_MUL); + a ^= a >> 47; + let mut b = (h ^ a).wrapping_mul(K_MUL); + b ^= b >> 47; + b = b.wrapping_mul(K_MUL); b } -const k0: u64 = 0xc3a5c85c97cb3127; -const k2: u64 = 0x9ae16a3b2f90404f; -const k1: u64 = 0xb492b66fbe98f273; -const k3: u64 = 0xc949d7c7509e6557; +const K0: u64 = 0xc3a5c85c97cb3127; +const K2: u64 = 0x9ae16a3b2f90404f; +const K1: u64 = 0xb492b66fbe98f273; +const K3: u64 = 0xc949d7c7509e6557; fn ch16(u: u64, v: u64) -> u64 { hash_128_to_64(u, v) } fn rot64(val: u64, shift: usize) -> u64 { if shift == 0 { - return val + return val; } - return (val >> shift) | val<<(64-shift) + return (val >> shift) | val << (64 - shift); } -fn shiftMix(val: u64) -> u64 { - return val ^ (val >> 47) +fn shift_mix(val: u64) -> u64 { + return val ^ (val >> 47); } fn hash16(u: u64, v: u64) -> u64 { @@ -51,89 +50,87 @@ fn fetch32(p: &[u8]) -> u32 { fn ch33to64(s: &[u8], length: usize) -> u64 { let mut z = read_uint64_le(&s[24..]); - let mut a = read_uint64_le(&s) + - (length as u64+read_uint64_le(&s[length-16..])).wrapping_mul(k0); - let mut b = rot64(a+z, 52); - let mut c= rot64(a, 37); + let mut a = + read_uint64_le(&s) + (length as u64 + read_uint64_le(&s[length - 16..])).wrapping_mul(K0); + let mut b = rot64(a + z, 52); + let mut c = rot64(a, 37); a += read_uint64_le(&s[8..]); c += rot64(a, 7); a += read_uint64_le(&s[16..]); - let vf= a + z; - let vs= b + rot64(a, 31) + c; + let vf = a + z; + let vs = b + rot64(a, 31) + c; - a = read_uint64_le(&s[16..]) + read_uint64_le(&s[length-32..]); - z = read_uint64_le(&s[length-8..]); - b = rot64(a+z, 52); + a = read_uint64_le(&s[16..]) + read_uint64_le(&s[length - 32..]); + z = read_uint64_le(&s[length - 8..]); + b = rot64(a + z, 52); c = rot64(a, 37); - a += read_uint64_le(&s[length-24..]); + a += read_uint64_le(&s[length - 24..]); c += rot64(a, 7); - a += read_uint64_le(&s[length-16..]); + a += read_uint64_le(&s[length - 16..]); - let wf= a + z; - let ws= b + rot64(a, 31) + c; - let r= shiftMix((vf+ws).wrapping_mul(k2) + (wf+vs).wrapping_mul(k0)); - return shiftMix(r.wrapping_mul(k0)+vs).wrapping_mul(k2) + let wf = a + z; + let ws = b + rot64(a, 31) + c; + let r = shift_mix((vf + ws).wrapping_mul(K2) + (wf + vs).wrapping_mul(K0)); + return shift_mix(r.wrapping_mul(K0) + vs).wrapping_mul(K2); } fn ch17to32(s: &[u8], length: usize) -> u64 { - let a = read_uint64_le(s).wrapping_mul(k1); - let b= read_uint64_le(&s[8..]); - let c= read_uint64_le(&s[length-8..]).wrapping_mul(k2); - let d= read_uint64_le(&s[length-16..]).wrapping_mul(k0); + let a = read_uint64_le(s).wrapping_mul(K1); + let b = read_uint64_le(&s[8..]); + let c = read_uint64_le(&s[length - 8..]).wrapping_mul(K2); + let d = read_uint64_le(&s[length - 16..]).wrapping_mul(K0); return hash16( - rot64(a-b, 43)+rot64(c, 30)+d, - a+rot64(b^k3, 20)-c+(length as u64), - ) + rot64(a - b, 43) + rot64(c, 30) + d, + a + rot64(b ^ K3, 20) - c + (length as u64), + ); } fn ch0to16(s: &[u8], length: usize) -> u64 { if length > 8 { let a = read_uint64_le(s); - let b= read_uint64_le(&s[length-8..]); - return ch16(a, rot64(b+(length as u64), (length))) ^ b; + let b = read_uint64_le(&s[length - 8..]); + return ch16(a, rot64(b + (length as u64), length)) ^ b; } if length >= 4 { - let a = (fetch32(s) as u64); - return ch16((length as u64)+(a<<3), (fetch32(&s[length-4..]) as u64)); + let a = fetch32(s) as u64; + return ch16((length as u64) + (a << 3), fetch32(&s[length - 4..]) as u64); } if length > 0 { let a = s[0]; - let b = s[length>>1]; - let c= s[length-1]; + let b = s[length >> 1]; + let c = s[length - 1]; let y = (a as u32) + ((b as u32) << 8); let z = (length as u32) + ((c as u32) << 2); - return shiftMix( - (y as u64).wrapping_mul(k2)^ - (z as u64).wrapping_mul(k3)) - .wrapping_mul(k2); + return shift_mix((y as u64).wrapping_mul(K2) ^ (z as u64).wrapping_mul(K3)) + .wrapping_mul(K2); } - return k2 + return K2; } -fn weakHash32Seeds(w: u64, x: u64, y: u64, z: u64, _a: u64, _b: u64) -> (u64, u64) { +fn weak_hash32_seeds(w: u64, x: u64, y: u64, z: u64, _a: u64, _b: u64) -> (u64, u64) { let mut a = _a + w; - let mut b = rot64(_b+a+z, 21); + let mut b = rot64(_b + a + z, 21); let c = a; a += x; a += y; b += rot64(a, 44); - return (a+z, b+c) + return (a + z, b + c); } // Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. -fn weakHash32SeedsByte(s: &[u8], a: u64, b: u64) -> (u64, u64) { +fn weak_hash32_seeds_byte(s: &[u8], a: u64, b: u64) -> (u64, u64) { _ = s[31]; - return weakHash32Seeds( - read_uint64_le(&s[0..0+8]), - read_uint64_le(&s[8..8+8]), - read_uint64_le(&s[16..16+8]), - read_uint64_le(&s[24..24+8]), + return weak_hash32_seeds( + read_uint64_le(&s[0..0 + 8]), + read_uint64_le(&s[8..8 + 8]), + read_uint64_le(&s[16..16 + 8]), + read_uint64_le(&s[24..24 + 8]), a, b, ); } -fn nearestMultiple64(b: &[u8]) -> usize { +fn nearest_multiple_64(b: &[u8]) -> usize { return ((b.len()) - 1) & !63; } @@ -141,41 +138,40 @@ fn nearestMultiple64(b: &[u8]) -> usize { pub fn city_hash_64(s: &[u8]) -> u64 { let length = s.len(); if length <= 16 { - return ch0to16(s, length) + return ch0to16(s, length); } if length <= 32 { - return ch17to32(s, length) + return ch17to32(s, length); } if length <= 64 { - return ch33to64(s, length) + return ch33to64(s, length); } - let x= read_uint64_le(s); - let y= read_uint64_le(&s[length-16..]) ^ k1; - let mut z = read_uint64_le(&s[length-56..]) ^ k0; + let x = read_uint64_le(s); + let y = read_uint64_le(&s[length - 16..]) ^ K1; + let mut z = read_uint64_le(&s[length - 56..]) ^ K0; - let mut v= weakHash32SeedsByte(&s[length-64..], (length as u64), y); - let mut w= weakHash32SeedsByte(&s[length-32..], (length as u64).wrapping_mul(k1), k0); - z += shiftMix(v.1).wrapping_mul(k1); - let mut x = rot64(z+x, 39).wrapping_mul(k1); - let mut y = rot64(y, 33).wrapping_mul(k1); + let mut v = weak_hash32_seeds_byte(&s[length - 64..], length as u64, y); + let mut w = weak_hash32_seeds_byte(&s[length - 32..], (length as u64).wrapping_mul(K1), K0); + z += shift_mix(v.1).wrapping_mul(K1); + let mut x = rot64(z + x, 39).wrapping_mul(K1); + let mut y = rot64(y, 33).wrapping_mul(K1); // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks. - let mut _s = &s[..nearestMultiple64(s)]; + let mut _s = &s[..nearest_multiple_64(s)]; while _s.len() > 0 { - x = rot64(x+y+v.0+read_uint64_le(&s[16..]), 37).wrapping_mul(k1); - y = rot64(y+v.1+read_uint64_le(&s[48..]), 42).wrapping_mul(k1); + x = rot64(x + y + v.0 + read_uint64_le(&s[16..]), 37).wrapping_mul(K1); + y = rot64(y + v.1 + read_uint64_le(&s[48..]), 42).wrapping_mul(K1); x ^= w.1; y ^= v.0; - z = rot64(z^w.0, 33); - v = weakHash32SeedsByte(s, v.1.wrapping_mul(k1), x+w.0); - w = weakHash32SeedsByte(&s[32..], z+w.1, y); + z = rot64(z ^ w.0, 33); + v = weak_hash32_seeds_byte(s, v.1.wrapping_mul(K1), x + w.0); + w = weak_hash32_seeds_byte(&s[32..], z + w.1, y); (z, x) = (x, z); _s = &_s[64..]; } return ch16( - ch16(v.0, w.0)+shiftMix(y).wrapping_mul(k1)+z, - ch16(v.1, w.1)+x, + ch16(v.0, w.0) + shift_mix(y).wrapping_mul(K1) + z, + ch16(v.1, w.1) + x, ); } - diff --git a/pyroscope/pprof-bin/src/lib.rs b/pyroscope/pprof-bin/src/lib.rs index ab28200c..02605ce0 100644 --- a/pyroscope/pprof-bin/src/lib.rs +++ b/pyroscope/pprof-bin/src/lib.rs @@ -1,23 +1,23 @@ #![allow(unused_assignments)] -mod utils; mod ch64; +mod merge; +use ch64::city_hash_64; +use ch64::read_uint64_le; use lazy_static::lazy_static; use pprof_pb::google::v1::Function; use pprof_pb::google::v1::Location; use pprof_pb::google::v1::Profile; -use pprof_pb::querier::v1::Level; +use pprof_pb::google::v1::Sample; use pprof_pb::querier::v1::FlameGraph; +use pprof_pb::querier::v1::Level; use pprof_pb::querier::v1::SelectMergeStacktracesResponse; -use std::panic; use prost::Message; use std::collections::{HashMap, HashSet}; -use std::slice::SliceIndex; +use std::panic; use std::sync::Mutex; use std::vec::Vec; use wasm_bindgen::prelude::*; -use ch64::city_hash_64; -use ch64::read_uint64_le; pub mod pprof_pb { @@ -39,20 +39,20 @@ pub mod pprof_pb { } struct TreeNodeV2 { - parent_id: u64, + //parent_id: u64, fn_id: u64, node_id: u64, - slf: u64, - total: u64 + slf: Vec, + total: Vec, } struct Tree { names: Vec, names_map: HashMap, - nodes: HashMap>, - sample_type: String, - max_self: i64, - nodes_num: i32 + nodes: HashMap>, + sample_types: Vec, + max_self: Vec, + nodes_num: i32, } fn find_node(id: u64, nodes: &Vec) -> i32 { @@ -73,7 +73,7 @@ fn get_node_id(parent_id: u64, name_hash: u64, level: u16) -> u64 { node_bytes[i] = ((parent_id >> (i * 8)) & 0xFF) as u8; } for i in 0..8 { - node_bytes[i+8] = ((name_hash >> (i * 8)) & 0xFF) as u8; + node_bytes[i + 8] = ((name_hash >> (i * 8)) & 0xFF) as u8; } let mut _level = level; if _level > 511 { @@ -82,6 +82,56 @@ fn get_node_id(parent_id: u64, name_hash: u64, level: u16) -> u64 { (city_hash_64(&node_bytes[0..]) >> 9) | ((_level as u64) << 55) } +struct MergeTotalsProcessor { + from_idx: Vec, +} + +impl MergeTotalsProcessor { + fn new(tree: &Tree, p: &Profile) -> MergeTotalsProcessor { + let mut from_idx: Vec = vec![-1; tree.sample_types.len()]; + for i in 0..tree.sample_types.len() { + let sample_type_to = &tree.sample_types[i]; + for j in 0..p.sample_type.len() { + let sample_type_from = format!( + "{}:{}", + p.string_table[p.sample_type[j].r#type as usize], + p.string_table[p.sample_type[j].unit as usize] + ); + if sample_type_from == *sample_type_to { + from_idx[i] = j as i32; + break; + } + } + } + MergeTotalsProcessor { from_idx } + } + + fn merge_totals( + &self, + node: &mut TreeNodeV2, + _max_self: &Vec, + sample: &Sample, + merge_self: bool, + ) -> Vec { + let mut max_self = _max_self.clone(); + for i in 0..self.from_idx.len() { + if self.from_idx[i] == -1 { + continue; + } + node.total[i] += sample.value[self.from_idx[i] as usize]; + if merge_self { + node.slf[i] += sample.value[self.from_idx[i] as usize]; + for i in 0..max_self.len() { + if max_self[i] < node.slf[i] { + max_self[i] = node.slf[i]; + } + } + } + } + max_self + } +} + fn merge(tree: &mut Tree, p: &Profile) { let mut functions: HashMap = HashMap::new(); for f in p.function.iter() { @@ -92,23 +142,7 @@ fn merge(tree: &mut Tree, p: &Profile) { locations.insert(l.id, &l); } - let mut value_idx: i32 = -1; - for i in 0..p.sample_type.len() { - let sample_type = format!( - "{}:{}", - p.string_table[p.sample_type[i].r#type as usize], - p.string_table[p.sample_type[i].unit as usize] - ); - if tree.sample_type == sample_type { - value_idx = i as i32; - break; - } - } - - if value_idx == -1 { - return; - } - let u_value_idx = value_idx as usize; + let m = MergeTotalsProcessor::new(tree, p); for l in p.location.iter() { let line = &p.string_table[functions[&l.line[0].function_id].name as usize]; let line_hash = city_hash_64(line.as_bytes()); @@ -125,38 +159,39 @@ fn merge(tree: &mut Tree, p: &Profile) { let location = locations[&s.location_id[i]]; let name = &p.string_table[functions[&location.line[0].function_id].name as usize]; let name_hash = city_hash_64(name.as_bytes()); - let node_id = get_node_id( - parent_id, name_hash,(s.location_id.len() - i) as u16 - ); - if !tree.nodes.contains_key(&parent_id) && tree.nodes_num < 2000000{ + let node_id = get_node_id(parent_id, name_hash, (s.location_id.len() - i) as u16); + if !tree.nodes.contains_key(&parent_id) && tree.nodes_num < 2000000 { tree.nodes.insert(parent_id, Vec::new()); } - let mut slf: u64 = 0; - if i == 0 { - slf = s.value[u_value_idx] as u64; - } - if tree.max_self < slf as i64 { - tree.max_self = slf as i64; - } let mut fake_children: Vec = Vec::new(); - let mut children = tree.nodes - .get_mut(&parent_id) - .unwrap_or(&mut fake_children); - let n = find_node(node_id, children); + let children = tree.nodes.get_mut(&parent_id).unwrap_or(&mut fake_children); + let mut n = find_node(node_id, children); if n == -1 { children.push(TreeNodeV2 { - parent_id, + //parent_id, fn_id: name_hash, node_id, - slf, - total: s.value[u_value_idx] as u64 + slf: vec![0; tree.sample_types.len()], + total: vec![0; tree.sample_types.len()], }); + let idx = children.len().clone() - 1; + let max_self = m.merge_totals( + children.get_mut(idx).unwrap(), + tree.max_self.as_ref(), + s, + i == 0, + ); + tree.max_self = max_self; + n = idx as i32; } else if tree.nodes_num < 2000000 { - children.get_mut(n as usize).unwrap().total += s.value[u_value_idx] as u64; - children.get_mut(n as usize).unwrap().slf += slf; + m.merge_totals( + children.get_mut(n as usize).unwrap(), + &tree.max_self, + s, + i == 0, + ); tree.nodes_num += 1; } - parent_id = node_id; } } @@ -176,38 +211,49 @@ fn read_uleb128(bytes: &[u8]) -> (usize, usize) { (result, shift) } - - -fn bfs(t: &Tree, res: &mut Vec) { - let mut total: u64 = 0; +fn bfs(t: &Tree, res: &mut Vec, sample_type: String) { + let mut total: i64 = 0; let mut root_children: &Vec = &Vec::new(); if t.nodes.contains_key(&(0u64)) { root_children = t.nodes.get(&(0u64)).unwrap(); } + + let mut _sample_type_index: i32 = -1; + for i in 0..t.sample_types.len() { + if t.sample_types[i] == sample_type { + _sample_type_index = i as i32; + break; + } + } + if _sample_type_index == -1 { + return; + } + let sample_type_index = _sample_type_index as usize; + for i in root_children.iter() { - total += i.total; + total += i.total[sample_type_index]; } let mut lvl = Level::default(); - lvl.values.extend([0, total as i64, 0, 0]); + lvl.values.extend([0, total, 0, 0]); res.push(lvl); - let totalNode: TreeNodeV2 = TreeNodeV2 { - slf: 0, - total: total, + let mut totals = vec![0; t.sample_types.len()]; + totals[sample_type_index] = total; + let total_node: TreeNodeV2 = TreeNodeV2 { + slf: vec![0; t.sample_types.len()], + total: totals, node_id: 0, fn_id: 0, - parent_id: 0 + //parent_id: 0 }; - let mut prepend_map: HashMap = HashMap::new(); + let mut prepend_map: HashMap = HashMap::new(); let mut reviewed: HashSet = HashSet::new(); - let mut refs: Vec<&TreeNodeV2> = vec![&totalNode]; - let mut refLen: usize = 1; - let mut i = 0; - while refLen > 0 { - i+=1; - let mut prepend: u64 = 0; + let mut refs: Vec<&TreeNodeV2> = vec![&total_node]; + let mut ref_len: usize = 1; + while ref_len > 0 { + let mut prepend: i64 = 0; let _refs = refs.clone(); refs.clear(); lvl = Level::default(); @@ -216,10 +262,9 @@ fn bfs(t: &Tree, res: &mut Vec) { let opt = t.nodes.get(&parent.node_id); if opt.is_none() { - prepend += parent.total; + prepend += parent.total[sample_type_index]; continue; } - let mut totalSum: u64 = 0; for n in opt.unwrap().iter() { if reviewed.contains(&n.node_id) { // PANIC!!! WE FOUND A LOOP @@ -229,21 +274,18 @@ fn bfs(t: &Tree, res: &mut Vec) { } prepend_map.insert(n.node_id, prepend); refs.push(n); - totalSum += n.total; - lvl.values.extend( - [ - prepend as i64, - n.total as i64, - n.slf as i64, - *t.names_map.get(&n.fn_id).unwrap_or(&1) as i64 - ] - ); + lvl.values.extend([ + prepend as i64, + n.total[sample_type_index], + n.slf[sample_type_index], + *t.names_map.get(&n.fn_id).unwrap_or(&1) as i64, + ]); prepend = 0; } - prepend += parent.slf; + prepend += parent.slf[sample_type_index]; } res.push(lvl.clone()); - refLen = refs.len(); + ref_len = refs.len(); } } @@ -251,144 +293,337 @@ lazy_static! { static ref CTX: Mutex> = Mutex::new(HashMap::new()); } -fn upsert_tree(ctx: &mut HashMap, id: u32) { +fn upsert_tree(ctx: &mut HashMap, id: u32, sample_types: Vec) { if !ctx.contains_key(&id) { + let _len = sample_types.len().clone(); ctx.insert( id, Tree { names: vec!["total".to_string(), "n/a".to_string()], names_map: HashMap::new(), nodes: HashMap::new(), - sample_type: "".to_string(), - max_self: 0, - nodes_num: 1 + sample_types, + max_self: vec![0; _len], + nodes_num: 1, }, ); } } -fn merge_trie(tree: &mut Tree, bytes: &[u8]) { - let mut size = 0; - let mut offs = 0; - (size, offs) = read_uleb128(bytes); +struct TrieReader { + bytes: Vec, + offs: usize, +} + +impl TrieReader { + fn new(bytes: &[u8]) -> TrieReader { + TrieReader { + bytes: bytes.to_vec(), + offs: 0, + } + } + + fn read_uint64_le(&mut self) -> u64 { + let res = read_uint64_le(&self.bytes[self.offs..]); + self.offs += 8; + res + } + + fn read_size(&mut self) -> usize { + let res = read_uleb128(&self.bytes[self.offs..]); + self.offs += res.1; + res.0 + } + + fn read_string(&mut self) -> String { + let size = self.read_size(); + let string = String::from_utf8_lossy(&self.bytes[self.offs..self.offs + size]).to_string(); + self.offs += size; + string + } + + /*fn read_blob(&mut self) -> &[u8] { + let size = self.read_size(); + let string = &self.bytes[self.offs..self.offs + size]; + self.offs += size; + string + } + + fn read_string_vec(&mut self) -> Vec { + let mut res = Vec::new(); + let size = self.read_size(); + for _ in 0..size { + res.push(self.read_string()); + } + res + }*/ + + fn read_blob_vec(&mut self) -> Vec<&[u8]> { + let mut res = Vec::new(); + let size = self.read_size(); + for _ in 0..size { + let uleb = read_uleb128(&self.bytes[self.offs..]); + self.offs += uleb.1; + let _size = uleb.0; + let string = &self.bytes[self.offs..self.offs + _size]; + self.offs += _size; + res.push(string); + } + res + } + /*fn end(&self) -> bool { + self.offs >= self.bytes.len() + }*/ +} + +fn merge_trie(tree: &mut Tree, bytes: &[u8], samples_type: &String) { + let _sample_type_index = tree.sample_types.iter().position(|x| x == samples_type); + if _sample_type_index.is_none() { + return; + } + let sample_type_index = _sample_type_index.unwrap(); + let mut reader = TrieReader::new(bytes); + let mut size = reader.read_size(); for _i in 0..size { - let id = read_uint64_le(&bytes[offs..]); - offs += 8; - let mut _offs: usize = 0; - let mut _size: usize = 0; - (_size, _offs) = read_uleb128(&bytes[offs..]); - offs += _offs; + let id = reader.read_uint64_le(); + let func = reader.read_string(); if !tree.names_map.contains_key(&id) && tree.names.len() < 2000000 { - tree.names.push(String::from_utf8_lossy(&bytes[offs..offs + _size]).to_string()); + tree.names.push(func); tree.names_map.insert(id, tree.names.len() - 1); } - offs += _size; } - let mut _offs: usize = 0; - (size, _offs) = read_uleb128(&bytes[offs..]); - offs += _offs; + size = reader.read_size(); for _i in 0..size { - let parent_id = read_uint64_le(&bytes[offs..]); - offs += 8; - let fn_id = read_uint64_le(&bytes[offs..]); - offs += 8; - let node_id = read_uint64_le(&bytes[offs..]); - offs += 8; - let slf = read_uint64_le(&bytes[offs..]); - offs += 8; - let total = read_uint64_le(&bytes[offs..]); - if tree.max_self < slf as i64 { - tree.max_self = slf as i64; + let parent_id = reader.read_uint64_le(); + let fn_id = reader.read_uint64_le(); + let node_id = reader.read_uint64_le(); + let _slf = reader.read_uint64_le() as i64; + let _total = reader.read_uint64_le() as i64; + if tree.max_self[sample_type_index] < _slf { + tree.max_self[sample_type_index] = _slf; } - offs += 8; + let mut slf = vec![0; tree.sample_types.len()]; + slf[sample_type_index] = _slf; + let mut total = vec![0; tree.sample_types.len()]; + total[sample_type_index] = _total; + let mut n: i32 = -1; if tree.nodes.contains_key(&parent_id) { - let n = find_node(node_id, tree.nodes.get(&parent_id).unwrap()); - if n != -1 { - tree.nodes.get_mut(&parent_id).unwrap().get_mut(n as usize).unwrap().total += total; - tree.nodes.get_mut(&parent_id).unwrap().get_mut(n as usize).unwrap().slf += slf; - } else if tree.nodes_num < 2000000 { - tree.nodes.get_mut(&parent_id).unwrap().push(TreeNodeV2 { - fn_id, - parent_id, - node_id, - slf, - total - }); - tree.nodes_num+=1; - } - - } else if tree.nodes_num < 2000000 { + n = find_node(node_id, tree.nodes.get(&parent_id).unwrap()); + } + if n != -1 { + tree.nodes + .get_mut(&parent_id) + .unwrap() + .get_mut(n as usize) + .unwrap() + .total[sample_type_index] += total[sample_type_index]; + tree.nodes + .get_mut(&parent_id) + .unwrap() + .get_mut(n as usize) + .unwrap() + .slf[sample_type_index] += slf[sample_type_index]; + } + if tree.nodes_num >= 2000000 { + return; + } + if !tree.nodes.contains_key(&parent_id) { tree.nodes.insert(parent_id, Vec::new()); - tree.nodes.get_mut(&parent_id).unwrap().push(TreeNodeV2 { - fn_id, - parent_id, - node_id, - slf, - total - }); - tree.nodes_num+=1; } + tree.nodes.get_mut(&parent_id).unwrap().push(TreeNodeV2 { + fn_id, + //parent_id, + node_id, + slf, + total, + }); + tree.nodes_num += 1; } } +/*fn upsert_string(prof: &mut Profile, s: String) -> i64 { + let mut idx = 0; + for i in 0..prof.string_table.len() { + if prof.string_table[i] == s { + idx = i as i64; + break; + } + } + if idx == 0 { + idx = prof.string_table.len() as i64; + prof.string_table.push(s); + } + idx +}*/ + +/*fn upsert_function(prof: &mut Profile, fn_id: u64, fn_name_id: i64) { + for f in prof.function.iter() { + if f.id == fn_id { + return; + } + } + let mut func = Function::default(); + func.name = fn_name_id; + func.id = fn_id; + func.filename = upsert_string(prof, "unknown".to_string()); + func.system_name = upsert_string(prof, "unknown".to_string()); + prof.function.push(func); +}*/ + +/*fn inject_locations(prof: &mut Profile, tree: &Tree) { + for n in tree.names_map.iter() { + let hash = *n.1 as u64; + let name = tree.names[hash as usize].clone(); + let fn_idx = upsert_string(prof, name); + upsert_function(prof, *n.0, fn_idx); + let mut loc = Location::default(); + let mut line = Line::default(); + line.function_id = *n.0; + loc.id = *n.0; + loc.line = vec![line]; + prof.location.push(loc) + } +}*/ + +/*fn upsert_sample(prof: &mut Profile, loc_id: Vec, val: i64, val_idx: i64) -> i64 { + let mut idx = -1; + for i in 0..prof.sample.len() { + if prof.sample[i].location_id.len() != loc_id.len() { + continue; + } + let mut found = true; + for j in 0..prof.sample[i].location_id.len() { + if prof.sample[i].location_id[j] != loc_id[j] { + found = false; + break; + } + } + if found { + idx = i as i64; + break; + } + } + if idx == -1 { + let mut sample = Sample::default(); + sample.location_id = loc_id.clone(); + sample.location_id.reverse(); + idx = prof.sample.len() as i64; + prof.sample.push(sample); + } + while prof.sample[idx as usize].value.len() <= val_idx as usize { + prof.sample[idx as usize].value.push(0) + } + prof.sample[idx as usize].value[val_idx as usize] += val; + idx +}*/ + +/*fn inject_functions( + prof: &mut Profile, + tree: &Tree, + parent_id: u64, + loc_ids: Vec, + val_idx: i64, +) { + if !tree.nodes.contains_key(&parent_id) { + return; + } + let children = tree.nodes.get(&parent_id).unwrap(); + for node in children.iter() { + let mut _loc_ids = loc_ids.clone(); + _loc_ids.push(node.fn_id); + //TODO: + upsert_sample(prof, _loc_ids.clone(), node.slf[0 /*TODO*/] as i64, val_idx); + if tree.nodes.contains_key(&node.node_id) { + inject_functions(prof, tree, node.node_id, _loc_ids, val_idx); + } + } +}*/ + +/*fn merge_profile(tree: &Tree, prof: &mut Profile, sample_type: String, sample_unit: String) { + let mut value_type = ValueType::default(); + value_type.r#type = upsert_string(prof, sample_type); + value_type.unit = upsert_string(prof, sample_unit); + prof.sample_type.push(value_type); + let type_idx = prof.sample_type.len() as i64 - 1; + inject_locations(prof, tree); + inject_functions(prof, tree, 0, vec![], type_idx); +}*/ + #[wasm_bindgen] pub fn merge_prof(id: u32, bytes: &[u8], sample_type: String) { let p = panic::catch_unwind(|| { let mut ctx = CTX.lock().unwrap(); - upsert_tree(&mut ctx, id); + upsert_tree(&mut ctx, id, vec![sample_type]); let mut tree = ctx.get_mut(&id).unwrap(); - tree.sample_type = sample_type; let prof = Profile::decode(bytes).unwrap(); merge(&mut tree, &prof); }); match p { - Ok(res) => {} - Err(err) => panic!(err) - + Ok(_) => {} + Err(err) => panic!("{:?}", err), } } #[wasm_bindgen] -pub fn merge_tree(id: u32, bytes: &[u8]) { +pub fn merge_tree(id: u32, bytes: &[u8], sample_type: String) { let result = panic::catch_unwind(|| { let mut ctx = CTX.lock().unwrap(); - upsert_tree(&mut ctx, id); + upsert_tree(&mut ctx, id, vec![sample_type.clone()]); let mut tree = ctx.get_mut(&id).unwrap(); - merge_trie(&mut tree, bytes); + merge_trie(&mut tree, bytes, &sample_type); 0 }); match result { - Ok(res) => {} - Err(err) => panic!(err) + Ok(_) => {} + Err(err) => panic!("{:?}", err), } } #[wasm_bindgen] -pub fn export_tree(id: u32) -> Vec { +pub fn export_tree(id: u32, sample_type: String) -> Vec { let p = panic::catch_unwind(|| { let mut ctx = CTX.lock().unwrap(); let mut res = SelectMergeStacktracesResponse::default(); - upsert_tree(&mut ctx, id); - let mut tree = ctx.get_mut(&id).unwrap(); + upsert_tree(&mut ctx, id, vec![sample_type.clone()]); + let tree = ctx.get_mut(&id).unwrap(); let mut fg = FlameGraph::default(); fg.names = tree.names.clone(); - fg.max_self = tree.max_self; + fg.max_self = tree.max_self[0 /* TODO */]; fg.total = 0; let mut root_children: &Vec = &vec![]; if tree.nodes.contains_key(&(0u64)) { root_children = tree.nodes.get(&(0u64)).unwrap(); } for n in root_children.iter() { - fg.total += n.total as i64; + fg.total += n.total[0 /*TODO*/] as i64; } - bfs(tree, &mut fg.levels); + bfs(tree, &mut fg.levels, sample_type.clone()); res.flamegraph = Some(fg); - return res.encode_to_vec(); + return res.encode_to_vec(); + }); + match p { + Ok(res) => return res, + Err(err) => panic!("{:?}", err), + } +} + +#[wasm_bindgen] +pub fn export_trees_pprof(payload: &[u8]) -> Vec { + let p = panic::catch_unwind(|| { + let mut reader = TrieReader::new(payload); + let bin_profs = reader.read_blob_vec(); + let mut merger = merge::ProfileMerge::new(); + for bin_prof in bin_profs { + let mut prof = Profile::decode(bin_prof).unwrap(); + merger.merge(&mut prof); + } + let res = merger.profile(); + res.encode_to_vec() }); match p { Ok(res) => return res, - Err(err) => panic!(err) + Err(err) => panic!("{:?}", err), } } diff --git a/pyroscope/pprof-bin/src/merge.rs b/pyroscope/pprof-bin/src/merge.rs new file mode 100644 index 00000000..7425f506 --- /dev/null +++ b/pyroscope/pprof-bin/src/merge.rs @@ -0,0 +1,679 @@ +use crate::ch64::city_hash_64; +use crate::pprof_pb::google::v1::Function; +use crate::pprof_pb::google::v1::Line; +use crate::pprof_pb::google::v1::Location; +use crate::pprof_pb::google::v1::Mapping; +use crate::pprof_pb::google::v1::Sample; +use crate::pprof_pb::google::v1::ValueType; +use crate::pprof_pb::google::v1::{Label, Profile}; +use bytemuck; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; + + +pub struct ProfileMerge { + prof: Option, + tmp: Vec, + + string_table: Option>, + function_table: Option>, + mapping_table: Option>, + location_table: Option>, + sample_table: Option>, +} + +impl ProfileMerge { + pub(crate) fn new() -> ProfileMerge { + ProfileMerge { + prof: Option::None, + tmp: Vec::new(), + + string_table: Option::None, + function_table: Option::None, + mapping_table: Option::None, + location_table: Option::None, + sample_table: Option::None, + } + } + pub fn merge(&mut self, p: &mut Profile) { + if p.sample.len() == 0 || p.string_table.len() < 2 { + return; + } + + sanitize_profile(&mut Some(p)); + + let mut initial = false; + if self.prof.is_none() { + self.init(p); + initial = true; + } + + self.tmp.resize(p.string_table.len(), 0); + self.string_table + .as_mut() + .unwrap() + .index(&mut self.tmp, &p.string_table); + + rewrite_strings(p, &mut self.tmp); + if initial { + rewrite_strings(self.prof.as_mut().unwrap(), &mut self.tmp) + } + + combine_headers(self.prof.as_mut().unwrap(), p); + + self.tmp.resize(p.function.len(), 0); + self.function_table + .as_mut() + .unwrap() + .index(&mut self.tmp, &p.function); + rewrite_functions(p, &mut self.tmp); + + self.tmp.resize(p.mapping.len(), 0); + self.mapping_table + .as_mut() + .unwrap() + .index(&mut self.tmp, &p.mapping); + rewrite_mappings(p, &mut self.tmp); + + self.tmp.resize(p.location.len(), 0); + self.location_table + .as_mut() + .unwrap() + .index(&mut self.tmp, &p.location); + rewrite_locations(p, &mut self.tmp); + + self.tmp.resize(p.sample.len(), 0); + self.sample_table + .as_mut() + .unwrap() + .index(&mut self.tmp, &p.sample); + + for i in 0..self.tmp.len() { + let idx = self.tmp[i]; + let dst = &mut self.sample_table.as_mut().unwrap().s[idx as usize].value; + let src = p.sample[i as usize].value.clone(); + for j in 0..src.len() { + dst[j] += src[j]; + } + } + } + + fn init(&mut self, p: &mut Profile) { + let factor = 2; + self.string_table = Some(RewriteTable::new( + factor * p.string_table.len(), + |s| s.clone(), + |s| s.clone(), + )); + self.function_table = Some(RewriteTable::new( + factor * p.function.len(), + FunctionKey::get, + |s| s.clone(), + )); + self.mapping_table = Some(RewriteTable::new( + factor * p.mapping.len(), + MappingKey::get, + |s| s.clone(), + )); + self.location_table = Some(RewriteTable::new( + factor * p.location.len(), + LocationKey::get, + |s| s.clone(), + )); + self.sample_table = Some(RewriteTable::new( + factor * p.sample.len(), + SampleKey::get, + |s| s.clone(), + )); + let mut _prof = Profile::default(); + _prof.sample_type = vec![]; + + _prof.drop_frames = p.drop_frames.clone(); + _prof.keep_frames = p.keep_frames.clone(); + _prof.time_nanos = p.time_nanos.clone(); + _prof.period_type = p.period_type.clone(); + _prof.period = p.period.clone(); + _prof.default_sample_type = p.default_sample_type.clone(); + for s in 0..p.sample_type.len() { + _prof.sample_type.push(p.sample_type[s].clone()); + } + self.prof = Some(_prof); + } + + pub fn profile(&mut self) -> Profile { + if self.prof.is_none() { + return Profile::default(); + } + let mut p = self.prof.as_mut().unwrap().clone(); + p.sample = self.sample_table.as_mut().unwrap().values().clone(); + p.location = self.location_table.as_mut().unwrap().values().clone(); + p.function = self.function_table.as_mut().unwrap().values().clone(); + p.mapping = self.mapping_table.as_mut().unwrap().values().clone(); + p.string_table = self.string_table.as_mut().unwrap().values().clone(); + for i in 0..p.location.len() { + p.location[i].id = i as u64 + 1; + } + for i in 0..p.function.len() { + p.function[i].id = i as u64 + 1; + } + for i in 0..p.mapping.len() { + p.mapping[i].id = i as u64 + 1; + } + return p; + } + +} + +fn rewrite_strings(p: &mut Profile, n: &Vec) { + for i in 0..p.sample_type.len() { + let t = &mut p.sample_type[i]; + if t.unit != 0 { + t.unit = n[t.unit as usize] as i64; + } + if t.r#type != 0 { + t.r#type = n[t.r#type as usize] as i64; + } + } + for i in 0..p.sample.len() { + let s = &mut p.sample[i]; + for j in 0..s.label.len() { + let l = &mut s.label[j]; + l.key = n[l.key as usize] as i64; + l.str = n[l.str as usize] as i64; + } + } + + for i in 0..p.mapping.len() { + let m = &mut p.mapping[i]; + m.filename = n[m.filename as usize] as i64; + m.build_id = n[m.build_id as usize] as i64; + } + + for i in 0..p.function.len() { + let f = &mut p.function[i]; + f.name = n[f.name as usize] as i64; + f.filename = n[f.filename as usize] as i64; + f.system_name = n[f.system_name as usize] as i64; + } + p.drop_frames = n[p.drop_frames as usize] as i64; + p.keep_frames = n[p.keep_frames as usize] as i64; + if !p.period_type.is_none() { + if p.period_type.as_mut().unwrap().r#type != 0 { + p.period_type.as_mut().unwrap().r#type = + n[p.period_type.as_mut().unwrap().r#type as usize] as i64; + } + if p.period_type.as_mut().unwrap().unit != 0 { + p.period_type.as_mut().unwrap().unit = + n[p.period_type.as_mut().unwrap().unit as usize] as i64; + } + } + + for i in 0..p.comment.len() { + let x = p.comment[i]; + p.comment[i] = n[x as usize] as i64; + } + p.default_sample_type = n[p.default_sample_type as usize] as i64; +} + +fn rewrite_functions(p: &mut Profile, n: &Vec) { + for i in 0..p.location.len() { + let loc = &mut p.location[i]; + for j in 0..loc.line.len() { + let line = &mut loc.line[j]; + if line.function_id > 0 { + line.function_id = n[line.function_id as usize - 1] as u64 + 1; + } + } + } +} + +fn rewrite_mappings(p: &mut Profile, n: &mut Vec) { + for i in 0..p.location.len() { + let loc = &mut p.location[i]; + if loc.mapping_id > 0 { + loc.mapping_id = n[loc.mapping_id as usize - 1] as u64 + 1; + } + } +} + +fn rewrite_locations(p: &mut Profile, n: &mut Vec) { + for i in 0..p.sample.len() { + let s = &mut p.sample[i]; + for j in 0..s.location_id.len() { + if s.location_id[j] > 0 { + s.location_id[j] = n[s.location_id[j] as usize - 1] as u64 + 1; + } + } + } +} + +fn sanitize_profile(_p: &mut Option<&mut Profile>) { + if _p.is_none() { + return; + } + let p = _p.as_mut().unwrap(); + let mut ms = p.string_table.len() as i64; + let mut z: i64 = -1; + for i in 0..p.string_table.len() { + let s = &p.string_table[i]; + if s == "" { + z = i as i64; + break; + } + } + if z == -1 { + z = ms; + p.string_table.push("".to_string()); + ms += 1; + } + let tmp = p.string_table[0].clone(); + p.string_table[0] = p.string_table[z as usize].clone(); + p.string_table[z as usize] = tmp; + + let str = |i: i64| -> i64 { + if i == 0 && z > 0 { + return z; + } + if i == z || i >= ms || i < 0 { + return 0; + } + return i; + }; + p.sample_type = remove_in_place(&mut p.sample_type, &mut |x, _| -> bool { + x.r#type = str(x.r#type); + x.unit = str(x.unit); + false + }); + + if !p.period_type.is_none() { + p.period_type.as_mut().unwrap().r#type = str(p.period_type.as_mut().unwrap().r#type); + p.period_type.as_mut().unwrap().unit = str(p.period_type.as_mut().unwrap().unit); + } + + p.default_sample_type = str(p.default_sample_type); + p.drop_frames = str(p.drop_frames); + p.keep_frames = str(p.keep_frames); + for i in 0..p.comment.len() { + p.comment[i] = str(p.comment[i]); + } + + let mut t: HashMap = HashMap::new(); + let mut j: u64 = 1; + p.mapping = remove_in_place(&mut p.mapping, &mut |x, _| -> bool { + x.build_id = str(x.build_id); + x.filename = str(x.filename); + t.insert(x.id, j); + x.id = j; + j += 1; + false + }); + + let mut mapping: Option = Option::None; + let p_mapping = &mut p.mapping; + p.location = remove_in_place(&mut p.location, &mut |x, _| -> bool { + if x.mapping_id == 0 { + if mapping.is_none() { + let mut _mapping = Mapping::default(); + _mapping.id = p_mapping.len() as u64 + 1; + mapping = Some(_mapping.clone()); + p_mapping.push(_mapping); + } + x.mapping_id = mapping.as_ref().unwrap().id; + return false; + } + x.mapping_id = t[&x.mapping_id]; + return x.mapping_id == 0; + }); + + t.clear(); + + j = 1; + p.function = remove_in_place(&mut p.function, &mut |x, _| -> bool { + x.name = str(x.name); + x.system_name = str(x.system_name); + x.filename = str(x.filename); + t.insert(x.id, j); + x.id = j; + j += 1; + false + }); + + p.location = remove_in_place(&mut p.location, &mut |x, _| -> bool { + for i in 0..x.line.len() { + let line = &mut x.line[i]; + line.function_id = t[&line.function_id]; + if line.function_id == 0 { + return true; + } + } + return false; + }); + + t.clear(); + j = 1; + for i in 0..p.location.len() { + let x = &mut p.location[i]; + t.insert(x.id, j); + x.id = j; + j += 1; + } + + let vs = p.sample_type.len(); + p.sample = remove_in_place(&mut p.sample, &mut |x, _| -> bool { + if x.value.len() != vs { + return true; + } + for i in 0..x.location_id.len() { + x.location_id[i] = t[&x.location_id[i]]; + if x.location_id[i] == 0 { + return true; + } + } + for i in 0..x.label.len() { + let l = &mut x.label[i]; + l.key = str(l.key); + l.str = str(l.str); + l.num_unit = str(l.num_unit); + } + false + }); +} + +fn remove_in_place bool>( + collection: &mut Vec, + predicate: &mut F, +) -> Vec { + let mut i: usize = 0; + for j in 0..collection.len() { + if !predicate(&mut collection[j], j as i64) { + let tmp = collection[i].clone(); + collection[i] = collection[j].clone(); + collection[j] = tmp; + i += 1; + } + } + return collection[..i].to_vec(); + /* + i := 0 + for j, x := range collection { + if !predicate(x, j) { + collection[j], collection[i] = collection[i], collection[j] + i++ + } + } + return collection[:i] + + */ +} + +fn combine_headers(a: &mut Profile, b: &Profile) { + compatible(a, b); + if a.time_nanos == 0 || b.time_nanos < a.time_nanos { + a.time_nanos = b.time_nanos + } + a.duration_nanos += b.duration_nanos; + if a.period == 0 || a.period < b.period { + a.period = b.period + } + if a.default_sample_type == 0 { + a.default_sample_type = b.default_sample_type + } +} +fn compatible(a: &Profile, b: &Profile) { + if !equal_value_type(&a.period_type, &b.period_type) { + panic!( + "incompatible period types {:?} and {:?}", + a.period_type, b.period_type + ); + } + if b.sample_type.len() != a.sample_type.len() { + panic!( + "incompatible sample types {:?} and {:?}", + a.sample_type, b.sample_type + ); + } + for i in 0..a.sample_type.len() { + if !equal_value_type( + &Some(a.sample_type[i].clone()), + &Some(b.sample_type[i].clone()), + ) { + panic!( + "incompatible sample types {:?} and {:?}", + a.sample_type, b.sample_type + ); + } + } +} + +fn equal_value_type(st1: &Option, st2: &Option) -> bool { + if st1.is_none() || st2.is_none() { + return false; + } + return st1.as_ref().unwrap().r#type == st2.as_ref().unwrap().r#type + && st1.as_ref().unwrap().unit == st2.as_ref().unwrap().unit; +} + +struct FunctionKey { + start_line: u32, + name: u32, + system_name: u32, + file_name: u32, +} + +impl FunctionKey { + fn get(f: &Function) -> FunctionKey { + return FunctionKey { + start_line: f.start_line as u32, + name: f.name as u32, + system_name: f.system_name as u32, + file_name: f.filename as u32, + }; + } +} + +impl PartialEq for FunctionKey { + fn eq(&self, other: &Self) -> bool { + return self.name == other.name + && self.system_name == other.system_name + && self.file_name == other.file_name + && self.start_line == other.start_line; + } +} + +impl Eq for FunctionKey {} + +impl Hash for FunctionKey { + fn hash(&self, state: &mut H) { + state.write_u32(self.name); + state.write_u32(self.system_name); + state.write_u32(self.file_name); + state.write_u32(self.start_line); + } +} + +struct MappingKey { + size: u64, + offset: u64, + build_id_or_file: i64, +} + +impl MappingKey { + fn get(m: &Mapping) -> MappingKey { + let mapsize_rounding = 0x1000; + let mut size = m.memory_limit - m.memory_start; + size = size + mapsize_rounding - 1; + size = size - (size % mapsize_rounding); + let mut k = MappingKey { + size: size, + offset: m.file_offset, + build_id_or_file: 0, + }; + if m.build_id != 0 { + k.build_id_or_file = m.build_id; + } + if m.filename != 0 { + k.build_id_or_file = m.filename; + } + k + } +} + +impl PartialEq for MappingKey { + fn eq(&self, other: &Self) -> bool { + return self.build_id_or_file == other.build_id_or_file + && self.offset == other.offset + && self.size == other.size; + } +} + +impl Eq for MappingKey {} + +impl Hash for MappingKey { + fn hash(&self, state: &mut H) { + state.write_i64(self.build_id_or_file); + state.write_u64(self.offset); + state.write_u64(self.size); + } +} + +struct LocationKey { + addr: u64, + lines: u64, + mapping_id: u64, +} + +impl LocationKey { + fn get(l: &Location) -> LocationKey { + return LocationKey { + addr: l.address, + lines: hash_lines(&l.line), + mapping_id: l.mapping_id, + }; + } +} + +impl PartialEq for LocationKey { + fn eq(&self, other: &Self) -> bool { + return self.lines == other.lines + && self.mapping_id == other.mapping_id + && self.addr == other.addr; + } +} + +impl Eq for LocationKey {} + +impl Hash for LocationKey { + fn hash(&self, state: &mut H) { + state.write_u64(self.lines); + state.write_u64(self.mapping_id); + state.write_u64(self.addr); + } +} + +fn hash_lines(s: &Vec) -> u64 { + let mut x = vec![0 as u64; s.len()]; + for i in 0..s.len() { + x[i] = s[i].function_id | ((s[i].line as u64) << 32) + } + let u64_arr = x.as_slice(); + let u8_arr: &[u8] = bytemuck::cast_slice(u64_arr); + return city_hash_64(u8_arr); +} + +struct SampleKey { + locations: u64, + labels: u64, +} + +impl SampleKey { + fn get(s: &Sample) -> SampleKey { + return SampleKey { + locations: hash_locations(&s.location_id), + labels: hash_labels(&s.label), + }; + } +} + +impl PartialEq for SampleKey { + fn eq(&self, other: &Self) -> bool { + return self.locations == other.locations && self.labels == other.labels; + } +} + +impl Eq for SampleKey {} + +impl Hash for SampleKey { + fn hash(&self, state: &mut H) { + state.write_u64(self.locations); + state.write_u64(self.labels); + } +} + +fn hash_labels(labels: &Vec