Show the Stage ID and Task ID that corresponds to the max metric
digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n6.2 s (248 ms, 405 ms, 1.8 s (stage 5.0: task 261))";
2 [labelType="html" label="<br><b>Project</b><br><br>"];
3 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 5"];
4 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 2,260,633<br>number of input batches: 600"];
}
5 [labelType="html" label="<b>Scan parquet itv024694_lending_club.customers</b><br><br>number of files read: 200<br>scan time total (min, med, max (stageId: taskId))<br>5.8 s (222 ms, 356 ms, 1.8 s (stage 5.0: task 261))<br>metadata time: 0 ms<br>size of files read: 184.9 MiB<br>number of output rows: 2,260,633"];
2->0;
3->2;
4->3;
5->4;
}
6
CollectLimit 21
Project [member_id#56, emp_title#57, cast(emp_length#58 as string) AS emp_length#106, home_ownership#59, cast(annual_income#60 as string) AS annual_income#108, address_state#61, address_zipcode#62, address_country#63, grade#64, sub_grade#65, verification_status#66, cast(total_high_credit_limit#67 as string) AS total_high_credit_limit#115, application_type#68, cast(join_annual_income#69 as string) AS join_annual_income#117, verification_status_joint#70, cast(ingest_date#71 as string) AS ingest_date#119]
Filter (isnotnull(member_id#56) AND StartsWith(member_id#56, e4c167053d5418230))