digraph G {
0 [labelType="html" label="<br><b>CollectLimit</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: total (min, med, max (stageId: taskId))\n2 ms (1 ms, 1 ms, 1 ms (stage 84.0: task 2317))";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 84.0: task 2317))<br>peak memory total (min, med, max (stageId: taskId))<br>32.5 MiB (16.3 MiB, 16.3 MiB, 16.3 MiB (stage 84.0: task 2317))<br>number of output rows: 33<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 84.0: task 2317))"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 6,519<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.6 s (0 ms, 1 ms, 20 ms (stage 78.1: task 1759))<br>records read: 68<br>local bytes read total (min, med, max (stageId: taskId))<br>3.8 KiB (1205.0 B, 2.6 KiB, 2.6 KiB (stage 79.0: task 2316))<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 84.0: task 2317))<br>remote bytes read: 1079.0 B<br>local blocks read: 5<br>remote blocks read: 1<br>data size total (min, med, max (stageId: taskId))<br>509.3 KiB (0.0 B, 1040.0 B, 1920.0 B (stage 78.1: task 2121))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>463.2 KiB (0.0 B, 947.0 B, 1690.0 B (stage 78.1: task 2121))"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: total (min, med, max (stageId: taskId))\n50.2 s (2 ms, 20 ms, 1.5 s (stage 78.1: task 1813))";
5 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>47.6 s (1 ms, 15 ms, 1.5 s (stage 78.1: task 1813))<br>peak memory total (min, med, max (stageId: taskId))<br>8.3 GiB (256.0 KiB, 16.3 MiB, 16.3 MiB (stage 78.1: task 1723))<br>number of output rows: 6,519<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1 (stage 78.1: task 1723))"];
}
6 [labelType="html" label="<br><b>Union</b><br><br>"];
subgraph cluster7 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n18.2 s (0 ms, 0 ms, 1.5 s (stage 78.1: task 1813))";
8 [labelType="html" label="<br><b>Project</b><br><br>"];
9 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 3,157"];
10 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>17.1 s (0 ms, 0 ms, 1.5 s (stage 78.1: task 1813))<br>peak memory total (min, med, max (stageId: taskId))<br>3.2 GiB (0.0 B, 0.0 B, 16.5 MiB (stage 78.1: task 1723))<br>number of output rows: 2,257,379<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1.4, 1.4, 1.4 (stage 78.1: task 1723))"];
}
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 226,406<br>shuffle write time total (min, med, max (stageId: taskId))<br>17 ms (0 ms, 0 ms, 17 ms (stage 77.1: task 1715))<br>records read: 2,260,298<br>local bytes read total (min, med, max (stageId: taskId))<br>81.2 MiB (0.0 B, 0.0 B, 515.2 KiB (stage 78.1: task 1890))<br>fetch wait time total (min, med, max (stageId: taskId))<br>15.7 s (0 ms, 0 ms, 1.5 s (stage 78.1: task 1813))<br>remote bytes read total (min, med, max (stageId: taskId))<br>57.7 MiB (0.0 B, 0.0 B, 510.6 KiB (stage 78.1: task 1804))<br>local blocks read: 1,168<br>remote blocks read: 832<br>data size total (min, med, max (stageId: taskId))<br>19.0 MiB (0.0 B, 0.0 B, 19.0 MiB (stage 77.1: task 1715))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>13.9 MiB (0.0 B, 0.0 B, 13.9 MiB (stage 77.1: task 1715))"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n387 ms (0 ms, 0 ms, 387 ms (stage 77.1: task 1715))";
13 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>298 ms (0 ms, 0 ms, 298 ms (stage 77.1: task 1715))<br>peak memory total (min, med, max (stageId: taskId))<br>40.0 MiB (0.0 B, 0.0 B, 40.0 MiB (stage 77.1: task 1715))<br>number of output rows: 226,406<br>avg hash probe bucket list iters: 1.6"];
14 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 226,436<br>number of input batches: 60"];
}
15 [labelType="html" label="<b>Scan parquet itv024694_lending_club.customers</b><br><br>number of files read: 200<br>scan time total (min, med, max (stageId: taskId))<br>187 ms (0 ms, 0 ms, 187 ms (stage 77.1: task 1715))<br>metadata time: 0 ms<br>size of files read: 184.9 MiB<br>number of output rows: 226,436"];
subgraph cluster16 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n5.5 s (0 ms, 0 ms, 639 ms (stage 78.1: task 2062))";
17 [labelType="html" label="<br><b>Project</b><br><br>"];
18 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 173"];
19 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>4.9 s (0 ms, 0 ms, 637 ms (stage 78.1: task 2062))<br>peak memory total (min, med, max (stageId: taskId))<br>3.2 GiB (0.0 B, 0.0 B, 16.3 MiB (stage 78.1: task 1915))<br>number of output rows: 484,136<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1, 1, 1.1 (stage 78.1: task 1949))"];
}
20 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 241,547<br>shuffle write time total (min, med, max (stageId: taskId))<br>19 ms (0 ms, 0 ms, 19 ms (stage 76.1: task 1714))<br>records read: 484,233<br>local bytes read total (min, med, max (stageId: taskId))<br>14.9 MiB (0.0 B, 0.0 B, 83.4 KiB (stage 78.1: task 2051))<br>fetch wait time total (min, med, max (stageId: taskId))<br>4.7 s (0 ms, 0 ms, 636 ms (stage 78.1: task 2062))<br>remote bytes read total (min, med, max (stageId: taskId))<br>14.9 MiB (0.0 B, 0.0 B, 83.4 KiB (stage 78.1: task 1985))<br>local blocks read: 200<br>remote blocks read: 200<br>data size total (min, med, max (stageId: taskId))<br>20.3 MiB (0.0 B, 0.0 B, 20.3 MiB (stage 76.1: task 1714))<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>14.9 MiB (0.0 B, 0.0 B, 14.9 MiB (stage 76.1: task 1714))"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n291 ms (0 ms, 12 ms, 222 ms (stage 76.1: task 1714))";
22 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>175 ms (0 ms, 9 ms, 125 ms (stage 76.1: task 1714))<br>peak memory total (min, med, max (stageId: taskId))<br>40.8 MiB (0.0 B, 256.0 KiB, 40.0 MiB (stage 76.1: task 1714))<br>number of output rows: 241,547<br>avg hash probe bucket list iters: 1.6"];
23 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 241,589<br>number of input batches: 59"];
}
24 [labelType="html" label="<b>Scan parquet itv024694_lending_club.loans_defaulters_delinq</b><br><br>number of files read: 2<br>scan time total (min, med, max (stageId: taskId))<br>66 ms (0 ms, 9 ms, 31 ms (stage 76.1: task 1711))<br>metadata time: 0 ms<br>size of files read: 31.1 MiB<br>number of output rows: 241,589"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: total (min, med, max (stageId: taskId))\n25.3 s (0 ms, 0 ms, 1.2 s (stage 78.1: task 2205))";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 3,189"];
28 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>24.6 s (0 ms, 0 ms, 1.2 s (stage 78.1: task 2205))<br>peak memory total (min, med, max (stageId: taskId))<br>3.2 GiB (0.0 B, 0.0 B, 16.5 MiB (stage 78.1: task 2120))<br>number of output rows: 2,257,384<br>avg hash probe bucket list iters (min, med, max (stageId: taskId)):<br>(1.4, 1.4, 1.4 (stage 78.1: task 2120))"];
}
29 [labelType="html" label="<b>Exchange</b><br><br>records read: 2,259,065<br>local bytes read total (min, med, max (stageId: taskId))<br>68.9 MiB (0.0 B, 0.0 B, 377.6 KiB (stage 78.1: task 2291))<br>fetch wait time total (min, med, max (stageId: taskId))<br>23.5 s (0 ms, 0 ms, 1.2 s (stage 78.1: task 2205))<br>remote bytes read total (min, med, max (stageId: taskId))<br>69.6 MiB (0.0 B, 0.0 B, 705.6 KiB (stage 78.1: task 2154))<br>local blocks read: 199<br>remote blocks read: 201"];
subgraph cluster30 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n13 ms (0 ms, 0 ms, 13 ms (stage 75.1: task 1710))";
31 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>10 ms (0 ms, 0 ms, 10 ms (stage 75.1: task 1710))<br>peak memory total (min, med, max (stageId: taskId))<br>256.0 KiB (0.0 B, 0.0 B, 256.0 KiB (stage 75.1: task 1710))"];
32 [labelType="html" label="<br><b>ColumnarToRow</b><br><br>"];
}
33 [labelType="html" label="<b>Scan parquet itv024694_lending_club.loans_defaulters_detail_rec_enq</b><br><br>number of files read: 2<br>scan time total (min, med, max (stageId: taskId))<br>10 ms (0 ms, 0 ms, 10 ms (stage 75.1: task 1710))<br>metadata time: 0 ms<br>size of files read: 144.3 MiB"];
2->0;
3->2;
5->3;
6->5;
8->6;
9->8;
10->9;
11->10;
13->11;
14->13;
15->14;
17->6;
18->17;
19->18;
20->19;
22->20;
23->22;
24->23;
26->6;
27->26;
28->27;
29->28;
31->29;
32->31;
33->32;
}
34
CollectLimit 21
HashAggregate(keys=[member_id#577], functions=[])
WholeStageCodegen (8)
Exchange hashpartitioning(member_id#577, 200), ENSURE_REQUIREMENTS, [id=#907]
HashAggregate(keys=[member_id#577], functions=[])
WholeStageCodegen (7)
Union
Project [member_id#577]
Filter (total_count#575L > 1)
HashAggregate(keys=[member_id#577], functions=[count(1)])
WholeStageCodegen (2)
Exchange hashpartitioning(member_id#577, 200), ENSURE_REQUIREMENTS, [id=#877]
HashAggregate(keys=[member_id#577], functions=[partial_count(1)])
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet itv024694_lending_club.customers[member_id#577] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[hdfs://m01.itversity.com:9000/public/trendytech/lendingclubproject/cleaned/cust..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<member_id:string>
Project [member_id#607]
Filter (total_count#605L > 1)
HashAggregate(keys=[member_id#607], functions=[count(1)])
WholeStageCodegen (4)
Exchange hashpartitioning(member_id#607, 200), ENSURE_REQUIREMENTS, [id=#887]
HashAggregate(keys=[member_id#607], functions=[partial_count(1)])
ColumnarToRow
WholeStageCodegen (3)
FileScan parquet itv024694_lending_club.loans_defaulters_delinq[member_id#607] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[hdfs://m01.itversity.com:9000/public/trendytech/lendingclubproject/cleaned/loan..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<member_id:string>
Project [member_id#625]
Filter (total_count#623L > 1)
HashAggregate(keys=[member_id#625], functions=[count(1)])
WholeStageCodegen (6)
Exchange hashpartitioning(member_id#625, 200), ENSURE_REQUIREMENTS, [id=#897]
HashAggregate(keys=[member_id#625], functions=[partial_count(1)])
ColumnarToRow
WholeStageCodegen (5)
FileScan parquet itv024694_lending_club.loans_defaulters_detail_rec_enq[member_id#625] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[hdfs://m01.itversity.com:9000/public/trendytech/lendingclubproject/cleaned/loan..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<member_id:string>