pyspark-shell - Details for Job 52

Status: SUCCEEDED
Submitted: 2026/05/09 01:05:02
Duration: 31 ms
Associated SQL Query: 21
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_80 { label="Stage 80 (skipped)"; subgraph cluster418 { label="Scan parquet itv024694_lending_club.loans_defaulters_detail_rec_enq"; 151 [labelType="html" label="FileScanRDD [151] getRowsToPython at NativeMethodAccessorImpl.java:0"]; 152 [labelType="html" label="MapPartitionsRDD [152] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster414 { label="WholeStageCodegen (5)"; 153 [labelType="html" label="MapPartitionsRDD [153] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster413 { label="Exchange"; 154 [labelType="html" label="MapPartitionsRDD [154] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } } 151->152; 152->153; 153->154; }

digraph G { subgraph clusterstage_81 { label="Stage 81 (skipped)"; subgraph cluster407 { label="Scan parquet itv024694_lending_club.loans_defaulters_delinq"; 145 [labelType="html" label="FileScanRDD [145] getRowsToPython at NativeMethodAccessorImpl.java:0"]; 146 [labelType="html" label="MapPartitionsRDD [146] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster403 { label="WholeStageCodegen (3)"; 147 [labelType="html" label="MapPartitionsRDD [147] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster402 { label="Exchange"; 148 [labelType="html" label="MapPartitionsRDD [148] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } } 145->146; 146->147; 147->148; }

digraph G { subgraph clusterstage_82 { label="Stage 82 (skipped)"; subgraph cluster396 { label="Scan parquet itv024694_lending_club.customers"; 139 [labelType="html" label="FileScanRDD [139] getRowsToPython at NativeMethodAccessorImpl.java:0"]; 140 [labelType="html" label="MapPartitionsRDD [140] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster392 { label="WholeStageCodegen (1)"; 141 [labelType="html" label="MapPartitionsRDD [141] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster391 { label="Exchange"; 142 [labelType="html" label="MapPartitionsRDD [142] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } } 139->140; 140->141; 141->142; }

digraph G { subgraph clusterstage_83 { label="Stage 83 (skipped)"; subgraph cluster391 { label="Exchange"; 143 [labelType="html" label="ShuffledRowRDD [143] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster386 { label="WholeStageCodegen (2)"; 144 [labelType="html" label="MapPartitionsRDD [144] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster402 { label="Exchange"; 149 [labelType="html" label="ShuffledRowRDD [149] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster397 { label="WholeStageCodegen (4)"; 150 [labelType="html" label="MapPartitionsRDD [150] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster413 { label="Exchange"; 155 [labelType="html" label="ShuffledRowRDD [155] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster408 { label="WholeStageCodegen (6)"; 156 [labelType="html" label="MapPartitionsRDD [156] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster385 { label="Union"; 157 [labelType="html" label="UnionRDD [157] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster382 { label="WholeStageCodegen (7)"; 158 [labelType="html" label="MapPartitionsRDD [158] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster381 { label="Exchange"; 159 [labelType="html" label="MapPartitionsRDD [159] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } } 143->144; 149->150; 155->156; 144->157; 150->157; 156->157; 157->158; 158->159; }

142,143

148,149

154,155

digraph G { subgraph clusterstage_84 { label="Stage 84"; subgraph cluster381 { label="Exchange"; 160 [labelType="html" label="ShuffledRowRDD [160] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster378 { label="WholeStageCodegen (8)"; 161 [labelType="html" label="MapPartitionsRDD [161] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } subgraph cluster419 { label="mapPartitionsInternal"; 162 [labelType="html" label="MapPartitionsRDD [162] getRowsToPython at NativeMethodAccessorImpl.java:0"]; } } 160->161; 161->162; }

159,160

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
84	getRowsToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2026/05/09 01:05:02	12 ms	1/1			2.2 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

getRowsToPython at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2026/05/09 01:05:02

12 ms

1/1

2.2 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
83	getRowsToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
82	getRowsToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/10
81	getRowsToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/8
80	getRowsToPython at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.getRowsToPython(Dataset.scala:3539) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/10

Page: