pyspark-shell - Details for Job 50

Status: SUCCEEDED
Submitted: 2026/05/09 01:04:51
Duration: 78 ms
Associated SQL Query: 20
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_70 { label="Stage 70 (skipped)"; subgraph cluster340 { label="Scan parquet itv024694_lending_club.loans_defaulters_delinq"; 121 [labelType="html" label="FileScanRDD [121] showString at NativeMethodAccessorImpl.java:0"]; 122 [labelType="html" label="MapPartitionsRDD [122] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster336 { label="WholeStageCodegen (3)"; 123 [labelType="html" label="MapPartitionsRDD [123] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster335 { label="Exchange"; 124 [labelType="html" label="MapPartitionsRDD [124] showString at NativeMethodAccessorImpl.java:0"]; } } 121->122; 122->123; 123->124; }

digraph G { subgraph clusterstage_71 { label="Stage 71 (skipped)"; subgraph cluster329 { label="Scan parquet itv024694_lending_club.customers"; 115 [labelType="html" label="FileScanRDD [115] showString at NativeMethodAccessorImpl.java:0"]; 116 [labelType="html" label="MapPartitionsRDD [116] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster325 { label="WholeStageCodegen (1)"; 117 [labelType="html" label="MapPartitionsRDD [117] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster324 { label="Exchange"; 118 [labelType="html" label="MapPartitionsRDD [118] showString at NativeMethodAccessorImpl.java:0"]; } } 115->116; 116->117; 117->118; }

digraph G { subgraph clusterstage_72 { label="Stage 72 (skipped)"; subgraph cluster351 { label="Scan parquet itv024694_lending_club.loans_defaulters_detail_rec_enq"; 127 [labelType="html" label="FileScanRDD [127] showString at NativeMethodAccessorImpl.java:0"]; 128 [labelType="html" label="MapPartitionsRDD [128] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster347 { label="WholeStageCodegen (5)"; 129 [labelType="html" label="MapPartitionsRDD [129] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster346 { label="Exchange"; 130 [labelType="html" label="MapPartitionsRDD [130] showString at NativeMethodAccessorImpl.java:0"]; } } 127->128; 128->129; 129->130; }

digraph G { subgraph clusterstage_73 { label="Stage 73 (skipped)"; subgraph cluster324 { label="Exchange"; 119 [labelType="html" label="ShuffledRowRDD [119] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster319 { label="WholeStageCodegen (2)"; 120 [labelType="html" label="MapPartitionsRDD [120] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster335 { label="Exchange"; 125 [labelType="html" label="ShuffledRowRDD [125] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster330 { label="WholeStageCodegen (4)"; 126 [labelType="html" label="MapPartitionsRDD [126] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster346 { label="Exchange"; 131 [labelType="html" label="ShuffledRowRDD [131] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster341 { label="WholeStageCodegen (6)"; 132 [labelType="html" label="MapPartitionsRDD [132] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster318 { label="Union"; 133 [labelType="html" label="UnionRDD [133] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster315 { label="WholeStageCodegen (7)"; 134 [labelType="html" label="MapPartitionsRDD [134] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster314 { label="Exchange"; 135 [labelType="html" label="MapPartitionsRDD [135] showString at NativeMethodAccessorImpl.java:0"]; } } 119->120; 125->126; 131->132; 120->133; 126->133; 132->133; 133->134; 134->135; }

118,119

124,125

130,131

digraph G { subgraph clusterstage_74 { label="Stage 74"; subgraph cluster314 { label="Exchange"; 136 [labelType="html" label="ShuffledRowRDD [136] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster311 { label="WholeStageCodegen (8)"; 137 [labelType="html" label="MapPartitionsRDD [137] showString at NativeMethodAccessorImpl.java:0"]; } subgraph cluster352 { label="mapPartitionsInternal"; 138 [labelType="html" label="MapPartitionsRDD [138] showString at NativeMethodAccessorImpl.java:0"]; } } 136->137; 137->138; }

135,136

Completed Stages (1)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
74	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:338) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	2026/05/09 01:04:51	69 ms	1/1			2.2 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

showString at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.Dataset.showString(Dataset.scala:338)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.GatewayConnection.run(GatewayConnection.java:238)
java.lang.Thread.run(Thread.java:750)

2026/05/09 01:04:51

69 ms

1/1

2.2 KiB

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
73	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:338) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/600
72	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:338) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/7
71	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:338) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/8
70	showString at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.Dataset.showString(Dataset.scala:338) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.GatewayConnection.run(GatewayConnection.java:238) java.lang.Thread.run(Thread.java:750)	Unknown	Unknown	0/6

Page: