pyspark-shell - Environment

Runtime Information

Name	Value
Java Home	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre
Java Version	1.8.0_282 (Red Hat, Inc.)
Scala Version	version 2.12.10

Spark Properties

Name	Value
spark.app.id	application_1753533637512_0663
spark.app.name	pyspark-shell
spark.app.startTime	1753591961220
spark.driver.appUIAddress	http://g02.itversity.com:35231
spark.driver.extraJavaOptions	-Dderby.system.home=/tmp/derby/
spark.driver.host	g02.itversity.com
spark.driver.port	33765
spark.dynamicAllocation.enabled	true
spark.dynamicAllocation.maxExecutors	10
spark.dynamicAllocation.minExecutors	2
spark.eventLog.dir	hdfs:///spark-logs
spark.eventLog.enabled	true
spark.executor.extraLibraryPath	/opt/hadoop/lib/native
spark.executor.id	driver
spark.executorEnv.PYTHONPATH	/opt/spark-3.1.2-bin-hadoop3.2/python/lib/py4j-0.10.9-src.zip:/opt/spark-3.1.2-bin-hadoop3.2/python<CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.10.9-src.zip
spark.history.fs.cleaner.enabled	true
spark.history.fs.logDirectory	hdfs:///spark-logs
spark.history.fs.update.interval	10s
spark.history.provider	org.apache.spark.deploy.history.FsHistoryProvider
spark.history.ui.port	18080
spark.master	yarn
spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS	m02.itversity.com
spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES	http://m02.itversity.com:19088/proxy/application_1753533637512_0663
spark.rdd.compress	True
spark.scheduler.mode	FIFO
spark.serializer.objectStreamReset	100
spark.shuffle.service.enabled	true
spark.sql.catalogImplementation	hive
spark.sql.repl.eagerEval.enabled	true
spark.sql.warehouse.dir	/user/itv018602/warehouse
spark.submit.deployMode	client
spark.submit.pyFiles
spark.ui.filters	org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter
spark.ui.port	0
spark.ui.showConsoleProgress	true
spark.yarn.historyServer.address	m02.itversity.com:18080
spark.yarn.isPython	true
spark.yarn.jars

Resource Profiles

Resource Profile Id	Resource Profile Contents
0	Executor Reqs: cores: [amount: 1] memory: [amount: 1024] offHeap: [amount: 0] Task Reqs: cpus: [amount: 1.0]

Hadoop Properties

Name	Value
adl.feature.ownerandgroup.enableupn	false
adl.http.timeout	-1
dfs.bytes-per-checksum	512
dfs.datanode.data.dir	/data2/hadoop/hadoop/dfs/dn/,/data1/hadoop/hadoop/dfs/dn/
dfs.ha.fencing.ssh.connect-timeout	30000
dfs.namenode.checkpoint.dir	/data1/hadoop/hadoop/dfs/snn/,/data2/hadoop/hadoop/dfs/snn/
dfs.namenode.name.dir	/data1/hadoop/hadoop/dfs/nn/,/data2/hadoop/hadoop/dfs/nn/
dfs.namenode.secondary.http-address	m02.itversity.com:50090
dfs.replication	3
dfs.webhdfs.enabled	true
file.blocksize	67108864
file.bytes-per-checksum	512
file.client-write-packet-size	65536
file.replication	1
file.stream-buffer-size	4096
fs.AbstractFileSystem.abfs.impl	org.apache.hadoop.fs.azurebfs.Abfs
fs.AbstractFileSystem.abfss.impl	org.apache.hadoop.fs.azurebfs.Abfss
fs.AbstractFileSystem.adl.impl	org.apache.hadoop.fs.adl.Adl
fs.AbstractFileSystem.file.impl	org.apache.hadoop.fs.local.LocalFs
fs.AbstractFileSystem.ftp.impl	org.apache.hadoop.fs.ftp.FtpFs
fs.AbstractFileSystem.har.impl	org.apache.hadoop.fs.HarFs
fs.AbstractFileSystem.hdfs.impl	org.apache.hadoop.fs.Hdfs
fs.AbstractFileSystem.s3a.impl	org.apache.hadoop.fs.s3a.S3A
fs.AbstractFileSystem.swebhdfs.impl	org.apache.hadoop.fs.SWebHdfs
fs.AbstractFileSystem.viewfs.impl	org.apache.hadoop.fs.viewfs.ViewFs
fs.AbstractFileSystem.wasb.impl	org.apache.hadoop.fs.azure.Wasb
fs.AbstractFileSystem.wasbs.impl	org.apache.hadoop.fs.azure.Wasbs
fs.AbstractFileSystem.webhdfs.impl	org.apache.hadoop.fs.WebHdfs
fs.abfs.impl	org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem
fs.abfss.impl	org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem
fs.adl.impl	org.apache.hadoop.fs.adl.AdlFileSystem
fs.adl.oauth2.access.token.provider.type	*********(redacted)
fs.automatic.close	true
fs.azure.authorization	false
fs.azure.authorization.caching.enable	true
fs.azure.local.sas.key.mode	false
fs.azure.sas.expiry.period	90d
fs.azure.saskey.usecontainersaskeyforallaccess	true
fs.azure.secure.mode	false
fs.azure.user.agent.prefix	unknown
fs.client.resolve.remote.symlinks	true
fs.client.resolve.topology.enabled	false
fs.defaultFS	hdfs://m01.itversity.com:9000
fs.df.interval	60000
fs.du.interval	600000
fs.ftp.data.connection.mode	ACTIVE_LOCAL_DATA_CONNECTION_MODE
fs.ftp.host	0.0.0.0
fs.ftp.host.port	21
fs.ftp.impl	org.apache.hadoop.fs.ftp.FTPFileSystem
fs.ftp.transfer.mode	BLOCK_TRANSFER_MODE
fs.har.impl.disable.cache	true
fs.permissions.umask-mode	022
fs.s3a.assumed.role.credentials.provider	org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
fs.s3a.assumed.role.session.duration	30m
fs.s3a.assumed.role.sts.endpoint.region	us-west-1
fs.s3a.attempts.maximum	20
fs.s3a.block.size	32M
fs.s3a.buffer.dir	${hadoop.tmp.dir}/s3a
fs.s3a.committer.magic.enabled	false
fs.s3a.committer.name	file
fs.s3a.committer.staging.abort.pending.uploads	true
fs.s3a.committer.staging.conflict-mode	fail
fs.s3a.committer.staging.tmp.path	tmp/staging
fs.s3a.committer.staging.unique-filenames	true
fs.s3a.committer.threads	8
fs.s3a.connection.establish.timeout	5000
fs.s3a.connection.maximum	15
fs.s3a.connection.ssl.enabled	true
fs.s3a.connection.timeout	200000
fs.s3a.etag.checksum.enabled	false
fs.s3a.fast.upload.active.blocks	4
fs.s3a.fast.upload.buffer	disk
fs.s3a.impl	org.apache.hadoop.fs.s3a.S3AFileSystem
fs.s3a.list.version	2
fs.s3a.max.total.tasks	5
fs.s3a.metadatastore.authoritative	false
fs.s3a.metadatastore.impl	org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore
fs.s3a.multiobjectdelete.enable	true
fs.s3a.multipart.purge	false
fs.s3a.multipart.purge.age	86400
fs.s3a.multipart.size	100M
fs.s3a.multipart.threshold	2147483647
fs.s3a.paging.maximum	5000
fs.s3a.path.style.access	false
fs.s3a.readahead.range	64K
fs.s3a.retry.interval	500ms
fs.s3a.retry.limit	${fs.s3a.attempts.maximum}
fs.s3a.retry.throttle.interval	1000ms
fs.s3a.retry.throttle.limit	${fs.s3a.attempts.maximum}
fs.s3a.s3guard.cli.prune.age	86400000
fs.s3a.s3guard.ddb.background.sleep	25ms
fs.s3a.s3guard.ddb.max.retries	9
fs.s3a.s3guard.ddb.table.capacity.read	500
fs.s3a.s3guard.ddb.table.capacity.write	100
fs.s3a.s3guard.ddb.table.create	false
fs.s3a.s3guard.ddb.throttle.retry.interval	100ms
fs.s3a.socket.recv.buffer	8192
fs.s3a.socket.send.buffer	8192
fs.s3a.threads.keepalivetime	60
fs.s3a.threads.max	10
fs.swift.impl	org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
fs.trash.checkpoint.interval	0
fs.trash.interval	86400
fs.viewfs.rename.strategy	SAME_MOUNTPOINT
fs.wasb.impl	org.apache.hadoop.fs.azure.NativeAzureFileSystem
fs.wasbs.impl	org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure
ftp.blocksize	67108864
ftp.bytes-per-checksum	512
ftp.client-write-packet-size	65536
ftp.replication	3
ftp.stream-buffer-size	4096
ha.failover-controller.cli-check.rpc-timeout.ms	20000
ha.failover-controller.graceful-fence.connection.retries	1
ha.failover-controller.graceful-fence.rpc-timeout.ms	5000
ha.failover-controller.new-active.rpc-timeout.ms	60000
ha.health-monitor.check-interval.ms	1000
ha.health-monitor.connect-retry-interval.ms	1000
ha.health-monitor.rpc-timeout.ms	45000
ha.health-monitor.sleep-after-disconnect.ms	1000
ha.zookeeper.acl	world:anyone:rwcda
ha.zookeeper.parent-znode	/hadoop-ha
ha.zookeeper.session-timeout.ms	10000
hadoop.caller.context.enabled	false
hadoop.caller.context.max.size	128
hadoop.caller.context.signature.max.size	40
hadoop.common.configuration.version	3.0.0
hadoop.http.authentication.kerberos.keytab	${user.home}/hadoop.keytab
hadoop.http.authentication.kerberos.principal	HTTP/_HOST@LOCALHOST
hadoop.http.authentication.signature.secret.file	*********(redacted)
hadoop.http.authentication.simple.anonymous.allowed	true
hadoop.http.authentication.token.validity	*********(redacted)
hadoop.http.authentication.type	simple
hadoop.http.cross-origin.allowed-headers	X-Requested-With,Content-Type,Accept,Origin
hadoop.http.cross-origin.allowed-methods	GET,POST,HEAD
hadoop.http.cross-origin.allowed-origins	*
hadoop.http.cross-origin.enabled	false
hadoop.http.cross-origin.max-age	1800
hadoop.http.filter.initializers	org.apache.hadoop.http.lib.StaticUserWebFilter
hadoop.http.logs.enabled	true
hadoop.http.staticuser.user	dr.who
hadoop.jetty.logs.serve.aliases	true
hadoop.kerberos.kinit.command	kinit
hadoop.kerberos.min.seconds.before.relogin	60
hadoop.proxyuser.hdfs.groups	*
hadoop.proxyuser.hdfs.hosts	*
hadoop.proxyuser.hive.groups	*
hadoop.proxyuser.hive.hosts	*
hadoop.proxyuser.hue.groups	*
hadoop.proxyuser.hue.hosts	*
hadoop.registry.jaas.context	Client
hadoop.registry.secure	false
hadoop.registry.system.acls	sasl:yarn@, sasl:mapred@, sasl:hdfs@
hadoop.registry.zk.connection.timeout.ms	15000
hadoop.registry.zk.quorum	localhost:2181
hadoop.registry.zk.retry.ceiling.ms	60000
hadoop.registry.zk.retry.interval.ms	1000
hadoop.registry.zk.retry.times	5
hadoop.registry.zk.root	/registry
hadoop.registry.zk.session.timeout.ms	60000
hadoop.rpc.protection	authentication
hadoop.rpc.socket.factory.class.default	org.apache.hadoop.net.StandardSocketFactory
hadoop.security.auth_to_local.mechanism	hadoop
hadoop.security.authentication	simple
hadoop.security.authorization	false
hadoop.security.credential.clear-text-fallback	true
hadoop.security.crypto.buffer.size	8192
hadoop.security.crypto.cipher.suite	AES/CTR/NoPadding
hadoop.security.crypto.codec.classes.aes.ctr.nopadding	org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec
hadoop.security.dns.log-slow-lookups.enabled	false
hadoop.security.dns.log-slow-lookups.threshold.ms	1000
hadoop.security.group.mapping	org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback
hadoop.security.group.mapping.ldap.connection.timeout.ms	60000
hadoop.security.group.mapping.ldap.conversion.rule	none
hadoop.security.group.mapping.ldap.directory.search.timeout	10000
hadoop.security.group.mapping.ldap.posix.attr.gid.name	gidNumber
hadoop.security.group.mapping.ldap.posix.attr.uid.name	uidNumber
hadoop.security.group.mapping.ldap.read.timeout.ms	60000
hadoop.security.group.mapping.ldap.search.attr.group.name	cn
hadoop.security.group.mapping.ldap.search.attr.member	member
hadoop.security.group.mapping.ldap.search.filter.group	(objectClass=group)
hadoop.security.group.mapping.ldap.search.filter.user	(&(objectClass=user)(sAMAccountName={0}))
hadoop.security.group.mapping.ldap.search.group.hierarchy.levels	0
hadoop.security.group.mapping.ldap.ssl	false
hadoop.security.group.mapping.providers.combined	true
hadoop.security.groups.cache.background.reload	false
hadoop.security.groups.cache.background.reload.threads	3
hadoop.security.groups.cache.secs	300
hadoop.security.groups.cache.warn.after.ms	5000
hadoop.security.groups.negative-cache.secs	30
hadoop.security.groups.shell.command.timeout	0s
hadoop.security.instrumentation.requires.admin	false
hadoop.security.java.secure.random.algorithm	SHA1PRNG
hadoop.security.key.default.bitlength	128
hadoop.security.key.default.cipher	AES/CTR/NoPadding
hadoop.security.kms.client.authentication.retry-count	1
hadoop.security.kms.client.encrypted.key.cache.expiry	43200000
hadoop.security.kms.client.encrypted.key.cache.low-watermark	0.3f
hadoop.security.kms.client.encrypted.key.cache.num.refill.threads	2
hadoop.security.kms.client.encrypted.key.cache.size	500
hadoop.security.kms.client.failover.sleep.base.millis	100
hadoop.security.kms.client.failover.sleep.max.millis	2000
hadoop.security.kms.client.timeout	60
hadoop.security.random.device.file.path	/dev/urandom
hadoop.security.sensitive-config-keys	*********(redacted)
hadoop.security.uid.cache.secs	14400
hadoop.service.shutdown.timeout	30s
hadoop.shell.missing.defaultFs.warning	false
hadoop.shell.safely.delete.limit.num.files	100
hadoop.ssl.client.conf	ssl-client.xml
hadoop.ssl.enabled	false
hadoop.ssl.enabled.protocols	TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2
hadoop.ssl.hostname.verifier	DEFAULT
hadoop.ssl.keystores.factory.class	org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory
hadoop.ssl.require.client.cert	false
hadoop.ssl.server.conf	ssl-server.xml
hadoop.system.tags	YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL
hadoop.tags.system	YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL
hadoop.tmp.dir	/tmp/hadoop-${user.name}
hadoop.user.group.static.mapping.overrides	dr.who=;
hadoop.util.hash.type	murmur
hadoop.workaround.non.threadsafe.getpwuid	true
hadoop.zk.acl	world:anyone:rwcda
hadoop.zk.num-retries	1000
hadoop.zk.retry-interval-ms	1000
hadoop.zk.timeout-ms	10000
io.compression.codec.bzip2.library	system-native
io.compression.codecs	org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec
io.erasurecode.codec.rs-legacy.rawcoders	rs-legacy_java
io.erasurecode.codec.rs.rawcoders	rs_native,rs_java
io.erasurecode.codec.xor.rawcoders	xor_native,xor_java
io.file.buffer.size	65536
io.map.index.interval	128
io.map.index.skip	0
io.mapfile.bloom.error.rate	0.005
io.mapfile.bloom.size	1048576
io.seqfile.compress.blocksize	1000000
io.seqfile.local.dir	${hadoop.tmp.dir}/io/local
io.serializations	org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization
io.skip.checksum.errors	false
ipc.client.bind.wildcard.addr	false
ipc.client.connect.max.retries	10
ipc.client.connect.max.retries.on.timeouts	45
ipc.client.connect.retry.interval	1000
ipc.client.connect.timeout	20000
ipc.client.connection.maxidletime	10000
ipc.client.fallback-to-simple-auth-allowed	false
ipc.client.idlethreshold	4000
ipc.client.kill.max	10
ipc.client.low-latency	false
ipc.client.ping	true
ipc.client.rpc-timeout.ms	0
ipc.client.tcpnodelay	true
ipc.maximum.data.length	67108864
ipc.maximum.response.length	134217728
ipc.ping.interval	60000
ipc.server.listen.queue.size	128
ipc.server.log.slow.rpc	false
ipc.server.max.connections	0
map.sort.class	org.apache.hadoop.util.QuickSort
mapreduce.am.max-attempts	2
mapreduce.app-submission.cross-platform	false
mapreduce.application.classpath	$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/
mapreduce.client.completion.pollinterval	5000
mapreduce.client.libjars.wildcard	true
mapreduce.client.output.filter	FAILED
mapreduce.client.progressmonitor.pollinterval	1000
mapreduce.client.submit.file.replication	10
mapreduce.cluster.acls.enabled	false
mapreduce.cluster.local.dir	${hadoop.tmp.dir}/mapred/local
mapreduce.fileoutputcommitter.algorithm.version	1
mapreduce.fileoutputcommitter.task.cleanup.enabled	false
mapreduce.framework.name	yarn
mapreduce.ifile.readahead	true
mapreduce.ifile.readahead.bytes	4194304
mapreduce.input.fileinputformat.list-status.num-threads	1
mapreduce.input.fileinputformat.split.minsize	0
mapreduce.input.lineinputformat.linespermap	1
mapreduce.job.acl-modify-job
mapreduce.job.acl-view-job
mapreduce.job.cache.limit.max-resources	0
mapreduce.job.cache.limit.max-resources-mb	0
mapreduce.job.cache.limit.max-single-resource-mb	0
mapreduce.job.classloader	false
mapreduce.job.committer.setup.cleanup.needed	true
mapreduce.job.complete.cancel.delegation.tokens	*********(redacted)
mapreduce.job.counters.max	120
mapreduce.job.emit-timeline-data	false
mapreduce.job.encrypted-intermediate-data	false
mapreduce.job.encrypted-intermediate-data-key-size-bits	128
mapreduce.job.encrypted-intermediate-data.buffer.kb	128
mapreduce.job.end-notification.max.attempts	5
mapreduce.job.end-notification.max.retry.interval	5000
mapreduce.job.end-notification.retry.attempts	0
mapreduce.job.end-notification.retry.interval	1000
mapreduce.job.finish-when-all-reducers-done	true
mapreduce.job.hdfs-servers	${fs.defaultFS}
mapreduce.job.heap.memory-mb.ratio	0.8
mapreduce.job.local-fs.single-disk-limit.bytes	-1
mapreduce.job.local-fs.single-disk-limit.check.interval-ms	5000
mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed	true
mapreduce.job.map.output.collector.class	org.apache.hadoop.mapred.MapTask$MapOutputBuffer
mapreduce.job.maps	2
mapreduce.job.max.map	-1
mapreduce.job.max.split.locations	15
mapreduce.job.maxtaskfailures.per.tracker	3
mapreduce.job.queuename	default
mapreduce.job.reduce.shuffle.consumer.plugin.class	org.apache.hadoop.mapreduce.task.reduce.Shuffle
mapreduce.job.reduce.slowstart.completedmaps	0.05
mapreduce.job.reducer.preempt.delay.sec	0
mapreduce.job.reducer.unconditional-preempt.delay.sec	300
mapreduce.job.reduces	1
mapreduce.job.running.map.limit	0
mapreduce.job.running.reduce.limit	0
mapreduce.job.sharedcache.mode	disabled
mapreduce.job.speculative.minimum-allowed-tasks	10
mapreduce.job.speculative.retry-after-no-speculate	1000
mapreduce.job.speculative.retry-after-speculate	15000
mapreduce.job.speculative.slowtaskthreshold	1.0
mapreduce.job.speculative.speculative-cap-running-tasks	0.1
mapreduce.job.speculative.speculative-cap-total-tasks	0.01
mapreduce.job.split.metainfo.maxsize	10000000
mapreduce.job.token.tracking.ids.enabled	*********(redacted)
mapreduce.job.ubertask.enable	false
mapreduce.job.ubertask.maxmaps	9
mapreduce.job.ubertask.maxreduces	1
mapreduce.jobhistory.address	m01.itversity.com:10020
mapreduce.jobhistory.admin.acl	*
mapreduce.jobhistory.admin.address	0.0.0.0:10033
mapreduce.jobhistory.always-scan-user-dir	false
mapreduce.jobhistory.bind-host	0.0.0.0
mapreduce.jobhistory.cleaner.enable	true
mapreduce.jobhistory.cleaner.interval-ms	86400000
mapreduce.jobhistory.client.thread-count	10
mapreduce.jobhistory.datestring.cache.size	200000
mapreduce.jobhistory.done-dir	/tmp/hadoop-yarn/staging/history/done
mapreduce.jobhistory.http.policy	HTTP_ONLY
mapreduce.jobhistory.intermediate-done-dir	/tmp/hadoop-yarn/staging/history/done_intermediate
mapreduce.jobhistory.intermediate-user-done-dir.permissions	770
mapreduce.jobhistory.jhist.format	binary
mapreduce.jobhistory.joblist.cache.size	20000
mapreduce.jobhistory.jobname.limit	50
mapreduce.jobhistory.keytab	/etc/security/keytab/jhs.service.keytab
mapreduce.jobhistory.loadedjob.tasks.max	-1
mapreduce.jobhistory.loadedjobs.cache.size	5
mapreduce.jobhistory.max-age-ms	604800000
mapreduce.jobhistory.minicluster.fixed.ports	false
mapreduce.jobhistory.move.interval-ms	180000
mapreduce.jobhistory.move.thread-count	3
mapreduce.jobhistory.principal	jhs/_HOST@REALM.TLD
mapreduce.jobhistory.recovery.enable	false
mapreduce.jobhistory.recovery.store.class	org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService
mapreduce.jobhistory.recovery.store.fs.uri	${hadoop.tmp.dir}/mapred/history/recoverystore
mapreduce.jobhistory.recovery.store.leveldb.path	${hadoop.tmp.dir}/mapred/history/recoverystore
mapreduce.jobhistory.webapp.address	m01.itversity.com:19888
mapreduce.jobhistory.webapp.https.address	0.0.0.0:19890
mapreduce.jobhistory.webapp.rest-csrf.custom-header	X-XSRF-Header
mapreduce.jobhistory.webapp.rest-csrf.enabled	false
mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
mapreduce.jobhistory.webapp.xfs-filter.xframe-options	SAMEORIGIN
mapreduce.jvm.system-properties-to-log	os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name
mapreduce.map.cpu.vcores	1
mapreduce.map.log.level	INFO
mapreduce.map.maxattempts	4
mapreduce.map.memory.mb	-1
mapreduce.map.output.compress	false
mapreduce.map.output.compress.codec	org.apache.hadoop.io.compress.DefaultCodec
mapreduce.map.skip.maxrecords	0
mapreduce.map.skip.proc-count.auto-incr	true
mapreduce.map.sort.spill.percent	0.80
mapreduce.map.speculative	true
mapreduce.output.fileoutputformat.compress	false
mapreduce.output.fileoutputformat.compress.codec	org.apache.hadoop.io.compress.DefaultCodec
mapreduce.output.fileoutputformat.compress.type	RECORD
mapreduce.outputcommitter.factory.scheme.s3a	org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
mapreduce.reduce.cpu.vcores	1
mapreduce.reduce.input.buffer.percent	0.0
mapreduce.reduce.log.level	INFO
mapreduce.reduce.markreset.buffer.percent	0.0
mapreduce.reduce.maxattempts	4
mapreduce.reduce.memory.mb	-1
mapreduce.reduce.merge.inmem.threshold	1000
mapreduce.reduce.shuffle.connect.timeout	180000
mapreduce.reduce.shuffle.fetch.retry.enabled	${yarn.nodemanager.recovery.enabled}
mapreduce.reduce.shuffle.fetch.retry.interval-ms	1000
mapreduce.reduce.shuffle.fetch.retry.timeout-ms	30000
mapreduce.reduce.shuffle.input.buffer.percent	0.70
mapreduce.reduce.shuffle.memory.limit.percent	0.25
mapreduce.reduce.shuffle.merge.percent	0.66
mapreduce.reduce.shuffle.parallelcopies	5
mapreduce.reduce.shuffle.read.timeout	180000
mapreduce.reduce.shuffle.retry-delay.max.ms	60000
mapreduce.reduce.skip.maxgroups	0
mapreduce.reduce.skip.proc-count.auto-incr	true
mapreduce.reduce.speculative	true
mapreduce.shuffle.connection-keep-alive.enable	false
mapreduce.shuffle.connection-keep-alive.timeout	5
mapreduce.shuffle.listen.queue.size	128
mapreduce.shuffle.max.connections	0
mapreduce.shuffle.max.threads	0
mapreduce.shuffle.port	13562
mapreduce.shuffle.ssl.enabled	false
mapreduce.shuffle.ssl.file.buffer.size	65536
mapreduce.shuffle.transfer.buffer.size	131072
mapreduce.task.combine.progress.records	10000
mapreduce.task.exit.timeout	60000
mapreduce.task.exit.timeout.check-interval-ms	20000
mapreduce.task.files.preserve.failedtasks	false
mapreduce.task.io.sort.factor	10
mapreduce.task.io.sort.mb	100
mapreduce.task.local-fs.write-limit.bytes	-1
mapreduce.task.merge.progress.records	10000
mapreduce.task.profile	false
mapreduce.task.profile.map.params	${mapreduce.task.profile.params}
mapreduce.task.profile.maps	0-2
mapreduce.task.profile.params	-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s
mapreduce.task.profile.reduce.params	${mapreduce.task.profile.params}
mapreduce.task.profile.reduces	0-2
mapreduce.task.skip.start.attempts	2
mapreduce.task.timeout	600000
mapreduce.task.userlog.limit.kb	0
net.topology.impl	org.apache.hadoop.net.NetworkTopology
net.topology.node.switch.mapping.impl	org.apache.hadoop.net.ScriptBasedMapping
net.topology.script.number.args	100
nfs.exports.allowed.hosts	* rw
rpc.metrics.quantile.enable	false
seq.io.sort.factor	100
seq.io.sort.mb	100
tfile.fs.input.buffer.size	262144
tfile.fs.output.buffer.size	262144
tfile.io.chunk.size	1048576
yarn.acl.enable	false
yarn.acl.reservation-enable	false
yarn.admin.acl	*
yarn.am.liveness-monitor.expiry-interval-ms	600000
yarn.app.attempt.diagnostics.limit.kc	64
yarn.app.mapreduce.am.command-opts	-Xmx1024m
yarn.app.mapreduce.am.container.log.backups	0
yarn.app.mapreduce.am.container.log.limit.kb	0
yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size	10
yarn.app.mapreduce.am.hard-kill-timeout-ms	10000
yarn.app.mapreduce.am.job.committer.cancel-timeout	60000
yarn.app.mapreduce.am.job.committer.commit-window	10000
yarn.app.mapreduce.am.job.task.listener.thread-count	30
yarn.app.mapreduce.am.log.level	INFO
yarn.app.mapreduce.am.resource.cpu-vcores	1
yarn.app.mapreduce.am.resource.mb	1536
yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms	1000
yarn.app.mapreduce.am.staging-dir	/tmp/hadoop-yarn/staging
yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled	false
yarn.app.mapreduce.client-am.ipc.max-retries	3
yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts	3
yarn.app.mapreduce.client.job.max-retries	3
yarn.app.mapreduce.client.job.retry-interval	2000
yarn.app.mapreduce.client.max-retries	3
yarn.app.mapreduce.shuffle.log.backups	0
yarn.app.mapreduce.shuffle.log.limit.kb	0
yarn.app.mapreduce.shuffle.log.separate	true
yarn.app.mapreduce.task.container.log.backups	0
yarn.client.application-client-protocol.poll-interval-ms	200
yarn.client.application-client-protocol.poll-timeout-ms	-1
yarn.client.failover-proxy-provider	org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider
yarn.client.failover-retries	0
yarn.client.failover-retries-on-socket-timeouts	0
yarn.client.load.resource-types.from-server	false
yarn.client.max-cached-nodemanagers-proxies	0
yarn.client.nodemanager-client-async.thread-pool-max-size	500
yarn.client.nodemanager-connect.max-wait-ms	180000
yarn.client.nodemanager-connect.retry-interval-ms	10000
yarn.cluster.max-application-priority	0
yarn.dispatcher.drain-events.timeout	300000
yarn.fail-fast	false
yarn.federation.cache-ttl.secs	300
yarn.federation.enabled	false
yarn.federation.registry.base-dir	yarnfederation/
yarn.federation.state-store.class	org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore
yarn.federation.subcluster-resolver.class	org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl
yarn.http.policy	HTTP_ONLY
yarn.intermediate-data-encryption.enable	false
yarn.ipc.rpc.class	org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC
yarn.is.minicluster	false
yarn.log-aggregation-enable	true
yarn.log-aggregation-status.time-out.ms	600000
yarn.log-aggregation.file-controller.TFile.class	org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController
yarn.log-aggregation.file-formats	TFile
yarn.log-aggregation.retain-check-interval-seconds	-1
yarn.log-aggregation.retain-seconds	2592000
yarn.minicluster.control-resource-monitoring	false
yarn.minicluster.fixed.ports	false
yarn.minicluster.use-rpc	false
yarn.minicluster.yarn.nodemanager.resource.memory-mb	4096
yarn.nm.liveness-monitor.expiry-interval-ms	600000
yarn.node-attribute.fs-store.impl.class	org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore
yarn.node-labels.configuration-type	centralized
yarn.node-labels.enabled	false
yarn.node-labels.fs-store.impl.class	org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore
yarn.nodemanager.address	${yarn.nodemanager.hostname}:0
yarn.nodemanager.admin-env	MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX
yarn.nodemanager.amrmproxy.address	0.0.0.0:8049
yarn.nodemanager.amrmproxy.client.thread-count	25
yarn.nodemanager.amrmproxy.enabled	false
yarn.nodemanager.amrmproxy.ha.enable	false
yarn.nodemanager.amrmproxy.interceptor-class.pipeline	org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor
yarn.nodemanager.aux-services	mapreduce_shuffle,spark_shuffle
yarn.nodemanager.aux-services.mapreduce_shuffle.class	org.apache.hadoop.mapred.ShuffleHandler
yarn.nodemanager.aux-services.spark_shuffle.class	org.apache.spark.network.yarn.YarnShuffleService
yarn.nodemanager.collector-service.address	${yarn.nodemanager.hostname}:8048
yarn.nodemanager.collector-service.thread-count	5
yarn.nodemanager.container-diagnostics-maximum-size	10000
yarn.nodemanager.container-executor.class	org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor
yarn.nodemanager.container-localizer.java.opts	-Xmx256m
yarn.nodemanager.container-localizer.log.level	INFO
yarn.nodemanager.container-manager.thread-count	20
yarn.nodemanager.container-metrics.enable	true
yarn.nodemanager.container-metrics.period-ms	-1
yarn.nodemanager.container-metrics.unregister-delay-ms	10000
yarn.nodemanager.container-monitor.enabled	true
yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled	false
yarn.nodemanager.container-retry-minimum-interval-ms	1000
yarn.nodemanager.container.stderr.pattern	{stderr,STDERR}
yarn.nodemanager.container.stderr.tail.bytes	4096
yarn.nodemanager.default-container-executor.log-dirs.permissions	710
yarn.nodemanager.delete.debug-delay-sec	0
yarn.nodemanager.delete.thread-count	4
yarn.nodemanager.disk-health-checker.enable	true
yarn.nodemanager.disk-health-checker.interval-ms	120000
yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage	90.0
yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb	0
yarn.nodemanager.disk-health-checker.min-healthy-disks	0.25
yarn.nodemanager.disk-validator	basic
yarn.nodemanager.distributed-scheduling.enabled	false
yarn.nodemanager.elastic-memory-control.enabled	false
yarn.nodemanager.elastic-memory-control.oom-handler	org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler
yarn.nodemanager.elastic-memory-control.timeout-sec	5
yarn.nodemanager.env-whitelist	JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
yarn.nodemanager.health-checker.interval-ms	600000
yarn.nodemanager.health-checker.script.timeout-ms	1200000
yarn.nodemanager.hostname	0.0.0.0
yarn.nodemanager.keytab	/etc/krb5.keytab
yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms	20
yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms	1000
yarn.nodemanager.linux-container-executor.cgroups.hierarchy	/hadoop-yarn
yarn.nodemanager.linux-container-executor.cgroups.mount	false
yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage	false
yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users	true
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user	nobody
yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern	^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$
yarn.nodemanager.linux-container-executor.resources-handler.class	org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler
yarn.nodemanager.local-cache.max-files-per-directory	8192
yarn.nodemanager.local-dirs	${hadoop.tmp.dir}/nm-local-dir
yarn.nodemanager.localizer.address	${yarn.nodemanager.hostname}:8040
yarn.nodemanager.localizer.cache.cleanup.interval-ms	600000
yarn.nodemanager.localizer.cache.target-size-mb	10240
yarn.nodemanager.localizer.client.thread-count	5
yarn.nodemanager.localizer.fetch.thread-count	4
yarn.nodemanager.log-aggregation.compression-type	none
yarn.nodemanager.log-aggregation.num-log-files-per-app	30
yarn.nodemanager.log-aggregation.policy.class	org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds	-1
yarn.nodemanager.log-container-debug-info.enabled	true
yarn.nodemanager.log-dirs	${yarn.log.dir}/userlogs
yarn.nodemanager.log.deletion-threads-count	4
yarn.nodemanager.log.retain-seconds	10800
yarn.nodemanager.logaggregation.threadpool-size-max	100
yarn.nodemanager.node-attributes.provider.fetch-interval-ms	600000
yarn.nodemanager.node-attributes.provider.fetch-timeout-ms	1200000
yarn.nodemanager.node-labels.provider.fetch-interval-ms	600000
yarn.nodemanager.node-labels.provider.fetch-timeout-ms	1200000
yarn.nodemanager.node-labels.resync-interval-ms	120000
yarn.nodemanager.numa-awareness.enabled	false
yarn.nodemanager.numa-awareness.numactl.cmd	/usr/bin/numactl
yarn.nodemanager.numa-awareness.read-topology	false
yarn.nodemanager.opportunistic-containers-max-queue-length	0
yarn.nodemanager.opportunistic-containers-use-pause-for-preemption	false
yarn.nodemanager.pmem-check-enabled	true
yarn.nodemanager.process-kill-wait.ms	5000
yarn.nodemanager.recovery.compaction-interval-secs	3600
yarn.nodemanager.recovery.dir	${hadoop.tmp.dir}/yarn-nm-recovery
yarn.nodemanager.recovery.enabled	false
yarn.nodemanager.recovery.supervised	false
yarn.nodemanager.remote-app-log-dir	/tmp/logs
yarn.nodemanager.remote-app-log-dir-suffix	logs
yarn.nodemanager.resource-monitor.interval-ms	3000
yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices	0,1
yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class	org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin
yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices	auto
yarn.nodemanager.resource-plugins.gpu.docker-plugin	nvidia-docker-v1
yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint	http://localhost:3476/v1.0/docker/cli
yarn.nodemanager.resource.count-logical-processors-as-cores	false
yarn.nodemanager.resource.cpu-vcores	30
yarn.nodemanager.resource.detect-hardware-capabilities	false
yarn.nodemanager.resource.memory-mb	51540
yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage	90.0
yarn.nodemanager.resource.memory.cgroups.swappiness	0
yarn.nodemanager.resource.memory.enabled	false
yarn.nodemanager.resource.memory.enforced	true
yarn.nodemanager.resource.pcores-vcores-multiplier	1.0
yarn.nodemanager.resource.percentage-physical-cpu-limit	80
yarn.nodemanager.resource.system-reserved-memory-mb	-1
yarn.nodemanager.resourcemanager.minimum.version	NONE
yarn.nodemanager.runtime.linux.allowed-runtimes	default
yarn.nodemanager.runtime.linux.docker.allowed-container-networks	host,none,bridge
yarn.nodemanager.runtime.linux.docker.capabilities	CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
yarn.nodemanager.runtime.linux.docker.default-container-network	host
yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed	false
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed	true
yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed	false
yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed	false
yarn.nodemanager.runtime.linux.docker.stop.grace-period	10
yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold	1
yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold	1
yarn.nodemanager.runtime.linux.sandbox-mode	disabled
yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions	read
yarn.nodemanager.sleep-delay-before-sigkill.ms	250
yarn.nodemanager.vmem-check-enabled	true
yarn.nodemanager.vmem-pmem-ratio	5
yarn.nodemanager.webapp.address	${yarn.nodemanager.hostname}:8042
yarn.nodemanager.webapp.cross-origin.enabled	false
yarn.nodemanager.webapp.https.address	0.0.0.0:8044
yarn.nodemanager.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.nodemanager.webapp.rest-csrf.enabled	false
yarn.nodemanager.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.nodemanager.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.nodemanager.windows-container.cpu-limit.enabled	false
yarn.nodemanager.windows-container.memory-limit.enabled	false
yarn.registry.class	org.apache.hadoop.registry.client.impl.FSRegistryOperationsService
yarn.resourcemanager.address	${yarn.resourcemanager.hostname}:8032
yarn.resourcemanager.admin.address	${yarn.resourcemanager.hostname}:8033
yarn.resourcemanager.admin.client.thread-count	1
yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.am.max-attempts	2
yarn.resourcemanager.amlauncher.thread-count	50
yarn.resourcemanager.application-timeouts.monitor.interval-ms	3000
yarn.resourcemanager.auto-update.containers	false
yarn.resourcemanager.bind-host	0.0.0.0
yarn.resourcemanager.client.thread-count	50
yarn.resourcemanager.configuration.file-system-based-store	/yarn/conf
yarn.resourcemanager.configuration.provider-class	org.apache.hadoop.yarn.LocalConfigurationProvider
yarn.resourcemanager.connect.max-wait.ms	900000
yarn.resourcemanager.connect.retry-interval.ms	30000
yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.container.liveness-monitor.interval-ms	600000
yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs	20
yarn.resourcemanager.delayed.delegation-token.removal-interval-ms	*********(redacted)
yarn.resourcemanager.delegation-token-renewer.thread-count	*********(redacted)
yarn.resourcemanager.delegation-token.max-conf-size-bytes	*********(redacted)
yarn.resourcemanager.delegation.key.update-interval	86400000
yarn.resourcemanager.delegation.token.max-lifetime	*********(redacted)
yarn.resourcemanager.delegation.token.renew-interval	*********(redacted)
yarn.resourcemanager.epoch.range	0
yarn.resourcemanager.fail-fast	${yarn.fail-fast}
yarn.resourcemanager.fs.state-store.num-retries	0
yarn.resourcemanager.fs.state-store.retry-interval-ms	1000
yarn.resourcemanager.fs.state-store.uri	${hadoop.tmp.dir}/yarn/system/rmstore
yarn.resourcemanager.ha.automatic-failover.embedded	true
yarn.resourcemanager.ha.automatic-failover.enabled	true
yarn.resourcemanager.ha.automatic-failover.zk-base-path	/yarn-leader-election
yarn.resourcemanager.ha.enabled	false
yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size	10
yarn.resourcemanager.hostname	m02.itversity.com
yarn.resourcemanager.keytab	/etc/krb5.keytab
yarn.resourcemanager.leveldb-state-store.compaction-interval-secs	3600
yarn.resourcemanager.leveldb-state-store.path	${hadoop.tmp.dir}/yarn/system/rmstore
yarn.resourcemanager.max-completed-applications	1000
yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory	10
yarn.resourcemanager.metrics.runtime.buckets	60,300,1440
yarn.resourcemanager.nm-container-queuing.load-comparator	QUEUE_LENGTH
yarn.resourcemanager.nm-container-queuing.max-queue-length	15
yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms	100
yarn.resourcemanager.nm-container-queuing.min-queue-length	5
yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms	10
yarn.resourcemanager.nm-container-queuing.queue-limit-stdev	1.0f
yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms	1000
yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs	*********(redacted)
yarn.resourcemanager.node-ip-cache.expiry-interval-secs	-1
yarn.resourcemanager.node-labels.provider.fetch-interval-ms	1800000
yarn.resourcemanager.node-removal-untracked.timeout-ms	60000
yarn.resourcemanager.nodemanager-connect-retries	10
yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs	3600
yarn.resourcemanager.nodemanager.minimum.version	NONE
yarn.resourcemanager.nodemanagers.heartbeat-interval-ms	1000
yarn.resourcemanager.opportunistic-container-allocation.enabled	false
yarn.resourcemanager.opportunistic-container-allocation.nodes-used	10
yarn.resourcemanager.placement-constraints.algorithm.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm
yarn.resourcemanager.placement-constraints.algorithm.iterator	SERIAL
yarn.resourcemanager.placement-constraints.algorithm.pool-size	1
yarn.resourcemanager.placement-constraints.handler	disabled
yarn.resourcemanager.placement-constraints.retry-attempts	3
yarn.resourcemanager.placement-constraints.scheduler.pool-size	1
yarn.resourcemanager.proxy-user-privileges.enabled	false
yarn.resourcemanager.recovery.enabled	false
yarn.resourcemanager.reservation-system.enable	false
yarn.resourcemanager.reservation-system.planfollower.time-step	1000
yarn.resourcemanager.resource-profiles.enabled	false
yarn.resourcemanager.resource-profiles.source-file	resource-profiles.json
yarn.resourcemanager.resource-tracker.address	${yarn.resourcemanager.hostname}:8031
yarn.resourcemanager.resource-tracker.client.thread-count	50
yarn.resourcemanager.rm.container-allocation.expiry-interval-ms	600000
yarn.resourcemanager.scheduler.address	${yarn.resourcemanager.hostname}:8030
yarn.resourcemanager.scheduler.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
yarn.resourcemanager.scheduler.client.thread-count	50
yarn.resourcemanager.scheduler.monitor.enable	false
yarn.resourcemanager.scheduler.monitor.policies	org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy
yarn.resourcemanager.state-store.max-completed-applications	${yarn.resourcemanager.max-completed-applications}
yarn.resourcemanager.store.class	org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size	10
yarn.resourcemanager.webapp.address	${yarn.resourcemanager.hostname}:19088
yarn.resourcemanager.webapp.cross-origin.enabled	false
yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled	*********(redacted)
yarn.resourcemanager.webapp.https.address	${yarn.resourcemanager.hostname}:8090
yarn.resourcemanager.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.resourcemanager.webapp.rest-csrf.enabled	false
yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.resourcemanager.webapp.ui-actions.enabled	true
yarn.resourcemanager.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.resourcemanager.work-preserving-recovery.enabled	true
yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms	10000
yarn.resourcemanager.zk-appid-node.split-index	0
yarn.resourcemanager.zk-delegation-token-node.split-index	*********(redacted)
yarn.resourcemanager.zk-max-znode-size.bytes	1048576
yarn.resourcemanager.zk-state-store.parent-path	/rmstore
yarn.rm.system-metrics-publisher.emit-container-events	false
yarn.router.clientrm.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor
yarn.router.interceptor.user.threadpool-size	5
yarn.router.pipeline.cache-max-size	25
yarn.router.rmadmin.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor
yarn.router.webapp.address	0.0.0.0:8089
yarn.router.webapp.https.address	0.0.0.0:8091
yarn.router.webapp.interceptor-class.pipeline	org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST
yarn.scheduler.configuration.fs.path	file://${hadoop.tmp.dir}/yarn/system/schedconf
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs	86400
yarn.scheduler.configuration.leveldb-store.path	${hadoop.tmp.dir}/yarn/system/confstore
yarn.scheduler.configuration.max.version	100
yarn.scheduler.configuration.mutation.acl-policy.class	org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
yarn.scheduler.configuration.store.class	file
yarn.scheduler.configuration.store.max-logs	1000
yarn.scheduler.configuration.zk-store.parent-path	/confstore
yarn.scheduler.include-port-in-node-name	false
yarn.scheduler.maximum-allocation-mb	8192
yarn.scheduler.maximum-allocation-vcores	4
yarn.scheduler.minimum-allocation-mb	1024
yarn.scheduler.minimum-allocation-vcores	1
yarn.scheduler.queue-placement-rules	user-group
yarn.sharedcache.admin.address	0.0.0.0:8047
yarn.sharedcache.admin.thread-count	1
yarn.sharedcache.app-checker.class	org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker
yarn.sharedcache.checksum.algo.impl	org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl
yarn.sharedcache.cleaner.initial-delay-mins	10
yarn.sharedcache.cleaner.period-mins	1440
yarn.sharedcache.cleaner.resource-sleep-ms	0
yarn.sharedcache.client-server.address	0.0.0.0:8045
yarn.sharedcache.client-server.thread-count	50
yarn.sharedcache.enabled	false
yarn.sharedcache.nested-level	3
yarn.sharedcache.nm.uploader.replication.factor	10
yarn.sharedcache.nm.uploader.thread-count	20
yarn.sharedcache.root-dir	/sharedcache
yarn.sharedcache.store.class	org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore
yarn.sharedcache.store.in-memory.check-period-mins	720
yarn.sharedcache.store.in-memory.initial-delay-mins	10
yarn.sharedcache.store.in-memory.staleness-period-mins	10080
yarn.sharedcache.uploader.server.address	0.0.0.0:8046
yarn.sharedcache.uploader.server.thread-count	50
yarn.sharedcache.webapp.address	0.0.0.0:8788
yarn.system-metrics-publisher.enabled	true
yarn.timeline-service.address	${yarn.timeline-service.hostname}:10200
yarn.timeline-service.app-aggregation-interval-secs	15
yarn.timeline-service.app-collector.linger-period.ms	60000
yarn.timeline-service.bind-host	0.0.0.0
yarn.timeline-service.client.best-effort	false
yarn.timeline-service.client.drain-entities.timeout.ms	2000
yarn.timeline-service.client.fd-clean-interval-secs	60
yarn.timeline-service.client.fd-flush-interval-secs	10
yarn.timeline-service.client.fd-retain-secs	300
yarn.timeline-service.client.internal-timers-ttl-secs	420
yarn.timeline-service.client.max-retries	30
yarn.timeline-service.client.retry-interval-ms	1000
yarn.timeline-service.enabled	true
yarn.timeline-service.entity-group-fs-store.active-dir	/tmp/entity-file-history/active
yarn.timeline-service.entity-group-fs-store.app-cache-size	10
yarn.timeline-service.entity-group-fs-store.cache-store-class	org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore
yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds	3600
yarn.timeline-service.entity-group-fs-store.done-dir	/tmp/entity-file-history/done/
yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size	10485760
yarn.timeline-service.entity-group-fs-store.retain-seconds	604800
yarn.timeline-service.entity-group-fs-store.scan-interval-seconds	60
yarn.timeline-service.entity-group-fs-store.summary-store	org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
yarn.timeline-service.entity-group-fs-store.with-user-dir	false
yarn.timeline-service.generic-application-history.enabled	true
yarn.timeline-service.generic-application-history.max-applications	10000
yarn.timeline-service.handler-thread-count	10
yarn.timeline-service.hbase-schema.prefix	prod.
yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds	259200000
yarn.timeline-service.hbase.coprocessor.jar.hdfs.location	/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar
yarn.timeline-service.hostname	m01.itversity.com
yarn.timeline-service.http-authentication.simple.anonymous.allowed	true
yarn.timeline-service.http-authentication.type	simple
yarn.timeline-service.http-cross-origin.enabled	false
yarn.timeline-service.keytab	/etc/krb5.keytab
yarn.timeline-service.leveldb-state-store.path	${hadoop.tmp.dir}/yarn/timeline
yarn.timeline-service.leveldb-timeline-store.path	${hadoop.tmp.dir}/yarn/timeline
yarn.timeline-service.leveldb-timeline-store.read-cache-size	104857600
yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size	10000
yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size	10000
yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms	300000
yarn.timeline-service.reader.class	org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl
yarn.timeline-service.reader.webapp.address	${yarn.timeline-service.webapp.address}
yarn.timeline-service.reader.webapp.https.address	${yarn.timeline-service.webapp.https.address}
yarn.timeline-service.recovery.enabled	false
yarn.timeline-service.state-store-class	org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore
yarn.timeline-service.store-class	org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
yarn.timeline-service.timeline-client.number-of-async-entities-to-merge	10
yarn.timeline-service.ttl-enable	true
yarn.timeline-service.ttl-ms	604800000
yarn.timeline-service.version	1.0f
yarn.timeline-service.webapp.address	${yarn.timeline-service.hostname}:8188
yarn.timeline-service.webapp.https.address	${yarn.timeline-service.hostname}:8190
yarn.timeline-service.webapp.rest-csrf.custom-header	X-XSRF-Header
yarn.timeline-service.webapp.rest-csrf.enabled	false
yarn.timeline-service.webapp.rest-csrf.methods-to-ignore	GET,OPTIONS,HEAD
yarn.timeline-service.webapp.xfs-filter.xframe-options	SAMEORIGIN
yarn.timeline-service.writer.class	org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl
yarn.timeline-service.writer.flush-interval-seconds	60
yarn.webapp.api-service.enable	false
yarn.webapp.filter-entity-list-by-user	false
yarn.webapp.ui2.enable	false
yarn.webapp.xfs-filter.enabled	true

System Properties

Name	Value
SPARK_SUBMIT	true
awt.toolkit	sun.awt.X11.XToolkit
derby.system.home	/tmp/derby/
file.encoding	UTF-8
file.encoding.pkg	sun.io
file.separator	/
java.awt.graphicsenv	sun.awt.X11GraphicsEnvironment
java.awt.printerjob	sun.print.PSPrinterJob
java.class.version	52.0
java.endorsed.dirs	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/endorsed
java.ext.dirs	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/ext:/usr/java/packages/lib/ext
java.home	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre
java.io.tmpdir	/tmp
java.library.path	/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
java.runtime.name	OpenJDK Runtime Environment
java.runtime.version	1.8.0_282-b08
java.specification.name	Java Platform API Specification
java.specification.vendor	Oracle Corporation
java.specification.version	1.8
java.vendor	Red Hat, Inc.
java.vendor.url	https://www.redhat.com/
java.vendor.url.bug	https://bugzilla.redhat.com/enter_bug.cgi?product=Red%20Hat%20Enterprise%20Linux%207&component=java-1.8.0-openjdk
java.version	1.8.0_282
java.vm.info	mixed mode
java.vm.name	OpenJDK 64-Bit Server VM
java.vm.specification.name	Java Virtual Machine Specification
java.vm.specification.vendor	Oracle Corporation
java.vm.specification.version	1.8
java.vm.vendor	Red Hat, Inc.
java.vm.version	25.282-b08
jetty.git.hash	b881a572662e1943a14ae12e7e1207989f218b74
line.separator
os.arch	amd64
os.name	Linux
os.version	4.19.62-mod-std-ipv6-64-rescue
path.separator	:
sun.arch.data.model	64
sun.boot.class.path	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/resources.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/rt.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/jsse.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/jce.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/charsets.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/jfr.jar:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/classes
sun.boot.library.path	/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.282.b08-1.el7_9.x86_64/jre/lib/amd64
sun.cpu.endian	little
sun.cpu.isalist
sun.io.unicode.encoding	UnicodeLittle
sun.java.command	org.apache.spark.deploy.SparkSubmit --conf spark.master=yarn --conf spark.sql.catalogImplementation=hive --conf spark.ui.port=0 --conf spark.sql.warehouse.dir=/user/itv018602/warehouse pyspark-shell
sun.java.launcher	SUN_STANDARD
sun.jnu.encoding	UTF-8
sun.management.compiler	HotSpot 64-Bit Tiered Compilers
sun.nio.ch.bugLevel
sun.os.patch.level	unknown
user.country	US
user.dir	/home/itv018602
user.home	/home/itv018602
user.language	en
user.name	itv018602
user.timezone	America/Toronto

Classpath Entries

Resource	Source
/opt/hadoop/etc/hadoop/	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2//conf/	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/HikariCP-2.5.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/JLargeArrays-1.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/JTransforms-3.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/ST4-4.0.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/accessors-smart-1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/activation-1.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/aircompressor-0.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-1.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/arrow-format-2.0.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/automaton-1.11-8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/avro-1.8.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/breeze_2.12-1.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/chill-java-0.9.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-cli-1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-codec-1.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-collections-3.2.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-compress-1.20.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-dbcp-1.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-httpclient-3.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-io-2.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-lang-2.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-lang3-3.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-logging-1.1.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-math3-3.4.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-net-3.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-pool-1.5.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/commons-text-1.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/core-1.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/curator-client-2.13.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/curator-framework-2.13.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/derby-10.12.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/dnsjava-2.1.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/ehcache-3.3.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/generex-1.0.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/gson-2.2.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/guava-14.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/guice-4.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/guice-servlet-4.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-cli-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-common-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-serde-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hk2-api-2.6.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/httpclient-4.5.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/httpcore-4.4.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/janino-3.0.16.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/javax.inject-1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/javolution-5.5.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jdo-api-3.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-client-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-common-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-hk2-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jersey-server-2.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jline-2.14.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/joda-time-2.10.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jodd-core-3.5.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jpam-1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json-1.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json-smart-2.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jsp-api-2.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jsr305-3.0.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jta-1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-client-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-common-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-core-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-server-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerb-util-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerby-config-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerby-util-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/libfb303-0.9.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/libthrift-0.12.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/log4j-1.2.17.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/lz4-java-1.7.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/metrics-core-4.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/metrics-json-4.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/minlog-1.3.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/objenesis-2.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/okhttp-2.7.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/okhttp-3.12.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/okio-1.14.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/opencsv-2.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/orc-core-1.5.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/orc-shims-1.5.12.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/oro-2.0.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/paranamer-2.8.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-column-1.10.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-common-1.10.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-format-2.4.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/py4j-0.10.9.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/pyrolite-4.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/re2j-1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-library-2.12.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/shims-0.9.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/snakeyaml-1.24.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-core_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-hive_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-repl_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-sql_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2-tests.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.2.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/stax-api-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/stax2-api-3.1.4.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/stream-2.9.6.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/super-csv-2.2.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/token-provider-1.0.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/transaction-api-1.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/velocity-1.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/xz-1.5.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/zookeeper-3.4.14.jar	System Classpath
/opt/spark-3.1.2-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar	System Classpath