log4j.properties

# Set root logger level to DEBUG and its only appender to console
log4j.rootLogger=WARN, console

# Console appender configuration
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

# Set the default spark-shell log level to WARN. When running the spark-shell, the
# log level for this class is used to overwrite the root logger's log level, so that
# the user can have different defaults for the shell and regular Spark apps.
log4j.logger.org.apache.spark.repl.Main=WARN

# Settings to quiet third party logs that are too verbose
log4j.logger.org.spark_project.jetty=WARN
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=WARN
log4j.logger.org.apache.parquet=ERROR
log4j.logger.parquet=ERROR

# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR

# Set the log level for your application's package
log4j.logger.your.application.package=DEBUG

# Additional settings to reduce logging
log4j.logger.org.apache.spark=WARN
log4j.logger.org.apache.hadoop=WARN
log4j.logger.org.apache.kafka=WARN
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.hive=WARN

# Hadoop metrics configuration to reduce warnings
log4j.logger.org.apache.hadoop.metrics2=ERROR
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=ERROR
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=ERROR

# S3 filesystem specific logging
log4j.logger.org.apache.hadoop.fs.s3a=WARN
log4j.logger.org.apache.hadoop.fs.s3a.impl=WARN

# AWS SDK logging
log4j.logger.com.amazonaws=WARN
log4j.logger.com.amazonaws.services.s3=WARN