Error

Spark Job was cancelled because it exceeded build time of 9000 secs: org.apache.spark.SparkException: Job 155 cancelled because SparkContext was shut down

Details

    Traceback (most recent call last):
  File "/opt/acc-bpt/release_2026_01/acc_bpt/shared/BptV2.py", line 284, in build
    rawdata = self.bptStatic.fetch(vars, times)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/acc_bpt/psb/shared/script_psb_h0hm.py", line 83, in fetch
    return super().fetch(var_defs, time_ranges, cache)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/acc_bpt/psb/shared/PsbBptStatic.py", line 35, in fetch
    pdf = self.nxcals.spark2pandas(sdf).drop('timestamp', axis='columns')
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/acc_bpt/shared/utils_nxcals.py", line 564, in spark2pandas
    pdf = spark_dataset.toPandas()
          ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/venv/lib/python3.11/site-packages/pyspark/sql/pandas/conversion.py", line 131, in toPandas
    batches = self._collect_as_arrow(split_batches=self_destruct)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/venv/lib/python3.11/site-packages/pyspark/sql/pandas/conversion.py", line 284, in _collect_as_arrow
    jsocket_auth_server.getResult()
  File "/opt/acc-bpt/release_2026_01/venv/lib/python3.11/site-packages/py4j/java_gateway.py", line 1322, in __call__
    return_value = get_return_value(
                   ^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/venv/lib/python3.11/site-packages/pyspark/errors/exceptions/captured.py", line 179, in deco
    return f(*a, **kw)
           ^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/venv/lib/python3.11/site-packages/py4j/protocol.py", line 326, in get_return_value
    raise Py4JJavaError(
py4j.protocol.Py4JJavaError: An error occurred while calling o1201.getResult.
: org.apache.spark.SparkException: Exception thrown in awaitResult: 
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:56)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:310)
	at org.apache.spark.security.SocketAuthServer.getResult(SocketAuthServer.scala:98)
	at org.apache.spark.security.SocketAuthServer.getResult(SocketAuthServer.scala:94)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: org.apache.spark.SparkException: Job 155 cancelled because SparkContext was shut down
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$cleanUpAfterSchedulerStop$1(DAGScheduler.scala:1253)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$cleanUpAfterSchedulerStop$1$adapted(DAGScheduler.scala:1251)
	at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
	at org.apache.spark.scheduler.DAGScheduler.cleanUpAfterSchedulerStop(DAGScheduler.scala:1251)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onStop(DAGScheduler.scala:3087)
	at org.apache.spark.util.EventLoop.stop(EventLoop.scala:84)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$stop$3(DAGScheduler.scala:2973)
	at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1375)
	at org.apache.spark.scheduler.DAGScheduler.stop(DAGScheduler.scala:2973)
	at org.apache.spark.SparkContext.$anonfun$stop$12(SparkContext.scala:2258)
	at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1375)
	at org.apache.spark.SparkContext.stop(SparkContext.scala:2258)
	at org.apache.spark.SparkContext.stop(SparkContext.scala:2211)
	at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$MonitorThread.run(YarnClientSchedulerBackend.scala:125)


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/acc-bpt/release_2026_01/jinja/build_static_plots.py", line 85, in _create_static_html_content
    output_html_file_path = driver.build(abs_plot_dest_file_html)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/acc-bpt/release_2026_01/acc_bpt/shared/BptV2.py", line 301, in build
    raise RuntimeError(f'Spark Job was cancelled because it exceeded build time of '
RuntimeError: Spark Job was cancelled because it exceeded build time of 9000 secs: org.apache.spark.SparkException: Job 155 cancelled because SparkContext was shut down

    

Generated 2026-04-30 23:34:16.994086+02:00