diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala index 5e923ad35a3ab..e9e4fffb4b1fb 100644 --- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala +++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala @@ -22,6 +22,8 @@ import java.net.URI import java.text.ParseException import java.util.UUID +import scala.jdk.CollectionConverters._ + import org.apache.ivy.Ivy import org.apache.ivy.core.LogOptions import org.apache.ivy.core.module.descriptor.{Artifact, DefaultDependencyDescriptor, DefaultExcludeRule, DefaultModuleDescriptor, ExcludeRule} @@ -29,10 +31,10 @@ import org.apache.ivy.core.module.id.{ArtifactId, ModuleId, ModuleRevisionId} import org.apache.ivy.core.report.{DownloadStatus, ResolveReport} import org.apache.ivy.core.resolve.ResolveOptions import org.apache.ivy.core.retrieve.RetrieveOptions -import org.apache.ivy.core.settings.IvySettings +import org.apache.ivy.core.settings.{IvySettings, NamedTimeoutConstraint} import org.apache.ivy.plugins.matcher.GlobPatternMatcher import org.apache.ivy.plugins.repository.file.FileRepository -import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver} +import org.apache.ivy.plugins.resolver.{AbstractResolver, ChainResolver, FileSystemResolver, IBiblioResolver} import org.apache.spark.SparkException import org.apache.spark.internal.{Logging, LogKeys} @@ -340,6 +342,28 @@ private[spark] object MavenUtils extends Logging { ivySettings } + /** Apply bounded network timeouts to every resolver in an Ivy settings graph. */ + private[spark] def setResolverTimeouts( + ivySettings: IvySettings, + connectTimeoutMs: Int, + readTimeoutMs: Int): Unit = { + require(connectTimeoutMs > 0, "The Ivy connection timeout must be positive") + require(readTimeoutMs > 0, "The Ivy read timeout must be positive") + + val name = s"spark-runtime-${UUID.randomUUID()}" + val timeout = new NamedTimeoutConstraint(name) + timeout.setConnectionTimeout(connectTimeoutMs) + timeout.setReadTimeout(readTimeoutMs) + ivySettings.addConfigured(timeout) + + ivySettings.getResolvers.asScala.foreach { + case resolver: AbstractResolver => + resolver.setTimeoutConstraint(name) + resolver.validate() + case _ => + } + } + /* Set ivy settings for location of cache, if option is supplied */ private[util] def processIvyPathArg(ivySettings: IvySettings, ivyPath: Option[String]): Unit = { val alternateIvyDir = ivyPath.filterNot(_.trim.isEmpty).getOrElse { diff --git a/common/utils/src/main/scala/org/apache/spark/util/RuntimeDependencyResolver.scala b/common/utils/src/main/scala/org/apache/spark/util/RuntimeDependencyResolver.scala new file mode 100644 index 0000000000000..eb624e6037922 --- /dev/null +++ b/common/utils/src/main/scala/org/apache/spark/util/RuntimeDependencyResolver.scala @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import java.io.PrintStream +import java.net.URI +import java.nio.file.{Path, Paths} +import java.util.concurrent.CancellationException +import java.util.concurrent.locks.ReentrantLock + +import org.apache.spark.util.ArrayImplicits._ + +/** Resolves runtime Ivy dependencies using one immutable Spark configuration snapshot. */ +private[spark] final class RuntimeDependencyResolver( + ivySettingsPath: Option[String], + configuredRepositories: Seq[String], + ivyPath: Option[String]) { + import RuntimeDependencyResolver._ + + /** Resolve an ivy URI. Calls are serialized because Ivy mutates process-wide state. */ + def resolve( + uri: URI, + repositoryPolicy: RepositoryPolicy = AllowRequestedRepositories, + connectTimeoutMs: Int = DefaultConnectTimeoutMs, + readTimeoutMs: Int = DefaultReadTimeoutMs, + isCancelled: () => Boolean = () => false): Seq[Path] = { + checkCancelled(isCancelled) + try { + ivyLock.lockInterruptibly() + } catch { + case _: InterruptedException => + Thread.currentThread().interrupt() + throw new CancellationException("Runtime Maven dependency resolution was cancelled") + } + + try { + require(uri.getScheme == "ivy", s"Expected an ivy URI, found: $uri") + val authority = Option(uri.getAuthority).getOrElse { + throw new IllegalArgumentException( + s"Invalid Ivy URI authority in uri $uri: Expected 'org:module:version', found null.") + } + if (authority.split(":").length != 3) { + throw new IllegalArgumentException( + s"Invalid Ivy URI authority in uri $uri: " + + s"Expected 'org:module:version', found $authority.") + } + + checkCancelled(isCancelled) + val (transitive, exclusions, requestedRepositories) = MavenUtils.parseQueryParams(uri) + val requested = requestedRepositories + .split(",") + .map(_.trim) + .filter(_.nonEmpty) + .toImmutableArraySeq + val repositories = (configuredRepositories ++ repositoryPolicy.validate(requested)) + .iterator + .map(_.trim) + .filter(_.nonEmpty) + .toSeq + .distinct + + implicit val printStream: PrintStream = System.err + val ivySettings = ivySettingsPath.filter(_.trim.nonEmpty) match { + case Some(path) => + MavenUtils.loadIvySettings(path, repositoriesOption(repositories), ivyPath) + case None => MavenUtils.buildIvySettings(repositoriesOption(repositories), ivyPath) + } + MavenUtils.setResolverTimeouts(ivySettings, connectTimeoutMs, readTimeoutMs) + + checkCancelled(isCancelled) + val exclusionsList = exclusions + .split(",") + .map(_.trim) + .filter(_.nonEmpty) + .toImmutableArraySeq + val result = MavenUtils.resolveMavenCoordinates( + authority, + ivySettings, + transitive = transitive, + exclusions = exclusionsList) + checkCancelled(isCancelled) + result.map(Paths.get(_)) + } finally { + ivyLock.unlock() + } + } +} + +private[spark] object RuntimeDependencyResolver { + val DefaultConnectTimeoutMs: Int = 30 * 1000 + val DefaultReadTimeoutMs: Int = 5 * 60 * 1000 + + private val ivyLock = new ReentrantLock() + + trait RepositoryPolicy { + def validate(requestedRepositories: Seq[String]): Seq[String] + } + + object AllowRequestedRepositories extends RepositoryPolicy { + override def validate(requestedRepositories: Seq[String]): Seq[String] = requestedRepositories + } + + object RejectRequestedRepositories extends RepositoryPolicy { + override def validate(requestedRepositories: Seq[String]): Seq[String] = { + if (requestedRepositories.nonEmpty) { + throw new IllegalArgumentException( + "Server-side Maven dependencies do not allow repositories from the ivy URI") + } + Nil + } + } + + private def repositoriesOption(repositories: Seq[String]): Option[String] = + Option(repositories.mkString(",")).filter(_.nonEmpty) + + private def checkCancelled(isCancelled: () => Boolean): Unit = { + if (isCancelled() || Thread.currentThread().isInterrupted) { + throw new CancellationException("Runtime Maven dependency resolution was cancelled") + } + } +} diff --git a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala index d30422ca8dd54..136f21d554174 100644 --- a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala +++ b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala @@ -21,6 +21,7 @@ import java.io.{File, OutputStream, PrintStream} import java.net.URI import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} +import java.util.concurrent.CancellationException import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ @@ -100,6 +101,21 @@ class MavenUtilsSuite } } + test("runtime resolver configures Ivy network timeouts") { + val settings = MavenUtils.buildIvySettings(None, Some(tempIvyPath)) + + MavenUtils.setResolverTimeouts(settings, connectTimeoutMs = 1234, readTimeoutMs = 5678) + + val resolvers = settings.getDefaultResolver + .asInstanceOf[ChainResolver] + .getResolvers + .asScala + .collect { case resolver: AbstractResolver => resolver } + assert(resolvers.nonEmpty) + assert(resolvers.forall(_.getTimeoutConstraint.getConnectionTimeout == 1234)) + assert(resolvers.forall(_.getTimeoutConstraint.getReadTimeout == 5678)) + } + test("add dependencies works correctly") { val md = MavenUtils.getModuleDescriptor val artifacts = MavenUtils.extractMavenCoordinates("com.databricks:spark-csv_2.12:0.1," + @@ -145,6 +161,51 @@ class MavenUtilsSuite } } + test("runtime dependency resolver loads custom Ivy settings") { + val main = MavenCoordinate("my.runtime.lib", "mylib", "0.1") + IvyTestUtils.withRepository(main, None, None) { repo => + val settings = Paths.get(tempIvyPath, "ivysettings.xml") + Files.writeString( + settings, + s""" + | + | + | + | + |""".stripMargin) + val resolver = new RuntimeDependencyResolver( + ivySettingsPath = Some(settings.toString), + configuredRepositories = Nil, + ivyPath = Some(tempIvyPath)) + + val resolved = resolver.resolve(URI.create(s"ivy://${main.toString}")) + + assert(resolved.exists(_.getFileName.toString.contains("my.runtime.lib_mylib-0.1"))) + assert(resolved.forall(_.startsWith(Paths.get(tempIvyPath)))) + } + } + + test("runtime dependency resolver applies the repository policy") { + val resolver = new RuntimeDependencyResolver(None, Nil, Some(tempIvyPath)) + val error = intercept[IllegalArgumentException] { + resolver.resolve( + URI.create("ivy://my.runtime.lib:mylib:0.1?repos=https://example.com/repository"), + RuntimeDependencyResolver.RejectRequestedRepositories) + } + + assert(error.getMessage.contains("do not allow repositories")) + } + + test("runtime dependency resolver honors cancellation before resolution") { + val resolver = new RuntimeDependencyResolver(None, Nil, Some(tempIvyPath)) + + intercept[CancellationException] { + resolver.resolve( + URI.create("ivy://my.runtime.lib:mylib:0.1"), + isCancelled = () => true) + } + } + test("search for artifact at local repositories") { val main = new MavenCoordinate("my.great.lib", "mylib", "0.1") val dep = "my.great.dep:mydep:0.5" diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 500229f0c9dac..6f4a8c9dc3f83 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -3042,7 +3042,8 @@ package object config { private[spark] val JAR_IVY_SETTING_PATH = ConfigBuilder(MavenUtils.JAR_IVY_SETTING_PATH_KEY) .doc("Path to an Ivy settings file to customize resolution of jars specified " + - "using spark.jars.packages instead of the built-in defaults, such as maven central. " + + "using spark.jars.packages or ivy:// URIs passed to SparkSession.addArtifact instead " + + "of the built-in defaults, such as maven central. " + "Additional repositories given by the command-line option --repositories " + "or spark.jars.repositories will also be included. " + "Useful for allowing Spark to resolve artifacts from behind a firewall " + @@ -3052,6 +3053,28 @@ package object config { .stringConf .createOptional + private[spark] val JAR_IVY_CONNECT_TIMEOUT = + ConfigBuilder("spark.jars.ivyConnectTimeout") + .doc("Connection timeout for Ivy repository requests made during runtime dependency " + + "resolution. This must be set before the SparkContext starts.") + .version("4.4.0") + .timeConf(TimeUnit.MILLISECONDS) + .checkValue( + timeout => timeout > 0 && timeout <= Int.MaxValue, + s"Timeout must be positive and no greater than ${Int.MaxValue} milliseconds.") + .createWithDefaultString("30s") + + private[spark] val JAR_IVY_READ_TIMEOUT = + ConfigBuilder("spark.jars.ivyReadTimeout") + .doc("Read timeout for Ivy repository requests made during runtime dependency resolution. " + + "This must be set before the SparkContext starts.") + .version("4.4.0") + .timeConf(TimeUnit.MILLISECONDS) + .checkValue( + timeout => timeout > 0 && timeout <= Int.MaxValue, + s"Timeout must be positive and no greater than ${Int.MaxValue} milliseconds.") + .createWithDefaultString("5m") + private[spark] val JAR_PACKAGES = ConfigBuilder("spark.jars.packages") .doc("Comma-separated list of Maven coordinates of jars to include " + @@ -3079,7 +3102,8 @@ package object config { private[spark] val JAR_REPOSITORIES = ConfigBuilder("spark.jars.repositories") .doc("Comma-separated list of additional remote repositories to search " + - "for the maven coordinates given with --packages or spark.jars.packages.") + "for the maven coordinates given with --packages, spark.jars.packages, or ivy:// URIs " + + "passed to SparkSession.addArtifact.") .version("2.3.0") .stringConf .toSequence diff --git a/docs/configuration.md b/docs/configuration.md index 683d1048dca60..7739cbe0768d0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1071,9 +1071,11 @@ Apart from these, the following properties are also available, and may be useful spark.jars.ivySettings - Path to an Ivy settings file to customize resolution of jars specified using spark.jars.packages - instead of the built-in defaults, such as maven central. Additional repositories given by the command-line - option --repositories or spark.jars.repositories will also be included. + Path to an Ivy settings file to customize resolution of jars specified using + spark.jars.packages or ivy:// URIs passed to + SparkSession.addArtifact instead of the built-in defaults, such as maven central. + Additional repositories given by the command-line option --repositories or + spark.jars.repositories will also be included. Useful for allowing Spark to resolve artifacts from behind a firewall e.g. via an in-house artifact server like Artifactory. Details on the settings file format can be found at Settings Files. @@ -1085,12 +1087,31 @@ Apart from these, the following properties are also available, and may be useful 2.2.0 - + + spark.jars.ivyConnectTimeout + 30s + + Connection timeout for Ivy repository requests made during runtime dependency resolution. + This must be set before the SparkContext starts. + + 4.4.0 + + + spark.jars.ivyReadTimeout + 5m + + Read timeout for Ivy repository requests made during runtime dependency resolution. + This must be set before the SparkContext starts. + + 4.4.0 + + spark.jars.repositories Comma-separated list of additional remote repositories to search for the maven coordinates - given with --packages or spark.jars.packages. + given with --packages, spark.jars.packages, or ivy:// URIs + passed to SparkSession.addArtifact. 2.3.0 diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index ee3ddb48ba5c6..432f50f8d4217 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -46,7 +46,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x81\x0f\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\x87\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommand\x12\\\n\x14\x63ompressed_operation\x18\x03 \x01(\x0b\x32\'.spark.connect.Plan.CompressedOperationH\x00R\x13\x63ompressedOperation\x1a\x8e\x02\n\x13\x43ompressedOperation\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12G\n\x07op_type\x18\x02 \x01(\x0e\x32..spark.connect.Plan.CompressedOperation.OpTypeR\x06opType\x12L\n\x11\x63ompression_codec\x18\x03 \x01(\x0e\x32\x1f.spark.connect.CompressionCodecR\x10\x63ompressionCodec"L\n\x06OpType\x12\x17\n\x13OP_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10OP_TYPE_RELATION\x10\x01\x12\x13\n\x0fOP_TYPE_COMMAND\x10\x02\x42\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x0f\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1aL\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x12"\n\x0c\x63\x61pabilities\x18\x02 \x03(\tR\x0c\x63\x61pabilities\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\x83\x06\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\x85\x02\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12^\n\x17result_chunking_options\x18\x02 \x01(\x0b\x32$.spark.connect.ResultChunkingOptionsH\x00R\x15resultChunkingOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\x87\x1c\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x8d\x01\n(pipeline_query_function_execution_signal\x18\x17 \x01(\x0b\x32\x33.spark.connect.PipelineQueryFunctionExecutionSignalH\x00R$pipelineQueryFunctionExecutionSignal\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a\xf8\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x12$\n\x0b\x63hunk_index\x18\x04 \x01(\x03H\x01R\nchunkIndex\x88\x01\x01\x12\x32\n\x13num_chunks_in_batch\x18\x05 \x01(\x03H\x02R\x10numChunksInBatch\x88\x01\x01\x42\x0f\n\r_start_offsetB\x0e\n\x0c_chunk_indexB\x16\n\x14_num_chunks_in_batch\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x93\x02\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x12)\n\x0eroot_error_idx\x18\x05 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x06 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrorsB\x11\n\x0f_root_error_idx\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xae\n\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a#\n\x0fMavenDependency\x12\x10\n\x03uri\x18\x01 \x01(\tR\x03uri\x1a\xcf\x01\n\rArtifactEntry\x12T\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactH\x00R\x08\x61rtifact\x12_\n\x10maven_dependency\x18\x02 \x01(\x0b\x32\x32.spark.connect.AddArtifactsRequest.MavenDependencyH\x00R\x0fmavenDependencyB\x07\n\x05value\x1a\xa9\x01\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x12J\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactEntryR\x07\x65ntries\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\xb5\x01\n\x15ResultChunkingOptions\x12;\n\x1a\x61llow_arrow_batch_chunking\x18\x01 \x01(\x08R\x17\x61llowArrowBatchChunking\x12@\n\x1apreferred_arrow_chunk_size\x18\x02 \x01(\x03H\x00R\x17preferredArrowChunkSize\x88\x01\x01\x42\x1d\n\x1b_preferred_arrow_chunk_size"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xd9\x0f\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\xa6\x04\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x12r\n\x14\x62reaking_change_info\x18\x05 \x01(\x0b\x32;.spark.connect.FetchErrorDetailsResponse.BreakingChangeInfoH\x02R\x12\x62reakingChangeInfo\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_stateB\x17\n\x15_breaking_change_info\x1a\xfa\x01\n\x12\x42reakingChangeInfo\x12+\n\x11migration_message\x18\x01 \x03(\tR\x10migrationMessage\x12k\n\x11mitigation_config\x18\x02 \x01(\x0b\x32\x39.spark.connect.FetchErrorDetailsResponse.MitigationConfigH\x00R\x10mitigationConfig\x88\x01\x01\x12$\n\x0bneeds_audit\x18\x03 \x01(\x08H\x01R\nneedsAudit\x88\x01\x01\x42\x14\n\x12_mitigation_configB\x0e\n\x0c_needs_audit\x1a:\n\x10MitigationConfig\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xea\x02\n\x13\x43loneSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12)\n\x0enew_session_id\x18\x04 \x01(\tH\x02R\x0cnewSessionId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x11\n\x0f_new_session_id"\xcc\x01\n\x14\x43loneSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12$\n\x0enew_session_id\x18\x03 \x01(\tR\x0cnewSessionId\x12:\n\x1anew_server_side_session_id\x18\x04 \x01(\tR\x16newServerSideSessionId"\xd3\x04\n\x10GetStatusRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12V\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12\x66\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02R\x0foperationStatus\x88\x01\x01\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1at\n\x16OperationStatusRequest\x12#\n\roperation_ids\x18\x01 \x03(\tR\x0coperationIds\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xad\x05\n\x11GetStatusResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId\x12_\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatusR\x11operationStatuses\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions\x1a\xab\x03\n\x0fOperationStatus\x12!\n\x0coperation_id\x18\x01 \x01(\tR\x0boperationId\x12U\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationStateR\x05state\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06*Q\n\x10\x43ompressionCodec\x12!\n\x1d\x43OMPRESSION_CODEC_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43OMPRESSION_CODEC_ZSTD\x10\x01\x32\xdf\x08\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x12Y\n\x0c\x43loneSession\x12".spark.connect.CloneSessionRequest\x1a#.spark.connect.CloneSessionResponse"\x00\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -71,8 +71,8 @@ _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" ]._serialized_options = b"8\001" - _globals["_COMPRESSIONCODEC"]._serialized_start = 20046 - _globals["_COMPRESSIONCODEC"]._serialized_end = 20127 + _globals["_COMPRESSIONCODEC"]._serialized_start = 20406 + _globals["_COMPRESSIONCODEC"]._serialized_end = 20487 _globals["_PLAN"]._serialized_start = 275 _globals["_PLAN"]._serialized_end = 758 _globals["_PLAN_COMPRESSEDOPERATION"]._serialized_start = 477 @@ -114,7 +114,7 @@ _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_start = 3448 _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_end = 3492 _globals["_ANALYZEPLANRESPONSE"]._serialized_start = 3565 - _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 5486 + _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 5522 _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_start = 4861 _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_end = 4918 _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_start = 4920 @@ -128,159 +128,163 @@ _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_start = 5105 _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_end = 5139 _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_start = 5141 - _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 5181 - _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 5183 - _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 5242 - _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 5244 - _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 5283 - _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 5285 - _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 5323 + _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 5217 + _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 5219 + _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 5278 + _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 5280 + _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 5319 + _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 5321 + _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 5359 _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_start = 3111 _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_end = 3120 _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_start = 3264 _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_end = 3275 - _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 5349 - _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 5432 - _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_start = 5434 - _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_end = 5476 - _globals["_EXECUTEPLANREQUEST"]._serialized_start = 5489 - _globals["_EXECUTEPLANREQUEST"]._serialized_end = 6260 - _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5923 - _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 6184 - _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 6263 - _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9854 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8363 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8434 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8437 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8685 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8688 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9205 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8783 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9115 + _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 5385 + _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 5468 + _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_start = 5470 + _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_end = 5512 + _globals["_EXECUTEPLANREQUEST"]._serialized_start = 5525 + _globals["_EXECUTEPLANREQUEST"]._serialized_end = 6296 + _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5959 + _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 6220 + _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 6299 + _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 9890 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 8399 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 8470 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 8473 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 8721 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 8724 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 9241 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 8819 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 9151 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_start = 8992 + ]._serialized_start = 9028 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_end = 9115 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9117 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9205 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9208 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9483 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9485 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9501 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9504 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9837 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9660 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9837 - _globals["_KEYVALUE"]._serialized_start = 9856 - _globals["_KEYVALUE"]._serialized_end = 9921 - _globals["_CONFIGREQUEST"]._serialized_start = 9924 - _globals["_CONFIGREQUEST"]._serialized_end = 11123 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10232 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10730 - _globals["_CONFIGREQUEST_SET"]._serialized_start = 10732 - _globals["_CONFIGREQUEST_SET"]._serialized_end = 10824 - _globals["_CONFIGREQUEST_GET"]._serialized_start = 10826 - _globals["_CONFIGREQUEST_GET"]._serialized_end = 10851 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10853 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10916 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10918 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10949 - _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10951 - _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 10999 - _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 11001 - _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 11028 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 11030 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11064 - _globals["_CONFIGRESPONSE"]._serialized_start = 11126 - _globals["_CONFIGRESPONSE"]._serialized_end = 11301 - _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11304 - _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12306 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11779 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11832 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11834 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11945 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11947 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 12040 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 12043 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12236 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12309 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12581 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12500 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12581 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12584 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 12910 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 12913 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13265 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13108 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13223 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13225 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13265 - _globals["_INTERRUPTREQUEST"]._serialized_start = 13268 - _globals["_INTERRUPTREQUEST"]._serialized_end = 13871 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 13671 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 13799 - _globals["_INTERRUPTRESPONSE"]._serialized_start = 13874 - _globals["_INTERRUPTRESPONSE"]._serialized_end = 14018 - _globals["_REATTACHOPTIONS"]._serialized_start = 14020 - _globals["_REATTACHOPTIONS"]._serialized_end = 14073 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14076 - _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14257 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14260 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 14666 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 14669 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15254 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15123 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15135 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15137 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15184 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15257 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15422 - _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15425 - _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15637 - _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15639 - _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 15747 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 15750 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16082 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16085 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18094 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16314 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16488 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16491 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 16859 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 16822 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 16859 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 16862 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17412 + ]._serialized_end = 9151 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9153 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9241 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9244 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 9519 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 9521 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 9537 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 9540 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 9873 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 9696 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 9873 + _globals["_KEYVALUE"]._serialized_start = 9892 + _globals["_KEYVALUE"]._serialized_end = 9957 + _globals["_CONFIGREQUEST"]._serialized_start = 9960 + _globals["_CONFIGREQUEST"]._serialized_end = 11159 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 10268 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 10766 + _globals["_CONFIGREQUEST_SET"]._serialized_start = 10768 + _globals["_CONFIGREQUEST_SET"]._serialized_end = 10860 + _globals["_CONFIGREQUEST_GET"]._serialized_start = 10862 + _globals["_CONFIGREQUEST_GET"]._serialized_end = 10887 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 10889 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 10952 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 10954 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10985 + _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10987 + _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 11035 + _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 11037 + _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 11064 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 11066 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 11100 + _globals["_CONFIGRESPONSE"]._serialized_start = 11162 + _globals["_CONFIGRESPONSE"]._serialized_end = 11337 + _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 11340 + _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 12666 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 11815 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 11868 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 11870 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11981 + _globals["_ADDARTIFACTSREQUEST_MAVENDEPENDENCY"]._serialized_start = 11983 + _globals["_ADDARTIFACTSREQUEST_MAVENDEPENDENCY"]._serialized_end = 12018 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTENTRY"]._serialized_start = 12021 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTENTRY"]._serialized_end = 12228 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 12231 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 12400 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 12403 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 12596 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 12669 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 12941 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 12860 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 12941 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 12944 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 13270 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 13273 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 13625 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 13468 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 13583 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 13585 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 13625 + _globals["_INTERRUPTREQUEST"]._serialized_start = 13628 + _globals["_INTERRUPTREQUEST"]._serialized_end = 14231 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 14031 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 14159 + _globals["_INTERRUPTRESPONSE"]._serialized_start = 14234 + _globals["_INTERRUPTRESPONSE"]._serialized_end = 14378 + _globals["_REATTACHOPTIONS"]._serialized_start = 14380 + _globals["_REATTACHOPTIONS"]._serialized_end = 14433 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_start = 14436 + _globals["_RESULTCHUNKINGOPTIONS"]._serialized_end = 14617 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 14620 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 15026 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 15029 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 15614 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 15483 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 15495 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 15497 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 15544 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 15617 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 15782 + _globals["_RELEASESESSIONREQUEST"]._serialized_start = 15785 + _globals["_RELEASESESSIONREQUEST"]._serialized_end = 15997 + _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 15999 + _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 16107 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 16110 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 16442 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 16445 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 18454 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 16674 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 16848 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 16851 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 17219 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 17182 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 17219 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 17222 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 17772 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_start = 17289 + ]._serialized_start = 17649 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_end = 17357 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17415 - _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17665 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17667 - _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 17725 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 17728 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18075 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18096 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18186 - _globals["_CLONESESSIONREQUEST"]._serialized_start = 18189 - _globals["_CLONESESSIONREQUEST"]._serialized_end = 18551 - _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18554 - _globals["_CLONESESSIONRESPONSE"]._serialized_end = 18758 - _globals["_GETSTATUSREQUEST"]._serialized_start = 18761 - _globals["_GETSTATUSREQUEST"]._serialized_end = 19356 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19160 - _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19276 - _globals["_GETSTATUSRESPONSE"]._serialized_start = 19359 - _globals["_GETSTATUSRESPONSE"]._serialized_end = 20044 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19617 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 20044 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 19814 - _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 20044 - _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20130 - _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21249 + ]._serialized_end = 17717 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17775 + _globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 18025 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 18027 + _globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_end = 18085 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 18088 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 18435 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 18456 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 18546 + _globals["_CLONESESSIONREQUEST"]._serialized_start = 18549 + _globals["_CLONESESSIONREQUEST"]._serialized_end = 18911 + _globals["_CLONESESSIONRESPONSE"]._serialized_start = 18914 + _globals["_CLONESESSIONRESPONSE"]._serialized_end = 19118 + _globals["_GETSTATUSREQUEST"]._serialized_start = 19121 + _globals["_GETSTATUSREQUEST"]._serialized_end = 19716 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 19520 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 19636 + _globals["_GETSTATUSRESPONSE"]._serialized_start = 19719 + _globals["_GETSTATUSRESPONSE"]._serialized_end = 20404 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 19977 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 20404 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_start = 20174 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE"]._serialized_end = 20404 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 20490 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 21609 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index 7750dc6726546..960f8eff181a5 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -896,14 +896,26 @@ class AnalyzePlanResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor VERSION_FIELD_NUMBER: builtins.int + CAPABILITIES_FIELD_NUMBER: builtins.int version: builtins.str + @property + def capabilities( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Capabilities supported by the server. Capability names are versioned so clients can + negotiate behavior without inferring it from the Spark version or a configuration value. + """ def __init__( self, *, version: builtins.str = ..., + capabilities: collections.abc.Iterable[builtins.str] | None = ..., ) -> None: ... def ClearField( - self, field_name: typing_extensions.Literal["version", b"version"] + self, + field_name: typing_extensions.Literal[ + "capabilities", b"capabilities", "version", b"version" + ], ) -> None: ... class DDLParse(google.protobuf.message.Message): @@ -2488,26 +2500,87 @@ class AddArtifactsRequest(google.protobuf.message.Message): self, field_name: typing_extensions.Literal["data", b"data", "name", b"name"] ) -> None: ... + class MavenDependency(google.protobuf.message.Message): + """A Maven dependency that must be resolved by the server. The URI uses Spark's existing + ivy://group:module:version syntax. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + URI_FIELD_NUMBER: builtins.int + uri: builtins.str + def __init__( + self, + *, + uri: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["uri", b"uri"]) -> None: ... + + class ArtifactEntry(google.protobuf.message.Message): + """An ordered artifact batch entry used by clients that support server-side Maven resolution.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ARTIFACT_FIELD_NUMBER: builtins.int + MAVEN_DEPENDENCY_FIELD_NUMBER: builtins.int + @property + def artifact(self) -> global___AddArtifactsRequest.SingleChunkArtifact: ... + @property + def maven_dependency(self) -> global___AddArtifactsRequest.MavenDependency: ... + def __init__( + self, + *, + artifact: global___AddArtifactsRequest.SingleChunkArtifact | None = ..., + maven_dependency: global___AddArtifactsRequest.MavenDependency | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "artifact", b"artifact", "maven_dependency", b"maven_dependency", "value", b"value" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "artifact", b"artifact", "maven_dependency", b"maven_dependency", "value", b"value" + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["value", b"value"] + ) -> typing_extensions.Literal["artifact", "maven_dependency"] | None: ... + class Batch(google.protobuf.message.Message): """A number of `SingleChunkArtifact` batched into a single RPC.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor ARTIFACTS_FIELD_NUMBER: builtins.int + ENTRIES_FIELD_NUMBER: builtins.int @property def artifacts( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___AddArtifactsRequest.SingleChunkArtifact - ]: ... + ]: + """Legacy artifact list used by clients that do not negotiate server-side Maven resolution.""" + @property + def entries( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___AddArtifactsRequest.ArtifactEntry + ]: + """Ordered mixture of uploaded artifacts and server-resolved Maven dependencies.""" def __init__( self, *, artifacts: collections.abc.Iterable[global___AddArtifactsRequest.SingleChunkArtifact] | None = ..., + entries: collections.abc.Iterable[global___AddArtifactsRequest.ArtifactEntry] + | None = ..., ) -> None: ... def ClearField( - self, field_name: typing_extensions.Literal["artifacts", b"artifacts"] + self, + field_name: typing_extensions.Literal["artifacts", b"artifacts", "entries", b"entries"], ) -> None: ... class BeginChunkedArtifact(google.protobuf.message.Message): diff --git a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala index d6444feffc0c6..fe7bc0b84532d 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -561,7 +561,11 @@ abstract class SparkSession extends Serializable with Closeable { /** * Add a single artifact to the current session. * - * Currently only local files with extensions .jar and .class are supported. + * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs. + * Ivy URIs are resolved using `spark.jars.ivySettings` when it is configured. When using Spark + * Connect, Ivy URIs with repositories in the `repos` query parameter, or sent to servers that do + * not support server-side Maven resolution, are resolved by the client without the server's Ivy + * settings. * * @since 4.0.0 */ @@ -572,6 +576,10 @@ abstract class SparkSession extends Serializable with Closeable { * Add a single artifact to the current session. * * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs. + * Ivy URIs are resolved using `spark.jars.ivySettings` when it is configured. When using Spark + * Connect, Ivy URIs with repositories in the `repos` query parameter, or sent to servers that do + * not support server-side Maven resolution, are resolved by the client without the server's Ivy + * settings. * * @since 4.0.0 */ @@ -622,7 +630,11 @@ abstract class SparkSession extends Serializable with Closeable { /** * Add one or more artifacts to the session. * - * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs + * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs. + * Ivy URIs are resolved using `spark.jars.ivySettings` when it is configured. When using Spark + * Connect, Ivy URIs with repositories in the `repos` query parameter, or sent to servers that do + * not support server-side Maven resolution, are resolved by the client without the server's Ivy + * settings. * * @since 4.0.0 */ diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala index bbe3fd16a9693..47f078ec3d216 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala @@ -23,8 +23,10 @@ import java.util.concurrent.TimeUnit import java.util.jar.{JarEntry, JarOutputStream} import java.util.zip.CRC32 +import scala.jdk.CollectionConverters._ + import com.google.protobuf.ByteString -import io.grpc.{ManagedChannel, Server} +import io.grpc.{ManagedChannel, Server, Status} import io.grpc.inprocess.{InProcessChannelBuilder, InProcessServerBuilder} import org.apache.spark.connect.proto.AddArtifactsRequest @@ -255,6 +257,103 @@ class ArtifactSuite extends ConnectFunSuite { assertFileDataEquality(artifacts.get(1).getData, Paths.get(file2)) } + test("server-side Maven dependencies preserve artifact order") { + val classFile = artifactFilePath.resolve("smallClassFile.class").toUri + val ivyUri = URI.create("ivy://my.artifactsuite.lib:mylib:0.1") + val jarFile = artifactFilePath.resolve("smallJar.jar").toUri + + artifactManager.addArtifacts( + Seq(classFile, ivyUri, jarFile), + serverSideMavenArtifacts = true) + + val requests = service.getAndClearLatestAddArtifactRequests() + assert(requests.size == 1) + val batch = requests.head.getBatch + assert(batch.getArtifactsCount == 0) + assert(batch.getEntriesCount == 3) + assert(batch.getEntries(0).getArtifact.getName == "classes/smallClassFile.class") + assert(batch.getEntries(1).getMavenDependency.getUri == ivyUri.toString) + assert(batch.getEntries(2).getArtifact.getName == "jars/smallJar.jar") + } + + test("server-side Maven mode resolves requested repositories on the client") { + val main = new MavenCoordinate("my.artifactsuite.client", "mylib", "0.1") + IvyTestUtils.withRepository(main, None, None) { repo => + val classFile = artifactFilePath.resolve("smallClassFile.class").toUri + val clientIvyUri = URI.create(s"ivy://${main.toString}?repos=$repo") + val serverIvyUri = URI.create("ivy://my.artifactsuite.server:mylib:0.1") + + artifactManager.addArtifacts( + Seq(classFile, clientIvyUri, serverIvyUri), + serverSideMavenArtifacts = true) + + val requests = service.getAndClearLatestAddArtifactRequests() + assert(requests.size == 1) + val batch = requests.head.getBatch + assert(batch.getEntriesCount == 3) + assert(batch.getEntries(0).getArtifact.getName == "classes/smallClassFile.class") + assert(batch.getEntries(1).getArtifact.getName.contains("my.artifactsuite.client_mylib-0.1")) + assert(batch.getEntries(2).getMavenDependency.getUri == serverIvyUri.toString) + } + } + + test("Spark Connect uses server-side Maven resolution when advertised") { + service.serverCapabilities = Seq( + SparkConnectClient.SERVER_SIDE_MAVEN_ARTIFACTS_CAPABILITY) + client = new SparkConnectClient(Configuration(), channel) + val ivyUri = URI.create("ivy://my.artifactsuite.lib:mylib:0.1") + + client.addArtifact(ivyUri) + + val requests = service.getAndClearLatestAddArtifactRequests() + assert(requests.size == 1) + assert(requests.head.getBatch.getEntriesCount == 1) + assert(requests.head.getBatch.getEntries(0).getMavenDependency.getUri == ivyUri.toString) + } + + test("Spark Connect resolves requested Maven repositories on the client") { + val main = new MavenCoordinate("my.artifactsuite.clientcapability", "mylib", "0.1") + IvyTestUtils.withRepository(main, None, None) { repo => + service.serverCapabilities = Seq( + SparkConnectClient.SERVER_SIDE_MAVEN_ARTIFACTS_CAPABILITY) + client = new SparkConnectClient(Configuration(), channel) + + client.addArtifact(URI.create(s"ivy://${main.toString}?repos=$repo")) + + val requests = service.getAndClearLatestAddArtifactRequests() + assert(requests.size == 1) + val batch = requests.head.getBatch + assert(batch.getEntriesCount == 1) + assert(batch.getEntries(0).hasArtifact) + assert( + batch.getEntries(0).getArtifact.getName.contains( + "my.artifactsuite.clientcapability_mylib-0.1")) + } + } + + test("Spark Connect falls back only when the Maven capability is absent") { + val main = new MavenCoordinate("my.artifactsuite.fallback", "mylib", "0.1") + IvyTestUtils.withRepository(main, None, None) { repo => + client = new SparkConnectClient(Configuration(), channel) + + client.addArtifact(URI.create(s"ivy://${main.toString}?repos=$repo")) + + val requests = service.getAndClearLatestAddArtifactRequests() + assert(requests.nonEmpty) + assert(requests.forall(_.getBatch.getEntriesCount == 0)) + assert(requests.flatMap(_.getBatch.getArtifactsList.asScala).exists { artifact => + artifact.getName.contains("my.artifactsuite.fallback_mylib-0.1") + }) + } + + service.errorToThrowOnAnalyze = Some(Status.UNAVAILABLE.asRuntimeException()) + client = new SparkConnectClient(Configuration(retryPolicies = Nil), channel) + intercept[Exception] { + client.addArtifact(URI.create("ivy://my.artifactsuite.fallback:mylib:0.1")) + } + assert(service.getAndClearLatestAddArtifactRequests().isEmpty) + } + test("Mix of SingleChunkArtifact and chunked artifact") { val path1 = artifactFilePath.resolve("smallClassFile.class") val file1 = path1.toUri diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala index 6a456a554c5c5..b37a0d08d8d67 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/SparkConnectClientSuite.scala @@ -1428,6 +1428,8 @@ class DummySparkConnectService() extends SparkConnectServiceGrpc.SparkConnectSer private val sparkConfigs = mutable.Map.empty[String, String] var errorToThrowOnExecute: Option[Throwable] = None + var errorToThrowOnAnalyze: Option[Throwable] = None + var serverCapabilities: Seq[String] = Nil private var errorToThrowOnConfig: Map[String, Throwable] = Map.empty @@ -1501,6 +1503,11 @@ class DummySparkConnectService() extends SparkConnectServiceGrpc.SparkConnectSer override def analyzePlan( request: AnalyzePlanRequest, responseObserver: StreamObserver[AnalyzePlanResponse]): Unit = { + errorToThrowOnAnalyze.foreach { error => + errorToThrowOnAnalyze = None + responseObserver.onError(error) + return + } // Reply with a dummy response using the same client ID val requestSessionId = request.getSessionId synchronized { @@ -1520,11 +1527,17 @@ class DummySparkConnectService() extends SparkConnectServiceGrpc.SparkConnectSer case _ => inputPlan = null } } - val response = AnalyzePlanResponse + val responseBuilder = AnalyzePlanResponse .newBuilder() .setSessionId(requestSessionId) - .build() - responseObserver.onNext(response) + if (request.hasSparkVersion) { + responseBuilder.setSparkVersion( + proto.AnalyzePlanResponse.SparkVersion + .newBuilder() + .setVersion("test-version") + .addAllCapabilities(serverCapabilities.asJava)) + } + responseObserver.onNext(responseBuilder.build()) responseObserver.onCompleted() } diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index be4ab641c7b5b..5c457747de528 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -283,6 +283,9 @@ message AnalyzePlanResponse { message SparkVersion { string version = 1; + // Capabilities supported by the server. Capability names are versioned so clients can + // negotiate behavior without inferring it from the Spark version or a configuration value. + repeated string capabilities = 2; } message DDLParse { @@ -679,9 +682,26 @@ message AddArtifactsRequest { ArtifactChunk data = 2; } + // A Maven dependency that must be resolved by the server. The URI uses Spark's existing + // ivy://group:module:version syntax. + message MavenDependency { + string uri = 1; + } + + // An ordered artifact batch entry used by clients that support server-side Maven resolution. + message ArtifactEntry { + oneof value { + SingleChunkArtifact artifact = 1; + MavenDependency maven_dependency = 2; + } + } + // A number of `SingleChunkArtifact` batched into a single RPC. message Batch { + // Legacy artifact list used by clients that do not negotiate server-side Maven resolution. repeated SingleChunkArtifact artifacts = 1; + // Ordered mixture of uploaded artifacts and server-resolved Maven dependencies. + repeated ArtifactEntry entries = 2; } // Signals the beginning/start of a chunked artifact. diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/ArtifactManager.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/ArtifactManager.scala index 1cfc9f170e896..df5102b34607a 100644 --- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/ArtifactManager.scala +++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/ArtifactManager.scala @@ -40,7 +40,7 @@ import org.apache.spark.connect.proto.AddArtifactsResponse.ArtifactSummary import org.apache.spark.network.util.JavaUtils.sha256Hex import org.apache.spark.sql.Artifact import org.apache.spark.sql.Artifact.{newCacheArtifact, newIvyArtifacts} -import org.apache.spark.util.{SparkFileUtils, SparkStringUtils, SparkThreadUtils} +import org.apache.spark.util.{MavenUtils, SparkFileUtils, SparkStringUtils, SparkThreadUtils} /** * The Artifact Manager is responsible for handling and transferring artifacts from the local @@ -66,6 +66,16 @@ class ArtifactManager( private[this] val classFinders = new CopyOnWriteArrayList[ClassFinder] private[this] val stubState = stub.stubState + private sealed trait ArtifactEntry { + def size: Long + } + private case class UploadedArtifact(artifact: Artifact) extends ArtifactEntry { + override def size: Long = artifact.size + } + private case class MavenDependency(uri: URI) extends ArtifactEntry { + override def size: Long = 0L + } + /** * Register a [[ClassFinder]] for dynamically generated classes. */ @@ -99,12 +109,20 @@ class ArtifactManager( } } + private def hasRequestedRepositories(uri: URI): Boolean = { + val (_, _, repositories) = MavenUtils.parseQueryParams(uri) + repositories.split(",").exists(_.trim.nonEmpty) + } + /** * Add a single artifact to the session. * * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs */ - def addArtifact(uri: URI): Unit = addArtifacts(parseArtifacts(uri)) + def addArtifact(uri: URI): Unit = addArtifact(uri, serverSideMavenArtifacts = false) + + private[client] def addArtifact(uri: URI, serverSideMavenArtifacts: Boolean): Unit = + addArtifacts(Seq(uri), serverSideMavenArtifacts) /** * Add a single in-memory artifact to the session while preserving the directory structure @@ -160,7 +178,26 @@ class ArtifactManager( * * Currently it supports local files with extensions .jar and .class and Apache Ivy URIs */ - def addArtifacts(uris: Seq[URI]): Unit = addArtifacts(uris.flatMap(parseArtifacts)) + def addArtifacts(uris: Seq[URI]): Unit = + addArtifacts(uris, serverSideMavenArtifacts = false) + + private[client] def addArtifacts( + uris: Seq[URI], + serverSideMavenArtifacts: Boolean): Unit = { + if (serverSideMavenArtifacts) { + val entries = uris.flatMap { uri => + uri.getScheme match { + // Resolve explicit repositories on the client so arbitrary repositories are never + // accessed from the server. + case "ivy" if !hasRequestedRepositories(uri) => MavenDependency(uri) :: Nil + case _ => parseArtifacts(uri).map(UploadedArtifact) + } + } + addArtifactEntries(entries, useOrderedEntries = true) + } else { + addArtifacts(uris.flatMap(parseArtifacts)) + } + } private[client] def isCachedArtifact(hash: String): Boolean = { val artifactName = s"${Artifact.CACHE_PREFIX}/$hash" @@ -293,13 +330,19 @@ class ArtifactManager( * Add a number of artifacts to the session. */ private[client] def addArtifacts(artifacts: Iterable[Artifact]): Unit = { - if (artifacts.isEmpty) { + addArtifactEntries(artifacts.map(UploadedArtifact), useOrderedEntries = false) + } + + private def addArtifactEntries( + entries: Iterable[ArtifactEntry], + useOrderedEntries: Boolean): Unit = { + if (entries.isEmpty) { return } try { stubState.retryHandler.retry { - addArtifactsImpl(artifacts) + addArtifactsImpl(entries, useOrderedEntries) } } catch { case ex: StatusRuntimeException => @@ -307,7 +350,9 @@ class ArtifactManager( } } - private[client] def addArtifactsImpl(artifacts: Iterable[Artifact]): Unit = { + private def addArtifactsImpl( + entries: Iterable[ArtifactEntry], + useOrderedEntries: Boolean): Unit = { val promise = Promise[Seq[ArtifactSummary]]() val responseHandler = new StreamObserver[proto.AddArtifactsResponse] { private val summaries = mutable.Buffer.empty[ArtifactSummary] @@ -329,36 +374,33 @@ class ArtifactManager( } } val stream = stub.addArtifacts(responseHandler) - val currentBatch = mutable.Buffer.empty[Artifact] + val currentBatch = mutable.Buffer.empty[ArtifactEntry] var currentBatchSize = 0L - def addToBatch(dep: Artifact, size: Long): Unit = { + def addToBatch(dep: ArtifactEntry): Unit = { currentBatch += dep - currentBatchSize += size + currentBatchSize += dep.size } def writeBatch(): Unit = { - addBatchedArtifacts(currentBatch.toSeq, stream) + addBatchedArtifacts(currentBatch.toSeq, stream, useOrderedEntries) currentBatch.clear() currentBatchSize = 0 } - artifacts.iterator.foreach { artifact => - val data = artifact.storage - val size = data.size - if (size > CHUNK_SIZE) { + entries.iterator.foreach { + case UploadedArtifact(artifact) if artifact.size > CHUNK_SIZE => // Payload can either be a batch OR a single chunked artifact. Write batch if non-empty // before chunking current artifact. if (currentBatch.nonEmpty) { writeBatch() } addChunkedArtifact(artifact, stream) - } else { - if (currentBatchSize + size > CHUNK_SIZE) { + case entry => + if (currentBatchSize + entry.size > CHUNK_SIZE) { writeBatch() } - addToBatch(artifact, size) - } + addToBatch(entry) } if (currentBatch.nonEmpty) { writeBatch() @@ -376,34 +418,47 @@ class ArtifactManager( * single [[proto.AddArtifactsRequest]]. */ private def addBatchedArtifacts( - artifacts: Seq[Artifact], - stream: StreamObserver[proto.AddArtifactsRequest]): Unit = { + entries: Seq[ArtifactEntry], + stream: StreamObserver[proto.AddArtifactsRequest], + useOrderedEntries: Boolean): Unit = { val builder = proto.AddArtifactsRequest .newBuilder() .setUserContext(clientConfig.userContext) .setClientType(clientConfig.userAgent) .setSessionId(sessionId) - artifacts.foreach { artifact => - val in = new CheckedInputStream(artifact.storage.stream, new CRC32) - try { - val data = proto.AddArtifactsRequest.ArtifactChunk - .newBuilder() - .setData(ByteString.readFrom(in)) - .setCrc(in.getChecksum.getValue) - + entries.foreach { + case UploadedArtifact(artifact) => + val in = new CheckedInputStream(artifact.storage.stream, new CRC32) + try { + val data = proto.AddArtifactsRequest.ArtifactChunk + .newBuilder() + .setData(ByteString.readFrom(in)) + .setCrc(in.getChecksum.getValue) + val uploaded = proto.AddArtifactsRequest.SingleChunkArtifact + .newBuilder() + .setName(artifact.path.toString) + .setData(data) + if (useOrderedEntries) { + builder.getBatchBuilder.addEntriesBuilder().setArtifact(uploaded) + } else { + builder.getBatchBuilder.addArtifacts(uploaded) + } + () + } catch { + case NonFatal(e) => + stream.onError(e) + throw e + } finally { + in.close() + } + case MavenDependency(uri) => + require(useOrderedEntries) builder.getBatchBuilder - .addArtifactsBuilder() - .setName(artifact.path.toString) - .setData(data) - .build() - } catch { - case NonFatal(e) => - stream.onError(e) - throw e - } finally { - in.close() + .addEntriesBuilder() + .getMavenDependencyBuilder + .setUri(uri.toString) + () } - } stream.onNext(builder.build()) } diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala index 5a3bd0b892c59..003bc33357992 100644 --- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala +++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/SparkConnectClient.scala @@ -43,7 +43,7 @@ import org.apache.spark.sql.connect.RuntimeConfig import org.apache.spark.sql.connect.common.ProtoUtils import org.apache.spark.sql.connect.common.config.ConnectCommon import org.apache.spark.sql.util.CloseableIterator -import org.apache.spark.util.SparkSystemUtils +import org.apache.spark.util.{SparkFileUtils, SparkSystemUtils} /** * Conceptually the remote spark session that communicates with the server. @@ -85,6 +85,25 @@ private[sql] class SparkConnectClient( private val conf: RuntimeConfig = new RuntimeConfig(this) + private lazy val serverCapabilities: Set[String] = { + val builder = proto.AnalyzePlanRequest + .newBuilder() + .setSparkVersion(proto.AnalyzePlanRequest.SparkVersion.newBuilder().build()) + .setUserContext(userContext) + .setSessionId(sessionId) + .setClientType(userAgent) + serverSideSessionId.foreach(builder.setClientObservedServerSideSessionId) + bstub + .analyzePlan(builder.build()) + .getSparkVersion + .getCapabilitiesList + .asScala + .toSet + } + + private def supportsServerSideMavenArtifacts: Boolean = + serverCapabilities.contains(SparkConnectClient.SERVER_SIDE_MAVEN_ARTIFACTS_CAPABILITY) + // Cached plan compression options. private var _planCompressionOptions: Option[Option[PlanCompressionOptions]] = None @@ -585,16 +604,19 @@ private[sql] class SparkConnectClient( /** * Add a single artifact to the client session. * - * Currently only local files with extensions .jar and .class are supported. + * Supports local .jar and .class files and Apache Ivy URIs. */ - def addArtifact(path: String): Unit = artifactManager.addArtifact(path) + def addArtifact(path: String): Unit = addArtifact(SparkFileUtils.resolveURI(path)) /** * Add a single artifact to the client session. * - * Currently only local files with extensions .jar and .class are supported. + * Supports local .jar and .class files and Apache Ivy URIs. */ - def addArtifact(uri: URI): Unit = artifactManager.addArtifact(uri) + def addArtifact(uri: URI): Unit = { + val serverSide = uri.getScheme == "ivy" && supportsServerSideMavenArtifacts + artifactManager.addArtifact(uri, serverSide) + } /** * Add a single in-memory artifact to the session while preserving the directory structure @@ -636,9 +658,12 @@ private[sql] class SparkConnectClient( /** * Add multiple artifacts to the session. * - * Currently only local files with extensions .jar and .class are supported. + * Supports local .jar and .class files and Apache Ivy URIs. */ - def addArtifacts(uri: Seq[URI]): Unit = artifactManager.addArtifacts(uri) + def addArtifacts(uri: Seq[URI]): Unit = { + val serverSide = uri.exists(_.getScheme == "ivy") && supportsServerSideMavenArtifacts + artifactManager.addArtifacts(uri, serverSide) + } /** * Register a [[ClassFinder]] for dynamically generated classes. @@ -710,6 +735,8 @@ private final class SparkConnectOperationIdException(val operationId: String) extends RuntimeException(s"Spark Connect operation ID: $operationId", null, false, false) object SparkConnectClient { + private[client] val SERVER_SIDE_MAVEN_ARTIFACTS_CAPABILITY = + "serverSideMavenArtifacts.v1" private[connect] val OPERATION_ID_HEADER = "spark-connect-operation-id" diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAddArtifactsHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAddArtifactsHandler.scala index 78970b7479567..843fe74e27a27 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAddArtifactsHandler.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAddArtifactsHandler.scala @@ -17,22 +17,30 @@ package org.apache.spark.sql.connect.service import java.io.File +import java.net.{URI, URISyntaxException} import java.nio.file.{Files, Path, Paths} +import java.security.MessageDigest +import java.util.Arrays +import java.util.concurrent.{Callable, CancellationException, ExecutionException, RejectedExecutionException, TimeoutException, TimeUnit} import java.util.zip.{CheckedOutputStream, CRC32} import scala.collection.mutable import scala.util.control.NonFatal +import com.google.common.util.concurrent.MoreExecutors +import io.grpc.Context import io.grpc.stub.StreamObserver -import org.apache.spark.SparkRuntimeException +import org.apache.spark.{SparkException, SparkRuntimeException} import org.apache.spark.connect.proto import org.apache.spark.connect.proto.{AddArtifactsRequest, AddArtifactsResponse} import org.apache.spark.connect.proto.AddArtifactsResponse.ArtifactSummary +import org.apache.spark.sql.Artifact import org.apache.spark.sql.artifact.ArtifactManager import org.apache.spark.sql.connect.utils.ErrorUtils import org.apache.spark.sql.util.ArtifactUtils import org.apache.spark.util.Utils +import org.apache.spark.util.RuntimeDependencyResolver.RejectRequestedRepositories /** * Handles [[AddArtifactsRequest]]s for the [[SparkConnectService]]. @@ -46,6 +54,13 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr protected val stagingDir: Path = Utils.createTempDir().toPath protected val stagedArtifacts: mutable.Buffer[StagedArtifact] = mutable.Buffer.empty[StagedArtifact] + private sealed trait PendingArtifact + private case class PendingStagedArtifact(artifact: StagedArtifact) extends PendingArtifact + private case class PendingMavenDependency(uri: URI) extends PendingArtifact + private val pendingArtifacts = mutable.Buffer.empty[PendingArtifact] + private val grpcContext = Context.current() + private var mavenDependencyCount = 0 + private var hasOrderedEntries = false // If not null, indicates the currently active chunked artifact that is being rebuilt from // several [[AddArtifactsRequest]]s. private var chunkedArtifact: StagedChunkedArtifact = _ @@ -78,8 +93,25 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr chunkedArtifact = null } } else if (req.hasBatch) { - // Each artifact in the batch is single-chunked. - req.getBatch.getArtifactsList.forEach(artifact => writeArtifactToFile(artifact).close()) + val batch = req.getBatch + if (batch.getArtifactsCount > 0 && batch.getEntriesCount > 0) { + throw SparkException.internalError( + "AddArtifacts batch cannot contain both legacy artifacts and ordered entries") + } + batch.getArtifactsList.forEach(artifact => writeArtifactToFile(artifact).close()) + if (batch.getEntriesCount > 0) { + hasOrderedEntries = true + } + batch.getEntriesList.forEach { entry => + entry.getValueCase match { + case proto.AddArtifactsRequest.ArtifactEntry.ValueCase.ARTIFACT => + writeArtifactToFile(entry.getArtifact).close() + case proto.AddArtifactsRequest.ArtifactEntry.ValueCase.MAVEN_DEPENDENCY => + addMavenDependency(entry.getMavenDependency) + case _ => + throw SparkException.internalError("AddArtifacts ordered entry has no value") + } + } } else { throw new UnsupportedOperationException(s"Unsupported data transfer request: $req") } @@ -101,11 +133,178 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr responseObserver.onError(throwable) } + private def addMavenDependency( + dependency: proto.AddArtifactsRequest.MavenDependency): Unit = { + val value = dependency.getUri + if (value.length > SparkConnectAddArtifactsHandler.MAX_MAVEN_URI_LENGTH) { + throw SparkException.internalError( + s"Maven dependency URI exceeds the ${ + SparkConnectAddArtifactsHandler.MAX_MAVEN_URI_LENGTH}-character limit") + } + val uri = try new URI(value) catch { + case _: URISyntaxException => null + } + if (uri == null || uri.getScheme != "ivy") { + throw SparkException.internalError(s"Maven dependency must be an ivy URI: $value") + } + mavenDependencyCount += 1 + if (mavenDependencyCount > SparkConnectAddArtifactsHandler.MAX_MAVEN_DEPENDENCIES) { + throw SparkException.internalError( + s"AddArtifacts accepts at most ${ + SparkConnectAddArtifactsHandler.MAX_MAVEN_DEPENDENCIES} Maven dependencies") + } + pendingArtifacts += PendingMavenDependency(uri) + } + protected def addStagedArtifactToArtifactManager(artifact: StagedArtifact): Unit = { require(holder != null) holder.addArtifact(artifact.path, artifact.stagedPath, artifact.fragment) } + protected def resolveMavenDependency( + uri: URI, + connectTimeoutMs: Int, + readTimeoutMs: Int, + isCancelled: () => Boolean): Seq[Artifact] = { + holder.artifactManager.resolveArtifacts( + uri, + RejectRequestedRepositories, + connectTimeoutMs, + readTimeoutMs, + isCancelled) + } + + private def resolveMavenDependencies( + dependencies: Seq[URI]): Map[URI, Seq[Artifact]] = { + if (dependencies.isEmpty) { + return Map.empty + } + + val timeoutMs = Option(grpcContext.getDeadline) + .map(_.timeRemaining(TimeUnit.MILLISECONDS)) + .map(math.min(_, SparkConnectAddArtifactsHandler.MAX_RESOLUTION_TIME_MS)) + .getOrElse(SparkConnectAddArtifactsHandler.MAX_RESOLUTION_TIME_MS) + if (timeoutMs <= 0) { + throw SparkException.internalError("AddArtifacts deadline expired before Maven resolution") + } + val timeoutAsInt = math.min(timeoutMs, Int.MaxValue.toLong).toInt + val connectTimeoutMs = math.min( + timeoutAsInt, + holder.artifactManager.ivyConnectTimeoutMs) + val readTimeoutMs = math.min(timeoutAsInt, holder.artifactManager.ivyReadTimeoutMs) + val task = try { + SparkConnectAddArtifactsHandler.mavenResolutionExecutor.submit( + new Callable[Map[URI, Seq[Artifact]]] { + override def call(): Map[URI, Seq[Artifact]] = dependencies.map { uri => + uri -> resolveMavenDependency( + uri, + connectTimeoutMs, + readTimeoutMs, + () => grpcContext.isCancelled || Thread.currentThread().isInterrupted) + }.toMap + }) + } catch { + case e: RejectedExecutionException => + throw SparkException.internalError( + "The server-side Maven resolution queue is full; retry the request", e) + } + + val listener = new Context.CancellationListener { + override def cancelled(context: Context): Unit = task.cancel(true) + } + grpcContext.addListener(listener, MoreExecutors.directExecutor()) + try { + task.get(timeoutMs, TimeUnit.MILLISECONDS) + } catch { + case e: CancellationException => + throw SparkException.internalError("Server-side Maven resolution was cancelled", e) + case e: TimeoutException => + task.cancel(true) + throw SparkException.internalError( + s"Server-side Maven resolution exceeded the ${timeoutMs}ms deadline", e) + case e: InterruptedException => + task.cancel(true) + Thread.currentThread().interrupt() + throw SparkException.internalError("Server-side Maven resolution was interrupted", e) + case e: ExecutionException => + e.getCause match { + case runtime: RuntimeException => throw runtime + case error: Error => throw error + case cause => + throw SparkException.internalError("Server-side Maven resolution failed", cause) + } + } finally { + grpcContext.removeListener(listener) + } + } + + private def stageResolvedArtifact(artifact: Artifact): StagedArtifact = { + val localFile = artifact.storage match { + case file: Artifact.LocalFile => file.path + case other => + throw SparkException.internalError( + s"Resolved Maven artifact has unexpected storage: ${other.getClass.getName}") + } + val physicalPath = Files.createTempFile(stagingDir, "resolved-maven-", ".jar") + val staged = new StagedArtifact(artifact.path.toString, Some(physicalPath)) + staged.writeFrom(localFile) + staged.close() + staged + } + + private def prepareArtifacts(): Seq[StagedArtifact] = { + if (!hasOrderedEntries) { + return stagedArtifacts.toSeq + } + + val dependencies = pendingArtifacts.collect { + case PendingMavenDependency(uri) => uri + }.distinct.toSeq + val resolved = resolveMavenDependencies(dependencies) + val resolvedArtifacts = pendingArtifacts.iterator.collect { + case PendingMavenDependency(uri) => resolved(uri) + }.flatten.toSeq + if (resolvedArtifacts.size > SparkConnectAddArtifactsHandler.MAX_RESOLVED_ARTIFACTS) { + throw SparkException.internalError( + s"Maven dependencies resolved to more than ${ + SparkConnectAddArtifactsHandler.MAX_RESOLVED_ARTIFACTS} artifacts") + } + val resolvedBytes = resolvedArtifacts.iterator.map(artifact => BigInt(artifact.size)).sum + if (resolvedBytes > BigInt(SparkConnectAddArtifactsHandler.MAX_RESOLVED_BYTES)) { + throw SparkException.internalError( + s"Maven dependencies resolved to more than ${ + SparkConnectAddArtifactsHandler.MAX_RESOLVED_BYTES} bytes") + } + + val uniqueByPath = mutable.LinkedHashMap.empty[Path, StagedArtifact] + val result = mutable.Buffer.empty[StagedArtifact] + def addChecked(artifact: StagedArtifact): Unit = { + uniqueByPath.get(artifact.path) match { + case None => + uniqueByPath.put(artifact.path, artifact) + result += artifact + case Some(existing) => + val sameContent = Arrays.equals(existing.getDigest, artifact.getDigest) && + existing.getSizeBytes == artifact.getSizeBytes && + Utils.contentEquals(existing.stagedPath.toFile, artifact.stagedPath.toFile) + if (!sameContent) { + throw new SparkRuntimeException( + "ARTIFACT_ALREADY_EXISTS", + Map("normalizedRemoteRelativePath" -> artifact.path.toString)) + } + result += artifact + } + } + + pendingArtifacts.foreach { + case PendingStagedArtifact(artifact) => addChecked(artifact) + case PendingMavenDependency(uri) => resolved(uri).foreach { artifact => + addChecked(stageResolvedArtifact(artifact)) + } + } + result.toSeq + } + /** * Process all the staged artifacts built in this stream. * @@ -114,8 +313,8 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr protected def flushStagedArtifacts(): Seq[ArtifactSummary] = { val failedArtifactExceptions = mutable.ListBuffer[SparkRuntimeException]() - // Non-lazy transformation when using Buffer. - val summaries = stagedArtifacts.map { artifact => + // Resolve and validate the complete ordered batch before mutating session state. + val summaries = prepareArtifacts().map { artifact => try { // We do not store artifacts that fail the CRC. The failure is reported in the artifact // summary and it is up to the client to decide whether to retry sending the artifact. @@ -177,6 +376,7 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr artifact: proto.AddArtifactsRequest.SingleChunkArtifact): StagedArtifact = { val stagedDep = new StagedArtifact(artifact.getName) stagedArtifacts += stagedDep + pendingArtifacts += PendingStagedArtifact(stagedDep) stagedDep.write(artifact.getData) stagedDep } @@ -190,6 +390,7 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr val stagedChunkedArtifact = new StagedChunkedArtifact(artifact.getName, artifact.getNumChunks, artifact.getTotalBytes) stagedArtifacts += stagedChunkedArtifact + pendingArtifacts += PendingStagedArtifact(stagedChunkedArtifact) stagedChunkedArtifact.write(artifact.getInitialChunk) stagedChunkedArtifact } @@ -197,7 +398,7 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr /** * Handles rebuilding an artifact from bytes sent over the wire. */ - class StagedArtifact(val name: String) { + class StagedArtifact(val name: String, physicalPath: Option[Path] = None) { // Workaround to keep the fragment. val (canonicalFileName: String, fragment: Option[String]) = if (name.startsWith(s"archives${File.separator}")) { @@ -213,8 +414,13 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr } val path: Path = Paths.get(canonicalFileName) - val stagedPath: Path = - try { + if (path.isAbsolute) { + throw new SparkRuntimeException( + errorClass = "INVALID_ARTIFACT_PATH", + messageParameters = Map("name" -> name)) + } + val stagedPath: Path = physicalPath.getOrElse { + val requestedPath = try { ArtifactUtils.concatenatePaths(stagingDir, path) } catch { case _: IllegalArgumentException => @@ -223,12 +429,20 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr messageParameters = Map("name" -> name)) case NonFatal(e) => throw e } + if (Files.exists(requestedPath)) { + Files.createTempFile(stagingDir, "duplicate-artifact-", ".tmp") + } else { + requestedPath + } + } Files.createDirectories(stagedPath.getParent) private val fileOut = Files.newOutputStream(stagedPath) private val checksumOut = new CheckedOutputStream(fileOut, new CRC32) private val overallChecksum = new CRC32() + private val digest = MessageDigest.getInstance("SHA-256") + private var contentDigest: Array[Byte] = _ private val builder = ArtifactSummary.newBuilder().setName(name) private var artifactSummary: ArtifactSummary = _ @@ -242,6 +456,13 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr def getCrc: Long = overallChecksum.getValue + def getDigest: Array[Byte] = { + require(contentDigest != null) + contentDigest.clone() + } + + def getSizeBytes: Long = Files.size(stagedPath) + def write(dataChunk: proto.AddArtifactsRequest.ArtifactChunk): Unit = { try dataChunk.getData.writeTo(checksumOut) catch { @@ -250,14 +471,34 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr throw e } - overallChecksum.update(dataChunk.getData.toByteArray) + val bytes = dataChunk.getData.toByteArray + overallChecksum.update(bytes) + digest.update(bytes) updateCrc(checksumOut.getChecksum.getValue == dataChunk.getCrc) checksumOut.getChecksum.reset() } + def writeFrom(file: Path): Unit = { + val in = Files.newInputStream(file) + try { + val buffer = new Array[Byte](64 * 1024) + var read = in.read(buffer) + while (read != -1) { + checksumOut.write(buffer, 0, read) + overallChecksum.update(buffer, 0, read) + digest.update(buffer, 0, read) + read = in.read(buffer) + } + updateCrc(isSuccess = true) + } finally { + in.close() + } + } + def close(): Unit = { if (artifactSummary == null) { checksumOut.close() + contentDigest = digest.digest() artifactSummary = builder .setName(name) .setIsCrcSuccessful(getCrcStatus.getOrElse(false)) @@ -316,3 +557,20 @@ class SparkConnectAddArtifactsHandler(val responseObserver: StreamObserver[AddAr } } } + +private[service] object SparkConnectAddArtifactsHandler { + val MAX_MAVEN_DEPENDENCIES: Int = 32 + val MAX_MAVEN_URI_LENGTH: Int = 8 * 1024 + val MAX_RESOLVED_ARTIFACTS: Int = 256 + val MAX_RESOLVED_BYTES: Long = 1024L * 1024L * 1024L + val MAX_RESOLUTION_TIME_MS: Long = 10L * 60L * 1000L + + private[service] val mavenResolutionExecutor = new java.util.concurrent.ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new java.util.concurrent.ArrayBlockingQueue[Runnable](8), + org.apache.spark.util.ThreadUtils.namedThreadFactory("spark-connect-maven-resolution"), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()) +} diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala index 49450f9387198..d66f38c775f0a 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala @@ -169,6 +169,7 @@ private[connect] class SparkConnectAnalyzeHandler( proto.AnalyzePlanResponse.SparkVersion .newBuilder() .setVersion(session.version) + .addCapabilities("serverSideMavenArtifacts.v1") .build()) case proto.AnalyzePlanRequest.AnalyzeCase.DDL_PARSE => diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala index 8094a084d3b65..ee69e24724f2b 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala @@ -17,7 +17,8 @@ package org.apache.spark.sql.connect.service import java.io.{FileOutputStream, InputStream} -import java.nio.file.{Files, Path} +import java.net.URI +import java.nio.file.{Files, Path, Paths} import java.util.UUID import java.util.jar.{JarEntry, JarOutputStream} import java.util.zip.CRC32 @@ -37,6 +38,7 @@ import io.grpc.stub.StreamObserver import org.apache.spark.SparkRuntimeException import org.apache.spark.connect.proto import org.apache.spark.connect.proto.{AddArtifactsRequest, AddArtifactsResponse} +import org.apache.spark.sql.Artifact import org.apache.spark.sql.connect.ResourceHelper import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.{ThreadUtils, Utils} @@ -63,7 +65,8 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper { class TestAddArtifactsHandler( responseObserver: StreamObserver[AddArtifactsResponse], - throwIfArtifactExists: Boolean = false) + throwIfArtifactExists: Boolean = false, + resolvedMavenArtifacts: Map[URI, Seq[Artifact]] = Map.empty) extends SparkConnectAddArtifactsHandler(responseObserver) { // Stop the staged artifacts from being automatically deleted @@ -88,6 +91,16 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper { artifactChecksums += (artifact.name -> artifact.getCrc) } + override protected def resolveMavenDependency( + uri: URI, + connectTimeoutMs: Int, + readTimeoutMs: Int, + isCancelled: () => Boolean): Seq[Artifact] = { + resolvedMavenArtifacts.getOrElse( + uri, + super.resolveMavenDependency(uri, connectTimeoutMs, readTimeoutMs, isCancelled)) + } + def getFinalArtifacts: Seq[String] = finalArtifacts.toSeq def stagingDirectory: Path = this.stagingDir def forceCleanUp(): Unit = super.cleanUpStagedArtifacts() @@ -315,6 +328,69 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper { } } + test("ordered entries resolve Maven dependencies before registration") { + val promise = Promise[AddArtifactsResponse]() + val ivyUri = URI.create("ivy://my.connect.lib:mylib:0.1") + val resolved = Artifact.newJarArtifact( + Paths.get("resolved.jar"), + new Artifact.LocalFile(inputFilePath.resolve("smallJar.jar"))) + val handler = new TestAddArtifactsHandler( + new DummyStreamObserver(promise), + resolvedMavenArtifacts = Map(ivyUri -> Seq(resolved))) + try { + def uploaded(name: String, path: Path): proto.AddArtifactsRequest.ArtifactEntry = { + val bytes = ByteString.copyFrom(Files.readAllBytes(path)) + val crc = new CRC32() + crc.update(bytes.toByteArray) + val artifact = proto.AddArtifactsRequest.SingleChunkArtifact + .newBuilder() + .setName(name) + .setData( + proto.AddArtifactsRequest.ArtifactChunk + .newBuilder() + .setData(bytes) + .setCrc(crc.getValue)) + proto.AddArtifactsRequest.ArtifactEntry.newBuilder().setArtifact(artifact).build() + } + + val request = AddArtifactsRequest + .newBuilder() + .setSessionId(sessionId) + .setUserContext(proto.UserContext.newBuilder().setUserId("c1")) + .setBatch( + proto.AddArtifactsRequest.Batch + .newBuilder() + .addEntries( + uploaded( + "classes/smallClassFile.class", + inputFilePath.resolve("smallClassFile.class"))) + .addEntries( + proto.AddArtifactsRequest.ArtifactEntry + .newBuilder() + .setMavenDependency( + proto.AddArtifactsRequest.MavenDependency + .newBuilder() + .setUri(ivyUri.toString))) + .addEntries(uploaded("jars/smallJar.jar", inputFilePath.resolve("smallJar.jar"))) + .build()) + .build() + + handler.onNext(request) + assert(handler.getFinalArtifacts.isEmpty) + handler.onCompleted() + + val response = ThreadUtils.awaitResult(promise.future, 5.seconds) + assert( + handler.getFinalArtifacts == Seq( + "classes/smallClassFile.class", + "jars/resolved.jar", + "jars/smallJar.jar")) + assert(response.getArtifactsList.asScala.map(_.getName) == handler.getFinalArtifacts) + } finally { + handler.forceCleanUp() + } + } + test("Multi chunk artifact") { val promise = Promise[AddArtifactsResponse]() val handler = new TestAddArtifactsHandler(new DummyStreamObserver(promise)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala index 9b22cf4d9dd6a..33570bfd7d2e5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala @@ -33,13 +33,14 @@ import org.apache.hadoop.fs.{LocalFileSystem, Path => FSPath} import org.apache.spark.{JobArtifactSet, JobArtifactState, SparkContext, SparkEnv, SparkException, SparkRuntimeException, SparkUnsupportedOperationException} import org.apache.spark.internal.{Logging, LogKeys} -import org.apache.spark.internal.config.{CONNECT_SCALA_UDF_STUB_PREFIXES, EXECUTOR_USER_CLASS_PATH_FIRST} +import org.apache.spark.internal.config.{CONNECT_SCALA_UDF_STUB_PREFIXES, EXECUTOR_USER_CLASS_PATH_FIRST, JAR_IVY_CONNECT_TIMEOUT, JAR_IVY_READ_TIMEOUT, JAR_IVY_SETTING_PATH, JAR_REPOSITORIES} import org.apache.spark.sql.Artifact import org.apache.spark.sql.classic.SparkSession import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.util.ArtifactUtils import org.apache.spark.storage.{BlockManager, CacheId, StorageLevel} -import org.apache.spark.util.{ChildFirstURLClassLoader, StubClassLoader, Utils} +import org.apache.spark.util.{ChildFirstURLClassLoader, RuntimeDependencyResolver, StubClassLoader, Utils} +import org.apache.spark.util.RuntimeDependencyResolver.{AllowRequestedRepositories, RepositoryPolicy} /** * This class handles the storage of artifacts as well as preparing the artifacts for use. @@ -70,6 +71,21 @@ class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging (ArtifactUtils.concatenatePaths(artifactRootPath, session.sessionUUID), s"$artifactRootURI/${session.sessionUUID}") + private lazy val runtimeDependencyResolver = { + val sparkConf = session.sparkContext.getConf + val ivyPath = Files.createDirectories(artifactPath.resolve(".ivy-cache")).toString + new RuntimeDependencyResolver( + ivySettingsPath = sparkConf.get(JAR_IVY_SETTING_PATH), + configuredRepositories = sparkConf.get(JAR_REPOSITORIES), + ivyPath = Some(ivyPath)) + } + + private[sql] lazy val ivyConnectTimeoutMs: Int = + session.sparkContext.getConf.get(JAR_IVY_CONNECT_TIMEOUT).toInt + + private[sql] lazy val ivyReadTimeoutMs: Int = + session.sparkContext.getConf.get(JAR_IVY_READ_TIMEOUT).toInt + // The base directory/URI where all class file artifacts are stored for this `sessionUUID`. protected[artifact] val (classDir, replClassURI): (Path, String) = (ArtifactUtils.concatenatePaths(artifactPath, "classes"), s"$artifactURI/classes/") @@ -323,6 +339,21 @@ class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging } } + private[sql] def resolveArtifacts( + uri: URI, + repositoryPolicy: RepositoryPolicy = AllowRequestedRepositories, + connectTimeoutMs: Int = ivyConnectTimeoutMs, + readTimeoutMs: Int = ivyReadTimeoutMs, + isCancelled: () => Boolean = () => false): Seq[Artifact] = { + uri.getScheme match { + case "ivy" => + runtimeDependencyResolver + .resolve(uri, repositoryPolicy, connectTimeoutMs, readTimeoutMs, isCancelled) + .map(path => Artifact.newJarArtifact(path.getFileName, new Artifact.LocalFile(path))) + case _ => Artifact.parseArtifacts(uri) + } + } + def classloader: ClassLoader = synchronized { cachedClassLoader.getOrElse { val loader = buildClassLoader diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala index ffbfed21bb29d..b42aa40294f25 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala @@ -724,7 +724,7 @@ class SparkSession private( /** @inheritdoc */ @Experimental override def addArtifact(uri: URI): Unit = { - artifactManager.addLocalArtifacts(Artifact.parseArtifacts(uri)) + artifactManager.addLocalArtifacts(artifactManager.resolveArtifacts(uri)) } /** @inheritdoc */ @@ -753,7 +753,7 @@ class SparkSession private( @Experimental @scala.annotation.varargs override def addArtifacts(uri: URI*): Unit = { - artifactManager.addLocalArtifacts(uri.flatMap(Artifact.parseArtifacts)) + artifactManager.addLocalArtifacts(uri.flatMap(artifactManager.resolveArtifacts(_))) } /** @inheritdoc */ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala new file mode 100644 index 0000000000000..889e37c6afc87 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.artifact + +import java.net.URI +import java.nio.file.Files + +import org.apache.spark.SparkConf +import org.apache.spark.internal.config.{JAR_IVY_CONNECT_TIMEOUT, JAR_IVY_READ_TIMEOUT} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.util.{IvyTestUtils, MavenUtils, Utils} +import org.apache.spark.util.MavenUtils.MavenCoordinate + +class ArtifactManagerIvySettingsSuite extends SharedSparkSession { + + private val ivySettingsDir = Utils.createTempDir(namePrefix = "artifact-ivy-settings") + private val coordinate = MavenCoordinate("my.runtime.lib", "mylib", "0.1") + private val repository = IvyTestUtils.createLocalRepositoryForTests( + coordinate, + dependencies = None, + rootDir = Some(ivySettingsDir.toPath.resolve("repository").toFile)) + private val ivySettings = ivySettingsDir.toPath.resolve("ivysettings.xml") + Files.writeString( + ivySettings, + s""" + | + | + | + | + |""".stripMargin) + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set(MavenUtils.JAR_IVY_SETTING_PATH_KEY, ivySettings.toString) + .set(JAR_IVY_CONNECT_TIMEOUT, 7000L) + .set(JAR_IVY_READ_TIMEOUT, 11000L) + } + + override def afterAll(): Unit = { + try { + super.afterAll() + } finally { + Utils.deleteRecursively(ivySettingsDir) + } + } + + test("Ivy timeout configurations have defaults") { + val conf = new SparkConf(loadDefaults = false) + assert(conf.get(JAR_IVY_CONNECT_TIMEOUT) == 30000L) + assert(conf.get(JAR_IVY_READ_TIMEOUT) == 300000L) + } + + test("SparkSession artifact APIs use spark.jars.ivySettings") { + val ivyUri = URI.create(s"ivy://${coordinate.toString}") + val otherSession = spark.newSession() + + spark.addArtifact(ivyUri) + val jarName = s"${coordinate.groupId}_${coordinate.artifactId}-${coordinate.version}.jar" + assert(Files.exists(spark.artifactManager.artifactPath.resolve("jars").resolve(jarName))) + assert( + !Files.exists(otherSession.artifactManager.artifactPath.resolve("jars").resolve(jarName))) + + otherSession.addArtifacts(ivyUri) + + assert(Files.exists( + otherSession.artifactManager.artifactPath.resolve("jars").resolve(jarName))) + assert( + spark.artifactManager.artifactPath.resolve(".ivy-cache") != + otherSession.artifactManager.artifactPath.resolve(".ivy-cache")) + assert(spark.artifactManager.ivyConnectTimeoutMs == 7000) + assert(spark.artifactManager.ivyReadTimeoutMs == 11000) + } +}