Add SPARQL proto text format, reduce dependencies

README.md

@@ -120,8 +120,6 @@ jelly-cli sparql from-jelly results.jellys --out-format=csv > results.csv
 
 Both commands handle SELECT results (bindings) and ASK results (a boolean). All standard result formats (JSON, XML, CSV and TSV) are supported, plus a plain text table (`text`) for output only.
 
-Jelly-SPARQL is an experimental draft and the format may still change.
-
 ### General tips
 
 Use the `--help` option to learn more about all the available settings:

build.sbt

@@ -11,7 +11,7 @@ resolvers +=
   "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots"
 
 lazy val jenaV = "6.2.0"
-lazy val jellyV = "3.7.3+63-e0758361-SNAPSHOT"
+lazy val jellyV = "3.7.3+64-36186011-SNAPSHOT"
 lazy val graalvmV = "25.4.4.1.1"
 
 addCommandAlias("fixAll", "scalafixAll; scalafmtAll")
@@ -34,8 +34,7 @@ lazy val graalOptions = Seq(
   // https://github.com/scala/scala3/pull/24109#issuecomment-3786629196
   "-J--sun-misc-unsafe-memory-access=allow",
   // Custom Graal features
-  "--features=eu.neverblink.jelly.cli.graal.ProtobufFeature," +
-    "eu.neverblink.jelly.cli.graal.JenaInternalsFeature," +
+  "--features=eu.neverblink.jelly.cli.graal.JenaInternalsFeature," +
     "eu.neverblink.jelly.cli.graal.LargeXmlFeature",
   "-H:ReflectionConfigurationFiles=" + file("graal.json").getAbsolutePath,
   // Needed to skip initializing all charsets.
@@ -69,13 +68,11 @@ lazy val root = (project in file("."))
       ("eu.neverblink.jelly" % "jelly-jena" % jellyV).excludeAll(ExclusionRule("org.apache.jena")),
       ("eu.neverblink.jelly" % "jelly-jena-sparql" % jellyV)
         .excludeAll(ExclusionRule("org.apache.jena")),
-      "eu.neverblink.jelly" % "jelly-core-protos-google" % jellyV,
       "com.github.alexarchambault" %% "case-app" % "2.1.0",
       "org.scalatest" %% "scalatest" % "3.2.20" % "test,test-serial",
       "org.yaml" % "snakeyaml" % "2.7" % Test,
       // For native-image reflection compatibility
       "org.graalvm.sdk" % "graal-sdk" % graalvmV % "provided",
-      "org.reflections" % "reflections" % "0.10.2",
     ),
     scalacOptions ++= Seq(
       "-Wunused:imports",

src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala

@@ -5,6 +5,7 @@ import eu.neverblink.jelly.cli.*
 import eu.neverblink.jelly.cli.command.rdf.util.*
 import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.*
 import eu.neverblink.jelly.cli.util.args.IndexRange
+import eu.neverblink.jelly.cli.util.io.ProtoText
 import eu.neverblink.jelly.cli.util.jena.{
   JenaSystemOptions,
   StreamRdfBatchWriter,
@@ -14,13 +15,13 @@ import eu.neverblink.jelly.convert.jena.JenaConverterFactory
 import eu.neverblink.jelly.core.JellyOptions
 import eu.neverblink.jelly.core.RdfHandler.AnyStatementHandler
 import eu.neverblink.jelly.core.proto.v1.RdfStreamFrame
-import eu.neverblink.jelly.core.proto.google.v1 as google
 import org.apache.jena.graph.{Node, Triple}
 import org.apache.jena.riot.system.StreamRDF
 import org.apache.jena.riot.system.StreamRDFWriter
 import org.apache.jena.sparql.core.Quad
 
 import java.io.{InputStream, OutputStream}
+import java.nio.charset.StandardCharsets.UTF_8
 import scala.jdk.CollectionConverters.*
 
 object RdfFromJellyPrint extends RdfCommandPrintUtil[RdfFormat.Writeable]:
@@ -187,10 +188,9 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ
     inline def writeFrameToOutput(f: RdfStreamFrame, frameIndex: Int): Unit =
       // we want to write a comment to the file before each frame
       val comment = f"# Frame $frameIndex\n"
-      outputStream.write(comment.getBytes)
-      val frame = google.RdfStreamFrame.parseFrom(f.toByteArray).toString
-      // the protoString is basically the jelly-txt format already
-      outputStream.write(frame.getBytes)
+      outputStream.write(comment.getBytes(UTF_8))
+      val frame = ProtoText.print(RdfStreamFrame.getDescriptor, f.toByteArray)
+      outputStream.write(frame.getBytes(UTF_8))
 
     try {
       val it = JellyUtil.iterateRdfStream(inputStream)

src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala

@@ -1,22 +1,22 @@
 package eu.neverblink.jelly.cli.command.rdf
 
 import caseapp.*
-import com.google.protobuf.TextFormat
 import eu.neverblink.jelly.cli.*
 import eu.neverblink.jelly.cli.command.rdf.util.*
 import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.*
+import eu.neverblink.jelly.cli.util.io.ProtoText
 import eu.neverblink.jelly.cli.util.jena.JenaSystemOptions
 import eu.neverblink.jelly.cli.util.jena.riot.{JellyStreamWriterGraphs, RiotParserUtil}
 import eu.neverblink.jelly.convert.jena.JenaConverterFactory
 import eu.neverblink.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage, JellyStreamWriter}
 import eu.neverblink.jelly.core.{JellyOptions, RdfProtoDeserializationError}
-import eu.neverblink.jelly.core.proto.google.v1 as google
 import eu.neverblink.jelly.core.proto.v1.*
 import eu.neverblink.jelly.core.utils.IoUtils
 import org.apache.jena.riot.system.StreamRDFWriter
 import org.apache.jena.riot.RIOT
 
 import java.io.{BufferedReader, FileInputStream, InputStream, InputStreamReader, OutputStream}
+import java.nio.charset.StandardCharsets.UTF_8
 import scala.util.Using
 
 object RdfToJellyPrint extends RdfCommandPrintUtil[RdfFormat.Readable]:
@@ -216,10 +216,10 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable
           "NEVER use it in production.\nUse --quiet to silence this warning.",
         true,
       )
-    Using.resource(InputStreamReader(inputStream)) { r1 =>
+    Using.resource(InputStreamReader(inputStream, UTF_8)) { r1 =>
       Using.resource(BufferedReader(r1)) { reader =>
         jellyTextStreamAsFrames(reader)
-          .map(txt => TextFormat.parse(txt, classOf[google.RdfStreamFrame]))
+          .map(txt => ProtoText.parse(RdfStreamFrame.getDescriptor, txt))
           .foreach(frame => {
             if getOptions.delimited then frame.writeDelimitedTo(outputStream)
             else frame.writeTo(outputStream)

src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala

@@ -1,12 +1,9 @@
 package eu.neverblink.jelly.cli.command.rdf.util
 
-import eu.neverblink.jelly.core.proto.google.v1 as google
+import eu.neverblink.jelly.cli.util.io.ProtoText
 import eu.neverblink.jelly.core.proto.v1.RdfStreamOptions
 
 object StreamOptionsUtil:
 
-  def prettyPrint(streamOptions: RdfStreamOptions): String = {
-    val streamOptionsBytes = streamOptions.toByteArray
-    val googleStreamOptions = google.RdfStreamOptions.parseFrom(streamOptionsBytes)
-    return googleStreamOptions.toString
-  }
+  def prettyPrint(streamOptions: RdfStreamOptions): String =
+    ProtoText.print(RdfStreamOptions.getDescriptor, streamOptions.toByteArray)

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala

@@ -14,6 +14,7 @@ object SparqlFromJellyPrint:
     "If no input file is specified, the input is read from stdin.\n" +
     "If no output file is specified, the output is written to stdout.\n" +
     "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" +
+    "The jelly-sparql-text format is a human-readable version of Jelly-SPARQL, for debugging.\n" +
     "If an error is detected, the program will exit with a non-zero code.\n" +
     "Otherwise, the program will exit with code 0.",
 )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala

@@ -1,14 +1,19 @@
 package eu.neverblink.jelly.cli.command.sparql
 
 import caseapp.*
-import com.google.protobuf.InvalidProtocolBufferException
+import com.google.protobuf.{InvalidProtocolBufferException, TextFormat}
 import eu.neverblink.jelly.cli.*
 import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat
+import eu.neverblink.jelly.cli.util.io.ProtoText
 import eu.neverblink.jelly.core.{RdfProtoDeserializationError, RdfProtoSerializationError}
+import eu.neverblink.jelly.core.proto.v1.sparql.SparqlResultsFrame
+import eu.neverblink.jelly.core.sparql.JellySparqlIoUtils
 import org.apache.jena.riot.{RIOT, RiotException}
 import org.apache.jena.riot.resultset.{ResultSetReaderRegistry, ResultSetWriterRegistry}
 
-import java.io.{InputStream, OutputStream}
+import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream}
+import java.nio.charset.StandardCharsets.UTF_8
+import scala.util.Using
 
 /** Common logic for the two SPARQL result set conversion commands.
   */
@@ -51,13 +56,15 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
       outputStream: OutputStream,
   ): Unit =
     try {
-      val context = RIOT.getContext.copy()
-      val reader = ResultSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang)
-      val writer = ResultSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang)
-      val result = reader.readAny(inputStream, context)
-      if result.isBoolean then
-        writer.write(outputStream, result.getBooleanResult.booleanValue, context)
-      else writer.write(outputStream, result.getResultSet, context)
+      (from, to) match
+        case (SparqlFormat.JellySparql, SparqlFormat.JellySparqlText) =>
+          jellyBinaryToText(inputStream, outputStream)
+        case (SparqlFormat.JellySparqlText, SparqlFormat.JellySparql) =>
+          jellyTextToBinary(inputStream, outputStream)
+        case (f: SparqlFormat.Jena, t: SparqlFormat.Jena) =>
+          jenaConvert(f, t, inputStream, outputStream)
+        case _ =>
+          throw CriticalException(f"Conversion from $from to $to is not supported.")
       outputStream.flush()
     } catch
       // The Jelly RowSet reader wraps I/O errors (including protobuf ones) in a RiotException,
@@ -72,3 +79,61 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}]
         throw JellyDeserializationError(e.getMessage)
       case e: RdfProtoSerializationError =>
         throw JellySerializationError(e.getMessage)
+      case e: TextFormat.ParseException =>
+        throw InvalidJellyFile(e)
+
+  private def jenaConvert(
+      from: SparqlFormat.Jena,
+      to: SparqlFormat.Jena,
+      inputStream: InputStream,
+      outputStream: OutputStream,
+  ): Unit =
+    val context = RIOT.getContext.copy()
+    val reader = ResultSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang)
+    val writer = ResultSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang)
+    val result = reader.readAny(inputStream, context)
+    if result.isBoolean then
+      writer.write(outputStream, result.getBooleanResult.booleanValue, context)
+    else writer.write(outputStream, result.getResultSet, context)
+
+  private val frameCommentPrefix = "# Frame"
+
+  private def jellyBinaryToText(inputStream: InputStream, outputStream: OutputStream): Unit =
+    val response = JellySparqlIoUtils.autodetectDelimiting(inputStream)
+    val input = response.newInput()
+    val frames =
+      if response.isDelimited then
+        Iterator.continually(SparqlResultsFrame.parseDelimitedFrom(input)).takeWhile(_ != null)
+      else Iterator(SparqlResultsFrame.parseFrom(input))
+    for (frame, i) <- frames.zipWithIndex do
+      outputStream.write(f"$frameCommentPrefix $i\n".getBytes(UTF_8))
+      val text = ProtoText.print(SparqlResultsFrame.getDescriptor, frame.toByteArray)
+      outputStream.write(text.getBytes(UTF_8))
+
+  /** Frames are split on the "# Frame" comments. Without them, the whole input is one frame. */
+  private def jellyTextToBinary(inputStream: InputStream, outputStream: OutputStream): Unit =
+    if !isQuietMode then
+      printLine(
+        "WARNING: The Jelly-SPARQL text format is not stable and may change in incompatible " +
+          "ways in the future.\nIt's only intended for testing and development.\n" +
+          "NEVER use it in production.\nUse --quiet to silence this warning.",
+        true,
+      )
+    def writeFrame(text: String): Unit =
+      ProtoText.parse(SparqlResultsFrame.getDescriptor, text).writeDelimitedTo(outputStream)
+
+    Using.resource(BufferedReader(InputStreamReader(inputStream, UTF_8))) { reader =>
+      val buffer = StringBuilder()
+      var hasContent = false
+      Iterator.continually(reader.readLine()).takeWhile(_ != null).foreach { line =>
+        if line.startsWith(frameCommentPrefix) then
+          if hasContent then writeFrame(buffer.toString)
+          buffer.clear()
+          hasContent = false
+        else
+          buffer.append(line).append('\n')
+          val trimmed = line.trim
+          if trimmed.nonEmpty && !trimmed.startsWith("#") then hasContent = true
+      }
+      if hasContent then writeFrame(buffer.toString)
+    }

src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala

@@ -14,6 +14,7 @@ object SparqlToJellyPrint:
     "If no input file is specified, the input is read from stdin.\n" +
     "If no output file is specified, the output is written to stdout.\n" +
     "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" +
+    "The input can also be in the jelly-sparql-text format, which is written by sparql from-jelly.\n" +
     "If an error is detected, the program will exit with a non-zero code.\n" +
     "Otherwise, the program will exit with code 0.",
 )

src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala

@@ -12,7 +12,6 @@ import org.apache.jena.riot.resultset.ResultSetLang
 sealed trait SparqlFormat:
   val fullName: String
   val cliOptions: List[String]
-  val jenaLang: Lang
   override final def toString: String = fullName
 
 object SparqlFormat:
@@ -23,41 +22,51 @@ object SparqlFormat:
   /** Formats we can write a result set to. */
   sealed trait Writeable extends SparqlFormat
 
-  case object Json extends SparqlFormat.Readable, SparqlFormat.Writeable:
+  /** Formats handled by Jena's result set readers and writers. */
+  sealed trait Jena extends SparqlFormat:
+    val jenaLang: Lang
+
+  case object Json extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable:
     override val fullName: String = "SPARQL results JSON"
     override val cliOptions: List[String] = List("json", "srj")
     override val jenaLang: Lang = ResultSetLang.RS_JSON
 
-  case object Xml extends SparqlFormat.Readable, SparqlFormat.Writeable:
+  case object Xml extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable:
     override val fullName: String = "SPARQL results XML"
     override val cliOptions: List[String] = List("xml", "srx")
     override val jenaLang: Lang = ResultSetLang.RS_XML
 
-  case object Csv extends SparqlFormat.Readable, SparqlFormat.Writeable:
+  case object Csv extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable:
     override val fullName: String = "CSV"
     override val cliOptions: List[String] = List("csv")
     override val jenaLang: Lang = ResultSetLang.RS_CSV
 
-  case object Tsv extends SparqlFormat.Readable, SparqlFormat.Writeable:
+  case object Tsv extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable:
     override val fullName: String = "TSV"
     override val cliOptions: List[String] = List("tsv")
     override val jenaLang: Lang = ResultSetLang.RS_TSV
 
   /** Jena's pretty-printed table. Jena registers no reader for it, so it's output-only. */
-  case object Text extends SparqlFormat.Writeable:
+  case object Text extends SparqlFormat.Jena, SparqlFormat.Writeable:
     override val fullName: String = "Text table"
     override val cliOptions: List[String] = List("text")
     override val jenaLang: Lang = ResultSetLang.RS_Text
 
   /** We never convert Jelly to Jelly, so this is neither Readable nor Writeable – it is only here
     * so that the other side of the conversion has a name.
     */
-  case object JellySparql extends SparqlFormat:
+  case object JellySparql extends SparqlFormat.Jena:
     override val fullName: String = "Jelly-SPARQL"
     override val cliOptions: List[String] = List("jelly-sparql")
     override val jenaLang: Lang = JellySparqlLanguage.JELLY_SPARQL
 
-  private val sparqlFormats: List[SparqlFormat] = List(Json, Xml, Csv, Tsv, Text, JellySparql)
+  case object JellySparqlText extends SparqlFormat.Readable, SparqlFormat.Writeable:
+    override val fullName: String = "Jelly-SPARQL text"
+    override val cliOptions: List[String] = List("jelly-sparql-text")
+    val extension = ".jellys.txt"
+
+  private val sparqlFormats: List[SparqlFormat] =
+    List(Json, Xml, Csv, Tsv, Text, JellySparql, JellySparqlText)
 
   def all: List[SparqlFormat] = sparqlFormats
 
@@ -86,5 +95,9 @@ object SparqlFormat:
   /** Infers the format based on the file name.
     */
   def inferFormat(fileName: String): Option[SparqlFormat] =
-    val guessType = RDFLanguages.guessContentType(fileName)
-    sparqlFormats.collectFirst { case f if f.jenaLang.getContentType == guessType => f }
+    if fileName.endsWith(JellySparqlText.extension) then Some(JellySparqlText)
+    else
+      val guessType = RDFLanguages.guessContentType(fileName)
+      sparqlFormats.collectFirst {
+        case f: SparqlFormat.Jena if f.jenaLang.getContentType == guessType => f
+      }

src/main/scala/eu/neverblink/jelly/cli/util/io/ProtoText.scala

@@ -0,0 +1,18 @@
+package eu.neverblink.jelly.cli.util.io
+
+import com.google.protobuf.Descriptors.Descriptor
+import com.google.protobuf.{DynamicMessage, TextFormat}
+
+/** Protobuf text format for Jelly messages, done through DynamicMessage and the message descriptor,
+  * so we don't need the Google-style proto classes.
+  */
+object ProtoText:
+  private val printer = TextFormat.printer().escapingNonAscii(false)
+
+  def print(descriptor: Descriptor, bytes: Array[Byte]): String =
+    printer.printToString(DynamicMessage.parseFrom(descriptor, bytes))
+
+  def parse(descriptor: Descriptor, text: String): DynamicMessage =
+    val builder = DynamicMessage.newBuilder(descriptor)
+    TextFormat.merge(text, builder)
+    builder.build()