diff --git a/README.md b/README.md index 89c19c2..80a4f9d 100644 --- a/README.md +++ b/README.md @@ -120,8 +120,6 @@ jelly-cli sparql from-jelly results.jellys --out-format=csv > results.csv Both commands handle SELECT results (bindings) and ASK results (a boolean). All standard result formats (JSON, XML, CSV and TSV) are supported, plus a plain text table (`text`) for output only. -Jelly-SPARQL is an experimental draft and the format may still change. - ### General tips Use the `--help` option to learn more about all the available settings: diff --git a/build.sbt b/build.sbt index 209b82c..e0f6665 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ resolvers += "Sonatype OSS Snapshots" at "https://central.sonatype.com/repository/maven-snapshots" lazy val jenaV = "6.2.0" -lazy val jellyV = "3.7.3+63-e0758361-SNAPSHOT" +lazy val jellyV = "3.7.3+64-36186011-SNAPSHOT" lazy val graalvmV = "25.4.4.1.1" addCommandAlias("fixAll", "scalafixAll; scalafmtAll") @@ -34,8 +34,7 @@ lazy val graalOptions = Seq( // https://github.com/scala/scala3/pull/24109#issuecomment-3786629196 "-J--sun-misc-unsafe-memory-access=allow", // Custom Graal features - "--features=eu.neverblink.jelly.cli.graal.ProtobufFeature," + - "eu.neverblink.jelly.cli.graal.JenaInternalsFeature," + + "--features=eu.neverblink.jelly.cli.graal.JenaInternalsFeature," + "eu.neverblink.jelly.cli.graal.LargeXmlFeature", "-H:ReflectionConfigurationFiles=" + file("graal.json").getAbsolutePath, // Needed to skip initializing all charsets. @@ -69,13 +68,11 @@ lazy val root = (project in file(".")) ("eu.neverblink.jelly" % "jelly-jena" % jellyV).excludeAll(ExclusionRule("org.apache.jena")), ("eu.neverblink.jelly" % "jelly-jena-sparql" % jellyV) .excludeAll(ExclusionRule("org.apache.jena")), - "eu.neverblink.jelly" % "jelly-core-protos-google" % jellyV, "com.github.alexarchambault" %% "case-app" % "2.1.0", "org.scalatest" %% "scalatest" % "3.2.20" % "test,test-serial", "org.yaml" % "snakeyaml" % "2.7" % Test, // For native-image reflection compatibility "org.graalvm.sdk" % "graal-sdk" % graalvmV % "provided", - "org.reflections" % "reflections" % "0.10.2", ), scalacOptions ++= Seq( "-Wunused:imports", diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala index 6487222..d6752e7 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfFromJelly.scala @@ -5,6 +5,7 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.rdf.util.* import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.* import eu.neverblink.jelly.cli.util.args.IndexRange +import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.cli.util.jena.{ JenaSystemOptions, StreamRdfBatchWriter, @@ -14,13 +15,13 @@ import eu.neverblink.jelly.convert.jena.JenaConverterFactory import eu.neverblink.jelly.core.JellyOptions import eu.neverblink.jelly.core.RdfHandler.AnyStatementHandler import eu.neverblink.jelly.core.proto.v1.RdfStreamFrame -import eu.neverblink.jelly.core.proto.google.v1 as google import org.apache.jena.graph.{Node, Triple} import org.apache.jena.riot.system.StreamRDF import org.apache.jena.riot.system.StreamRDFWriter import org.apache.jena.sparql.core.Quad import java.io.{InputStream, OutputStream} +import java.nio.charset.StandardCharsets.UTF_8 import scala.jdk.CollectionConverters.* object RdfFromJellyPrint extends RdfCommandPrintUtil[RdfFormat.Writeable]: @@ -187,10 +188,9 @@ object RdfFromJelly extends RdfSerDesCommand[RdfFromJellyOptions, RdfFormat.Writ inline def writeFrameToOutput(f: RdfStreamFrame, frameIndex: Int): Unit = // we want to write a comment to the file before each frame val comment = f"# Frame $frameIndex\n" - outputStream.write(comment.getBytes) - val frame = google.RdfStreamFrame.parseFrom(f.toByteArray).toString - // the protoString is basically the jelly-txt format already - outputStream.write(frame.getBytes) + outputStream.write(comment.getBytes(UTF_8)) + val frame = ProtoText.print(RdfStreamFrame.getDescriptor, f.toByteArray) + outputStream.write(frame.getBytes(UTF_8)) try { val it = JellyUtil.iterateRdfStream(inputStream) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala index 87a7f05..1986d13 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala @@ -1,22 +1,22 @@ package eu.neverblink.jelly.cli.command.rdf import caseapp.* -import com.google.protobuf.TextFormat import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.rdf.util.* import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.* +import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.cli.util.jena.JenaSystemOptions import eu.neverblink.jelly.cli.util.jena.riot.{JellyStreamWriterGraphs, RiotParserUtil} import eu.neverblink.jelly.convert.jena.JenaConverterFactory import eu.neverblink.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage, JellyStreamWriter} import eu.neverblink.jelly.core.{JellyOptions, RdfProtoDeserializationError} -import eu.neverblink.jelly.core.proto.google.v1 as google import eu.neverblink.jelly.core.proto.v1.* import eu.neverblink.jelly.core.utils.IoUtils import org.apache.jena.riot.system.StreamRDFWriter import org.apache.jena.riot.RIOT import java.io.{BufferedReader, FileInputStream, InputStream, InputStreamReader, OutputStream} +import java.nio.charset.StandardCharsets.UTF_8 import scala.util.Using object RdfToJellyPrint extends RdfCommandPrintUtil[RdfFormat.Readable]: @@ -216,10 +216,10 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable "NEVER use it in production.\nUse --quiet to silence this warning.", true, ) - Using.resource(InputStreamReader(inputStream)) { r1 => + Using.resource(InputStreamReader(inputStream, UTF_8)) { r1 => Using.resource(BufferedReader(r1)) { reader => jellyTextStreamAsFrames(reader) - .map(txt => TextFormat.parse(txt, classOf[google.RdfStreamFrame])) + .map(txt => ProtoText.parse(RdfStreamFrame.getDescriptor, txt)) .foreach(frame => { if getOptions.delimited then frame.writeDelimitedTo(outputStream) else frame.writeTo(outputStream) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala index aa89c8d..57471c9 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/rdf/util/StreamOptionsUtil.scala @@ -1,12 +1,9 @@ package eu.neverblink.jelly.cli.command.rdf.util -import eu.neverblink.jelly.core.proto.google.v1 as google +import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.core.proto.v1.RdfStreamOptions object StreamOptionsUtil: - def prettyPrint(streamOptions: RdfStreamOptions): String = { - val streamOptionsBytes = streamOptions.toByteArray - val googleStreamOptions = google.RdfStreamOptions.parseFrom(streamOptionsBytes) - return googleStreamOptions.toString - } + def prettyPrint(streamOptions: RdfStreamOptions): String = + ProtoText.print(RdfStreamOptions.getDescriptor, streamOptions.toByteArray) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala index a8ecdac..baf5ab8 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlFromJelly.scala @@ -14,6 +14,7 @@ object SparqlFromJellyPrint: "If no input file is specified, the input is read from stdin.\n" + "If no output file is specified, the output is written to stdout.\n" + "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" + + "The jelly-sparql-text format is a human-readable version of Jelly-SPARQL, for debugging.\n" + "If an error is detected, the program will exit with a non-zero code.\n" + "Otherwise, the program will exit with code 0.", ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala index 49f9fd5..c21f2b5 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesCommand.scala @@ -1,14 +1,19 @@ package eu.neverblink.jelly.cli.command.sparql import caseapp.* -import com.google.protobuf.InvalidProtocolBufferException +import com.google.protobuf.{InvalidProtocolBufferException, TextFormat} import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat +import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.core.{RdfProtoDeserializationError, RdfProtoSerializationError} +import eu.neverblink.jelly.core.proto.v1.sparql.SparqlResultsFrame +import eu.neverblink.jelly.core.sparql.JellySparqlIoUtils import org.apache.jena.riot.{RIOT, RiotException} import org.apache.jena.riot.resultset.{ResultSetReaderRegistry, ResultSetWriterRegistry} -import java.io.{InputStream, OutputStream} +import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream} +import java.nio.charset.StandardCharsets.UTF_8 +import scala.util.Using /** Common logic for the two SPARQL result set conversion commands. */ @@ -51,13 +56,15 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}] outputStream: OutputStream, ): Unit = try { - val context = RIOT.getContext.copy() - val reader = ResultSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang) - val writer = ResultSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang) - val result = reader.readAny(inputStream, context) - if result.isBoolean then - writer.write(outputStream, result.getBooleanResult.booleanValue, context) - else writer.write(outputStream, result.getResultSet, context) + (from, to) match + case (SparqlFormat.JellySparql, SparqlFormat.JellySparqlText) => + jellyBinaryToText(inputStream, outputStream) + case (SparqlFormat.JellySparqlText, SparqlFormat.JellySparql) => + jellyTextToBinary(inputStream, outputStream) + case (f: SparqlFormat.Jena, t: SparqlFormat.Jena) => + jenaConvert(f, t, inputStream, outputStream) + case _ => + throw CriticalException(f"Conversion from $from to $to is not supported.") outputStream.flush() } catch // The Jelly RowSet reader wraps I/O errors (including protobuf ones) in a RiotException, @@ -72,3 +79,61 @@ abstract class SparqlSerDesCommand[T <: HasJellyCommandOptions: {Parser, Help}] throw JellyDeserializationError(e.getMessage) case e: RdfProtoSerializationError => throw JellySerializationError(e.getMessage) + case e: TextFormat.ParseException => + throw InvalidJellyFile(e) + + private def jenaConvert( + from: SparqlFormat.Jena, + to: SparqlFormat.Jena, + inputStream: InputStream, + outputStream: OutputStream, + ): Unit = + val context = RIOT.getContext.copy() + val reader = ResultSetReaderRegistry.getFactory(from.jenaLang).create(from.jenaLang) + val writer = ResultSetWriterRegistry.getFactory(to.jenaLang).create(to.jenaLang) + val result = reader.readAny(inputStream, context) + if result.isBoolean then + writer.write(outputStream, result.getBooleanResult.booleanValue, context) + else writer.write(outputStream, result.getResultSet, context) + + private val frameCommentPrefix = "# Frame" + + private def jellyBinaryToText(inputStream: InputStream, outputStream: OutputStream): Unit = + val response = JellySparqlIoUtils.autodetectDelimiting(inputStream) + val input = response.newInput() + val frames = + if response.isDelimited then + Iterator.continually(SparqlResultsFrame.parseDelimitedFrom(input)).takeWhile(_ != null) + else Iterator(SparqlResultsFrame.parseFrom(input)) + for (frame, i) <- frames.zipWithIndex do + outputStream.write(f"$frameCommentPrefix $i\n".getBytes(UTF_8)) + val text = ProtoText.print(SparqlResultsFrame.getDescriptor, frame.toByteArray) + outputStream.write(text.getBytes(UTF_8)) + + /** Frames are split on the "# Frame" comments. Without them, the whole input is one frame. */ + private def jellyTextToBinary(inputStream: InputStream, outputStream: OutputStream): Unit = + if !isQuietMode then + printLine( + "WARNING: The Jelly-SPARQL text format is not stable and may change in incompatible " + + "ways in the future.\nIt's only intended for testing and development.\n" + + "NEVER use it in production.\nUse --quiet to silence this warning.", + true, + ) + def writeFrame(text: String): Unit = + ProtoText.parse(SparqlResultsFrame.getDescriptor, text).writeDelimitedTo(outputStream) + + Using.resource(BufferedReader(InputStreamReader(inputStream, UTF_8))) { reader => + val buffer = StringBuilder() + var hasContent = false + Iterator.continually(reader.readLine()).takeWhile(_ != null).foreach { line => + if line.startsWith(frameCommentPrefix) then + if hasContent then writeFrame(buffer.toString) + buffer.clear() + hasContent = false + else + buffer.append(line).append('\n') + val trimmed = line.trim + if trimmed.nonEmpty && !trimmed.startsWith("#") then hasContent = true + } + if hasContent then writeFrame(buffer.toString) + } diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala index 1cb692b..1bb02d0 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/SparqlToJelly.scala @@ -14,6 +14,7 @@ object SparqlToJellyPrint: "If no input file is specified, the input is read from stdin.\n" + "If no output file is specified, the output is written to stdout.\n" + "Both SELECT results (bindings) and ASK results (a boolean) are supported.\n" + + "The input can also be in the jelly-sparql-text format, which is written by sparql from-jelly.\n" + "If an error is detected, the program will exit with a non-zero code.\n" + "Otherwise, the program will exit with code 0.", ) diff --git a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala index b491081..e4c70af 100644 --- a/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala +++ b/src/main/scala/eu/neverblink/jelly/cli/command/sparql/util/SparqlFormat.scala @@ -12,7 +12,6 @@ import org.apache.jena.riot.resultset.ResultSetLang sealed trait SparqlFormat: val fullName: String val cliOptions: List[String] - val jenaLang: Lang override final def toString: String = fullName object SparqlFormat: @@ -23,28 +22,32 @@ object SparqlFormat: /** Formats we can write a result set to. */ sealed trait Writeable extends SparqlFormat - case object Json extends SparqlFormat.Readable, SparqlFormat.Writeable: + /** Formats handled by Jena's result set readers and writers. */ + sealed trait Jena extends SparqlFormat: + val jenaLang: Lang + + case object Json extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable: override val fullName: String = "SPARQL results JSON" override val cliOptions: List[String] = List("json", "srj") override val jenaLang: Lang = ResultSetLang.RS_JSON - case object Xml extends SparqlFormat.Readable, SparqlFormat.Writeable: + case object Xml extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable: override val fullName: String = "SPARQL results XML" override val cliOptions: List[String] = List("xml", "srx") override val jenaLang: Lang = ResultSetLang.RS_XML - case object Csv extends SparqlFormat.Readable, SparqlFormat.Writeable: + case object Csv extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable: override val fullName: String = "CSV" override val cliOptions: List[String] = List("csv") override val jenaLang: Lang = ResultSetLang.RS_CSV - case object Tsv extends SparqlFormat.Readable, SparqlFormat.Writeable: + case object Tsv extends SparqlFormat.Jena, SparqlFormat.Readable, SparqlFormat.Writeable: override val fullName: String = "TSV" override val cliOptions: List[String] = List("tsv") override val jenaLang: Lang = ResultSetLang.RS_TSV /** Jena's pretty-printed table. Jena registers no reader for it, so it's output-only. */ - case object Text extends SparqlFormat.Writeable: + case object Text extends SparqlFormat.Jena, SparqlFormat.Writeable: override val fullName: String = "Text table" override val cliOptions: List[String] = List("text") override val jenaLang: Lang = ResultSetLang.RS_Text @@ -52,12 +55,18 @@ object SparqlFormat: /** We never convert Jelly to Jelly, so this is neither Readable nor Writeable – it is only here * so that the other side of the conversion has a name. */ - case object JellySparql extends SparqlFormat: + case object JellySparql extends SparqlFormat.Jena: override val fullName: String = "Jelly-SPARQL" override val cliOptions: List[String] = List("jelly-sparql") override val jenaLang: Lang = JellySparqlLanguage.JELLY_SPARQL - private val sparqlFormats: List[SparqlFormat] = List(Json, Xml, Csv, Tsv, Text, JellySparql) + case object JellySparqlText extends SparqlFormat.Readable, SparqlFormat.Writeable: + override val fullName: String = "Jelly-SPARQL text" + override val cliOptions: List[String] = List("jelly-sparql-text") + val extension = ".jellys.txt" + + private val sparqlFormats: List[SparqlFormat] = + List(Json, Xml, Csv, Tsv, Text, JellySparql, JellySparqlText) def all: List[SparqlFormat] = sparqlFormats @@ -86,5 +95,9 @@ object SparqlFormat: /** Infers the format based on the file name. */ def inferFormat(fileName: String): Option[SparqlFormat] = - val guessType = RDFLanguages.guessContentType(fileName) - sparqlFormats.collectFirst { case f if f.jenaLang.getContentType == guessType => f } + if fileName.endsWith(JellySparqlText.extension) then Some(JellySparqlText) + else + val guessType = RDFLanguages.guessContentType(fileName) + sparqlFormats.collectFirst { + case f: SparqlFormat.Jena if f.jenaLang.getContentType == guessType => f + } diff --git a/src/main/scala/eu/neverblink/jelly/cli/graal/ProtobufFeature.scala b/src/main/scala/eu/neverblink/jelly/cli/graal/ProtobufFeature.scala deleted file mode 100644 index ab0ce91..0000000 --- a/src/main/scala/eu/neverblink/jelly/cli/graal/ProtobufFeature.scala +++ /dev/null @@ -1,27 +0,0 @@ -package eu.neverblink.jelly.cli.graal - -import org.graalvm.nativeimage.hosted.{Feature, RuntimeReflection} -import org.reflections.Reflections -import org.reflections.scanners.Scanners - -import scala.jdk.CollectionConverters.* - -class ProtobufFeature extends Feature: - import Feature.* - - override def getDescription: String = - "Registers Google-style Protobuf classes for reflection. Needed for jelly-text support." - - override def beforeAnalysis(access: BeforeAnalysisAccess): Unit = - val reflections = Reflections("eu.neverblink.jelly.core.proto.google.v1", Scanners.SubTypes) - val classesForSubtyping: Seq[Class[?]] = Seq( - classOf[com.google.protobuf.GeneratedMessage], - classOf[com.google.protobuf.GeneratedMessage.Builder[?]], - classOf[com.google.protobuf.ProtocolMessageEnum], - ) - val classes: Seq[Class[?]] = - classesForSubtyping.flatMap(reflections.getSubTypesOf(_).asScala.toSeq) - classes.foreach(clazz => { - RuntimeReflection.register(clazz) - RuntimeReflection.register(clazz.getDeclaredMethods*) - }) diff --git a/src/main/scala/eu/neverblink/jelly/cli/util/io/ProtoText.scala b/src/main/scala/eu/neverblink/jelly/cli/util/io/ProtoText.scala new file mode 100644 index 0000000..9f5499a --- /dev/null +++ b/src/main/scala/eu/neverblink/jelly/cli/util/io/ProtoText.scala @@ -0,0 +1,18 @@ +package eu.neverblink.jelly.cli.util.io + +import com.google.protobuf.Descriptors.Descriptor +import com.google.protobuf.{DynamicMessage, TextFormat} + +/** Protobuf text format for Jelly messages, done through DynamicMessage and the message descriptor, + * so we don't need the Google-style proto classes. + */ +object ProtoText: + private val printer = TextFormat.printer().escapingNonAscii(false) + + def print(descriptor: Descriptor, bytes: Array[Byte]): String = + printer.printToString(DynamicMessage.parseFrom(descriptor, bytes)) + + def parse(descriptor: Descriptor, text: String): DynamicMessage = + val builder = DynamicMessage.newBuilder(descriptor) + TextFormat.merge(text, builder) + builder.build() diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala index df96898..b2692d2 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/helpers/DataGenHelper.scala @@ -1,7 +1,8 @@ package eu.neverblink.jelly.cli.command.helpers import eu.neverblink.jelly.convert.jena.riot.JellyLanguage -import eu.neverblink.jelly.core.proto.google.v1 as google +import eu.neverblink.jelly.cli.util.io.ProtoText +import eu.neverblink.jelly.core.proto.v1.RdfStreamFrame import org.apache.jena.query.{Dataset, DatasetFactory} import org.apache.jena.rdf.model.{Model, ModelFactory, ResourceFactory} import org.apache.jena.riot.{Lang, RDFDataMgr, RDFLanguages} @@ -75,8 +76,8 @@ object DataGenHelper: */ def generateJellyText(nTriples: Int): String = val bytes = generateJellyBytes(nTriples) - val frame = google.RdfStreamFrame.parseDelimitedFrom(ByteArrayInputStream(bytes)) - frame.toString + val frame = RdfStreamFrame.parseDelimitedFrom(ByteArrayInputStream(bytes)) + ProtoText.print(RdfStreamFrame.getDescriptor, frame.toByteArray) /** This method generates a Jelly byte input stream with nTriples * @param nTriples diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala index a3d9961..ed7bfb7 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala @@ -6,7 +6,7 @@ import eu.neverblink.jelly.cli.* import eu.neverblink.jelly.convert.jena.riot.JellyLanguage import eu.neverblink.jelly.core.proto.v1.{LogicalStreamType, PhysicalStreamType, RdfStreamFrame} import eu.neverblink.jelly.core.JellyOptions -import eu.neverblink.jelly.core.proto.google.v1 as google +import eu.neverblink.jelly.cli.util.io.ProtoText import eu.neverblink.jelly.core.utils.IoUtils import org.apache.jena.irix.IRIs import org.apache.jena.rdf.model.{Model, ModelFactory} @@ -776,9 +776,10 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers: initialJellyFile => { val initialFrames = readJellyFile(new FileInputStream(initialJellyFile)) val initialOpts = initialFrames.head.getRows.asScala.head.getOptions - val jellyText = google.RdfStreamFrame.parseDelimitedFrom( - new FileInputStream(initialJellyFile), - ).toString + val jellyText = ProtoText.print( + RdfStreamFrame.getDescriptor, + RdfStreamFrame.parseDelimitedFrom(new FileInputStream(initialJellyFile)).toByteArray, + ) val bytes = ByteArrayInputStream(jellyText.getBytes()) RdfToJelly.testMode(true) RdfToJelly.setStdIn(bytes) diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala index 5cfb2b6..cd0954b 100644 --- a/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala +++ b/src/test/scala/eu/neverblink/jelly/cli/command/sparql/SparqlSerDesSpec.scala @@ -5,13 +5,13 @@ import eu.neverblink.jelly.cli.command.helpers.TestFixtureHelper import eu.neverblink.jelly.cli.command.sparql.util.SparqlFormat import eu.neverblink.jelly.convert.jena.sparql.JellySparqlLanguage import org.apache.jena.query.{ResultSet, ResultSetFactory} -import org.apache.jena.riot.{Lang, ResultSetMgr} -import org.apache.jena.riot.resultset.ResultSetLang +import org.apache.jena.riot.{Lang, RIOT, ResultSetMgr} +import org.apache.jena.riot.resultset.{ResultSetLang, ResultSetWriterRegistry} import org.apache.jena.sparql.resultset.ResultsCompare import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import java.io.ByteArrayInputStream +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Files, Path} import java.util.UUID.randomUUID @@ -42,6 +42,17 @@ object SparqlSerDesSpec: def parse(s: String, lang: Lang): ResultSet = parse(s.getBytes(UTF_8), lang) + /** Writes the SELECT results to Jelly-SPARQL, with at most `valuesPerFrame` cells per frame. */ + def toSmallFrames(json: String, valuesPerFrame: Int): Array[Byte] = + val out = ByteArrayOutputStream() + val context = RIOT.getContext.copy() + .set(JellySparqlLanguage.SYMBOL_MAX_VALUES_PER_FRAME, valuesPerFrame) + ResultSetWriterRegistry + .getFactory(JellySparqlLanguage.JELLY_SPARQL) + .create(JellySparqlLanguage.JELLY_SPARQL) + .write(out, parse(json, ResultSetLang.RS_JSON), context) + out.toByteArray + class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: import SparqlSerDesSpec.* @@ -204,3 +215,106 @@ class SparqlSerDesSpec extends AnyWordSpec with TestFixtureHelper with Matchers: } } } + + /** Runs `sparql from-jelly` to the text format and returns the text. */ + private def toText(jelly: Array[Byte]): String = + SparqlFromJelly.setStdIn(ByteArrayInputStream(jelly)) + SparqlFromJelly.runTestCommand( + List("sparql", "from-jelly", "--out-format", "jelly-sparql-text"), + )._1 + + /** Runs `sparql to-jelly` over the text format and returns the Jelly-SPARQL bytes. */ + private def fromText(text: String): Array[Byte] = + SparqlToJelly.setStdIn(ByteArrayInputStream(text.getBytes(UTF_8))) + SparqlToJelly.runTestCommand( + List("sparql", "to-jelly", "--in-format", "jelly-sparql-text", "--quiet"), + ) + SparqlToJelly.getOutBytes + + "Jelly-SPARQL text format" should { + "be written by from-jelly, one commented block per frame" in { + val text = toText(toJelly(selectJson, ".srj")) + text should startWith("# Frame 0\n") + text should not include "# Frame 1" + for expected <- Seq("options {", "variables {", "name: \"label\"", "row_count: 3") do + text should include(expected) + // Non-ASCII text is written without escaping + text should include("cześć") + } + + "round trip a SELECT result set through text and back" in { + val text = toText(toJelly(selectJson, ".srj")) + ResultsCompare.equalsByTermAndOrder( + parse(fromText(text), JellySparqlLanguage.JELLY_SPARQL), + parse(selectJson, ResultSetLang.RS_JSON), + ) shouldBe true + } + + "round trip an ASK result through text and back" in { + for value <- Seq(true, false) do + val text = toText(toJelly(askJson(value), ".srj")) + ResultSetMgr.readBoolean( + ByteArrayInputStream(fromText(text)), + JellySparqlLanguage.JELLY_SPARQL, + ) shouldBe value + } + + "keep frame boundaries in a multi-frame stream" in { + val jelly = toSmallFrames(selectJson, 4) + val text = toText(jelly) + val frameCount = text.linesIterator.count(_.startsWith("# Frame")) + frameCount should be > 1 + // Converting back to binary and to text again must give the same frames + val backToJelly = fromText(text) + toText(backToJelly) shouldBe text + ResultsCompare.equalsByTermAndOrder( + parse(backToJelly, JellySparqlLanguage.JELLY_SPARQL), + parse(selectJson, ResultSetLang.RS_JSON), + ) shouldBe true + } + + "read text without frame comments as a single frame" in { + val text = toText(toJelly(selectJson, ".srj")) + .linesIterator.filterNot(_.startsWith("#")).mkString("\n") + ResultsCompare.equalsByTermAndOrder( + parse(fromText(text), JellySparqlLanguage.JELLY_SPARQL), + parse(selectJson, ResultSetLang.RS_JSON), + ) shouldBe true + } + + "infer the format from the .jellys.txt extension on both sides" in { + val jelly = toJelly(selectJson, ".srj") + withEmptyFile(SparqlFormat.JellySparqlText.extension) { target => + SparqlFromJelly.setStdIn(ByteArrayInputStream(jelly)) + SparqlFromJelly.runTestCommand(List("sparql", "from-jelly", "--to", target)) + val text = String(Files.readAllBytes(Path.of(target)), UTF_8) + text should startWith("# Frame 0") + SparqlToJelly.runTestCommand(List("sparql", "to-jelly", target, "--quiet")) + ResultsCompare.equalsByTermAndOrder( + parse(SparqlToJelly.getOutBytes, JellySparqlLanguage.JELLY_SPARQL), + parse(selectJson, ResultSetLang.RS_JSON), + ) shouldBe true + } + } + + "warn that the text format is unstable, unless --quiet is set" in { + val text = toText(toJelly(selectJson, ".srj")) + SparqlToJelly.setStdIn(ByteArrayInputStream(text.getBytes(UTF_8))) + val (_, err) = SparqlToJelly.runTestCommand( + List("sparql", "to-jelly", "--in-format", "jelly-sparql-text"), + ) + err should include("WARNING") + SparqlToJelly.setStdIn(ByteArrayInputStream(text.getBytes(UTF_8))) + val (_, quietErr) = SparqlToJelly.runTestCommand( + List("sparql", "to-jelly", "--in-format", "jelly-sparql-text", "--quiet"), + ) + quietErr shouldBe empty + } + + "report malformed text" in { + val e = intercept[ExitException] { + fromText("this is { definitely not a frame") + } + e.getCause shouldBe a[InvalidJellyFile] + } + }