contexts = new HashMap<>();
+ for (String group : touchedGroups) {
+ contexts.put(
+ group,
+ LogRecordReadContext.createReadContext(
+ tableInfo.getTableId(),
+ LogFormat.ARROW,
+ tableInfo.getSchemaId(),
+ groupRowTypes.get(group),
+ false,
+ LogRecordReadContext.SchemaResolution.TARGET,
+ null,
+ ColumnGroupSchemaGetter.group(schemaGetter, group),
+ allocationManagerFactory));
+ }
+ return contexts;
+ }
+
+ /** Field getters over the physical row of {@code group}. */
+ public InternalRow.FieldGetter[] groupFieldGetters(String group) {
+ return InternalRow.createFieldGetters(groupRowTypes.get(group));
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ColumnGroupStitcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ColumnGroupStitcher.java
new file mode 100644
index 00000000000..716fb4f7b46
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ColumnGroupStitcher.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.scanner.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.record.LogRecord;
+import org.apache.fluss.record.LogRecordBatch;
+import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.record.LogRecords;
+import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.rpc.entity.ColumnGroupFetchResult;
+import org.apache.fluss.utils.CloseableIterator;
+
+import javax.annotation.Nullable;
+
+import java.io.Closeable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Stitches column-group rows onto base rows by offset for one completed fetch (FIP-45).
+ *
+ * The base batches and every group's batches are ordered by base offset, so the stitch is a
+ * merge-join: for each base record the cursor of every touched group is advanced to the same
+ * offset. Batch boundaries need not line up between the base log and the group logs.
+ */
+@Internal
+final class ColumnGroupStitcher implements Closeable {
+
+ private final ColumnGroupReadPlan plan;
+ private final GroupCursor[] cursors;
+ private final InternalRow.FieldGetter[][] groupFieldGetters;
+
+ ColumnGroupStitcher(
+ ColumnGroupReadPlan plan,
+ Map groupReadContexts,
+ Map groupResults,
+ boolean checkCrcs) {
+ this.plan = plan;
+ List groups = plan.touchedGroups();
+ this.cursors = new GroupCursor[groups.size()];
+ this.groupFieldGetters = new InternalRow.FieldGetter[groups.size()][];
+ for (int i = 0; i < groups.size(); i++) {
+ String group = groups.get(i);
+ ColumnGroupFetchResult result = groupResults.get(group);
+ LogRecords records = result == null ? MemoryLogRecords.EMPTY : result.getRecords();
+ cursors[i] = new GroupCursor(records, groupReadContexts.get(group), checkCrcs);
+ groupFieldGetters[i] = plan.groupFieldGetters(group);
+ }
+ }
+
+ /**
+ * Positions every group cursor at {@code offset}. Returns false when some group has no row for
+ * it, which means the fetch ran out of group records: the caller must stop consuming this fetch
+ * at {@code offset} and fetch again from there.
+ */
+ boolean prepare(long offset) {
+ for (GroupCursor cursor : cursors) {
+ if (!cursor.advanceTo(offset)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Builds the output row of {@code baseRecord} after a successful {@link #prepare}. */
+ InternalRow stitch(LogRecord baseRecord, InternalRow.FieldGetter[] baseFieldGetters) {
+ GenericRow row = new GenericRow(plan.outputCount());
+ InternalRow baseRow = baseRecord.getRow();
+ for (int i = 0; i < plan.outputCount(); i++) {
+ int source = plan.outputSource(i);
+ int field = plan.outputField(i);
+ if (source < 0) {
+ row.setField(i, baseFieldGetters[field].getFieldOrNull(baseRow));
+ } else {
+ InternalRow groupRow = cursors[source].currentRow();
+ row.setField(i, groupFieldGetters[source][field].getFieldOrNull(groupRow));
+ }
+ }
+ return row;
+ }
+
+ @Override
+ public void close() {
+ for (GroupCursor cursor : cursors) {
+ cursor.close();
+ }
+ }
+
+ /** A cursor over the records of one column group, ordered by base offset. */
+ private static final class GroupCursor implements Closeable {
+ private final Iterator batches;
+ private final LogRecordReadContext readContext;
+ private final boolean checkCrcs;
+ @Nullable private CloseableIterator records;
+ @Nullable private LogRecord current;
+
+ GroupCursor(LogRecords logRecords, LogRecordReadContext readContext, boolean checkCrcs) {
+ this.batches = logRecords.batches().iterator();
+ this.readContext = readContext;
+ this.checkCrcs = checkCrcs;
+ }
+
+ boolean advanceTo(long offset) {
+ while (current == null || current.logOffset() < offset) {
+ if (records != null && records.hasNext()) {
+ current = records.next();
+ continue;
+ }
+ if (records != null) {
+ records.close();
+ records = null;
+ }
+ if (!batches.hasNext()) {
+ current = null;
+ return false;
+ }
+ LogRecordBatch batch = batches.next();
+ if (checkCrcs) {
+ batch.ensureValid();
+ }
+ records = batch.records(readContext);
+ }
+ return current.logOffset() == offset;
+ }
+
+ InternalRow currentRow() {
+ if (current == null) {
+ throw new IllegalStateException("No column group row prepared.");
+ }
+ return current.getRow();
+ }
+
+ @Override
+ public void close() {
+ if (records != null) {
+ records.close();
+ records = null;
+ }
+ current = null;
+ }
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
index de9cb1d3f61..4f67982168f 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java
@@ -38,6 +38,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collections;
@@ -79,6 +81,8 @@ public abstract class CompletedFetch {
private long nextFetchOffset;
private boolean isConsumed = false;
private boolean initialized = false;
+ // FIP-45: stitches column-group rows onto base rows; null when no group is projected
+ @Nullable private final ColumnGroupStitcher columnGroupStitcher;
public CompletedFetch(
TableBucket tableBucket,
@@ -92,6 +96,35 @@ public CompletedFetch(
boolean isCheckCrcs,
long fetchOffset,
long filteredEndOffset) {
+ this(
+ tableBucket,
+ tablePath,
+ error,
+ sizeInBytes,
+ highWatermark,
+ batches,
+ readContext,
+ logScannerStatus,
+ isCheckCrcs,
+ fetchOffset,
+ filteredEndOffset,
+ null);
+ }
+
+ public CompletedFetch(
+ TableBucket tableBucket,
+ TablePath tablePath,
+ ApiError error,
+ int sizeInBytes,
+ long highWatermark,
+ Iterator batches,
+ LogRecordReadContext readContext,
+ LogScannerStatus logScannerStatus,
+ boolean isCheckCrcs,
+ long fetchOffset,
+ long filteredEndOffset,
+ @Nullable ColumnGroupStitcher columnGroupStitcher) {
+ this.columnGroupStitcher = columnGroupStitcher;
this.tableBucket = tableBucket;
this.tablePath = tablePath;
this.error = error;
@@ -122,10 +155,17 @@ ScanRecord toScanRecord(LogRecord record) {
InternalRow.FieldGetter[] selectedFieldGetters =
readContext.getSelectedFieldGetters(schemaId);
- GenericRow newRow = new GenericRow(selectedFieldGetters.length);
- InternalRow internalRow = record.getRow();
- for (int i = 0; i < selectedFieldGetters.length; i++) {
- newRow.setField(i, selectedFieldGetters[i].getFieldOrNull(internalRow));
+ final InternalRow newRow;
+ if (columnGroupStitcher != null) {
+ // FIP-45: splice the column-group values onto the base row by offset
+ newRow = columnGroupStitcher.stitch(record, selectedFieldGetters);
+ } else {
+ GenericRow row = new GenericRow(selectedFieldGetters.length);
+ InternalRow internalRow = record.getRow();
+ for (int i = 0; i < selectedFieldGetters.length; i++) {
+ row.setField(i, selectedFieldGetters[i].getFieldOrNull(internalRow));
+ }
+ newRow = row;
}
return new ScanRecord(
@@ -190,6 +230,9 @@ void setInitialized() {
void drain() {
if (!isConsumed) {
maybeCloseRecordStream();
+ if (columnGroupStitcher != null) {
+ columnGroupStitcher.close();
+ }
cachedRecordException = null;
isConsumed = true;
@@ -295,6 +338,10 @@ List fetchArrowBatches(int maxRecords) {
if (isConsumed) {
return Collections.emptyList();
}
+ if (columnGroupStitcher != null) {
+ throw new UnsupportedOperationException(
+ "Arrow batch polling is not supported for projections touching column groups.");
+ }
List arrowBatches = new ArrayList<>();
int recordsFetched = 0;
@@ -362,6 +409,14 @@ private LogRecord nextFetchedRecord() throws Exception {
LogRecord record = records.next();
// skip any records out of range.
if (record.logOffset() >= nextFetchOffset) {
+ if (columnGroupStitcher != null
+ && !columnGroupStitcher.prepare(record.logOffset())) {
+ // FIP-45: the fetch carried fewer column-group rows than base rows;
+ // stop here and fetch again from this offset.
+ nextFetchOffset = record.logOffset();
+ drain();
+ return null;
+ }
return record;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
index 0384389c283..b255fbf7604 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java
@@ -50,6 +50,28 @@ public DefaultCompletedFetch(
boolean isCheckCrc,
Long fetchOffset,
@Nullable ByteBuf parsedByteBuf) {
+ this(
+ tableBucket,
+ tablePath,
+ fetchLogResultForBucket,
+ readContext,
+ logScannerStatus,
+ isCheckCrc,
+ fetchOffset,
+ parsedByteBuf,
+ null);
+ }
+
+ public DefaultCompletedFetch(
+ TableBucket tableBucket,
+ TablePath tablePath,
+ FetchLogResultForBucket fetchLogResultForBucket,
+ LogRecordReadContext readContext,
+ LogScannerStatus logScannerStatus,
+ boolean isCheckCrc,
+ Long fetchOffset,
+ @Nullable ByteBuf parsedByteBuf,
+ @Nullable ColumnGroupStitcher columnGroupStitcher) {
super(
tableBucket,
tablePath,
@@ -61,7 +83,8 @@ public DefaultCompletedFetch(
logScannerStatus,
isCheckCrc,
fetchOffset,
- fetchLogResultForBucket.getFilteredEndOffset());
+ fetchLogResultForBucket.getFilteredEndOffset(),
+ columnGroupStitcher);
this.parsedByteBuf = parsedByteBuf;
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index 77a8504ca33..8d54ef94ee6 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -486,7 +486,9 @@ private synchronized void handleFetchLogResponse(
// the data is pruned
isCheckCrcs,
fetchOffset,
- hasRecords ? parsedByteBuf : null));
+ hasRecords ? parsedByteBuf : null,
+ trc.newColumnGroupStitcher(
+ fetchResultForBucket, isCheckCrcs)));
}
}
}
@@ -741,6 +743,9 @@ static class TableReadContext {
@Nullable final Projection projection;
@Nullable final org.apache.fluss.rpc.messages.PbPredicate cachedPbPredicate;
final int filterSchemaId;
+ // FIP-45: how the projection maps onto the base log and the column groups
+ @Nullable final ColumnGroupReadPlan columnGroupPlan;
+ final Map groupReadContexts;
TableReadContext(
TableInfo tableInfo,
@@ -751,24 +756,39 @@ static class TableReadContext {
ChunkedAllocationManager.ChunkedFactory chunkedFactory) {
this.tablePath = tableInfo.getTablePath();
this.isPartitioned = tableInfo.isPartitioned();
+ this.columnGroupPlan = ColumnGroupReadPlan.of(tableInfo.getSchema(), projection);
// Share the LogFetcher-owned chunked factory so all tables reuse one chunk
// pool and LogFetcher.close() releases the chunks after contexts are closed.
- this.readContext =
- LogRecordReadContext.createReadContext(
- tableInfo,
- false,
- schemaResolution,
- projection,
- schemaGetter,
- chunkedFactory);
- this.remoteReadContext =
- LogRecordReadContext.createReadContext(
- tableInfo,
- true,
- schemaResolution,
- projection,
- schemaGetter,
- chunkedFactory);
+ if (columnGroupPlan != null) {
+ // the base log holds only the default-group columns; group columns are stitched
+ this.readContext =
+ columnGroupPlan.createBaseContext(
+ tableInfo, false, schemaResolution, schemaGetter, chunkedFactory);
+ this.remoteReadContext =
+ columnGroupPlan.createBaseContext(
+ tableInfo, true, schemaResolution, schemaGetter, chunkedFactory);
+ this.groupReadContexts =
+ columnGroupPlan.createGroupContexts(
+ tableInfo, schemaGetter, chunkedFactory);
+ } else {
+ this.readContext =
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ false,
+ schemaResolution,
+ projection,
+ schemaGetter,
+ chunkedFactory);
+ this.remoteReadContext =
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ true,
+ schemaResolution,
+ projection,
+ schemaGetter,
+ chunkedFactory);
+ this.groupReadContexts = Collections.emptyMap();
+ }
this.projection = projection;
this.cachedPbPredicate =
recordBatchFilter != null
@@ -778,9 +798,23 @@ static class TableReadContext {
this.filterSchemaId = tableInfo.getSchemaId();
}
+ /** A stitcher for one fetch result, or null when no column group is projected. */
+ @Nullable
+ ColumnGroupStitcher newColumnGroupStitcher(
+ FetchLogResultForBucket fetchResult, boolean checkCrcs) {
+ if (columnGroupPlan == null || columnGroupPlan.touchedGroups().isEmpty()) {
+ return null;
+ }
+ return new ColumnGroupStitcher(
+ columnGroupPlan, groupReadContexts, fetchResult.columnGroups(), checkCrcs);
+ }
+
void close() {
readContext.close();
remoteReadContext.close();
+ for (LogRecordReadContext groupReadContext : groupReadContexts.values()) {
+ groupReadContext.close();
+ }
}
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendColumnsResult.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendColumnsResult.java
new file mode 100644
index 00000000000..b9bdd83b2d2
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendColumnsResult.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table.writer;
+
+import org.apache.fluss.annotation.PublicEvolving;
+
+/**
+ * The result of {@link AppendWriter#appendColumns}: the state of the column group on the bucket
+ * after the rows were appended.
+ *
+ * @since 0.9
+ */
+@PublicEvolving
+public final class AppendColumnsResult {
+ private final long logEndOffset;
+ private final long highWatermark;
+
+ public AppendColumnsResult(long logEndOffset, long highWatermark) {
+ this.logEndOffset = logEndOffset;
+ this.highWatermark = highWatermark;
+ }
+
+ /**
+ * The enrichment watermark of the column group on the bucket: the exclusive base-log offset up
+ * to which the group is contiguously filled on the leader.
+ */
+ public long getLogEndOffset() {
+ return logEndOffset;
+ }
+
+ /**
+ * The committed enrichment watermark of the column group on the bucket: the exclusive base-log
+ * offset up to which the group is filled on every in-sync replica. Readers projecting the group
+ * never see rows at or beyond it.
+ */
+ public long getHighWatermark() {
+ return highWatermark;
+ }
+
+ @Override
+ public String toString() {
+ return "AppendColumnsResult{"
+ + "logEndOffset="
+ + logEndOffset
+ + ", highWatermark="
+ + highWatermark
+ + '}';
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriter.java
index 1b5cee055fc..a380b143e09 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriter.java
@@ -18,8 +18,10 @@
package org.apache.fluss.client.table.writer;
import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.row.InternalRow;
+import java.util.List;
import java.util.concurrent.CompletableFuture;
/**
@@ -37,4 +39,24 @@ public interface AppendWriter extends TableWriter {
* @return A {@link CompletableFuture} that always returns append result when complete normally.
*/
CompletableFuture append(InternalRow record);
+
+ /**
+ * Appends the columns of one column group for rows that already exist in the log (FIP-45 log
+ * enrichment via append columns).
+ *
+ * {@code rows} carry only the group's columns, in schema order, and fill the contiguous
+ * base-log offsets {@code [firstSourceOffset, firstSourceOffset + rows.size())} of {@code
+ * bucket}. The first offset must equal the group's current log end offset (its enrichment
+ * watermark) on the bucket; a batch entirely below it is acknowledged without effect, so
+ * replaying after a restart is safe, while a gap or a batch running past the base high
+ * watermark fails with {@link org.apache.fluss.exception.InvalidColumnGroupOffsetException}.
+ *
+ * @param columnGroup the column group to fill
+ * @param bucket the bucket whose rows are enriched
+ * @param firstSourceOffset the base-log offset filled by the first row
+ * @param rows the group rows, one per consecutive offset
+ * @return the column group's watermarks on the bucket after the append
+ */
+ CompletableFuture appendColumns(
+ String columnGroup, TableBucket bucket, long firstSourceOffset, List rows);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriterImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriterImpl.java
index 6aa3b9311ca..c3ca1fe4611 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriterImpl.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AppendWriterImpl.java
@@ -17,14 +17,18 @@
package org.apache.fluss.client.table.writer;
+import org.apache.fluss.client.write.ColumnGroupWriter;
import org.apache.fluss.client.write.WriteRecord;
import org.apache.fluss.client.write.WriterClient;
+import org.apache.fluss.exception.UnknownColumnGroupException;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.InternalRow.FieldGetter;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.compacted.CompactedRow;
import org.apache.fluss.row.encode.CompactedRowEncoder;
import org.apache.fluss.row.encode.IndexedRowEncoder;
@@ -48,6 +52,9 @@ class AppendWriterImpl extends AbstractTableWriter implements AppendWriter {
private final CompactedRowEncoder compactedRowEncoder;
private final FieldGetter[] fieldGetters;
private final TableInfo tableInfo;
+ // FIP-45: the base log stores only the default-group columns; null when there are no groups
+ @Nullable private final int[] baseColumnIndices;
+ @Nullable private volatile ColumnGroupWriter columnGroupWriter;
AppendWriterImpl(TablePath tablePath, TableInfo tableInfo, WriterClient writerClient) {
super(tablePath, tableInfo, writerClient);
@@ -70,6 +77,18 @@ class AppendWriterImpl extends AbstractTableWriter implements AppendWriter {
this.compactedRowEncoder = new CompactedRowEncoder(fieldDataTypes);
this.fieldGetters = InternalRow.createFieldGetters(tableInfo.getRowType());
this.tableInfo = tableInfo;
+ if (tableInfo.getSchema().hasColumnGroups()) {
+ if (logFormat != LogFormat.ARROW) {
+ throw new IllegalArgumentException(
+ "Column groups require ARROW log format, but table '"
+ + tablePath
+ + "' uses "
+ + logFormat);
+ }
+ this.baseColumnIndices = tableInfo.getSchema().getDefaultGroupColumnIndices();
+ } else {
+ this.baseColumnIndices = null;
+ }
}
/**
@@ -94,12 +113,36 @@ record =
WriteRecord.forCompactedAppend(
tableInfo, physicalPath, compactedRow, bucketKey);
} else {
- // ARROW format supports general internal row
- record = WriteRecord.forArrowAppend(tableInfo, physicalPath, row, bucketKey);
+ // ARROW format supports general internal row. For a column-group table only the
+ // default-group columns are written to the base log (FIP-45); the enrichment columns
+ // of the row are ignored here and filled later through appendColumns.
+ InternalRow physicalRow =
+ baseColumnIndices == null
+ ? row
+ : ProjectedRow.from(baseColumnIndices).replaceRow(row);
+ record = WriteRecord.forArrowAppend(tableInfo, physicalPath, physicalRow, bucketKey);
}
return send(record).thenApply(ignored -> APPEND_SUCCESS);
}
+ @Override
+ public CompletableFuture appendColumns(
+ String columnGroup,
+ TableBucket bucket,
+ long firstSourceOffset,
+ List rows) {
+ if (baseColumnIndices == null) {
+ throw new UnknownColumnGroupException(
+ "Table " + tablePath + " does not declare any column group.");
+ }
+ ColumnGroupWriter writer = columnGroupWriter;
+ if (writer == null) {
+ writer = writerClient.getOrCreateColumnGroupWriter(tablePath, tableInfo);
+ columnGroupWriter = writer;
+ }
+ return writer.appendColumns(columnGroup, bucket, firstSourceOffset, rows);
+ }
+
private CompactedRow encodeCompactedRow(InternalRow row) {
if (row instanceof CompactedRow) {
return (CompactedRow) row;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/ColumnGroupWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/write/ColumnGroupWriter.java
new file mode 100644
index 00000000000..1fdcf113683
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/ColumnGroupWriter.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.write;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.client.table.writer.AppendColumnsResult;
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
+import org.apache.fluss.exception.LeaderNotAvailableException;
+import org.apache.fluss.exception.UnknownColumnGroupException;
+import org.apache.fluss.memory.UnmanagedPagedOutputView;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.ChangeType;
+import org.apache.fluss.record.MemoryLogRecordsArrowBuilder;
+import org.apache.fluss.record.bytesview.BytesView;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.arrow.ArrowWriter;
+import org.apache.fluss.row.arrow.ArrowWriterPool;
+import org.apache.fluss.rpc.gateway.TabletServerGateway;
+import org.apache.fluss.rpc.messages.PbProduceLogColumnsReqForBucket;
+import org.apache.fluss.rpc.messages.PbProduceLogColumnsRespForBucket;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
+import org.apache.fluss.rpc.protocol.Errors;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil;
+import org.apache.fluss.types.RowType;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Writes column-group rows (FIP-45 log enrichment via append columns) for one table.
+ *
+ * Each call encodes the rows of one column group for a contiguous range of base-log offsets into
+ * a standard Arrow log record batch and sends it to the bucket leader with a {@link
+ * ProduceLogColumnsRequest}. The server stamps the base offsets into the batch header, so the bytes
+ * written here are exactly the bytes later served to readers.
+ *
+ *
TODO (WP7): batching, one in-flight request per bucket, leader-change retries and resync on
+ * {@code expected_source_offset} are not implemented; every call is one request.
+ */
+@Internal
+@ThreadSafe
+public final class ColumnGroupWriter implements AutoCloseable {
+
+ private final TablePath tablePath;
+ private final TableInfo tableInfo;
+ private final MetadataUpdater metadataUpdater;
+ private final int acks;
+ private final int requestTimeoutMs;
+ private final int batchBufferSize;
+ private final BufferAllocator allocator;
+ private final Map writerPools = new ConcurrentHashMap<>();
+
+ public ColumnGroupWriter(
+ TablePath tablePath,
+ TableInfo tableInfo,
+ MetadataUpdater metadataUpdater,
+ Configuration conf,
+ int acks) {
+ this.tablePath = tablePath;
+ this.tableInfo = tableInfo;
+ this.metadataUpdater = metadataUpdater;
+ this.acks = acks;
+ this.requestTimeoutMs = (int) conf.get(ConfigOptions.CLIENT_REQUEST_TIMEOUT).toMillis();
+ this.batchBufferSize =
+ (int) conf.get(ConfigOptions.CLIENT_WRITER_REQUEST_MAX_SIZE).getBytes();
+ this.allocator = BufferAllocatorUtil.createBufferAllocator(null);
+ }
+
+ /** Appends {@code rows} of {@code columnGroup} at base offsets starting at {@code first}. */
+ public CompletableFuture appendColumns(
+ String columnGroup,
+ TableBucket bucket,
+ long firstSourceOffset,
+ List rows) {
+ Schema schema = tableInfo.getSchema();
+ if (!schema.getColumnGroups().containsKey(columnGroup)) {
+ throw new UnknownColumnGroupException(
+ "Unknown column group '" + columnGroup + "' on table " + tablePath);
+ }
+ RowType groupRowType = schema.getColumnGroupRowType(columnGroup);
+ for (InternalRow row : rows) {
+ if (row.getFieldCount() != groupRowType.getFieldCount()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Column group '%s' has %d columns but a row with %d fields was given.",
+ columnGroup, groupRowType.getFieldCount(), row.getFieldCount()));
+ }
+ }
+ if (rows.isEmpty()) {
+ throw new IllegalArgumentException("No rows to append for column group " + columnGroup);
+ }
+
+ final BytesView records;
+ try {
+ records = encode(columnGroup, groupRowType, rows);
+ } catch (Exception e) {
+ CompletableFuture failed = new CompletableFuture<>();
+ failed.completeExceptionally(
+ new FlussRuntimeException("Failed to encode column group rows.", e));
+ return failed;
+ }
+
+ TabletServerGateway gateway;
+ try {
+ gateway = leaderGateway(bucket);
+ } catch (Exception e) {
+ CompletableFuture failed = new CompletableFuture<>();
+ failed.completeExceptionally(e);
+ return failed;
+ }
+
+ ProduceLogColumnsRequest request =
+ new ProduceLogColumnsRequest()
+ .setAcks(acks)
+ .setTableId(tableInfo.getTableId())
+ .setTimeoutMs(requestTimeoutMs)
+ .setColumnGroup(columnGroup);
+ PbProduceLogColumnsReqForBucket bucketReq =
+ request.addBucketsReq()
+ .setBucketId(bucket.getBucket())
+ .setFirstSourceOffset(firstSourceOffset);
+ if (bucket.getPartitionId() != null) {
+ bucketReq.setPartitionId(bucket.getPartitionId());
+ }
+ bucketReq.setRecordsBytesView(records);
+
+ return gateway.produceLogColumns(request).thenApply(response -> toResult(bucket, response));
+ }
+
+ private TabletServerGateway leaderGateway(TableBucket bucket) {
+ int leader;
+ try {
+ leader = metadataUpdater.leaderFor(tablePath, bucket);
+ } catch (Exception e) {
+ metadataUpdater.checkAndUpdateMetadata(tablePath, bucket);
+ leader = metadataUpdater.leaderFor(tablePath, bucket);
+ }
+ TabletServerGateway gateway = metadataUpdater.newTabletServerClientForNode(leader);
+ if (gateway == null) {
+ throw new LeaderNotAvailableException(
+ "No tablet server gateway for leader " + leader + " of bucket " + bucket);
+ }
+ return gateway;
+ }
+
+ private BytesView encode(String columnGroup, RowType groupRowType, List rows)
+ throws Exception {
+ ArrowWriterPool pool =
+ writerPools.computeIfAbsent(columnGroup, g -> new ArrowWriterPool(allocator));
+ ArrowWriter arrowWriter =
+ pool.getOrCreateWriter(
+ tableInfo.getTableId(),
+ tableInfo.getSchemaId(),
+ batchBufferSize,
+ groupRowType,
+ tableInfo.getTableConfig().getArrowCompressionInfo());
+ int pageSize = Math.max(4096, Math.min(batchBufferSize, 1024 * 1024));
+ UnmanagedPagedOutputView outputView = new UnmanagedPagedOutputView(pageSize);
+ MemoryLogRecordsArrowBuilder builder =
+ MemoryLogRecordsArrowBuilder.builder(
+ tableInfo.getSchemaId(), arrowWriter, outputView, true, null);
+ try {
+ for (InternalRow row : rows) {
+ if (builder.isFull()) {
+ throw new IllegalArgumentException(
+ "Too many column group rows for one request; split the rows into "
+ + "smaller batches (limit "
+ + ConfigOptions.CLIENT_WRITER_REQUEST_MAX_SIZE.key()
+ + ").");
+ }
+ builder.append(ChangeType.APPEND_ONLY, row);
+ }
+ builder.close();
+ return builder.build();
+ } finally {
+ builder.recycleArrowWriter();
+ }
+ }
+
+ private static AppendColumnsResult toResult(
+ TableBucket bucket, ProduceLogColumnsResponse response) {
+ if (response.getBucketsRespsCount() == 0) {
+ throw new FlussRuntimeException(
+ "Empty produceLogColumns response for bucket " + bucket);
+ }
+ PbProduceLogColumnsRespForBucket resp = response.getBucketsRespAt(0);
+ if (resp.hasErrorCode() && resp.getErrorCode() != Errors.NONE.code()) {
+ Errors error = Errors.forCode(resp.getErrorCode());
+ String message =
+ "Column group write for bucket "
+ + bucket
+ + " failed: "
+ + resp.getErrorMessage();
+ if (error == Errors.INVALID_COLUMN_GROUP_OFFSET) {
+ long expected =
+ resp.hasExpectedSourceOffset() ? resp.getExpectedSourceOffset() : -1L;
+ throw new InvalidColumnGroupOffsetException(message, expected);
+ }
+ throw error.exception(message);
+ }
+ return new AppendColumnsResult(resp.getLogEndOffset(), resp.getHighWatermark());
+ }
+
+ @Override
+ public void close() {
+ for (ArrowWriterPool pool : writerPools.values()) {
+ pool.close();
+ }
+ writerPools.clear();
+ allocator.close();
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index ffa18ad9455..eccd4841a5f 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -668,15 +668,20 @@ private WriteBatch createWriteBatch(
clock.milliseconds());
case ARROW_LOG:
+ // FIP-45: the base log of a column-group table physically holds only the
+ // default-group columns; statistics are keyed by table columns and are skipped.
+ boolean hasColumnGroups = tableInfo.getSchema().hasColumnGroups();
ArrowWriter arrowWriter =
arrowWriterPool.getOrCreateWriter(
tableInfo.getTableId(),
schemaId,
outputView.getPreAllocatedSize(),
- tableInfo.getRowType(),
+ hasColumnGroups
+ ? tableInfo.getSchema().getBaseRowType()
+ : tableInfo.getRowType(),
tableInfo.getTableConfig().getArrowCompressionInfo());
LogRecordBatchStatisticsCollector statisticsCollector = null;
- if (tableInfo.isStatisticsEnabled()) {
+ if (tableInfo.isStatisticsEnabled() && !hasColumnGroups) {
statisticsCollector =
new LogRecordBatchStatisticsCollector(
tableInfo.getRowType(), tableInfo.getStatsIndexMapping());
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
index ad8c7870547..4565c43203e 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
@@ -30,6 +30,7 @@
import org.apache.fluss.exception.IllegalConfigurationException;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.metrics.ClientMetricGroup;
import org.apache.fluss.utils.CopyOnWriteMap;
@@ -93,6 +94,9 @@ public class WriterClient {
private final IdempotenceManager idempotenceManager;
private final WriterMetricGroup writerMetricGroup;
private final DynamicPartitionCreator dynamicPartitionCreator;
+ // FIP-45: column-group writers per table, created on first appendColumns
+ private final Map columnGroupWriters = new CopyOnWriteMap<>();
+ private short acks = -1;
public WriterClient(
Configuration conf,
@@ -120,6 +124,7 @@ public WriterClient(
this.idempotenceManager = idempotenceManagerLocal;
short acks = configureAcks(idempotenceManager.idempotenceEnabled());
+ this.acks = acks;
int retries = configureRetries(idempotenceManager.idempotenceEnabled());
this.accumulator =
new RecordAccumulator(
@@ -167,6 +172,14 @@ public void send(WriteRecord record, WriteCallback callback) {
* call to complete, however no guarantee is made about the completion of records sent after the
* flush call begins.
*/
+ /** The column-group writer of {@code tablePath} (FIP-45), created on first use. */
+ public ColumnGroupWriter getOrCreateColumnGroupWriter(
+ TablePath tablePath, TableInfo tableInfo) {
+ throwIfWriterClosed();
+ return columnGroupWriters.computeIfAbsent(
+ tablePath, tp -> new ColumnGroupWriter(tp, tableInfo, metadataUpdater, conf, acks));
+ }
+
public void flush() {
LOG.trace("Flushing accumulated records in writer.");
long start = System.currentTimeMillis();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/ColumnGroupITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/ColumnGroupITCase.java
new file mode 100644
index 00000000000..2a34d72cae4
--- /dev/null
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/ColumnGroupITCase.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.table;
+
+import org.apache.fluss.client.admin.ClientToServerITCaseBase;
+import org.apache.fluss.client.table.scanner.ScanRecord;
+import org.apache.fluss.client.table.scanner.log.LogScanner;
+import org.apache.fluss.client.table.scanner.log.ScanRecords;
+import org.apache.fluss.client.table.writer.AppendColumnsResult;
+import org.apache.fluss.client.table.writer.AppendWriter;
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
+import org.apache.fluss.exception.UnknownColumnGroupException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.row.BinaryString;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.server.replica.ReplicaManager;
+import org.apache.fluss.server.tablet.TabletServer;
+import org.apache.fluss.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.fluss.testutils.DataTestUtils.row;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * End-to-end test of FIP-45 column groups on the revised design: a column group is a shadow log
+ * sharing the base log's offsets, the server ships base and group bytes zero-copy, the client
+ * stitches rows, and reads projecting a group are gated at the group's high watermark.
+ */
+class ColumnGroupITCase extends ClientToServerITCaseBase {
+
+ private static final Schema SCHEMA =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("geo", DataTypes.STRING())
+ .columnGroup("geo_group")
+ .column("score", DataTypes.DOUBLE())
+ .columnGroup("risk_group")
+ .build();
+
+ private static final int BASE_ROWS = 10;
+
+ private TablePath createColumnGroupTable(String name) throws Exception {
+ TablePath tablePath = TablePath.of("test_db_cg", name);
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(SCHEMA)
+ .distributedBy(1)
+ // WP2 (follower replication of column groups) is not in the POC, so the
+ // committed enrichment watermark only advances with a single replica.
+ .property(ConfigOptions.TABLE_REPLICATION_FACTOR.key(), "1")
+ .build();
+ createTable(tablePath, descriptor, false);
+ return tablePath;
+ }
+
+ private static void writeBaseRows(Table table, int count) throws Exception {
+ AppendWriter writer = table.newAppend().createWriter();
+ for (int i = 0; i < count; i++) {
+ // the writer takes the full-width row; enrichment columns are ignored here
+ writer.append(row(i, "name" + i, null, null));
+ }
+ writer.flush();
+ }
+
+ private static List geoRows(int from, int toExclusive) {
+ List rows = new ArrayList<>();
+ for (int i = from; i < toExclusive; i++) {
+ rows.add(row("geo" + i));
+ }
+ return rows;
+ }
+
+ private static List scoreRows(int from, int toExclusive) {
+ List rows = new ArrayList<>();
+ for (int i = from; i < toExclusive; i++) {
+ rows.add(row(i * 0.5d));
+ }
+ return rows;
+ }
+
+ private static List pollUpTo(LogScanner scanner, int expected, Duration timeout) {
+ List records = new ArrayList<>();
+ long deadline = System.currentTimeMillis() + timeout.toMillis();
+ while (records.size() < expected && System.currentTimeMillis() < deadline) {
+ ScanRecords scanRecords = scanner.poll(Duration.ofMillis(200));
+ for (ScanRecord record : scanRecords) {
+ records.add(record);
+ }
+ }
+ return records;
+ }
+
+ @Test
+ void testBaseOnlyReadsAreNotGatedByColumnGroups() throws Exception {
+ TablePath tablePath = createColumnGroupTable("base_only");
+ try (Table table = conn.getTable(tablePath)) {
+ writeBaseRows(table, BASE_ROWS);
+
+ // a projection touching only base columns reads up to the high watermark as today,
+ // even though no enrichment was written yet
+ try (LogScanner scanner = createLogScanner(table, new int[] {0, 1})) {
+ scanner.subscribe(0, 0L);
+ List records = pollUpTo(scanner, BASE_ROWS, Duration.ofSeconds(10));
+ assertThat(records).hasSize(BASE_ROWS);
+ for (int i = 0; i < BASE_ROWS; i++) {
+ ScanRecord record = records.get(i);
+ assertThat(record.logOffset()).isEqualTo(i);
+ assertThat(record.timestamp()).isGreaterThan(0L);
+ assertThat(record.getRow().getInt(0)).isEqualTo(i);
+ assertThat(record.getRow().getString(1).toString()).isEqualTo("name" + i);
+ }
+ }
+
+ // a projection touching a column group is gated at the group's high watermark (0)
+ try (LogScanner scanner = createLogScanner(table, new int[] {0, 2})) {
+ scanner.subscribe(0, 0L);
+ assertThat(pollUpTo(scanner, 1, Duration.ofSeconds(2))).isEmpty();
+ }
+ }
+ }
+
+ @Test
+ void testEnrichmentIsStitchedAndGatedPerGroup() throws Exception {
+ TablePath tablePath = createColumnGroupTable("stitch");
+ try (Table table = conn.getTable(tablePath)) {
+ writeBaseRows(table, BASE_ROWS);
+ long tableId = table.getTableInfo().getTableId();
+ TableBucket bucket = new TableBucket(tableId, 0);
+ AppendWriter writer = table.newAppend().createWriter();
+
+ // remember the base batch timestamps: stitched rows must carry the same ones
+ Map baseTimestamps = new HashMap<>();
+ try (LogScanner scanner = createLogScanner(table, new int[] {0})) {
+ scanner.subscribe(0, 0L);
+ for (ScanRecord record : pollUpTo(scanner, BASE_ROWS, Duration.ofSeconds(10))) {
+ baseTimestamps.put(record.logOffset(), record.timestamp());
+ }
+ }
+ assertThat(baseTimestamps).hasSize(BASE_ROWS);
+
+ // fill the first half of geo_group
+ AppendColumnsResult result =
+ writer.appendColumns("geo_group", bucket, 0L, geoRows(0, 5)).get();
+ assertThat(result.getLogEndOffset()).isEqualTo(5L);
+ assertThat(result.getHighWatermark()).isEqualTo(5L);
+
+ // a read projecting geo (in a different order than the schema) sees exactly the
+ // filled prefix, with base values, group values and base timestamps
+ try (LogScanner scanner = createLogScanner(table, new int[] {2, 0})) {
+ scanner.subscribe(0, 0L);
+ List records = pollUpTo(scanner, 5, Duration.ofSeconds(10));
+ assertThat(records).hasSize(5);
+ for (int i = 0; i < 5; i++) {
+ ScanRecord record = records.get(i);
+ assertThat(record.logOffset()).isEqualTo(i);
+ assertThat(record.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(record.getRow().getString(0).toString()).isEqualTo("geo" + i);
+ assertThat(record.getRow().getInt(1)).isEqualTo(i);
+ assertThat(record.timestamp()).isEqualTo(baseTimestamps.get((long) i));
+ }
+ // the rest is gated until enrichment lands
+ assertThat(pollUpTo(scanner, 1, Duration.ofSeconds(1))).isEmpty();
+
+ writer.appendColumns("geo_group", bucket, 5L, geoRows(5, BASE_ROWS)).get();
+ records = pollUpTo(scanner, 5, Duration.ofSeconds(10));
+ assertThat(records).hasSize(5);
+ for (int i = 0; i < 5; i++) {
+ assertThat(records.get(i).logOffset()).isEqualTo(5 + i);
+ assertThat(records.get(i).getRow().getString(0).toString())
+ .isEqualTo("geo" + (5 + i));
+ }
+ }
+
+ // SELECT * touches risk_group too, which is still empty: nothing is visible yet
+ try (LogScanner scanner = createLogScanner(table)) {
+ scanner.subscribe(0, 0L);
+ assertThat(pollUpTo(scanner, 1, Duration.ofSeconds(1))).isEmpty();
+
+ // fill risk_group in batches that do not line up with the base batch
+ writer.appendColumns("risk_group", bucket, 0L, scoreRows(0, 3)).get();
+ writer.appendColumns("risk_group", bucket, 3L, scoreRows(3, 6)).get();
+ writer.appendColumns("risk_group", bucket, 6L, scoreRows(6, 9)).get();
+ AppendColumnsResult last =
+ writer.appendColumns("risk_group", bucket, 9L, scoreRows(9, 10)).get();
+ assertThat(last.getLogEndOffset()).isEqualTo(BASE_ROWS);
+
+ List records = pollUpTo(scanner, BASE_ROWS, Duration.ofSeconds(10));
+ assertThat(records).hasSize(BASE_ROWS);
+ for (int i = 0; i < BASE_ROWS; i++) {
+ InternalRow row = records.get(i).getRow();
+ assertThat(records.get(i).logOffset()).isEqualTo(i);
+ assertThat(row.getFieldCount()).isEqualTo(4);
+ assertThat(row.getInt(0)).isEqualTo(i);
+ assertThat(row.getString(1).toString()).isEqualTo("name" + i);
+ assertThat(row.getString(2).toString()).isEqualTo("geo" + i);
+ assertThat(row.getDouble(3)).isEqualTo(i * 0.5d);
+ assertThat(records.get(i).timestamp()).isEqualTo(baseTimestamps.get((long) i));
+ }
+ }
+
+ // a scan starting inside the base batch and inside a group batch stitches correctly
+ try (LogScanner scanner = createLogScanner(table, new int[] {3, 2, 1})) {
+ scanner.subscribe(0, 4L);
+ List records = pollUpTo(scanner, 6, Duration.ofSeconds(10));
+ assertThat(records).hasSize(6);
+ for (int i = 0; i < 6; i++) {
+ int offset = 4 + i;
+ InternalRow row = records.get(i).getRow();
+ assertThat(records.get(i).logOffset()).isEqualTo(offset);
+ assertThat(row.getDouble(0)).isEqualTo(offset * 0.5d);
+ assertThat(row.getString(1).toString()).isEqualTo("geo" + offset);
+ assertThat(row.getString(2).toString()).isEqualTo("name" + offset);
+ }
+ }
+
+ // a projection with only group columns still advances (a carrier base column is
+ // fetched and dropped)
+ try (LogScanner scanner = createLogScanner(table, new int[] {2})) {
+ scanner.subscribe(0, 0L);
+ List records = pollUpTo(scanner, BASE_ROWS, Duration.ofSeconds(10));
+ assertThat(records).hasSize(BASE_ROWS);
+ assertThat(records.get(7).getRow().getFieldCount()).isEqualTo(1);
+ assertThat(records.get(7).getRow().getString(0).toString()).isEqualTo("geo7");
+ }
+ }
+ }
+
+ @Test
+ void testColumnGroupSurvivesTabletServerRestart() throws Exception {
+ TablePath tablePath = createColumnGroupTable("restart");
+ try (Table table = conn.getTable(tablePath)) {
+ writeBaseRows(table, BASE_ROWS);
+ TableBucket bucket = new TableBucket(table.getTableInfo().getTableId(), 0);
+ AppendWriter writer = table.newAppend().createWriter();
+ writer.appendColumns("geo_group", bucket, 0L, geoRows(0, 7)).get();
+
+ // the (only) replica is the leader; restart the tablet server hosting it
+ int leaderServer = -1;
+ for (TabletServer server : FLUSS_CLUSTER_EXTENSION.getTabletServers()) {
+ if (server.getReplicaManager().getReplica(bucket)
+ instanceof ReplicaManager.OnlineReplica) {
+ leaderServer = server.getServerId();
+ }
+ }
+ assertThat(leaderServer).isNotNegative();
+ FLUSS_CLUSTER_EXTENSION.stopTabletServer(leaderServer);
+ FLUSS_CLUSTER_EXTENSION.startTabletServer(leaderServer);
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllReplicaReady(bucket);
+
+ // the group log was recovered from disk and the watermark re-established on
+ // leadership, so the filled prefix is visible and appends continue after it
+ try (LogScanner scanner = createLogScanner(table, new int[] {0, 2})) {
+ scanner.subscribe(0, 0L);
+ List records = pollUpTo(scanner, 7, Duration.ofSeconds(20));
+ assertThat(records).hasSize(7);
+ assertThat(records.get(6).getRow().getString(1).toString()).isEqualTo("geo6");
+ assertThat(pollUpTo(scanner, 1, Duration.ofSeconds(1))).isEmpty();
+
+ AppendColumnsResult result =
+ writer.appendColumns("geo_group", bucket, 7L, geoRows(7, BASE_ROWS)).get();
+ assertThat(result.getLogEndOffset()).isEqualTo(BASE_ROWS);
+ assertThat(pollUpTo(scanner, 3, Duration.ofSeconds(10))).hasSize(3);
+ }
+ }
+ }
+
+ @Test
+ void testAppendColumnsValidation() throws Exception {
+ TablePath tablePath = createColumnGroupTable("validation");
+ try (Table table = conn.getTable(tablePath)) {
+ writeBaseRows(table, BASE_ROWS);
+ TableBucket bucket = new TableBucket(table.getTableInfo().getTableId(), 0);
+ AppendWriter writer = table.newAppend().createWriter();
+
+ // unknown group is rejected on the client
+ assertThatThrownBy(
+ () -> writer.appendColumns("no_such_group", bucket, 0L, geoRows(0, 1)))
+ .isInstanceOf(UnknownColumnGroupException.class);
+
+ // a gap: the group expects offset 0
+ assertThatThrownBy(
+ () ->
+ writer.appendColumns("geo_group", bucket, 2L, geoRows(2, 4))
+ .get())
+ .isInstanceOf(ExecutionException.class)
+ .hasCauseInstanceOf(InvalidColumnGroupOffsetException.class)
+ .satisfies(
+ e ->
+ assertThat(
+ ((InvalidColumnGroupOffsetException)
+ e.getCause())
+ .getExpectedSourceOffset())
+ .isEqualTo(0L));
+
+ // running past the base high watermark
+ assertThatThrownBy(
+ () ->
+ writer.appendColumns(
+ "geo_group",
+ bucket,
+ 0L,
+ geoRows(0, BASE_ROWS + 1))
+ .get())
+ .hasCauseInstanceOf(InvalidColumnGroupOffsetException.class)
+ .hasMessageContaining("high watermark");
+
+ // a valid write, then a whole-batch replay is acknowledged without effect
+ assertThat(
+ writer.appendColumns("geo_group", bucket, 0L, geoRows(0, 5))
+ .get()
+ .getLogEndOffset())
+ .isEqualTo(5L);
+ assertThat(
+ writer.appendColumns("geo_group", bucket, 0L, geoRows(0, 5))
+ .get()
+ .getLogEndOffset())
+ .isEqualTo(5L);
+ assertThat(
+ writer.appendColumns("geo_group", bucket, 1L, geoRows(1, 3))
+ .get()
+ .getLogEndOffset())
+ .isEqualTo(5L);
+
+ // a batch straddling the log end offset is rejected with the expected offset so the
+ // client can re-slice
+ assertThatThrownBy(
+ () ->
+ writer.appendColumns("geo_group", bucket, 3L, geoRows(3, 8))
+ .get())
+ .hasCauseInstanceOf(InvalidColumnGroupOffsetException.class)
+ .satisfies(
+ e ->
+ assertThat(
+ ((InvalidColumnGroupOffsetException)
+ e.getCause())
+ .getExpectedSourceOffset())
+ .isEqualTo(5L));
+
+ // wrong arity is rejected on the client
+ assertThatThrownBy(
+ () ->
+ writer.appendColumns(
+ "geo_group",
+ bucket,
+ 5L,
+ Arrays.asList(row(BinaryString.fromString("x"), 1.0d))))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/ColumnGroupSourceOffsetTruncatedException.java b/fluss-common/src/main/java/org/apache/fluss/exception/ColumnGroupSourceOffsetTruncatedException.java
new file mode 100644
index 00000000000..220523c9c65
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/ColumnGroupSourceOffsetTruncatedException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.exception;
+
+import org.apache.fluss.annotation.PublicEvolving;
+
+/**
+ * Thrown when an {@code appendColumns} write targets a source offset that has already fallen below
+ * the bucket's local log start — the base record has aged out under retention and the row can no
+ * longer be enriched. This is terminal for that offset; the client should record the loss and
+ * advance, not retry.
+ *
+ * @since 0.10
+ */
+@PublicEvolving
+public class ColumnGroupSourceOffsetTruncatedException extends ApiException {
+ public ColumnGroupSourceOffsetTruncatedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ColumnGroupSourceOffsetTruncatedException(String message) {
+ super(message);
+ }
+
+ public ColumnGroupSourceOffsetTruncatedException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupConfigException.java b/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupConfigException.java
new file mode 100644
index 00000000000..7c71101a28d
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupConfigException.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.exception;
+
+import org.apache.fluss.annotation.PublicEvolving;
+
+/**
+ * Thrown at {@code CREATE TABLE} time when a column-group declaration is structurally invalid — for
+ * example, when a partition-key column is included in a {@code column-groups.} property, when a
+ * group references an unknown column, when membership is duplicated across groups, or when a group
+ * is paired with a primary key. Detected before the table is created; the error message identifies
+ * the offending column or group so the user can correct the DDL.
+ *
+ * @since 0.10
+ */
+@PublicEvolving
+public class InvalidColumnGroupConfigException extends ApiException {
+ public InvalidColumnGroupConfigException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public InvalidColumnGroupConfigException(String message) {
+ super(message);
+ }
+
+ public InvalidColumnGroupConfigException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupOffsetException.java b/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupOffsetException.java
new file mode 100644
index 00000000000..592b8d8e6fd
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/InvalidColumnGroupOffsetException.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.exception;
+
+import org.apache.fluss.annotation.PublicEvolving;
+
+/**
+ * Thrown when an {@code appendColumns} write violates the strict-from-EWM ordering invariant — the
+ * supplied {@code source_offset} does not equal the next slot after the current per-bucket
+ * Enrichment Watermark (EWM). The client typically reaches this state when its EWM cache is stale
+ * relative to the leader; on receiving this error, the client batching layer should refresh the EWM
+ * and drop any in-flight batches whose first offset is no longer the leader's expected next slot
+ * rather than blindly retrying.
+ *
+ * @since 0.10
+ */
+@PublicEvolving
+public class InvalidColumnGroupOffsetException extends ApiException {
+ private static final long serialVersionUID = 1L;
+
+ /** The source offset the server expected, or -1 when unknown. */
+ private final long expectedSourceOffset;
+
+ public InvalidColumnGroupOffsetException(String message, long expectedSourceOffset) {
+ super(message);
+ this.expectedSourceOffset = expectedSourceOffset;
+ }
+
+ public InvalidColumnGroupOffsetException(String message, Throwable cause) {
+ super(message, cause);
+ this.expectedSourceOffset = -1L;
+ }
+
+ public long getExpectedSourceOffset() {
+ return expectedSourceOffset;
+ }
+
+ public InvalidColumnGroupOffsetException(String message) {
+ super(message);
+ this.expectedSourceOffset = -1L;
+ }
+
+ public InvalidColumnGroupOffsetException(Throwable cause) {
+ super(cause);
+ this.expectedSourceOffset = -1L;
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/UnknownColumnGroupException.java b/fluss-common/src/main/java/org/apache/fluss/exception/UnknownColumnGroupException.java
new file mode 100644
index 00000000000..0b743b36ed3
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/UnknownColumnGroupException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.exception;
+
+import org.apache.fluss.annotation.PublicEvolving;
+
+/**
+ * Thrown when an {@code appendColumns} request or a fetch projection references a column group name
+ * that is not declared on the table. Usually indicates a client / server schema-version skew; the
+ * client should refresh table metadata before surfacing the error to the user.
+ *
+ * @since 0.10
+ */
+@PublicEvolving
+public class UnknownColumnGroupException extends ApiException {
+ public UnknownColumnGroupException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public UnknownColumnGroupException(String message) {
+ super(message);
+ }
+
+ public UnknownColumnGroupException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/ColumnGroupSchemaGetter.java b/fluss-common/src/main/java/org/apache/fluss/metadata/ColumnGroupSchemaGetter.java
new file mode 100644
index 00000000000..2fec1f2d77a
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/ColumnGroupSchemaGetter.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.metadata;
+
+import org.apache.fluss.annotation.Internal;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+
+/**
+ * A {@link SchemaGetter} view of a column-group table (FIP-45) that answers the physical
+ * schema of one of its logs: either the base log (the default-group columns only) or the shadow log
+ * of one column group (that group's columns only). Anything that decodes or projects physical
+ * batches of such a table must resolve schemas through one of these views.
+ */
+@Internal
+public final class ColumnGroupSchemaGetter implements SchemaGetter {
+
+ private final SchemaGetter delegate;
+ private final Function mapper;
+ private final Map cache = new ConcurrentHashMap<>();
+
+ private ColumnGroupSchemaGetter(SchemaGetter delegate, Function mapper) {
+ this.delegate = delegate;
+ this.mapper = mapper;
+ }
+
+ /** A view answering the base physical schema (default-group columns). */
+ public static ColumnGroupSchemaGetter base(SchemaGetter delegate) {
+ return new ColumnGroupSchemaGetter(delegate, ColumnGroupSchemaGetter::toBaseSchema);
+ }
+
+ /** A view answering the physical schema of column group {@code groupName}. */
+ public static ColumnGroupSchemaGetter group(SchemaGetter delegate, String groupName) {
+ return new ColumnGroupSchemaGetter(delegate, schema -> toGroupSchema(schema, groupName));
+ }
+
+ /** The base physical schema of {@code schema}: only its default-group columns. */
+ public static Schema toBaseSchema(Schema schema) {
+ if (!schema.hasColumnGroups()) {
+ return schema;
+ }
+ return subset(schema, schema.getDefaultGroupColumnIndices());
+ }
+
+ /** The physical schema of column group {@code groupName}: only that group's columns. */
+ public static Schema toGroupSchema(Schema schema, String groupName) {
+ return subset(schema, schema.getColumnGroupColumnIndices(groupName));
+ }
+
+ private static Schema subset(Schema schema, int[] indices) {
+ List columns = new ArrayList<>(indices.length);
+ for (int index : indices) {
+ // drop the group tag: the physical schema of a log has no groups of its own
+ columns.add(schema.getColumns().get(index).withColumnGroup(null));
+ }
+ return Schema.newBuilder().fromColumns(columns).build();
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ return cache.computeIfAbsent(schemaId, id -> mapper.apply(delegate.getSchema(id)));
+ }
+
+ @Override
+ public CompletableFuture getSchemaInfoAsync(int schemaId) {
+ return delegate.getSchemaInfoAsync(schemaId)
+ .thenApply(
+ info -> new SchemaInfo(mapper.apply(info.getSchema()), info.getSchemaId()));
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ SchemaInfo latest = delegate.getLatestSchemaInfo();
+ return new SchemaInfo(mapper.apply(latest.getSchema()), latest.getSchemaId());
+ }
+
+ @Override
+ public void release() {
+ // the delegate is owned by its creator
+ cache.clear();
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
index 92b675c428c..4757dbfc7ef 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
@@ -19,6 +19,7 @@
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.annotation.PublicStable;
+import org.apache.fluss.exception.InvalidColumnGroupConfigException;
import org.apache.fluss.types.ArrayType;
import org.apache.fluss.types.DataField;
import org.apache.fluss.types.DataType;
@@ -37,12 +38,15 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -224,6 +228,81 @@ public int getHighestFieldId() {
return highestFieldId;
}
+ /**
+ * Returns a map of column group names to their column indices. Only includes columns that have
+ * a column group assigned.
+ */
+ public Map> getColumnGroups() {
+ Map> groups = new HashMap<>();
+ for (int i = 0; i < columns.size(); i++) {
+ Optional group = columns.get(i).getColumnGroup();
+ if (group.isPresent()) {
+ groups.computeIfAbsent(group.get(), k -> new ArrayList<>()).add(i);
+ }
+ }
+ return groups;
+ }
+
+ /** Returns the set of column group names defined in this schema. */
+ public Set getColumnGroupNames() {
+ return getColumnGroups().keySet();
+ }
+
+ /** Returns indices of columns that are NOT in any column group (the default group). */
+ public int[] getDefaultGroupColumnIndices() {
+ return IntStream.range(0, columns.size())
+ .filter(i -> !columns.get(i).getColumnGroup().isPresent())
+ .toArray();
+ }
+
+ /** Whether this schema declares at least one non-default column group. */
+ public boolean hasColumnGroups() {
+ for (Column column : columns) {
+ if (column.getColumnGroup().isPresent()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * The physical row type of the base log: only the columns of the default group, in schema
+ * order. For a schema without column groups this equals {@link #getRowType()}.
+ */
+ public RowType getBaseRowType() {
+ return rowType.project(getDefaultGroupColumnIndices());
+ }
+
+ /**
+ * The physical row type of the column-group log for {@code groupName}: the group's columns in
+ * schema order.
+ *
+ * @throws IllegalArgumentException if the group is not declared on this schema
+ */
+ public RowType getColumnGroupRowType(String groupName) {
+ return rowType.project(getColumnGroupColumnIndices(groupName));
+ }
+
+ /**
+ * Indices (in schema order) of the columns belonging to {@code groupName}.
+ *
+ * @throws IllegalArgumentException if the group is not declared on this schema
+ */
+ public int[] getColumnGroupColumnIndices(String groupName) {
+ List indices = getColumnGroups().get(groupName);
+ if (indices == null) {
+ throw new IllegalArgumentException(
+ String.format("Column group '%s' does not exist in schema.", groupName));
+ }
+ return indices.stream().mapToInt(Integer::intValue).toArray();
+ }
+
+ /** The column group of the column at {@code columnIndex}, or null for the default group. */
+ @Nullable
+ public String getColumnGroupOf(int columnIndex) {
+ return columns.get(columnIndex).getColumnGroup().orElse(null);
+ }
+
@Override
public String toString() {
return "Schema{"
@@ -260,6 +339,55 @@ public int hashCode() {
// --------------------------------------------------------------------------------------------
+ /**
+ * Max length of a column group name. Unlike database/table names (see {@link TablePath}), a
+ * column group name never becomes part of a filesystem path, so this is a plain sanity bound
+ * rather than a path-length budget. Group names only live in the schema and in the {@code
+ * column_group} RPC field.
+ */
+ private static final int MAX_COLUMN_GROUP_NAME_LENGTH = 64;
+
+ /**
+ * Validates a column group name, throwing {@link InvalidColumnGroupConfigException} when it is
+ * invalid. Column group names obey the same identifier rules as Fluss database/table names: a
+ * non-empty string of ASCII alphanumerics, {@code '_'} and {@code '-'}, not starting with the
+ * reserved {@code "__"} prefix, and no longer than {@link #MAX_COLUMN_GROUP_NAME_LENGTH}
+ * characters.
+ */
+ private static void validateColumnGroupName(String groupName) {
+ checkNotNull(groupName, "Column group name must not be null.");
+ String error = detectInvalidColumnGroupName(groupName);
+ if (error != null) {
+ throw new InvalidColumnGroupConfigException(
+ "Column group name '" + groupName + "' is invalid: " + error);
+ }
+ }
+
+ /**
+ * Returns a human-readable reason why {@code groupName} is not a valid column group name, or
+ * {@code null} if it is valid. Shared with callers (e.g. the Flink DDL parser) that need to
+ * surface the same constraint under their own exception type. {@code groupName} must be
+ * non-null.
+ */
+ public static @Nullable String detectInvalidColumnGroupName(String groupName) {
+ if (groupName.isEmpty()) {
+ return "the empty string is not allowed";
+ }
+ if (groupName.length() > MAX_COLUMN_GROUP_NAME_LENGTH) {
+ return "the length is longer than the max allowed length "
+ + MAX_COLUMN_GROUP_NAME_LENGTH;
+ }
+ if (groupName.startsWith(TablePath.INTERNAL_NAME_PREFIX)) {
+ return "'"
+ + TablePath.INTERNAL_NAME_PREFIX
+ + "' is not allowed as prefix, since it is reserved for internal names in Fluss";
+ }
+ if (TablePath.containsInvalidPattern(groupName)) {
+ return "it contains one or more characters other than ASCII alphanumerics, '_' and '-'";
+ }
+ return null;
+ }
+
/** Builder for configuring and creating instances of {@link Schema}. */
public static Schema.Builder newBuilder() {
return new Builder();
@@ -365,7 +493,8 @@ public Builder fromColumns(List inputColumns) {
column.dataType,
column.comment,
newColumnId,
- column.aggFunction));
+ column.aggFunction,
+ column.columnGroup));
}
}
@@ -475,6 +604,23 @@ public Builder column(String columnName, DataType dataType, AggFunction aggFunct
return this;
}
+ /**
+ * Declares a column that is appended to this schema and assigned to a named column group.
+ *
+ * Equivalent to {@link #column(String, DataType)} immediately followed by {@link
+ * #columnGroup(String)}.
+ *
+ * @param columnName column name
+ * @param dataType column data type
+ * @param columnGroup the column group this column belongs to
+ * @return this builder for fluent API
+ */
+ public Builder column(String columnName, DataType dataType, String columnGroup) {
+ checkNotNull(columnGroup, "Column group name must not be null.");
+ column(columnName, dataType);
+ return columnGroup(columnGroup);
+ }
+
/** Apply comment to the previous column. */
public Builder withComment(@Nullable String comment) {
if (!columns.isEmpty()) {
@@ -488,6 +634,53 @@ public Builder withComment(@Nullable String comment) {
return this;
}
+ /** Assign the previous column to a column group. */
+ public Builder columnGroup(String groupName) {
+ validateColumnGroupName(groupName);
+ if (!columns.isEmpty()) {
+ columns.set(
+ columns.size() - 1,
+ columns.get(columns.size() - 1).withColumnGroup(groupName));
+ } else {
+ throw new IllegalArgumentException(
+ "Method 'columnGroup(...)' must be called after a column definition, "
+ + "but there is no preceding column defined.");
+ }
+ return this;
+ }
+
+ /**
+ * Scopes a block of column declarations so each column added inside the block is assigned
+ * to the given column group. Columns added inside the block that explicitly specify a
+ * different group (e.g. via {@link #column(String, DataType, String)}) keep their explicit
+ * group; columns added before the block are not affected.
+ *
+ *
Example:
+ *
+ *
{@code
+ * .columnGroup("enriched_risk", g -> g
+ * .column("risk_score", DataTypes.DOUBLE())
+ * .column("risk_classification", DataTypes.STRING()))
+ * }
+ *
+ * @param groupName the column group name
+ * @param block a consumer that adds the group's columns to this builder
+ * @return this builder for fluent API
+ */
+ public Builder columnGroup(String groupName, Consumer block) {
+ validateColumnGroupName(groupName);
+ checkNotNull(block, "Column group block must not be null.");
+ int startIndex = columns.size();
+ block.accept(this);
+ for (int i = startIndex; i < columns.size(); i++) {
+ Column current = columns.get(i);
+ if (!current.getColumnGroup().isPresent()) {
+ columns.set(i, current.withColumnGroup(groupName));
+ }
+ }
+ return this;
+ }
+
/**
* Declares a primary key constraint for a set of given columns. Primary key uniquely
* identify a row in a table. Neither of columns in a primary can be nullable. Adding a
@@ -589,18 +782,19 @@ public static final class Column implements Serializable {
private final DataType dataType;
private final @Nullable String comment;
private final @Nullable AggFunction aggFunction;
+ private final @Nullable String columnGroup;
public Column(String columnName, DataType dataType) {
- this(columnName, dataType, null, UNKNOWN_COLUMN_ID, null);
+ this(columnName, dataType, null, UNKNOWN_COLUMN_ID, null, null);
}
public Column(String columnName, DataType dataType, @Nullable String comment) {
- this(columnName, dataType, comment, UNKNOWN_COLUMN_ID, null);
+ this(columnName, dataType, comment, UNKNOWN_COLUMN_ID, null, null);
}
public Column(
String columnName, DataType dataType, @Nullable String comment, int columnId) {
- this(columnName, dataType, comment, columnId, null);
+ this(columnName, dataType, comment, columnId, null, null);
}
public Column(
@@ -609,11 +803,22 @@ public Column(
@Nullable String comment,
int columnId,
@Nullable AggFunction aggFunction) {
+ this(columnName, dataType, comment, columnId, aggFunction, null);
+ }
+
+ public Column(
+ String columnName,
+ DataType dataType,
+ @Nullable String comment,
+ int columnId,
+ @Nullable AggFunction aggFunction,
+ @Nullable String columnGroup) {
this.columnName = columnName;
this.dataType = dataType;
this.comment = comment;
this.columnId = columnId;
this.aggFunction = aggFunction;
+ this.columnGroup = columnGroup;
}
public String getName() {
@@ -641,12 +846,21 @@ public Optional getAggFunction() {
return Optional.ofNullable(aggFunction);
}
+ /** Returns the column group name, if any. */
+ public Optional getColumnGroup() {
+ return Optional.ofNullable(columnGroup);
+ }
+
public Column withComment(String comment) {
- return new Column(columnName, dataType, comment, columnId, aggFunction);
+ return new Column(columnName, dataType, comment, columnId, aggFunction, columnGroup);
}
public Column withAggFunction(@Nullable AggFunction aggFunction) {
- return new Column(columnName, dataType, comment, columnId, aggFunction);
+ return new Column(columnName, dataType, comment, columnId, aggFunction, columnGroup);
+ }
+
+ public Column withColumnGroup(@Nullable String columnGroup) {
+ return new Column(columnName, dataType, comment, columnId, aggFunction, columnGroup);
}
@Override
@@ -660,6 +874,7 @@ public String toString() {
sb.append(EncodingUtils.escapeSingleQuotes(c));
sb.append("'");
});
+ getColumnGroup().ifPresent(g -> sb.append(" COLUMN GROUP '").append(g).append("'"));
return sb.toString();
}
@@ -676,12 +891,13 @@ public boolean equals(Object o) {
&& Objects.equals(dataType, that.dataType)
&& Objects.equals(comment, that.comment)
&& Objects.equals(columnId, that.columnId)
- && Objects.equals(aggFunction, that.aggFunction);
+ && Objects.equals(aggFunction, that.aggFunction)
+ && Objects.equals(columnGroup, that.columnGroup);
}
@Override
public int hashCode() {
- return Objects.hash(columnName, dataType, comment, columnId, aggFunction);
+ return Objects.hash(columnName, dataType, comment, columnId, aggFunction, columnGroup);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableDescriptor.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableDescriptor.java
index fc19f6b7eab..f4c9f4858f0 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableDescriptor.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableDescriptor.java
@@ -22,6 +22,7 @@
import org.apache.fluss.config.ConfigOption;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.ConfigurationUtils;
+import org.apache.fluss.exception.InvalidColumnGroupConfigException;
import org.apache.fluss.utils.json.JsonSerdeUtils;
import org.apache.fluss.utils.json.TableDescriptorJsonSerde;
@@ -117,6 +118,31 @@ private TableDescriptor(
f));
}
+ // Phase M.2: partition keys must belong to the default column group. Enrichment-group
+ // columns are NULL at base-write time and cannot determine the row's partition.
+ Map> columnGroups = schema.getColumnGroups();
+ if (!columnGroups.isEmpty() && !partitionKeys.isEmpty()) {
+ Set groupedColumnNames =
+ columnGroups.values().stream()
+ .flatMap(List::stream)
+ .map(idx -> schema.getColumns().get(idx).getName())
+ .collect(Collectors.toSet());
+ List illegal =
+ partitionKeys.stream()
+ .filter(groupedColumnNames::contains)
+ .collect(Collectors.toList());
+ if (!illegal.isEmpty()) {
+ throw new InvalidColumnGroupConfigException(
+ String.format(
+ "Partition keys must belong to the default column group "
+ + "(enrichment-group columns are NULL at base-write "
+ + "time and cannot determine the partition). "
+ + "Offending partition keys declared in non-default "
+ + "groups: %s",
+ illegal));
+ }
+ }
+
if (this.tableDistribution != null) {
this.tableDistribution
.getBucketKeys()
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TablePath.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TablePath.java
index eef9e2adea8..1092e4e613f 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TablePath.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TablePath.java
@@ -61,7 +61,9 @@ public class TablePath implements Serializable {
// are not
// allowed to start with this prefix to prevent conflicts with system-generated identifiers.
// This convention aligns with and maintains compatibility with Apache Kafka's naming standards.
- private static final String INTERNAL_NAME_PREFIX = "__";
+ // Package-private so other identifier validators in this package (e.g. column group names in
+ // Schema) can share the same reserved-prefix convention.
+ static final String INTERNAL_NAME_PREFIX = "__";
public TablePath(String databaseName, String tableName) {
this.databaseName = databaseName;
@@ -196,8 +198,13 @@ public static String detectInvalidName(String identifier) {
return null;
}
- /** Valid characters for Fluss table names are the ASCII alphanumerics, '_' and '-'. */
- private static boolean containsInvalidPattern(String identifier) {
+ /**
+ * Valid characters for Fluss table names are the ASCII alphanumerics, '_' and '-'.
+ *
+ * Package-private so sibling identifier validators in this package (e.g. column group names
+ * in {@link Schema}) can apply the same character whitelist.
+ */
+ static boolean containsInvalidPattern(String identifier) {
for (int i = 0; i < identifier.length(); ++i) {
char c = identifier.charAt(i);
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
index 35f4202e02c..c001546e52c 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
@@ -56,6 +56,7 @@
import java.util.List;
import static org.apache.fluss.record.DefaultLogRecordBatch.APPEND_ONLY_FLAG_MASK;
+import static org.apache.fluss.record.LogRecordBatchFormat.BASE_OFFSET_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V0;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
@@ -66,6 +67,7 @@
import static org.apache.fluss.record.LogRecordBatchFormat.V1_RECORD_BATCH_HEADER_SIZE;
import static org.apache.fluss.record.LogRecordBatchFormat.V2_RECORD_BATCH_HEADER_SIZE;
import static org.apache.fluss.record.LogRecordBatchFormat.attributeOffset;
+import static org.apache.fluss.record.LogRecordBatchFormat.lastOffsetDeltaOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize;
import static org.apache.fluss.record.LogRecordBatchFormat.recordsCountOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.schemaIdOffset;
@@ -116,6 +118,17 @@ public FileLogProjection(ProjectionPushdownCache projectionsCache) {
this.arrowHeaderBuffer.order(ByteOrder.LITTLE_ENDIAN);
}
+ /** The last offset of the last batch included by the most recent {@link #project} call. */
+ private long lastProjectedOffset = -1L;
+
+ /**
+ * The last offset of the last non-empty batch included by the most recent {@link #project}
+ * call, or -1 if none. Used to size the column-group companion read (FIP-45).
+ */
+ public long lastProjectedOffset() {
+ return lastProjectedOffset;
+ }
+
public void setCurrentProjection(
long tableId,
SchemaGetter schemaGetter,
@@ -185,6 +198,7 @@ public BytesViewLogRecords project(FileChannel channel, int start, int end, int
MultiBytesView.Builder builder = MultiBytesView.builder();
int position = start;
+ lastProjectedOffset = -1L;
ProjectionInfo currentProjection = null;
short prevSchemaId = -1;
@@ -234,6 +248,9 @@ public BytesViewLogRecords project(FileChannel channel, int start, int end, int
// the projected batch exceeds the remaining budget, stop here
return new BytesViewLogRecords(builder.build());
}
+ lastProjectedOffset =
+ logHeaderBuffer.getLong(BASE_OFFSET_OFFSET)
+ + logHeaderBuffer.getInt(lastOffsetDeltaOffset(magic));
maxBytes -= newBatchSizeInBytes;
position += batchSizeInBytes;
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index cc5bcbdbae1..fb9a78ba4ad 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -118,10 +118,36 @@ public static LogRecordReadContext createReadContext(
@Nullable Projection projection,
SchemaGetter schemaGetter,
AllocationManager.Factory allocationManagerFactory) {
+ return createReadContext(
+ tableInfo.getTableId(),
+ tableInfo.getTableConfig().getLogFormat(),
+ tableInfo.getSchemaId(),
+ tableInfo.getRowType(),
+ readFromRemote,
+ schemaResolution,
+ projection,
+ schemaGetter,
+ allocationManagerFactory);
+ }
+
+ /**
+ * Creates a {@link LogRecordReadContext} for a log whose physical row type is {@code rowType}.
+ * Used for the base log and the column-group logs of a column-group table (FIP-45), whose
+ * physical rows are subsets of the table row; {@code schemaGetter} must answer the matching
+ * physical schemas (see {@link org.apache.fluss.metadata.ColumnGroupSchemaGetter}).
+ */
+ public static LogRecordReadContext createReadContext(
+ long tableId,
+ LogFormat logFormat,
+ int schemaId,
+ RowType rowType,
+ boolean readFromRemote,
+ SchemaResolution schemaResolution,
+ @Nullable Projection projection,
+ SchemaGetter schemaGetter,
+ AllocationManager.Factory allocationManagerFactory) {
checkNotNull(schemaResolution, "schemaResolution");
boolean readAsTargetSchema = schemaResolution == SchemaResolution.TARGET;
- RowType rowType = tableInfo.getRowType();
- LogFormat logFormat = tableInfo.getTableConfig().getLogFormat();
// only for arrow log format, the projection can be push downed to the server side
boolean projectionPushDowned =
logFormat == LogFormat.ARROW && !readFromRemote && projection != null;
@@ -131,7 +157,6 @@ public static LogRecordReadContext createReadContext(
// the reader dynamically adapts to each batch's schema.
ReadTarget target = null;
if (readAsTargetSchema || projection != null) {
- int schemaId = tableInfo.getSchemaId();
if (projection == null) {
// set a default dummy projection to simplify code
projection = Projection.of(IntStream.range(0, rowType.getFieldCount()).toArray());
@@ -150,15 +175,11 @@ public static LogRecordReadContext createReadContext(
if (logFormat == LogFormat.ARROW) {
return createArrowReadContext(
- tableInfo.getTableId(),
- target,
- projectionPushDowned,
- schemaGetter,
- allocationManagerFactory);
+ tableId, target, projectionPushDowned, schemaGetter, allocationManagerFactory);
} else if (logFormat == LogFormat.INDEXED) {
- return createIndexedReadContext(tableInfo.getTableId(), target, schemaGetter);
+ return createIndexedReadContext(tableId, target, schemaGetter);
} else if (logFormat == LogFormat.COMPACTED) {
- return createCompactedRowReadContext(tableInfo.getTableId(), target, schemaGetter);
+ return createCompactedRowReadContext(tableId, target, schemaGetter);
} else {
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java
index fef00c6c558..a33a59b6ce3 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java
@@ -27,6 +27,8 @@
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.utils.types.Tuple2;
+import javax.annotation.Nullable;
+
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -71,6 +73,31 @@ public class FlussPaths {
/** Suffix of a writer snapshot file. */
public static final String WRITER_SNAPSHOT_FILE_SUFFIX = ".writer_snapshot";
+ /** Infix for column group segment files. */
+ public static final String COLUMN_GROUP_INFIX = ".col.";
+
+ /** Prefix of the per-column-group log directory under a log tablet directory (FIP-45). */
+ public static final String COLUMN_GROUP_DIR_PREFIX = "col-";
+
+ /**
+ * The directory holding the shadow log segments of column group {@code groupName}: {@code
+ * {logTabletDir}/col-{groupName}/}.
+ */
+ public static File columnGroupLogDir(File logTabletDir, String groupName) {
+ return new File(logTabletDir, COLUMN_GROUP_DIR_PREFIX + groupName);
+ }
+
+ /** The column group name of a column-group log directory, or null if it is not one. */
+ @Nullable
+ public static String columnGroupNameFromDir(File dir) {
+ String name = dir.getName();
+ if (name.startsWith(COLUMN_GROUP_DIR_PREFIX)
+ && name.length() > COLUMN_GROUP_DIR_PREFIX.length()) {
+ return name.substring(COLUMN_GROUP_DIR_PREFIX.length());
+ }
+ return null;
+ }
+
/** The directory name for storing remote log index files. */
public static final String REMOTE_LOG_INDEX_LOCAL_CACHE = "remote-log-index-cache";
@@ -346,6 +373,39 @@ public static File timeIndexFile(File dir, long offset) {
return new File(dir, filenamePrefixFromOffset(offset) + TIME_INDEX_FILE_SUFFIX);
}
+ /**
+ * Construct a column group log file name in the given dir with the given base offset and group
+ * name.
+ *
+ * @param logTabletDir The log tablet directory
+ * @param offset The base offset of the log file
+ * @param groupName The column group name
+ */
+ public static File columnGroupLogFile(File logTabletDir, long offset, String groupName) {
+ return new File(
+ logTabletDir,
+ filenamePrefixFromOffset(offset)
+ + COLUMN_GROUP_INFIX
+ + groupName
+ + LOG_FILE_SUFFIX);
+ }
+
+ /**
+ * Construct a column group offset index file name in the given dir.
+ *
+ * @param dir The directory
+ * @param offset The base offset
+ * @param groupName The column group name
+ */
+ public static File columnGroupOffsetIndexFile(File dir, long offset, String groupName) {
+ return new File(
+ dir,
+ filenamePrefixFromOffset(offset)
+ + COLUMN_GROUP_INFIX
+ + groupName
+ + INDEX_FILE_SUFFIX);
+ }
+
/**
* Returns a File instance with parent directory as logDir and the file name as writer snapshot
* file for the given offset.
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
index cbddfaceada..4c56f1964b6 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
@@ -46,6 +46,7 @@ public class ColumnJsonSerde
static final String AGG_FUNCTION = "agg_function";
static final String AGG_FUNCTION_TYPE = "type";
static final String AGG_FUNCTION_PARAMS = "parameters";
+ static final String COLUMN_GROUP = "column_group";
@Override
public void serialize(Schema.Column column, JsonGenerator generator) throws IOException {
@@ -72,6 +73,9 @@ public void serialize(Schema.Column column, JsonGenerator generator) throws IOEx
generator.writeEndObject();
}
generator.writeNumberField(ID, column.getColumnId());
+ if (column.getColumnGroup().isPresent()) {
+ generator.writeStringField(COLUMN_GROUP, column.getColumnGroup().get());
+ }
generator.writeEndObject();
}
@@ -105,11 +109,14 @@ public Schema.Column deserialize(JsonNode node) {
}
}
+ String columnGroup = node.hasNonNull(COLUMN_GROUP) ? node.get(COLUMN_GROUP).asText() : null;
+
return new Schema.Column(
columnName,
dataType,
node.hasNonNull(COMMENT) ? node.get(COMMENT).asText() : null,
node.has(ID) ? node.get(ID).asInt() : UNKNOWN_COLUMN_ID,
- aggFunction);
+ aggFunction,
+ columnGroup);
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/TableDescriptorTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/TableDescriptorTest.java
index bd996107522..2ec3405bcef 100644
--- a/fluss-common/src/test/java/org/apache/fluss/metadata/TableDescriptorTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/metadata/TableDescriptorTest.java
@@ -19,6 +19,7 @@
import org.apache.fluss.config.ConfigBuilder;
import org.apache.fluss.config.ConfigOption;
+import org.apache.fluss.exception.InvalidColumnGroupConfigException;
import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.api.Test;
@@ -319,6 +320,51 @@ void testPartitionedTable() {
"Bucket key [f0, f3] shouldn't include any column in partition keys [f0].");
}
+ @Test
+ void testPartitionKeyMustBelongToDefaultGroup() {
+ // Phase M.2: partition keys must live in the default group, not in a named
+ // column group — enrichment-group columns are NULL at base-write time and can't
+ // determine the row's partition.
+ Schema logSchema =
+ Schema.newBuilder()
+ .column("dt", DataTypes.STRING())
+ .column("device_id", DataTypes.INT())
+ .column("payload", DataTypes.STRING())
+ .column("geo_region", DataTypes.STRING())
+ .columnGroup("enriched")
+ .column("risk_score", DataTypes.DOUBLE())
+ .columnGroup("enriched")
+ .build();
+
+ // Allowed: partition key (dt) is in the default group; enriched group has other cols.
+ TableDescriptor ok =
+ TableDescriptor.builder()
+ .schema(logSchema)
+ .partitionedBy("dt")
+ .distributedBy(1, "device_id")
+ .build();
+ assertThat(ok.getPartitionKeys()).containsExactly("dt");
+
+ // Rejected: partition key (geo_region) is declared in the 'enriched' group.
+ Schema bad =
+ Schema.newBuilder()
+ .column("dt", DataTypes.STRING())
+ .column("device_id", DataTypes.INT())
+ .column("geo_region", DataTypes.STRING())
+ .columnGroup("enriched")
+ .build();
+ assertThatThrownBy(
+ () ->
+ TableDescriptor.builder()
+ .schema(bad)
+ .partitionedBy("geo_region")
+ .distributedBy(1, "device_id")
+ .build())
+ .isInstanceOf(InvalidColumnGroupConfigException.class)
+ .hasMessageContaining("Partition keys must belong to the default column group")
+ .hasMessageContaining("geo_region");
+ }
+
@Test
void testInvalidListaggParameterEmptyDelimiter() {
// LISTAGG with empty delimiter - should fail
diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/TableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/TableSchemaTest.java
index a5ef1fed7ee..0eaf4b616d1 100644
--- a/fluss-common/src/test/java/org/apache/fluss/metadata/TableSchemaTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/metadata/TableSchemaTest.java
@@ -426,4 +426,113 @@ void testListaggWithCustomDelimiter() {
assertThat(schemaStringAgg.getAggFunction("items").get().getParameter("delimiter"))
.isEqualTo(", ");
}
+
+ @Test
+ void testColumnGroupBuilderForms() {
+ // Three forms of declaring the same two-group device_logs schema must produce equal
+ // schemas: retroactive .columnGroup(name), inline .column(name, type, group), and the
+ // .columnGroup(name, block) lambda form.
+ Schema retroactive =
+ Schema.newBuilder()
+ .column("device_id", DataTypes.STRING())
+ .column("ip", DataTypes.STRING())
+ .column("payload", DataTypes.STRING())
+ .column("geo_region", DataTypes.STRING())
+ .columnGroup("enriched_geo")
+ .column("risk_score", DataTypes.DOUBLE())
+ .columnGroup("enriched_risk")
+ .column("risk_classification", DataTypes.STRING())
+ .columnGroup("enriched_risk")
+ .build();
+
+ Schema inline =
+ Schema.newBuilder()
+ .column("device_id", DataTypes.STRING())
+ .column("ip", DataTypes.STRING())
+ .column("payload", DataTypes.STRING())
+ .column("geo_region", DataTypes.STRING(), "enriched_geo")
+ .column("risk_score", DataTypes.DOUBLE(), "enriched_risk")
+ .column("risk_classification", DataTypes.STRING(), "enriched_risk")
+ .build();
+
+ Schema block =
+ Schema.newBuilder()
+ .column("device_id", DataTypes.STRING())
+ .column("ip", DataTypes.STRING())
+ .column("payload", DataTypes.STRING())
+ .columnGroup(
+ "enriched_geo", g -> g.column("geo_region", DataTypes.STRING()))
+ .columnGroup(
+ "enriched_risk",
+ g ->
+ g.column("risk_score", DataTypes.DOUBLE())
+ .column("risk_classification", DataTypes.STRING()))
+ .build();
+
+ assertThat(inline).isEqualTo(retroactive);
+ assertThat(block).isEqualTo(retroactive);
+ assertThat(retroactive.getColumnGroupNames())
+ .containsExactlyInAnyOrder("enriched_geo", "enriched_risk");
+ assertThat(retroactive.getColumnGroups().get("enriched_geo")).containsExactly(3);
+ assertThat(retroactive.getColumnGroups().get("enriched_risk")).containsExactly(4, 5);
+ assertThat(retroactive.getDefaultGroupColumnIndices()).containsExactly(0, 1, 2);
+ }
+
+ @Test
+ void testColumnGroupBlockPreservesExplicitGroup() {
+ // An explicit .column(name, type, "other") inside a block keeps its explicit group.
+ Schema schema =
+ Schema.newBuilder()
+ .column("device_id", DataTypes.STRING())
+ .columnGroup(
+ "enriched_geo",
+ g ->
+ g.column("geo_region", DataTypes.STRING())
+ .column(
+ "risk_score",
+ DataTypes.DOUBLE(),
+ "enriched_risk"))
+ .build();
+
+ assertThat(schema.getColumns().get(1).getColumnGroup()).hasValue("enriched_geo");
+ assertThat(schema.getColumns().get(2).getColumnGroup()).hasValue("enriched_risk");
+ }
+
+ @Test
+ void testColumnGroupBlockDoesNotAffectColumnsAddedBefore() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("device_id", DataTypes.STRING())
+ .column("ip", DataTypes.STRING())
+ .columnGroup(
+ "enriched_geo", g -> g.column("geo_region", DataTypes.STRING()))
+ .build();
+
+ assertThat(schema.getColumns().get(0).getColumnGroup()).isEmpty();
+ assertThat(schema.getColumns().get(1).getColumnGroup()).isEmpty();
+ assertThat(schema.getColumns().get(2).getColumnGroup()).hasValue("enriched_geo");
+ }
+
+ @Test
+ void testColumnGroupInlineAndBlockNullChecks() {
+ assertThatThrownBy(
+ () ->
+ Schema.newBuilder()
+ .column("f0", DataTypes.STRING(), (String) null)
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("Column group name must not be null.");
+
+ assertThatThrownBy(
+ () ->
+ Schema.newBuilder()
+ .columnGroup(null, g -> g.column("f0", DataTypes.STRING()))
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("Column group name must not be null.");
+
+ assertThatThrownBy(() -> Schema.newBuilder().columnGroup("g", null).build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("Column group block must not be null.");
+ }
}
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ColumnGroupFetchResult.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ColumnGroupFetchResult.java
new file mode 100644
index 00000000000..8c4c478392b
--- /dev/null
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ColumnGroupFetchResult.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.rpc.entity;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.record.LogRecords;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/**
+ * The records of one column group returned by a fetch (FIP-45). The batches are standard log record
+ * batches whose base offsets are base-log offsets, covering at least the offset range of the base
+ * records returned in the same fetch, so a reader can stitch them onto the base rows by offset.
+ */
+@Internal
+public final class ColumnGroupFetchResult {
+ private final String groupName;
+ private final long highWatermark;
+ private final LogRecords records;
+
+ public ColumnGroupFetchResult(String groupName, long highWatermark, LogRecords records) {
+ this.groupName = checkNotNull(groupName, "groupName");
+ this.highWatermark = highWatermark;
+ this.records = checkNotNull(records, "records");
+ }
+
+ public String getGroupName() {
+ return groupName;
+ }
+
+ /** The column group's high watermark (committed enrichment watermark). */
+ public long getHighWatermark() {
+ return highWatermark;
+ }
+
+ public LogRecords getRecords() {
+ return records;
+ }
+}
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java
index e239d8e0521..d858fa724a1 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java
@@ -27,6 +27,9 @@
import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.Map;
+
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** Result of {@link FetchLogRequest} for each table bucket. */
@@ -36,6 +39,8 @@ public class FetchLogResultForBucket extends ResultForBucket {
private final @Nullable LogRecords records;
private final long highWatermark;
private final long filteredEndOffset;
+ /** FIP-45: column-group records shipped alongside {@link #records}, keyed by group name. */
+ private final Map columnGroups;
public FetchLogResultForBucket(
TableBucket tableBucket, LogRecords records, long highWatermark) {
@@ -94,11 +99,48 @@ private FetchLogResultForBucket(
long highWatermark,
long filteredEndOffset,
ApiError error) {
+ this(
+ tableBucket,
+ remoteLogFetchInfo,
+ records,
+ highWatermark,
+ filteredEndOffset,
+ error,
+ Collections.emptyMap());
+ }
+
+ private FetchLogResultForBucket(
+ TableBucket tableBucket,
+ @Nullable RemoteLogFetchInfo remoteLogFetchInfo,
+ @Nullable LogRecords records,
+ long highWatermark,
+ long filteredEndOffset,
+ ApiError error,
+ Map columnGroups) {
super(tableBucket, error);
this.remoteLogFetchInfo = remoteLogFetchInfo;
this.records = records;
this.highWatermark = highWatermark;
this.filteredEndOffset = filteredEndOffset;
+ this.columnGroups = columnGroups;
+ }
+
+ /** Returns a copy of this result carrying the given column-group records (FIP-45). */
+ public FetchLogResultForBucket withColumnGroups(
+ Map columnGroups) {
+ return new FetchLogResultForBucket(
+ tableBucket,
+ remoteLogFetchInfo,
+ records,
+ highWatermark,
+ filteredEndOffset,
+ getError(),
+ columnGroups);
+ }
+
+ /** Column-group records keyed by group name; empty when no column group was touched. */
+ public Map columnGroups() {
+ return columnGroups;
}
/**
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogColumnsResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogColumnsResultForBucket.java
new file mode 100644
index 00000000000..d4581ecc1ef
--- /dev/null
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/ProduceLogColumnsResultForBucket.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.rpc.entity;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.rpc.protocol.ApiError;
+
+/** Result of a produce-log-columns (FIP-45 column-group append) request for one bucket. */
+@Internal
+public class ProduceLogColumnsResultForBucket extends ResultForBucket {
+ private final long logEndOffset;
+ private final long highWatermark;
+ private final long expectedSourceOffset;
+
+ public ProduceLogColumnsResultForBucket(
+ TableBucket tableBucket, long logEndOffset, long highWatermark) {
+ this(tableBucket, logEndOffset, highWatermark, -1L, ApiError.NONE);
+ }
+
+ public ProduceLogColumnsResultForBucket(TableBucket tableBucket, ApiError error) {
+ this(tableBucket, -1L, -1L, -1L, error);
+ }
+
+ public ProduceLogColumnsResultForBucket(
+ TableBucket tableBucket, ApiError error, long expectedSourceOffset) {
+ this(tableBucket, -1L, -1L, expectedSourceOffset, error);
+ }
+
+ private ProduceLogColumnsResultForBucket(
+ TableBucket tableBucket,
+ long logEndOffset,
+ long highWatermark,
+ long expectedSourceOffset,
+ ApiError error) {
+ super(tableBucket, error);
+ this.logEndOffset = logEndOffset;
+ this.highWatermark = highWatermark;
+ this.expectedSourceOffset = expectedSourceOffset;
+ }
+
+ /** The column group's log end offset (enrichment watermark) after the append. */
+ public long getLogEndOffset() {
+ return logEndOffset;
+ }
+
+ /** The column group's high watermark (committed enrichment watermark) after the append. */
+ public long getHighWatermark() {
+ return highWatermark;
+ }
+
+ /** The source offset the server expected, or -1 if not applicable. */
+ public long getExpectedSourceOffset() {
+ return expectedSourceOffset;
+ }
+}
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/TabletServerGateway.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/TabletServerGateway.java
index 34bcd361ef9..746b3225bc3 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/TabletServerGateway.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/TabletServerGateway.java
@@ -40,6 +40,8 @@
import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsResponse;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.PrefixLookupResponse;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
import org.apache.fluss.rpc.messages.PutKvRequest;
@@ -92,6 +94,17 @@ CompletableFuture notifyLeaderAndIsr(
@RPC(api = ApiKeys.PRODUCE_LOG)
CompletableFuture produceLog(ProduceLogRequest request);
+ /**
+ * Produce the columns of one column group for a contiguous range of existing base-log offsets
+ * (FIP-45 log enrichment via append columns).
+ *
+ * @return the produce log columns response, including the per-bucket column-group log end
+ * offset and high watermark after the append.
+ */
+ @RPC(api = ApiKeys.PRODUCE_LOG_COLUMNS)
+ CompletableFuture produceLogColumns(
+ ProduceLogColumnsRequest request);
+
/**
* Fetch log data from the specified table bucket. The request can send by the client scanner or
* other tablet server.
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java
index baf4256650e..2c99dff2c31 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java
@@ -110,7 +110,8 @@ public enum ApiKeys {
SCAN_KV(1061, 0, 0, PUBLIC),
GET_CLUSTER_HEALTH(1062, 0, 0, PUBLIC),
LIST_REMOTE_LOG_MANIFESTS(1063, 0, 0, PUBLIC),
- LIST_KV_SNAPSHOTS(1064, 0, 0, PUBLIC);
+ LIST_KV_SNAPSHOTS(1064, 0, 0, PUBLIC),
+ PRODUCE_LOG_COLUMNS(1065, 0, 0, PUBLIC);
private static final Map ID_TO_TYPE =
Arrays.stream(ApiKeys.values())
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/Errors.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/Errors.java
index c7d074ed113..74d584d4276 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/Errors.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/Errors.java
@@ -20,6 +20,7 @@
import org.apache.fluss.exception.ApiException;
import org.apache.fluss.exception.AuthenticationException;
import org.apache.fluss.exception.AuthorizationException;
+import org.apache.fluss.exception.ColumnGroupSourceOffsetTruncatedException;
import org.apache.fluss.exception.ConfigException;
import org.apache.fluss.exception.CorruptMessageException;
import org.apache.fluss.exception.CorruptRecordException;
@@ -35,6 +36,8 @@
import org.apache.fluss.exception.IneligibleReplicaException;
import org.apache.fluss.exception.InsufficientKvLeaderReplicaCapacityException;
import org.apache.fluss.exception.InvalidAlterTableException;
+import org.apache.fluss.exception.InvalidColumnGroupConfigException;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
import org.apache.fluss.exception.InvalidColumnProjectionException;
import org.apache.fluss.exception.InvalidConfigException;
import org.apache.fluss.exception.InvalidCoordinatorException;
@@ -87,6 +90,7 @@
import org.apache.fluss.exception.TooManyBucketsException;
import org.apache.fluss.exception.TooManyPartitionsException;
import org.apache.fluss.exception.TooManyScannersException;
+import org.apache.fluss.exception.UnknownColumnGroupException;
import org.apache.fluss.exception.UnknownScannerIdException;
import org.apache.fluss.exception.UnknownServerException;
import org.apache.fluss.exception.UnknownTableOrBucketException;
@@ -285,7 +289,25 @@ public enum Errors {
HISTORICAL_PARTITION_THROTTLED(
73,
"Historical partition request is throttled because too many historical requests are in flight.",
- HistoricalPartitionThrottledException::new);
+ HistoricalPartitionThrottledException::new),
+ INVALID_COLUMN_GROUP_OFFSET(
+ 74,
+ "The appendColumns first source offset does not match the column group's log end "
+ + "offset, or the batch runs past the base log high watermark.",
+ InvalidColumnGroupOffsetException::new),
+ COLUMN_GROUP_SOURCE_OFFSET_TRUNCATED(
+ 75,
+ "The appendColumns source offset is below the base log start offset; the base "
+ + "records have aged out under retention.",
+ ColumnGroupSourceOffsetTruncatedException::new),
+ UNKNOWN_COLUMN_GROUP(
+ 76,
+ "The referenced column group is not declared on the table.",
+ UnknownColumnGroupException::new),
+ INVALID_COLUMN_GROUP_CONFIG(
+ 77,
+ "The column-group declaration is structurally invalid.",
+ InvalidColumnGroupConfigException::new);
private static final Logger LOG = LoggerFactory.getLogger(Errors.class);
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java
index 219b51087a4..a192d739f58 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java
@@ -25,10 +25,12 @@
import org.apache.fluss.record.MemoryLogRecords;
import org.apache.fluss.remote.RemoteLogFetchInfo;
import org.apache.fluss.remote.RemoteLogSegment;
+import org.apache.fluss.rpc.entity.ColumnGroupFetchResult;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.rpc.messages.LookupRequest;
import org.apache.fluss.rpc.messages.PbAclFilter;
import org.apache.fluss.rpc.messages.PbAclInfo;
+import org.apache.fluss.rpc.messages.PbColumnGroupRecords;
import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket;
import org.apache.fluss.rpc.messages.PbKeyValue;
import org.apache.fluss.rpc.messages.PbPartitionSpec;
@@ -51,7 +53,9 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -239,6 +243,24 @@ public static FetchLogResultForBucket getFetchLogResultForBucket(
new FetchLogResultForBucket(
tb, records, respForBucket.getHighWatermark());
}
+ if (respForBucket.getColumnGroupsCount() > 0) {
+ Map columnGroups = new HashMap<>();
+ for (PbColumnGroupRecords pbGroup : respForBucket.getColumnGroupsList()) {
+ LogRecords groupRecords =
+ pbGroup.hasRecords()
+ ? MemoryLogRecords.pointToByteBuffer(
+ toByteBuffer(pbGroup.getRecordsSlice()))
+ : MemoryLogRecords.EMPTY;
+ columnGroups.put(
+ pbGroup.getGroupName(),
+ new ColumnGroupFetchResult(
+ pbGroup.getGroupName(),
+ pbGroup.getHighWatermark(),
+ groupRecords));
+ }
+ fetchLogResultForBucket =
+ fetchLogResultForBucket.withColumnGroups(columnGroups);
+ }
}
}
diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto
index 2a05018d73e..a092c728d57 100644
--- a/fluss-rpc/src/main/proto/FlussApi.proto
+++ b/fluss-rpc/src/main/proto/FlussApi.proto
@@ -228,6 +228,45 @@ message ProduceLogResponse {
repeated PbProduceLogRespForBucket buckets_resp = 1;
}
+// FIP-45: produce log columns (column-group / enrichment) request and response.
+// Writes the columns of one column group for a contiguous range of existing base-log offsets.
+// `records` is a MemoryLogRecords (ARROW format) carrying only the group's columns in schema
+// order; the server stamps `first_source_offset` as the base offset of the first batch and
+// requires it to equal the column group's current log end offset (enrichment watermark).
+message ProduceLogColumnsRequest {
+ required int32 acks = 1;
+ required int64 table_id = 2;
+ required int32 timeout_ms = 3;
+ required string column_group = 4;
+ repeated PbProduceLogColumnsReqForBucket buckets_req = 5;
+}
+
+message ProduceLogColumnsResponse {
+ repeated PbProduceLogColumnsRespForBucket buckets_resp = 1;
+}
+
+message PbProduceLogColumnsReqForBucket {
+ optional int64 partition_id = 1;
+ required int32 bucket_id = 2;
+ // the base-log offset enriched by the first row of `records`; rows are contiguous from here.
+ required int64 first_source_offset = 3;
+ required bytes records = 4;
+}
+
+message PbProduceLogColumnsRespForBucket {
+ optional int64 partition_id = 1;
+ required int32 bucket_id = 2;
+ optional int32 error_code = 3;
+ optional string error_message = 4;
+ // the column group's log end offset (enrichment watermark) after the append.
+ optional int64 log_end_offset = 5;
+ // the column group's high watermark (committed enrichment watermark) after the append.
+ optional int64 high_watermark = 6;
+ // set on INVALID_COLUMN_GROUP_OFFSET: the offset the server expected, so the client can
+ // re-slice a batch that straddles the group's log end offset after a replay.
+ optional int64 expected_source_offset = 7;
+}
+
// fetch log request and response
message FetchLogRequest {
required int32 follower_server_id = 1; // value -1 indicate the request from client.
@@ -399,6 +438,10 @@ message ListOffsetsRequest {
optional int64 partition_id = 4;
repeated int32 bucket_id = 5 [packed = true]; // it is recommended to use packed for repeated numerics to get more efficient encoding
optional int64 startTimestamp = 6;
+ // FIP-45: when set, offsets are answered for the named column group instead of the base log:
+ // LATEST returns the group's high watermark (committed enrichment watermark) and
+ // LEADER_END_OFFSET_SNAPSHOT returns the group's log end offset (enrichment watermark).
+ optional string column_group = 7;
}
message ListOffsetsResponse {
repeated PbListOffsetsRespForBucket buckets_resp = 1;
@@ -917,6 +960,15 @@ message PbFetchLogReqForBucket {
// TODO leader epoch
required int64 fetch_offset = 3;
required int32 max_fetch_bytes = 4;
+ // Follower-only (FIP-45): the follower's log end offset per column group, so the leader can
+ // ship the column-group records the follower is missing. Absent for client fetches.
+ repeated PbColumnGroupFetch column_group_fetches = 5;
+}
+
+// FIP-45: per column-group fetch cursor advertised by a follower.
+message PbColumnGroupFetch {
+ required string group_name = 1;
+ required int64 fetch_offset = 2;
}
message PbFetchLogRespForTable {
@@ -936,6 +988,20 @@ message PbFetchLogRespForBucket {
// non-empty records field when only trailing batches were filtered. The client should start
// its next fetch from the later of this offset and the end of the records it received.
optional int64 filtered_end_offset = 9;
+ // FIP-45: column-group records covering the same offset range as `records`, one entry per
+ // column group touched by the projection (or every group when there is no projection). The
+ // batches are standard log record batches whose base offsets are base-log offsets; the client
+ // stitches them onto the base rows by offset. Absent for tables without column groups and for
+ // projections that touch only base columns.
+ repeated PbColumnGroupRecords column_groups = 10;
+}
+
+// FIP-45: records of one column group shipped alongside the base records of a fetch.
+message PbColumnGroupRecords {
+ required string group_name = 1;
+ // The committed enrichment watermark (column-group high watermark) of the group.
+ required int64 high_watermark = 2;
+ optional bytes records = 3;
}
message PbPutKvReqForBucket {
diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java
index f465bb4a69a..8abcfdeca1a 100644
--- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java
+++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java
@@ -71,6 +71,8 @@
import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsResponse;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.PrefixLookupResponse;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
import org.apache.fluss.rpc.messages.PutKvRequest;
@@ -117,6 +119,12 @@ public CompletableFuture produceLog(ProduceLogRequest reques
return null;
}
+ @Override
+ public CompletableFuture produceLogColumns(
+ ProduceLogColumnsRequest request) {
+ return null;
+ }
+
@Override
public CompletableFuture fetchLog(FetchLogRequest request) {
return null;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/ColumnGroupWriteData.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/ColumnGroupWriteData.java
new file mode 100644
index 00000000000..8e2da51d1d8
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/ColumnGroupWriteData.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.entity;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.record.MemoryLogRecords;
+
+/** The column-group rows to append for one bucket (FIP-45 produce log columns). */
+@Internal
+public final class ColumnGroupWriteData {
+ private final long firstSourceOffset;
+ private final MemoryLogRecords records;
+
+ public ColumnGroupWriteData(long firstSourceOffset, MemoryLogRecords records) {
+ this.firstSourceOffset = firstSourceOffset;
+ this.records = records;
+ }
+
+ /** The base-log offset filled by the first row of {@link #getRecords()}. */
+ public long getFirstSourceOffset() {
+ return firstSourceOffset;
+ }
+
+ public MemoryLogRecords getRecords() {
+ return records;
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupAppendInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupAppendInfo.java
new file mode 100644
index 00000000000..a5f37b620fe
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupAppendInfo.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.log;
+
+import org.apache.fluss.annotation.Internal;
+
+/** Result of appending a batch of column-group rows to a {@link ColumnGroupLog}. */
+@Internal
+public final class ColumnGroupAppendInfo {
+ private final long firstOffset;
+ private final long lastOffset;
+ private final int rowCount;
+ private final boolean duplicated;
+
+ public ColumnGroupAppendInfo(
+ long firstOffset, long lastOffset, int rowCount, boolean duplicated) {
+ this.firstOffset = firstOffset;
+ this.lastOffset = lastOffset;
+ this.rowCount = rowCount;
+ this.duplicated = duplicated;
+ }
+
+ public static ColumnGroupAppendInfo duplicated(long firstOffset, long lastOffset) {
+ return new ColumnGroupAppendInfo(
+ firstOffset, lastOffset, (int) (lastOffset - firstOffset + 1), true);
+ }
+
+ public long firstOffset() {
+ return firstOffset;
+ }
+
+ public long lastOffset() {
+ return lastOffset;
+ }
+
+ public int rowCount() {
+ return rowCount;
+ }
+
+ /** True when every row of the batch was already filled and the append was skipped. */
+ public boolean isDuplicated() {
+ return duplicated;
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnGroupAppendInfo("
+ + "firstOffset="
+ + firstOffset
+ + ", lastOffset="
+ + lastOffset
+ + ", rowCount="
+ + rowCount
+ + ", duplicated="
+ + duplicated
+ + ')';
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupFetchPlan.java b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupFetchPlan.java
new file mode 100644
index 00000000000..55ad708456c
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupFetchPlan.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * How a fetch projection over a column-group table maps onto the physical logs (FIP-45): which
+ * base-log columns to project (in base physical positions) and which column groups it touches.
+ */
+@Internal
+public final class ColumnGroupFetchPlan {
+
+ private static final ColumnGroupFetchPlan NONE =
+ new ColumnGroupFetchPlan(null, Collections.emptyList(), null);
+
+ /** Base physical column positions to project, or null for the whole base row. */
+ @Nullable private final int[] baseFields;
+
+ /** Column groups touched by the projection, in schema order of first appearance. */
+ private final List groups;
+
+ /** Schema getter answering the base physical schema, or null when the table has no groups. */
+ @Nullable private final SchemaGetter baseSchemaGetter;
+
+ private ColumnGroupFetchPlan(
+ @Nullable int[] baseFields,
+ List groups,
+ @Nullable SchemaGetter baseSchemaGetter) {
+ this.baseFields = baseFields;
+ this.groups = groups;
+ this.baseSchemaGetter = baseSchemaGetter;
+ }
+
+ /**
+ * Plans a fetch against {@code schema}.
+ *
+ * @param schema the latest table schema
+ * @param projectedFields the projected table column positions, or null for all columns
+ * @param baseSchemaGetter schema getter answering the base physical schema
+ */
+ public static ColumnGroupFetchPlan plan(
+ Schema schema, @Nullable int[] projectedFields, SchemaGetter baseSchemaGetter) {
+ if (!schema.hasColumnGroups()) {
+ return NONE;
+ }
+ int[] baseIndices = schema.getDefaultGroupColumnIndices();
+ if (projectedFields == null) {
+ // SELECT *: every group is touched, whole base row is shipped.
+ List allGroups = new ArrayList<>();
+ for (int i = 0; i < schema.getColumns().size(); i++) {
+ String group = schema.getColumnGroupOf(i);
+ if (group != null && !allGroups.contains(group)) {
+ allGroups.add(group);
+ }
+ }
+ return new ColumnGroupFetchPlan(null, allGroups, baseSchemaGetter);
+ }
+ List baseFields = new ArrayList<>();
+ Set groups = new LinkedHashSet<>();
+ for (int field : projectedFields) {
+ String group = schema.getColumnGroupOf(field);
+ if (group == null) {
+ baseFields.add(basePosition(baseIndices, field));
+ } else {
+ groups.add(group);
+ }
+ }
+ if (baseFields.isEmpty()) {
+ // The server cannot project zero columns; carry the first base column so the fetch
+ // still advances offsets. The client drops it when building the output row.
+ baseFields.add(0);
+ }
+ int[] base = baseFields.stream().mapToInt(Integer::intValue).toArray();
+ return new ColumnGroupFetchPlan(base, new ArrayList<>(groups), baseSchemaGetter);
+ }
+
+ private static int basePosition(int[] baseIndices, int tableColumn) {
+ for (int i = 0; i < baseIndices.length; i++) {
+ if (baseIndices[i] == tableColumn) {
+ return i;
+ }
+ }
+ throw new IllegalArgumentException("Column " + tableColumn + " is not a base column.");
+ }
+
+ public boolean hasColumnGroups() {
+ return baseSchemaGetter != null;
+ }
+
+ /** The projection to apply to the base log, in base physical positions. */
+ @Nullable
+ public int[] baseProjection(@Nullable int[] projectedFields) {
+ return hasColumnGroups() ? baseFields : projectedFields;
+ }
+
+ /** The schema getter to resolve the base log's physical schema with. */
+ public SchemaGetter schemaGetter(SchemaGetter tableSchemaGetter) {
+ return baseSchemaGetter != null ? baseSchemaGetter : tableSchemaGetter;
+ }
+
+ public List touchedGroups() {
+ return groups;
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupLog.java
new file mode 100644
index 00000000000..a0207b83652
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/ColumnGroupLog.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.log;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.record.BytesViewLogRecords;
+import org.apache.fluss.record.DefaultLogRecordBatch;
+import org.apache.fluss.record.FileLogInputStream;
+import org.apache.fluss.record.FileLogRecords;
+import org.apache.fluss.record.LogRecordBatch;
+import org.apache.fluss.record.LogRecords;
+import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.bytesview.MultiBytesView;
+import org.apache.fluss.utils.FlussPaths;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * The shadow log of one column group of a log bucket (FIP-45).
+ *
+ * A column group is stored as a chain of standard {@link LogSegment}s that live in their own
+ * directory next to the base segments ({@code {tabletDir}/col-{group}/}). Every batch holds only
+ * the group's columns and is addressed by the base-log offsets it fills: the batch base
+ * offset is the source offset of its first row and rows are contiguous from there. This makes the
+ * group's log end offset the enrichment watermark (the exclusive offset up to which the
+ * group is filled) and the group's high watermark the committed enrichment watermark, so
+ * the base log's offset index, recovery, truncation and zero-copy read machinery apply unchanged.
+ *
+ *
All mutating methods must be called while holding the owning {@link LogTablet}'s lock.
+ */
+@Internal
+@NotThreadSafe
+public final class ColumnGroupLog implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ColumnGroupLog.class);
+
+ private final File groupDir;
+ private final String groupName;
+ private final Configuration conf;
+ private final LogSegments segments;
+ private final int maxSegmentFileSize;
+ private final TableBucket tableBucket;
+
+ /** Exclusive end offset of the filled range: the enrichment watermark of the group. */
+ private volatile long logEndOffset;
+
+ /** Committed enrichment watermark: filled range replicated to the ISR (min over ISR LEOs). */
+ private volatile long highWatermark;
+
+ /** The first offset the group log holds data for. */
+ private volatile long logStartOffset;
+
+ private ColumnGroupLog(
+ File groupDir,
+ String groupName,
+ Configuration conf,
+ LogSegments segments,
+ TableBucket tableBucket,
+ long logStartOffset,
+ long logEndOffset) {
+ this.groupDir = groupDir;
+ this.groupName = groupName;
+ this.conf = conf;
+ this.segments = segments;
+ this.tableBucket = tableBucket;
+ this.maxSegmentFileSize = (int) conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes();
+ this.logStartOffset = logStartOffset;
+ this.logEndOffset = logEndOffset;
+ this.highWatermark = 0L;
+ }
+
+ /** Names of the column groups that have a log directory under {@code tabletDir}. */
+ public static Set discoverGroups(File tabletDir) {
+ Set groups = new HashSet<>();
+ File[] children = tabletDir.listFiles();
+ if (children == null) {
+ return groups;
+ }
+ for (File child : children) {
+ if (child.isDirectory()) {
+ String group = FlussPaths.columnGroupNameFromDir(child);
+ if (group != null) {
+ groups.add(group);
+ }
+ }
+ }
+ return groups;
+ }
+
+ /** Opens (or creates) the column-group log for {@code groupName}, recovering it if needed. */
+ public static ColumnGroupLog load(
+ File tabletDir,
+ String groupName,
+ Configuration conf,
+ TableBucket tableBucket,
+ boolean isCleanShutdown)
+ throws IOException {
+ File groupDir = FlussPaths.columnGroupLogDir(tabletDir, groupName);
+ Files.createDirectories(groupDir.toPath());
+ LogSegments segments = new LogSegments(tableBucket);
+
+ File[] files = groupDir.listFiles();
+ if (files != null) {
+ Arrays.sort(files, Comparator.comparing(File::getName));
+ for (File file : files) {
+ if (!file.isFile()) {
+ continue;
+ }
+ if (LocalLog.isIndexFile(file)) {
+ long offset = FlussPaths.offsetFromFile(file);
+ if (!FlussPaths.logFile(groupDir, offset).exists()) {
+ Files.deleteIfExists(file.toPath());
+ }
+ } else if (LocalLog.isLogFile(file)) {
+ long baseOffset = FlussPaths.offsetFromFile(file);
+ LogSegment segment =
+ LogSegment.open(groupDir, baseOffset, conf, true, 0, LogFormat.ARROW);
+ try {
+ segment.sanityCheck();
+ } catch (NoSuchFileException e) {
+ LOG.warn(
+ "Rebuilding index of column group '{}' segment {} for bucket {}",
+ groupName,
+ baseOffset,
+ tableBucket);
+ segment.recover();
+ }
+ segments.add(segment);
+ }
+ }
+ }
+
+ long logEndOffset = 0L;
+ long logStartOffset = 0L;
+ if (!segments.isEmpty()) {
+ LogSegment last = segments.lastSegment().get();
+ if (!isCleanShutdown) {
+ int truncated = last.recover();
+ if (truncated > 0) {
+ LOG.warn(
+ "Truncated {} bytes of column group '{}' for bucket {} during recovery",
+ truncated,
+ groupName,
+ tableBucket);
+ }
+ }
+ logEndOffset = last.readNextOffset();
+ logStartOffset = segments.firstSegmentBaseOffset().get();
+ }
+ return new ColumnGroupLog(
+ groupDir, groupName, conf, segments, tableBucket, logStartOffset, logEndOffset);
+ }
+
+ public String getGroupName() {
+ return groupName;
+ }
+
+ /** The enrichment watermark: exclusive end of the contiguously filled offset range. */
+ public long logEndOffset() {
+ return logEndOffset;
+ }
+
+ /** The committed enrichment watermark of the group. */
+ public long highWatermark() {
+ return highWatermark;
+ }
+
+ public long logStartOffset() {
+ return logStartOffset;
+ }
+
+ public List segments() {
+ return segments.values();
+ }
+
+ /** Raises the high watermark to {@code newHighWatermark}, bounded by the log end offset. */
+ public boolean maybeIncrementHighWatermark(long newHighWatermark) {
+ long bounded = Math.min(newHighWatermark, logEndOffset);
+ if (bounded > highWatermark) {
+ highWatermark = bounded;
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Appends {@code records} whose first row fills base offset {@code firstOffset}. The caller has
+ * validated that {@code firstOffset == logEndOffset()}. Batch base offsets and commit
+ * timestamps are stamped in place; the CRC does not cover them.
+ */
+ public ColumnGroupAppendInfo append(
+ MemoryLogRecords records, long firstOffset, long commitTimestamp) throws IOException {
+ long nextOffset = firstOffset;
+ int rowCount = 0;
+ for (LogRecordBatch batch : records.batches()) {
+ if (!(batch instanceof DefaultLogRecordBatch)) {
+ throw new FlussRuntimeException(
+ "Currently, we only support DefaultLogRecordBatch.");
+ }
+ DefaultLogRecordBatch defaultBatch = (DefaultLogRecordBatch) batch;
+ defaultBatch.setBaseLogOffset(nextOffset);
+ defaultBatch.setCommitTimestamp(commitTimestamp);
+ rowCount += batch.getRecordCount();
+ nextOffset = batch.nextLogOffset();
+ }
+ if (rowCount == 0) {
+ return new ColumnGroupAppendInfo(firstOffset, firstOffset - 1, 0, false);
+ }
+ long lastOffset = nextOffset - 1;
+
+ if (segments.isEmpty()) {
+ segments.add(LogSegment.open(groupDir, firstOffset, conf, LogFormat.ARROW));
+ logStartOffset = firstOffset;
+ }
+ LogSegment active = segments.activeSegment();
+ if (active.shouldRoll(
+ new RollParams(maxSegmentFileSize, lastOffset, records.sizeInBytes()))) {
+ active.onBecomeInactiveSegment();
+ active = LogSegment.open(groupDir, firstOffset, conf, LogFormat.ARROW);
+ segments.add(active);
+ LOG.info(
+ "Rolled new segment for column group '{}' of bucket {} at offset {}",
+ groupName,
+ tableBucket,
+ firstOffset);
+ }
+ active.append(lastOffset, commitTimestamp, firstOffset, records);
+ logEndOffset = nextOffset;
+ return new ColumnGroupAppendInfo(firstOffset, lastOffset, rowCount, false);
+ }
+
+ /**
+ * Reads the group records covering base offsets {@code [startOffset, endOffsetInclusive]} as
+ * zero-copy file slices. The result may start at a batch containing {@code startOffset} (thus
+ * include earlier rows) and, when {@code maxBytes} is exhausted, may end before {@code
+ * endOffsetInclusive}; readers stitch by offset and stop at the last offset covered.
+ */
+ public LogRecords read(long startOffset, long endOffsetInclusive, int maxBytes)
+ throws IOException {
+ if (segments.isEmpty() || endOffsetInclusive < startOffset) {
+ return MemoryLogRecords.EMPTY;
+ }
+ MultiBytesView.Builder builder = MultiBytesView.builder();
+ boolean wroteAny = false;
+ int budget = maxBytes;
+ Optional segmentOpt = segments.floorSegment(startOffset);
+ if (!segmentOpt.isPresent()) {
+ segmentOpt = segments.firstSegment();
+ }
+ long cursor = startOffset;
+ while (segmentOpt.isPresent() && cursor <= endOffsetInclusive) {
+ LogSegment segment = segmentOpt.get();
+ FileLogRecords fileRecords = segment.getFileLogRecords();
+ FileLogRecords.LogOffsetPosition startPos =
+ segment.translateOffset(Math.max(cursor, segment.getBaseOffset()));
+ if (startPos == null) {
+ segmentOpt = segments.higherSegment(segment.getBaseOffset());
+ continue;
+ }
+ FileLogRecords.LogOffsetPosition endPos =
+ fileRecords.searchForOffsetWithSize(endOffsetInclusive, startPos.getPosition());
+ int endPosition =
+ endPos == null
+ ? fileRecords.sizeInBytes()
+ : endPos.getPosition() + endPos.getSize();
+ int length = boundedLength(fileRecords, startPos.getPosition(), endPosition, budget);
+ if (length <= 0) {
+ break;
+ }
+ builder.addBytes(fileRecords.channel(), startPos.getPosition(), length);
+ wroteAny = true;
+ budget -= length;
+ if (endPos != null || length < endPosition - startPos.getPosition()) {
+ break;
+ }
+ cursor = segment.readNextOffset();
+ segmentOpt = segments.higherSegment(segment.getBaseOffset());
+ }
+ return wroteAny ? new BytesViewLogRecords(builder.build()) : MemoryLogRecords.EMPTY;
+ }
+
+ /** Length of whole batches in {@code [start, end)} that fit {@code budget} (at least one). */
+ private static int boundedLength(FileLogRecords fileRecords, int start, int end, int budget) {
+ if (end - start <= budget) {
+ return end - start;
+ }
+ int length = 0;
+ for (FileLogInputStream.FileChannelLogRecordBatch batch :
+ (Iterable)
+ () -> fileRecords.batchIterator(start, end)) {
+ int size = batch.sizeInBytes();
+ if (length > 0 && length + size > budget) {
+ break;
+ }
+ length += size;
+ if (length >= budget) {
+ break;
+ }
+ }
+ return length;
+ }
+
+ /** Truncates so that the log ends with the greatest offset below {@code targetOffset}. */
+ public void truncateTo(long targetOffset) throws IOException {
+ if (targetOffset >= logEndOffset) {
+ return;
+ }
+ if (targetOffset <= logStartOffset || segments.isEmpty()) {
+ truncateFullyAndStartAt(targetOffset);
+ return;
+ }
+ List deletable = new ArrayList<>();
+ for (LogSegment segment : segments.values()) {
+ if (segment.getBaseOffset() > targetOffset) {
+ deletable.add(segment);
+ }
+ }
+ for (LogSegment segment : deletable) {
+ segments.remove(segment.getBaseOffset());
+ }
+ LocalLog.deleteSegmentFiles(deletable, LocalLog.SegmentDeletionReason.LOG_TRUNCATION);
+ // like the base log: batches are truncated whole, but the log end offset becomes the
+ // requested offset so that the group stays aligned with the base log
+ segments.activeSegment().truncateTo(targetOffset);
+ logEndOffset = targetOffset;
+ highWatermark = Math.min(highWatermark, logEndOffset);
+ LOG.info(
+ "Truncated column group '{}' of bucket {} to offset {}",
+ groupName,
+ tableBucket,
+ targetOffset);
+ }
+
+ /** Deletes all data and restarts the log at {@code newOffset}. */
+ public void truncateFullyAndStartAt(long newOffset) throws IOException {
+ List all = segments.values();
+ for (LogSegment segment : all) {
+ segments.remove(segment.getBaseOffset());
+ }
+ LocalLog.deleteSegmentFiles(all, LocalLog.SegmentDeletionReason.LOG_TRUNCATION);
+ logStartOffset = newOffset;
+ logEndOffset = newOffset;
+ highWatermark = Math.min(highWatermark, newOffset);
+ }
+
+ /**
+ * Retention advance: base offsets below {@code newStartOffset} no longer exist, so the group is
+ * trivially complete up to there. Moves the log start, log end and high watermark forward when
+ * they are behind.
+ */
+ public void advanceStartOffsetTo(long newStartOffset) throws IOException {
+ if (newStartOffset <= logEndOffset) {
+ return;
+ }
+ LOG.info(
+ "Advancing column group '{}' of bucket {} from {} to base log start offset {}",
+ groupName,
+ tableBucket,
+ logEndOffset,
+ newStartOffset);
+ List all = segments.values();
+ for (LogSegment segment : all) {
+ segments.remove(segment.getBaseOffset());
+ }
+ LocalLog.deleteSegmentFiles(all, LocalLog.SegmentDeletionReason.LOG_RETENTION);
+ logStartOffset = newStartOffset;
+ logEndOffset = newStartOffset;
+ highWatermark = Math.max(highWatermark, newStartOffset);
+ }
+
+ public void flush() throws IOException {
+ for (LogSegment segment : segments.values()) {
+ segment.flush();
+ }
+ }
+
+ @Override
+ public void close() {
+ segments.close();
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnGroupLog("
+ + "group="
+ + groupName
+ + ", bucket="
+ + tableBucket
+ + ", logStartOffset="
+ + logStartOffset
+ + ", logEndOffset="
+ + logEndOffset
+ + ", highWatermark="
+ + highWatermark
+ + ')';
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
index 76a30a1ddc7..a6684836d78 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
@@ -27,6 +27,8 @@
import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -68,6 +70,8 @@ public final class FetchParams {
@Nullable private final Map tableFilterInfoMap;
// the lazily initialized projection util to read and project file logs
@Nullable private FileLogProjection fileLogProjection;
+ // FIP-45: the column groups touched by the current fetch, empty for base-only fetches
+ private List currentColumnGroups = Collections.emptyList();
private final int minFetchBytes;
private final long maxWaitMs;
private final FetchLogReadPreference readPreference;
@@ -170,6 +174,16 @@ public FileLogProjection projection() {
}
}
+ /** Sets the column groups touched by the current fetch (FIP-45). */
+ public void setCurrentColumnGroups(List columnGroups) {
+ this.currentColumnGroups = columnGroups;
+ }
+
+ /** The column groups touched by the current fetch; empty when none. */
+ public List currentColumnGroups() {
+ return currentColumnGroups;
+ }
+
/** Returns the filter info for the given table, or null if no filter is registered. */
@Nullable
public FilterInfo getFilterInfo(long tableId) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/ListOffsetsParam.java b/fluss-server/src/main/java/org/apache/fluss/server/log/ListOffsetsParam.java
index d4fb211b04f..902f1ae4cc8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/ListOffsetsParam.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/ListOffsetsParam.java
@@ -48,12 +48,29 @@ public class ListOffsetsParam {
private final int followerServerId;
private final Integer offsetType;
private @Nullable final Long startTimestamp;
+ // FIP-45: answer offsets for this column group instead of the base log when set
+ private @Nullable final String columnGroup;
public ListOffsetsParam(
int followerServerId, Integer offsetType, @Nullable Long startTimestamp) {
+ this(followerServerId, offsetType, startTimestamp, null);
+ }
+
+ public ListOffsetsParam(
+ int followerServerId,
+ Integer offsetType,
+ @Nullable Long startTimestamp,
+ @Nullable String columnGroup) {
this.followerServerId = followerServerId;
this.offsetType = offsetType;
this.startTimestamp = startTimestamp;
+ this.columnGroup = columnGroup;
+ }
+
+ /** The column group the offsets are asked for, or null for the base log. */
+ @Nullable
+ public String getColumnGroup() {
+ return columnGroup;
}
public int getFollowerServerId() {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
index 8548b1015d6..b4e159f4e90 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
@@ -27,6 +27,7 @@
import org.apache.fluss.metrics.Counter;
import org.apache.fluss.metrics.Histogram;
import org.apache.fluss.record.FileLogProjection;
+import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.MemoryLogRecords;
import org.apache.fluss.server.metrics.group.TabletServerMetricGroup;
import org.apache.fluss.utils.FileUtils;
@@ -351,6 +352,29 @@ LogSegment createAndDeleteSegment(
return newSegment;
}
+ /**
+ * The offset metadata of the end of the batch containing {@code offset}: the message offset
+ * right after that batch and the file position right after its bytes. Used to clamp a read at
+ * an offset that may sit inside a batch (FIP-45 committed enrichment watermark), since batches
+ * are never split on the wire. Returns the log end offset metadata when {@code offset} is at or
+ * past the end of the log.
+ */
+ LogOffsetMetadata convertToBatchEndOffsetMetadata(long offset) throws IOException {
+ Optional segmentOpt = segments.floorSegment(offset);
+ while (segmentOpt.isPresent()) {
+ LogSegment segment = segmentOpt.get();
+ FileLogRecords.LogOffsetPosition position = segment.translateOffset(offset);
+ if (position != null) {
+ return new LogOffsetMetadata(
+ position.getOffset() + 1,
+ segment.getBaseOffset(),
+ position.getPosition() + position.getSize());
+ }
+ segmentOpt = segments.higherSegment(segment.getBaseOffset());
+ }
+ return nextOffsetMetadata;
+ }
+
/**
* Given a message offset, find its corresponding offset metadata in the log. If the message
* offset is out of range, throw an OffsetOutOfRangeException.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java
index 150c3c38ff3..d709cd1ffaa 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java
@@ -18,6 +18,10 @@
package org.apache.fluss.server.log;
import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.rpc.entity.ColumnGroupFetchResult;
+
+import java.util.Collections;
+import java.util.Map;
/** Structure used for lower level reads. */
@Internal
@@ -26,11 +30,26 @@ public class LogReadInfo {
private final FetchDataInfo fetchedData;
private final long highWatermark;
private final long logEndOffset;
+ private final Map columnGroups;
public LogReadInfo(FetchDataInfo fetchedData, long highWatermark, long logEndOffset) {
+ this(fetchedData, highWatermark, logEndOffset, Collections.emptyMap());
+ }
+
+ public LogReadInfo(
+ FetchDataInfo fetchedData,
+ long highWatermark,
+ long logEndOffset,
+ Map columnGroups) {
this.fetchedData = fetchedData;
this.highWatermark = highWatermark;
this.logEndOffset = logEndOffset;
+ this.columnGroups = columnGroups;
+ }
+
+ /** Column-group records covering the fetched base range (FIP-45), keyed by group name. */
+ public Map getColumnGroups() {
+ return columnGroups;
}
public FetchDataInfo getFetchedData() {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java
index 610ed4bf245..d01a7c2ab4e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java
@@ -21,9 +21,11 @@
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.config.TableConfig;
+import org.apache.fluss.exception.ColumnGroupSourceOffsetTruncatedException;
import org.apache.fluss.exception.CorruptRecordException;
import org.apache.fluss.exception.DuplicateSequenceException;
import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
import org.apache.fluss.exception.InvalidTimestampException;
import org.apache.fluss.exception.LogOffsetOutOfRangeException;
import org.apache.fluss.exception.LogStorageException;
@@ -64,6 +66,8 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -143,6 +147,12 @@ public final class LogTablet {
// Metric reads are allowed to observe transient intermediate states.
private volatile long estimatedPendingStartTimeMs = -1L;
+ // FIP-45: the shadow logs of the column groups of this bucket, keyed by group name.
+ @GuardedBy("lock")
+ private final Map columnGroupLogs = new ConcurrentHashMap<>();
+
+ private final Configuration logConf;
+
private LogTablet(
File dataDir,
PhysicalTablePath physicalPath,
@@ -157,6 +167,7 @@ private LogTablet(
this.dataDir = dataDir;
this.physicalPath = physicalPath;
this.localLog = localLog;
+ this.logConf = conf;
this.maxSegmentFileSize = (int) conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes();
this.logFlushIntervalMessages = conf.get(ConfigOptions.LOG_FLUSH_INTERVAL_MESSAGES);
int writerExpirationCheckIntervalMs =
@@ -410,17 +421,221 @@ public static LogTablet create(
tableBucket,
logFormat);
- return new LogTablet(
- dataDir,
- tablePath,
- log,
- conf,
- rollExpiredActiveSegmentEnabled,
- scheduler,
- writerStateManager,
- tableConfig,
- isChangelog,
- clock);
+ LogTablet logTablet =
+ new LogTablet(
+ dataDir,
+ tablePath,
+ log,
+ conf,
+ rollExpiredActiveSegmentEnabled,
+ scheduler,
+ writerStateManager,
+ tableConfig,
+ isChangelog,
+ clock);
+ logTablet.loadColumnGroupLogs(tabletDir, isCleanShutdown);
+ return logTablet;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // FIP-45: column-group shadow logs
+ // ------------------------------------------------------------------------------------------
+
+ private void loadColumnGroupLogs(File tabletDir, boolean isCleanShutdown) throws IOException {
+ Set groups = ColumnGroupLog.discoverGroups(tabletDir);
+ for (String group : groups) {
+ ColumnGroupLog columnGroupLog =
+ ColumnGroupLog.load(
+ tabletDir, group, logConf, getTableBucket(), isCleanShutdown);
+ columnGroupLogs.put(group, columnGroupLog);
+ LOG.info("Loaded {} for bucket {}", columnGroupLog, getTableBucket());
+ }
+ }
+
+ /** The column-group logs of this bucket, keyed by group name. */
+ public Map getColumnGroupLogs() {
+ return Collections.unmodifiableMap(columnGroupLogs);
+ }
+
+ /** The column-group log of {@code groupName}, or null if it has no data yet. */
+ @Nullable
+ public ColumnGroupLog getColumnGroupLog(String groupName) {
+ return columnGroupLogs.get(groupName);
+ }
+
+ /** Gets or creates the column-group log of {@code groupName}. */
+ public ColumnGroupLog getOrCreateColumnGroupLog(String groupName) throws IOException {
+ ColumnGroupLog existing = columnGroupLogs.get(groupName);
+ if (existing != null) {
+ return existing;
+ }
+ synchronized (lock) {
+ existing = columnGroupLogs.get(groupName);
+ if (existing != null) {
+ return existing;
+ }
+ ColumnGroupLog created =
+ ColumnGroupLog.load(
+ localLog.getLogTabletDir(), groupName, logConf, getTableBucket(), true);
+ columnGroupLogs.put(groupName, created);
+ return created;
+ }
+ }
+
+ /**
+ * The log end offset (enrichment watermark) of {@code groupName}; offsets below the base log
+ * start are trivially complete, so an absent group answers the base log start offset.
+ */
+ public long getColumnGroupLogEndOffset(String groupName) {
+ ColumnGroupLog columnGroupLog = columnGroupLogs.get(groupName);
+ long logStart = logStartOffset();
+ return columnGroupLog == null
+ ? logStart
+ : Math.max(columnGroupLog.logEndOffset(), logStart);
+ }
+
+ /** The high watermark (committed enrichment watermark) of {@code groupName}. */
+ public long getColumnGroupHighWatermark(String groupName) {
+ ColumnGroupLog columnGroupLog = columnGroupLogs.get(groupName);
+ long logStart = logStartOffset();
+ return columnGroupLog == null
+ ? logStart
+ : Math.max(columnGroupLog.highWatermark(), logStart);
+ }
+
+ /**
+ * Appends the rows of column group {@code groupName} for the contiguous base offsets starting
+ * at {@code firstSourceOffset}.
+ *
+ *
+ * - {@code firstSourceOffset} must equal the group's log end offset; a batch entirely below
+ * it is acknowledged as a duplicate (client replay), a batch straddling it or leaving a
+ * gap is rejected with the expected offset.
+ *
- the last row must be below the base high watermark: enrichment never runs ahead of
+ * replicated base rows.
+ *
- offsets that base retention already removed are trivially complete: the group log is
+ * advanced to the base log start offset before validation.
+ *
+ */
+ public ColumnGroupAppendInfo appendColumnsAsLeader(
+ String groupName, MemoryLogRecords records, long firstSourceOffset) throws IOException {
+ synchronized (lock) {
+ localLog.checkIfMemoryMappedBufferClosed();
+ ColumnGroupLog columnGroupLog = getOrCreateColumnGroupLog(groupName);
+ long baseLogStart = logStartOffset();
+ long baseHighWatermark = getHighWatermark();
+ if (columnGroupLog.logEndOffset() < baseLogStart) {
+ columnGroupLog.advanceStartOffsetTo(baseLogStart);
+ }
+ long expected = columnGroupLog.logEndOffset();
+
+ int rowCount = 0;
+ for (LogRecordBatch batch : records.batches()) {
+ rowCount += batch.getRecordCount();
+ }
+ if (rowCount == 0) {
+ return new ColumnGroupAppendInfo(
+ firstSourceOffset, firstSourceOffset - 1, 0, false);
+ }
+ long lastSourceOffset = firstSourceOffset + rowCount - 1;
+
+ if (lastSourceOffset < expected) {
+ LOG.debug(
+ "Skipping already filled column group '{}' rows [{}, {}] for bucket {}",
+ groupName,
+ firstSourceOffset,
+ lastSourceOffset,
+ getTableBucket());
+ return ColumnGroupAppendInfo.duplicated(firstSourceOffset, lastSourceOffset);
+ }
+ if (firstSourceOffset < baseLogStart) {
+ throw new ColumnGroupSourceOffsetTruncatedException(
+ String.format(
+ "Column group '%s' source offset %d of bucket %s is below the base "
+ + "log start offset %d; the base rows have been deleted.",
+ groupName, firstSourceOffset, getTableBucket(), baseLogStart));
+ }
+ if (firstSourceOffset != expected) {
+ throw new InvalidColumnGroupOffsetException(
+ String.format(
+ "Column group '%s' of bucket %s expects rows to start at offset %d "
+ + "(its log end offset), but the batch starts at %d.",
+ groupName, getTableBucket(), expected, firstSourceOffset),
+ expected);
+ }
+ if (lastSourceOffset >= baseHighWatermark) {
+ throw new InvalidColumnGroupOffsetException(
+ String.format(
+ "Column group '%s' rows [%d, %d] of bucket %s run past the base "
+ + "log high watermark %d; enrichment cannot run ahead of "
+ + "replicated base rows.",
+ groupName,
+ firstSourceOffset,
+ lastSourceOffset,
+ getTableBucket(),
+ baseHighWatermark),
+ expected);
+ }
+ return columnGroupLog.append(records, firstSourceOffset, clock.milliseconds());
+ }
+ }
+
+ /**
+ * Reads the records of column group {@code groupName} covering base offsets {@code
+ * [startOffset, endOffsetInclusive]}, as zero-copy file slices.
+ */
+ public LogRecords readColumnGroup(
+ String groupName, long startOffset, long endOffsetInclusive, int maxBytes)
+ throws IOException {
+ ColumnGroupLog columnGroupLog = columnGroupLogs.get(groupName);
+ if (columnGroupLog == null) {
+ return MemoryLogRecords.EMPTY;
+ }
+ return columnGroupLog.read(startOffset, endOffsetInclusive, maxBytes);
+ }
+
+ /**
+ * Reads messages from the local log like {@link #read(long, int, FetchIsolation, boolean,
+ * FileLogProjection, FilterContext)} but never past {@code upperBoundOffset} (exclusive). Used
+ * to clamp fetches touching column groups at the committed enrichment watermark (FIP-45).
+ */
+ public FetchDataInfo read(
+ long readOffset,
+ int maxLength,
+ FetchIsolation fetchIsolation,
+ boolean minOneMessage,
+ @Nullable FileLogProjection projection,
+ @Nullable FilterContext filterContext,
+ long upperBoundOffset)
+ throws IOException {
+ LogOffsetMetadata maxOffsetMetadata;
+ if (fetchIsolation == FetchIsolation.LOG_END) {
+ maxOffsetMetadata = localLog.getLocalLogEndOffsetMetadata();
+ } else {
+ maxOffsetMetadata = fetchHighWatermarkMetadata();
+ }
+ if (upperBoundOffset <= readOffset) {
+ // nothing below the bound is left to read: answer an empty fetch at the read offset
+ // (the client will poll again once the column group's high watermark advances)
+ long bounded = Math.min(readOffset, maxOffsetMetadata.getMessageOffset());
+ return localLog.read(
+ bounded,
+ maxLength,
+ minOneMessage,
+ convertToOffsetMetadataOrThrow(bounded),
+ projection,
+ filterContext);
+ }
+ if (upperBoundOffset < maxOffsetMetadata.getMessageOffset()) {
+ // batches are never split: include the whole batch containing the last visible
+ // offset; column-group rows are only shipped up to the bound, so the client stops
+ // exactly there and re-fetches from it
+ synchronized (lock) {
+ maxOffsetMetadata = localLog.convertToBatchEndOffsetMetadata(upperBoundOffset - 1);
+ }
+ }
+ return localLog.read(
+ readOffset, maxLength, minOneMessage, maxOffsetMetadata, projection, filterContext);
}
@VisibleForTesting
@@ -1130,6 +1345,10 @@ boolean truncateTo(long targetOffset) throws LogStorageException {
truncateFullyAndStartAt(targetOffset);
} else {
List deletedSegments = localLog.truncateTo(targetOffset);
+ // group logs never run past the base log: follow its actual end offset
+ for (ColumnGroupLog columnGroupLog : columnGroupLogs.values()) {
+ columnGroupLog.truncateTo(localLog.getLocalLogEndOffset());
+ }
deleteWriterSnapshots(deletedSegments, writerStateManager);
rebuildWriterState(targetOffset, writerStateManager);
@@ -1157,6 +1376,9 @@ void truncateFullyAndStartAt(long newOffset) throws LogStorageException {
synchronized (lock) {
try {
localLog.truncateFullyAndStartAt(newOffset);
+ for (ColumnGroupLog columnGroupLog : columnGroupLogs.values()) {
+ columnGroupLog.truncateFullyAndStartAt(newOffset);
+ }
writerStateManager.truncateFullyAndStartAt(newOffset);
rebuildWriterState(newOffset, writerStateManager);
updateHighWatermark(localLog.getLocalLogEndOffset());
@@ -1210,6 +1432,9 @@ public void close() {
} catch (IOException e) {
LOG.error("Error while taking writer snapshot for bucket {}.", getTableBucket(), e);
}
+ for (ColumnGroupLog columnGroupLog : columnGroupLogs.values()) {
+ columnGroupLog.close();
+ }
localLog.close();
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index ee28567756f..0d5c2de280d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -34,8 +34,10 @@
import org.apache.fluss.exception.NotEnoughReplicasException;
import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.exception.TooManyScannersException;
+import org.apache.fluss.exception.UnknownColumnGroupException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.ChangelogImage;
+import org.apache.fluss.metadata.ColumnGroupSchemaGetter;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
@@ -49,10 +51,14 @@
import org.apache.fluss.metrics.groups.MetricGroup;
import org.apache.fluss.predicate.Predicate;
import org.apache.fluss.record.DefaultValueRecordBatch;
+import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.record.KvRecordBatch;
+import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.LogRecords;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.rpc.entity.ColumnGroupFetchResult;
+import org.apache.fluss.rpc.entity.ProduceLogColumnsResultForBucket;
import org.apache.fluss.rpc.protocol.Errors;
import org.apache.fluss.rpc.protocol.MergeMode;
import org.apache.fluss.rpc.util.PredicateMessageUtils;
@@ -80,6 +86,9 @@
import org.apache.fluss.server.kv.snapshot.PeriodicSnapshotManager;
import org.apache.fluss.server.kv.snapshot.RocksIncrementalSnapshot;
import org.apache.fluss.server.kv.snapshot.SnapshotContext;
+import org.apache.fluss.server.log.ColumnGroupAppendInfo;
+import org.apache.fluss.server.log.ColumnGroupFetchPlan;
+import org.apache.fluss.server.log.ColumnGroupLog;
import org.apache.fluss.server.log.FetchDataInfo;
import org.apache.fluss.server.log.FetchIsolation;
import org.apache.fluss.server.log.FetchParams;
@@ -512,6 +521,9 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException {
bucketEpoch = requestBucketEpoch;
+ // FIP-45: column-group high watermarks follow the same rule.
+ maybeIncrementColumnGroupHWs();
+
// We may need to increment high watermark since ISR could be down to 1.
return maybeIncrementLeaderHW(logTablet, currentTimeMs);
});
@@ -1833,6 +1845,9 @@ public long getOffset(RemoteLogManager remoteLogManager, ListOffsetsParam listOf
return inReadLock(
leaderIsrUpdateLock,
() -> {
+ if (listOffsetsParam.getColumnGroup() != null) {
+ return getColumnGroupOffset(listOffsetsParam);
+ }
int offsetType = listOffsetsParam.getOffsetType();
if (offsetType == ListOffsetsParam.TIMESTAMP_OFFSET_TYPE) {
return getOffsetByTimestamp(remoteLogManager, listOffsetsParam);
@@ -1930,16 +1945,37 @@ private LogReadInfo readRecords(FetchParams fetchParams, LogTablet logTablet)
FilterContext filterContext = createFilterContext(fetchParams);
+ // FIP-45: a fetch touching column groups is clamped at min(HW, CEW_g) over the groups.
+ List touchedGroups = fetchParams.currentColumnGroups();
+ long upperBound = Long.MAX_VALUE;
+ if (!touchedGroups.isEmpty() && !fetchParams.isFromFollower()) {
+ for (String group : touchedGroups) {
+ upperBound = Math.min(upperBound, logTablet.getColumnGroupHighWatermark(group));
+ }
+ }
+
FetchDataInfo fetchDataInfo;
try {
- fetchDataInfo =
- logTablet.read(
- readOffset,
- fetchParams.maxFetchBytes(),
- fetchParams.isolation(),
- fetchParams.minOneMessage(),
- fetchParams.projection(),
- filterContext);
+ if (upperBound == Long.MAX_VALUE) {
+ fetchDataInfo =
+ logTablet.read(
+ readOffset,
+ fetchParams.maxFetchBytes(),
+ fetchParams.isolation(),
+ fetchParams.minOneMessage(),
+ fetchParams.projection(),
+ filterContext);
+ } else {
+ fetchDataInfo =
+ logTablet.read(
+ readOffset,
+ fetchParams.maxFetchBytes(),
+ fetchParams.isolation(),
+ fetchParams.minOneMessage(),
+ fetchParams.projection(),
+ filterContext,
+ upperBound);
+ }
} finally {
// Close readContext eagerly — it is only used for statistics extraction during
// batch filtering and is NOT referenced by the returned FetchDataInfo records.
@@ -1947,7 +1983,209 @@ private LogReadInfo readRecords(FetchParams fetchParams, LogTablet logTablet)
IOUtils.closeQuietly(filterContext.getReadContext());
}
}
- return new LogReadInfo(fetchDataInfo, initialHighWatermark, initialLogEndOffset);
+
+ if (touchedGroups.isEmpty() || fetchParams.isFromFollower()) {
+ return new LogReadInfo(fetchDataInfo, initialHighWatermark, initialLogEndOffset);
+ }
+
+ // Ship the column-group records covering the base range that was just read. The base
+ // bytes are untouched file slices; the client stitches group rows onto them by offset.
+ Map groupResults = new HashMap<>();
+ // the base read may include a batch running past the bound; group rows never do
+ long lastBaseOffset =
+ Math.min(lastOffsetOf(fetchDataInfo, fetchParams.projection()), upperBound - 1);
+ for (String group : touchedGroups) {
+ LogRecords groupRecords =
+ lastBaseOffset < readOffset
+ ? MemoryLogRecords.EMPTY
+ : logTablet.readColumnGroup(
+ group, readOffset, lastBaseOffset, fetchParams.maxFetchBytes());
+ groupResults.put(
+ group,
+ new ColumnGroupFetchResult(
+ group, logTablet.getColumnGroupHighWatermark(group), groupRecords));
+ }
+ return new LogReadInfo(
+ fetchDataInfo, initialHighWatermark, initialLogEndOffset, groupResults);
+ }
+
+ /** The last base offset contained in a read result, or -1 when it holds no records. */
+ private static long lastOffsetOf(
+ FetchDataInfo fetchDataInfo, @Nullable FileLogProjection projection) {
+ LogRecords records = fetchDataInfo.getRecords();
+ if (records.sizeInBytes() == 0) {
+ return -1L;
+ }
+ if (projection != null) {
+ // the projection walked the batch headers already
+ return projection.lastProjectedOffset();
+ }
+ long last = -1L;
+ for (LogRecordBatch batch : records.batches()) {
+ last = batch.lastLogOffset();
+ }
+ return last;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // FIP-45: column groups
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Follower log end offsets per column group, keyed by follower id then group name. Populated by
+ * follower fetches carrying column-group cursors.
+ */
+ private final Map> followerColumnGroupEndOffsets =
+ new ConcurrentHashMap<>();
+
+ @Nullable private volatile ColumnGroupSchemaGetter baseSchemaGetter;
+
+ /** Schema getter answering the base physical schema of this column-group table. */
+ private ColumnGroupSchemaGetter baseSchemaGetter() {
+ ColumnGroupSchemaGetter getter = baseSchemaGetter;
+ if (getter == null) {
+ getter = ColumnGroupSchemaGetter.base(schemaGetter);
+ baseSchemaGetter = getter;
+ }
+ return getter;
+ }
+
+ /** Plans how a fetch projection maps onto the base log and the column groups. */
+ public ColumnGroupFetchPlan planColumnGroupFetch(@Nullable int[] projectedFields) {
+ Schema schema = schemaGetter.getLatestSchemaInfo().getSchema();
+ if (!schema.hasColumnGroups()) {
+ return ColumnGroupFetchPlan.plan(schema, projectedFields, schemaGetter);
+ }
+ return ColumnGroupFetchPlan.plan(schema, projectedFields, baseSchemaGetter());
+ }
+
+ /** Appends the rows of one column group at existing base offsets (leader only). */
+ public ProduceLogColumnsResultForBucket appendColumnsAsLeader(
+ String groupName, MemoryLogRecords records, long firstSourceOffset) throws Exception {
+ return inReadLock(
+ leaderIsrUpdateLock,
+ () -> {
+ if (!isLeader()) {
+ throw new NotLeaderOrFollowerException(
+ String.format(
+ "Leader not local for bucket %s on tabletServer %d",
+ tableBucket, localTabletServerId));
+ }
+ Schema schema = schemaGetter.getLatestSchemaInfo().getSchema();
+ if (!schema.getColumnGroups().containsKey(groupName)) {
+ throw new UnknownColumnGroupException(
+ String.format(
+ "Column group '%s' is not declared on table %s.",
+ groupName, physicalPath.getTablePath()));
+ }
+ ColumnGroupAppendInfo appendInfo;
+ try {
+ appendInfo =
+ logTablet.appendColumnsAsLeader(
+ groupName, records, firstSourceOffset);
+ } catch (IOException e) {
+ LOG.error(
+ "Error while appending column group '{}' to {}",
+ groupName,
+ tableBucket,
+ e);
+ fatalErrorHandler.onFatalError(e);
+ throw new LogStorageException(
+ "Error while appending column group records to " + tableBucket, e);
+ }
+ maybeIncrementColumnGroupHW(groupName);
+ LOG.trace(
+ "Appended {} of column group '{}' to {}",
+ appendInfo,
+ groupName,
+ tableBucket);
+ return new ProduceLogColumnsResultForBucket(
+ tableBucket,
+ logTablet.getColumnGroupLogEndOffset(groupName),
+ logTablet.getColumnGroupHighWatermark(groupName));
+ });
+ }
+
+ private void maybeIncrementColumnGroupHWs() {
+ for (String group : logTablet.getColumnGroupLogs().keySet()) {
+ maybeIncrementColumnGroupHW(group);
+ }
+ }
+
+ /**
+ * Advances the high watermark of a column group to the smallest log end offset of the group
+ * among the leader and the in-sync followers, mirroring {@link #maybeIncrementLeaderHW}.
+ *
+ * TODO (WP2): followers do not replicate column groups yet, so a follower with no reported
+ * cursor is not counted. Once follower fetches carry {@code PbColumnGroupFetch} cursors and
+ * followers append the shipped group records, an in-sync follower without a cursor must hold
+ * the watermark back exactly like the base high watermark.
+ */
+ private boolean maybeIncrementColumnGroupHW(String groupName) {
+ if (isUnderMinIsr()) {
+ return false;
+ }
+ ColumnGroupLog columnGroupLog = logTablet.getColumnGroupLog(groupName);
+ if (columnGroupLog == null) {
+ return false;
+ }
+ long newHighWatermark = columnGroupLog.logEndOffset();
+ for (FollowerReplica follower : followerReplicasMap.values()) {
+ int followerId = follower.getFollowerId();
+ if (!isrState.maximalIsr().contains(followerId)) {
+ continue;
+ }
+ Map cursors = followerColumnGroupEndOffsets.get(followerId);
+ Long followerEndOffset = cursors == null ? null : cursors.get(groupName);
+ if (followerEndOffset != null && followerEndOffset < newHighWatermark) {
+ newHighWatermark = followerEndOffset;
+ }
+ }
+ boolean incremented = columnGroupLog.maybeIncrementHighWatermark(newHighWatermark);
+ if (incremented) {
+ LOG.debug(
+ "Column group '{}' high watermark of bucket {} advanced to {}",
+ groupName,
+ tableBucket,
+ columnGroupLog.highWatermark());
+ }
+ return incremented;
+ }
+
+ /** Records a follower's column-group log end offsets reported in a fetch (WP2 hook). */
+ public void updateFollowerColumnGroupEndOffsets(int followerId, Map endOffsets) {
+ followerColumnGroupEndOffsets.put(followerId, endOffsets);
+ for (String group : endOffsets.keySet()) {
+ maybeIncrementColumnGroupHW(group);
+ }
+ }
+
+ /** Answers list-offsets for a column group (FIP-45). */
+ private long getColumnGroupOffset(ListOffsetsParam listOffsetsParam) {
+ String group = listOffsetsParam.getColumnGroup();
+ Schema schema = schemaGetter.getLatestSchemaInfo().getSchema();
+ if (!schema.getColumnGroups().containsKey(group)) {
+ throw new UnknownColumnGroupException(
+ String.format(
+ "Column group '%s' is not declared on table %s.",
+ group, physicalPath.getTablePath()));
+ }
+ int offsetType = listOffsetsParam.getOffsetType();
+ if (offsetType == ListOffsetsParam.LATEST_OFFSET_TYPE) {
+ return listOffsetsParam.getFollowerServerId() < 0
+ ? logTablet.getColumnGroupHighWatermark(group)
+ : logTablet.getColumnGroupLogEndOffset(group);
+ } else if (offsetType == ListOffsetsParam.LEADER_END_OFFSET_SNAPSHOT_TYPE) {
+ return logTablet.getColumnGroupLogEndOffset(group);
+ } else if (offsetType == ListOffsetsParam.EARLIEST_OFFSET_TYPE) {
+ ColumnGroupLog columnGroupLog = logTablet.getColumnGroupLog(group);
+ return columnGroupLog == null
+ ? logTablet.logStartOffset()
+ : Math.max(columnGroupLog.logStartOffset(), logTablet.logStartOffset());
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported list offset type " + offsetType + " for column group " + group);
+ }
}
/**
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index b3357019230..7bf6cf90431 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -25,6 +25,7 @@
import org.apache.fluss.exception.ConfigException;
import org.apache.fluss.exception.FencedLeaderEpochException;
import org.apache.fluss.exception.HistoricalPartitionThrottledException;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
import org.apache.fluss.exception.InvalidColumnProjectionException;
import org.apache.fluss.exception.InvalidCoordinatorException;
import org.apache.fluss.exception.InvalidPartitionException;
@@ -61,6 +62,7 @@
import org.apache.fluss.rpc.entity.ListOffsetsResultForBucket;
import org.apache.fluss.rpc.entity.LookupResultForBucket;
import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket;
+import org.apache.fluss.rpc.entity.ProduceLogColumnsResultForBucket;
import org.apache.fluss.rpc.entity.ProduceLogResultForBucket;
import org.apache.fluss.rpc.entity.PutKvResultForBucket;
import org.apache.fluss.rpc.entity.TableStatsResultForBucket;
@@ -74,6 +76,7 @@
import org.apache.fluss.rpc.protocol.Errors;
import org.apache.fluss.rpc.protocol.MergeMode;
import org.apache.fluss.server.coordinator.CoordinatorContext;
+import org.apache.fluss.server.entity.ColumnGroupWriteData;
import org.apache.fluss.server.entity.FetchReqInfo;
import org.apache.fluss.server.entity.LakeBucketOffset;
import org.apache.fluss.server.entity.LookupDataForBucket;
@@ -91,6 +94,7 @@
import org.apache.fluss.server.kv.scan.ScannerManager;
import org.apache.fluss.server.kv.snapshot.CompletedKvSnapshotCommitter;
import org.apache.fluss.server.kv.snapshot.DefaultSnapshotContext;
+import org.apache.fluss.server.log.ColumnGroupFetchPlan;
import org.apache.fluss.server.log.FetchDataInfo;
import org.apache.fluss.server.log.FetchParams;
import org.apache.fluss.server.log.ListOffsetsParam;
@@ -688,6 +692,49 @@ public void appendRecordsToLog(
timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback);
}
+ /**
+ * Appends the rows of one column group to the leader replicas of the given buckets (FIP-45).
+ *
+ * TODO (WP2/WP7): acks are not honoured yet; the response is sent once the leader has
+ * appended locally. With {@code acks=all} the response should wait until the group's high
+ * watermark covers the appended rows, like {@link #appendRecordsToLog} does for base rows.
+ */
+ public void appendColumnsToLog(
+ int timeoutMs,
+ int requiredAcks,
+ String columnGroup,
+ Map entriesPerBucket,
+ Consumer> responseCallback) {
+ if (isRequiredAcksInvalid(requiredAcks)) {
+ throw new InvalidRequiredAcksException("Invalid required acks: " + requiredAcks);
+ }
+ localDiskManager.ensureWritable();
+ List results = new ArrayList<>();
+ for (Map.Entry entry : entriesPerBucket.entrySet()) {
+ TableBucket tb = entry.getKey();
+ ColumnGroupWriteData writeData = entry.getValue();
+ try {
+ Replica replica = getReplicaOrException(tb);
+ results.add(
+ replica.appendColumnsAsLeader(
+ columnGroup,
+ writeData.getRecords(),
+ writeData.getFirstSourceOffset()));
+ } catch (InvalidColumnGroupOffsetException e) {
+ results.add(
+ new ProduceLogColumnsResultForBucket(
+ tb, ApiError.fromThrowable(e), e.getExpectedSourceOffset()));
+ } catch (Exception e) {
+ if (isUnexpectedException(e)) {
+ LOG.error(
+ "Error appending column group '{}' on replica {}", columnGroup, tb, e);
+ }
+ results.add(new ProduceLogColumnsResultForBucket(tb, ApiError.fromThrowable(e)));
+ }
+ }
+ responseCallback.accept(results);
+ }
+
/**
* Fetch records from a replica. Currently, we will return the fetched records immediately.
*
@@ -1752,14 +1799,18 @@ public Map readFromLog(
replica.getTablePath(), replica.getLogFormat()));
}
+ // FIP-45: map the projection onto the base physical log and the column groups.
+ ColumnGroupFetchPlan columnGroupPlan =
+ replica.planColumnGroupFetch(fetchReqInfo.getProjectFields());
fetchParams.setCurrentFetch(
tb.getTableId(),
fetchOffset,
adjustedMaxBytes,
- replica.getSchemaGetter(),
+ columnGroupPlan.schemaGetter(replica.getSchemaGetter()),
replica.getArrowCompressionInfo(),
- fetchReqInfo.getProjectFields(),
+ columnGroupPlan.baseProjection(fetchReqInfo.getProjectFields()),
projectionsCache);
+ fetchParams.setCurrentColumnGroups(columnGroupPlan.touchedGroups());
// If the client prefers remote reads and the offset is covered, return remote fetch
// info.
@@ -1799,6 +1850,9 @@ public Map readFromLog(
new FetchLogResultForBucket(
tb, fetchedData.getRecords(), readInfo.getHighWatermark());
}
+ if (!readInfo.getColumnGroups().isEmpty()) {
+ fetchLogResult = fetchLogResult.withColumnGroups(readInfo.getColumnGroups());
+ }
logReadResult.put(
tb,
new LogReadResult(fetchLogResult, fetchedData.getFetchOffsetMetadata()));
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java
index bd3ef49b35a..5b6010bdf75 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java
@@ -66,6 +66,8 @@
import org.apache.fluss.rpc.messages.PbScanReqForBucket;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.PrefixLookupResponse;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
import org.apache.fluss.rpc.messages.PutKvRequest;
@@ -86,6 +88,7 @@
import org.apache.fluss.server.RpcServiceBase;
import org.apache.fluss.server.authorizer.Authorizer;
import org.apache.fluss.server.coordinator.MetadataManager;
+import org.apache.fluss.server.entity.ColumnGroupWriteData;
import org.apache.fluss.server.entity.FetchReqInfo;
import org.apache.fluss.server.entity.LookupDataForBucket;
import org.apache.fluss.server.entity.NotifyKvSnapshotOffsetData;
@@ -137,6 +140,7 @@
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyLeaderAndIsrRequestData;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifyRemoteLogOffsetsData;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getNotifySnapshotOffsetData;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogColumnsData;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getProduceLogData;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getStopReplicaData;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getTableFilterInfoMap;
@@ -151,6 +155,7 @@
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeLookupResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeNotifyLeaderAndIsrResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePrefixLookupResponse;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogColumnsResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeProduceLogResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makePutKvResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeStopReplicaResponse;
@@ -226,6 +231,21 @@ public CompletableFuture produceLog(ProduceLogRequest reques
return response;
}
+ @Override
+ public CompletableFuture produceLogColumns(
+ ProduceLogColumnsRequest request) {
+ authorizeTable(WRITE, request.getTableId());
+ CompletableFuture response = new CompletableFuture<>();
+ Map writeData = getProduceLogColumnsData(request);
+ replicaManager.appendColumnsToLog(
+ request.getTimeoutMs(),
+ request.getAcks(),
+ request.getColumnGroup(),
+ writeData,
+ bucketResults -> response.complete(makeProduceLogColumnsResponse(bucketResults)));
+ return response;
+ }
+
@Override
public CompletableFuture fetchLog(FetchLogRequest request) {
Map fetchLogData = getFetchLogData(request);
@@ -480,7 +500,8 @@ public CompletableFuture listOffsets(ListOffsetsRequest req
new ListOffsetsParam(
request.getFollowerServerId(),
request.hasOffsetType() ? request.getOffsetType() : null,
- request.hasStartTimestamp() ? request.getStartTimestamp() : null),
+ request.hasStartTimestamp() ? request.getStartTimestamp() : null,
+ request.hasColumnGroup() ? request.getColumnGroup() : null),
tableBuckets,
(responseList) -> response.complete(makeListOffsetsResponse(responseList)));
return response;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
index 9952d485711..0df1eaf5506 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
@@ -54,11 +54,13 @@
import org.apache.fluss.record.MemoryLogRecords;
import org.apache.fluss.remote.RemoteLogFetchInfo;
import org.apache.fluss.remote.RemoteLogSegment;
+import org.apache.fluss.rpc.entity.ColumnGroupFetchResult;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.rpc.entity.LimitScanResultForBucket;
import org.apache.fluss.rpc.entity.ListOffsetsResultForBucket;
import org.apache.fluss.rpc.entity.LookupResultForBucket;
import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket;
+import org.apache.fluss.rpc.entity.ProduceLogColumnsResultForBucket;
import org.apache.fluss.rpc.entity.ProduceLogResultForBucket;
import org.apache.fluss.rpc.entity.PutKvResultForBucket;
import org.apache.fluss.rpc.entity.TableStatsResultForBucket;
@@ -108,6 +110,7 @@
import org.apache.fluss.rpc.messages.PbAlterConfig;
import org.apache.fluss.rpc.messages.PbBucketMetadata;
import org.apache.fluss.rpc.messages.PbBucketOffset;
+import org.apache.fluss.rpc.messages.PbColumnGroupRecords;
import org.apache.fluss.rpc.messages.PbCreateAclRespInfo;
import org.apache.fluss.rpc.messages.PbDatabaseSummary;
import org.apache.fluss.rpc.messages.PbDescribeConfig;
@@ -138,6 +141,8 @@
import org.apache.fluss.rpc.messages.PbPhysicalTablePath;
import org.apache.fluss.rpc.messages.PbPrefixLookupReqForBucket;
import org.apache.fluss.rpc.messages.PbPrefixLookupRespForBucket;
+import org.apache.fluss.rpc.messages.PbProduceLogColumnsReqForBucket;
+import org.apache.fluss.rpc.messages.PbProduceLogColumnsRespForBucket;
import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket;
import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket;
import org.apache.fluss.rpc.messages.PbProducerTableOffsets;
@@ -162,6 +167,8 @@
import org.apache.fluss.rpc.messages.PbValueList;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.PrefixLookupResponse;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
import org.apache.fluss.rpc.messages.PutKvRequest;
@@ -177,6 +184,7 @@
import org.apache.fluss.server.authorizer.AclCreateResult;
import org.apache.fluss.server.authorizer.AclDeleteResult;
import org.apache.fluss.server.entity.AdjustIsrResultForBucket;
+import org.apache.fluss.server.entity.ColumnGroupWriteData;
import org.apache.fluss.server.entity.CommitLakeTableSnapshotsData;
import org.apache.fluss.server.entity.CommitRemoteLogManifestData;
import org.apache.fluss.server.entity.FetchReqInfo;
@@ -923,6 +931,48 @@ public static Map getProduceLogData(
return produceEntryData;
}
+ /** Decodes a produce-log-columns request into per-bucket column-group rows (FIP-45). */
+ public static Map getProduceLogColumnsData(
+ ProduceLogColumnsRequest request) {
+ long tableId = request.getTableId();
+ Map data = new HashMap<>();
+ for (PbProduceLogColumnsReqForBucket bucketReq : request.getBucketsReqsList()) {
+ ByteBuffer recordBuffer = toByteBuffer(bucketReq.getRecordsSlice());
+ MemoryLogRecords logRecords = MemoryLogRecords.pointToByteBuffer(recordBuffer);
+ TableBucket tb =
+ new TableBucket(
+ tableId,
+ bucketReq.hasPartitionId() ? bucketReq.getPartitionId() : null,
+ bucketReq.getBucketId());
+ data.put(tb, new ColumnGroupWriteData(bucketReq.getFirstSourceOffset(), logRecords));
+ }
+ return data;
+ }
+
+ public static ProduceLogColumnsResponse makeProduceLogColumnsResponse(
+ Collection results) {
+ ProduceLogColumnsResponse response = new ProduceLogColumnsResponse();
+ for (ProduceLogColumnsResultForBucket result : results) {
+ PbProduceLogColumnsRespForBucket bucketResp =
+ response.addBucketsResp().setBucketId(result.getBucketId());
+ TableBucket tableBucket = result.getTableBucket();
+ if (tableBucket.getPartitionId() != null) {
+ bucketResp.setPartitionId(tableBucket.getPartitionId());
+ }
+ if (result.failed()) {
+ bucketResp.setError(result.getErrorCode(), result.getErrorMessage());
+ if (result.getExpectedSourceOffset() >= 0) {
+ bucketResp.setExpectedSourceOffset(result.getExpectedSourceOffset());
+ }
+ } else {
+ bucketResp
+ .setLogEndOffset(result.getLogEndOffset())
+ .setHighWatermark(result.getHighWatermark());
+ }
+ }
+ return response;
+ }
+
public static ProduceLogResponse makeProduceLogResponse(
Collection appendLogResultForBucketList) {
ProduceLogResponse produceResponse = new ProduceLogResponse();
@@ -1007,6 +1057,28 @@ public static Map getFetchLogData(FetchLogRequest req
return fetchDataMap;
}
+ private static void setRecords(PbColumnGroupRecords pbGroup, LogRecords records) {
+ if (records instanceof FileLogRecords) {
+ FileChannelChunk chunk = ((FileLogRecords) records).toChunk();
+ pbGroup.setRecords(chunk.getFileChannel(), chunk.getPosition(), chunk.getSize());
+ } else if (records instanceof BytesViewLogRecords) {
+ pbGroup.setRecordsBytesView(((BytesViewLogRecords) records).getBytesView());
+ } else if (records instanceof MemoryLogRecords) {
+ if (records == MemoryLogRecords.EMPTY) {
+ pbGroup.setRecords(new byte[0]);
+ } else {
+ MemoryLogRecords logRecords = (MemoryLogRecords) records;
+ pbGroup.setRecords(
+ logRecords.getMemorySegment(),
+ logRecords.getPosition(),
+ logRecords.sizeInBytes());
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "Not supported log records type: " + records.getClass().getName());
+ }
+ }
+
public static FetchLogResponse makeFetchLogResponse(
Map fetchLogResult,
Map fetchLogErrors) {
@@ -1090,6 +1162,15 @@ public static FetchLogResponse makeFetchLogResponse(
"Not supported log records type: " + records.getClass().getName());
}
}
+ // FIP-45: column-group records ride alongside the base records, zero-copy too.
+ for (ColumnGroupFetchResult groupResult : bucketResult.columnGroups().values()) {
+ PbColumnGroupRecords pbGroup =
+ fetchLogRespForBucket
+ .addColumnGroup()
+ .setGroupName(groupResult.getGroupName())
+ .setHighWatermark(groupResult.getHighWatermark());
+ setRecords(pbGroup, groupResult.getRecords());
+ }
}
if (fetchLogRespMap.containsKey(tb.getTableId())) {
fetchLogRespMap.get(tb.getTableId()).add(fetchLogRespForBucket);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/ColumnGroupLogTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/ColumnGroupLogTest.java
new file mode 100644
index 00000000000..f05fede8b89
--- /dev/null
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/ColumnGroupLogTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.log;
+
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.config.MemorySize;
+import org.apache.fluss.exception.InvalidColumnGroupOffsetException;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.record.LogRecordBatch;
+import org.apache.fluss.record.LogRecords;
+import org.apache.fluss.record.LogTestBase;
+import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.server.metrics.group.TestingMetricGroups;
+import org.apache.fluss.utils.FlussPaths;
+import org.apache.fluss.utils.clock.SystemClock;
+import org.apache.fluss.utils.concurrent.FlussScheduler;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
+import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
+import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for the column-group shadow log ({@link ColumnGroupLog}) hosted by a {@link LogTablet}. */
+class ColumnGroupLogTest extends LogTestBase {
+
+ private static final String GROUP = "geo";
+
+ private @TempDir File tempDir;
+ private File logDir;
+ private FlussScheduler scheduler;
+ private LogTablet logTablet;
+
+ @BeforeEach
+ void setup() throws Exception {
+ super.before();
+ // small segments so group logs roll within a test
+ conf.set(ConfigOptions.LOG_SEGMENT_FILE_SIZE, MemorySize.parse("2kb"));
+ conf.set(ConfigOptions.LOG_INDEX_INTERVAL_SIZE, MemorySize.parse("256b"));
+ logDir =
+ LogTestUtils.makeRandomLogTabletDir(
+ tempDir,
+ DATA1_TABLE_PATH.getDatabaseName(),
+ DATA1_TABLE_ID,
+ DATA1_TABLE_PATH.getTableName());
+ scheduler = new FlussScheduler(1);
+ scheduler.startup();
+ logTablet = createLogTablet(true);
+ }
+
+ @AfterEach
+ void teardown() throws Exception {
+ if (logTablet != null) {
+ logTablet.close();
+ }
+ scheduler.shutdown();
+ }
+
+ private LogTablet createLogTablet(boolean cleanShutdown) throws Exception {
+ return LogTablet.create(
+ tempDir,
+ PhysicalTablePath.of(DATA1_TABLE_PATH),
+ logDir,
+ conf,
+ new AtomicBoolean(false),
+ TestingMetricGroups.TABLET_SERVER_METRICS,
+ 0,
+ scheduler,
+ LogFormat.ARROW,
+ 1,
+ false,
+ SystemClock.getInstance(),
+ cleanShutdown);
+ }
+
+ private static MemoryLogRecords rows(int from, int toExclusive) throws Exception {
+ List objects = new ArrayList<>();
+ for (int i = from; i < toExclusive; i++) {
+ objects.add(new Object[] {i, "row-" + i});
+ }
+ return genMemoryLogRecordsByObject(objects);
+ }
+
+ /** Appends {@code count} base rows in one batch and moves the high watermark to the end. */
+ private void appendBase(int count) throws Exception {
+ long from = logTablet.localLogEndOffset();
+ logTablet.appendAsLeader(rows((int) from, (int) from + count));
+ logTablet.updateHighWatermark(logTablet.localLogEndOffset());
+ }
+
+ private static List batchRanges(LogRecords records) {
+ List ranges = new ArrayList<>();
+ for (LogRecordBatch batch : records.batches()) {
+ ranges.add(new long[] {batch.baseLogOffset(), batch.lastLogOffset()});
+ }
+ return ranges;
+ }
+
+ @Test
+ void testAppendStampsBaseOffsetsAndAdvancesWatermark() throws Exception {
+ appendBase(10);
+
+ ColumnGroupAppendInfo info = logTablet.appendColumnsAsLeader(GROUP, rows(0, 4), 0L);
+ assertThat(info.firstOffset()).isEqualTo(0L);
+ assertThat(info.lastOffset()).isEqualTo(3L);
+ assertThat(info.rowCount()).isEqualTo(4);
+ assertThat(info.isDuplicated()).isFalse();
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(4L);
+ // the high watermark is the replica's business; the log alone leaves it at 0
+ assertThat(logTablet.getColumnGroupHighWatermark(GROUP)).isEqualTo(0L);
+
+ info = logTablet.appendColumnsAsLeader(GROUP, rows(4, 10), 4L);
+ assertThat(info.lastOffset()).isEqualTo(9L);
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(10L);
+
+ // batches carry the base offsets they fill and a real commit timestamp
+ LogRecords all = logTablet.readColumnGroup(GROUP, 0L, 9L, Integer.MAX_VALUE);
+ assertThat(batchRanges(all)).containsExactly(new long[] {0, 3}, new long[] {4, 9});
+ for (LogRecordBatch batch : all.batches()) {
+ assertThat(batch.commitTimestamp()).isGreaterThan(0L);
+ assertThat(batch.schemaId()).isEqualTo(schemaId);
+ }
+ assertThat(logTablet.getColumnGroupLog(GROUP).highWatermark()).isEqualTo(0L);
+ assertThat(FlussPaths.columnGroupLogDir(logDir, GROUP)).isDirectory();
+ }
+
+ @Test
+ void testValidation() throws Exception {
+ appendBase(10);
+
+ // gap
+ assertThatThrownBy(() -> logTablet.appendColumnsAsLeader(GROUP, rows(2, 4), 2L))
+ .isInstanceOf(InvalidColumnGroupOffsetException.class)
+ .satisfies(
+ e ->
+ assertThat(
+ ((InvalidColumnGroupOffsetException) e)
+ .getExpectedSourceOffset())
+ .isEqualTo(0L));
+ // past the base high watermark
+ assertThatThrownBy(() -> logTablet.appendColumnsAsLeader(GROUP, rows(0, 11), 0L))
+ .isInstanceOf(InvalidColumnGroupOffsetException.class)
+ .hasMessageContaining("high watermark");
+
+ logTablet.appendColumnsAsLeader(GROUP, rows(0, 5), 0L);
+
+ // whole-batch replay is a no-op
+ ColumnGroupAppendInfo dup = logTablet.appendColumnsAsLeader(GROUP, rows(1, 3), 1L);
+ assertThat(dup.isDuplicated()).isTrue();
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(5L);
+
+ // straddling batch carries the expected offset
+ assertThatThrownBy(() -> logTablet.appendColumnsAsLeader(GROUP, rows(3, 8), 3L))
+ .isInstanceOf(InvalidColumnGroupOffsetException.class)
+ .satisfies(
+ e ->
+ assertThat(
+ ((InvalidColumnGroupOffsetException) e)
+ .getExpectedSourceOffset())
+ .isEqualTo(5L));
+ }
+
+ @Test
+ void testRollAndRangedReadAcrossSegments() throws Exception {
+ appendBase(200);
+ for (int i = 0; i < 200; i += 10) {
+ logTablet.appendColumnsAsLeader(GROUP, rows(i, i + 10), i);
+ }
+ ColumnGroupLog columnGroupLog = logTablet.getColumnGroupLog(GROUP);
+ assertThat(columnGroupLog.logEndOffset()).isEqualTo(200L);
+ assertThat(columnGroupLog.segments().size()).isGreaterThan(1);
+
+ // a range inside one batch returns just that batch
+ assertThat(batchRanges(logTablet.readColumnGroup(GROUP, 42L, 47L, Integer.MAX_VALUE)))
+ .containsExactly(new long[] {40, 49});
+ // a range spanning segments returns every batch covering it and nothing beyond
+ List ranges =
+ batchRanges(logTablet.readColumnGroup(GROUP, 95L, 133L, Integer.MAX_VALUE));
+ assertThat(ranges.get(0)).containsExactly(90L, 99L);
+ assertThat(ranges.get(ranges.size() - 1)).containsExactly(130L, 139L);
+ assertThat(ranges).hasSize(5);
+ // a byte budget cuts at whole batches but always returns at least one
+ List budgeted = batchRanges(logTablet.readColumnGroup(GROUP, 0L, 199L, 1));
+ assertThat(budgeted).hasSize(1);
+ assertThat(budgeted.get(0)).containsExactly(0L, 9L);
+ // nothing beyond the log end
+ assertThat(logTablet.readColumnGroup(GROUP, 200L, 250L, Integer.MAX_VALUE).sizeInBytes())
+ .isEqualTo(0);
+ }
+
+ @Test
+ void testTruncateFollowsBaseLog() throws Exception {
+ appendBase(50);
+ for (int i = 0; i < 50; i += 10) {
+ logTablet.appendColumnsAsLeader(GROUP, rows(i, i + 10), i);
+ }
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(50L);
+
+ // like the base log, batches are truncated whole (the batch [20, 29] is dropped) while
+ // the end offset becomes the requested one, so both logs stay aligned
+ logTablet.truncateTo(25L);
+ assertThat(logTablet.localLogEndOffset()).isEqualTo(25L);
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(25L);
+ assertThat(batchRanges(logTablet.readColumnGroup(GROUP, 0L, 49L, Integer.MAX_VALUE)))
+ .containsExactly(new long[] {0, 9}, new long[] {10, 19});
+
+ // the group continues from the truncation point
+ logTablet.appendAsLeader(rows(25, 30));
+ logTablet.updateHighWatermark(30L);
+ logTablet.appendColumnsAsLeader(GROUP, rows(25, 30), 25L);
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(30L);
+ }
+
+ @Test
+ void testReopenRecoversGroupLog() throws Exception {
+ appendBase(60);
+ for (int i = 0; i < 60; i += 10) {
+ logTablet.appendColumnsAsLeader(GROUP, rows(i, i + 10), i);
+ }
+ logTablet.flush(true);
+ logTablet.close();
+
+ // an unclean reopen re-scans the last segment and restores the watermark
+ logTablet = createLogTablet(false);
+ assertThat(logTablet.getColumnGroupLogs()).containsKey(GROUP);
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(60L);
+ assertThat(batchRanges(logTablet.readColumnGroup(GROUP, 55L, 59L, Integer.MAX_VALUE)))
+ .containsExactly(new long[] {50, 59});
+
+ // and appends continue where they stopped: a replay of filled rows is a no-op, a gap is
+ // rejected with the recovered offset
+ logTablet.updateHighWatermark(60L);
+ assertThat(logTablet.appendColumnsAsLeader(GROUP, rows(0, 5), 0L).isDuplicated()).isTrue();
+ assertThat(logTablet.getColumnGroupLogEndOffset(GROUP)).isEqualTo(60L);
+ assertThatThrownBy(() -> logTablet.appendColumnsAsLeader(GROUP, rows(0, 5), 70L))
+ .isInstanceOf(InvalidColumnGroupOffsetException.class)
+ .hasMessageContaining("expects rows to start at offset 60")
+ .satisfies(
+ e ->
+ assertThat(
+ ((InvalidColumnGroupOffsetException) e)
+ .getExpectedSourceOffset())
+ .isEqualTo(60L));
+ }
+}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java
index 06ff7078e20..5165cf7f1b6 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TestTabletServerGateway.java
@@ -81,6 +81,8 @@
import org.apache.fluss.rpc.messages.PbTableBucket;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.PrefixLookupResponse;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsRequest;
+import org.apache.fluss.rpc.messages.ProduceLogColumnsResponse;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
import org.apache.fluss.rpc.messages.PutKvRequest;
@@ -175,6 +177,12 @@ public CompletableFuture produceLog(ProduceLogRequest reques
return response;
}
+ @Override
+ public CompletableFuture produceLogColumns(
+ ProduceLogColumnsRequest request) {
+ return null;
+ }
+
@Override
public CompletableFuture fetchLog(FetchLogRequest request) {
Map fetchLogData = getFetchLogData(request);