Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/job-arrow-extension.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ jobs:

- name: Upload to Codecov
if: ${{ !cancelled() && matrix.php == '8.3' && matrix.os == 'ubuntu-latest' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./var/phpunit/coverage/clover
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/job-extension-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:

- name: Upload to Codecov
if: ${{ !cancelled() && matrix.php-version == '8.3' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./var/phpunit/coverage/clover
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/job-flow-php-extension.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ jobs:

- name: Upload to Codecov
if: ${{ !cancelled() && matrix.php == '8.3' && matrix.os == 'ubuntu-latest' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./var/phpunit/coverage/clover
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/job-phpunit-telemetry-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:

- name: Upload to Codecov
if: ${{ !cancelled() && matrix.php-version == '8.3' && matrix.phpunit-version == '11' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./var/phpunit/coverage/clover
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

use function array_diff;
use function array_keys;
use function array_map;
use function array_values;
use function iterator_to_array;
use function sprintf;
Expand Down Expand Up @@ -121,7 +122,10 @@ public function extract(FlowContext $context): Generator

foreach ($reader->batches($source, $batchSize) as $rawBatch) {
if ($columns === null) {
$columns = array_values(array_diff(array_keys($rawBatch[0]->values), $tail));
$columns = array_map(
strval(...),
array_values(array_diff(array_keys($rawBatch[0]->values), $tail)),
);

if (
$this->schema === null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
use function array_key_exists;

/**
* @implements Encoder<array<string, mixed>>
* @implements Encoder<array<array-key, mixed>>
*/
final class ParquetEncoder implements Encoder
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,9 @@ private function openWriter(DestinationStream $stream): Writer

private function schema(): Schema
{
return ($this->schema ?? $this->inferredSchema ?? throw new RuntimeException(
return ($this->schema
?? $this->inferredSchema
?? throw new RuntimeException(
'Schema has not been inferred yet. Load at least one batch of rows first.',
))->gracefulRemove(...$this->router->droppedNames());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public static function columnNames(Filesystem $filesystem, string $uri): array
}

/**
* @return list<array<string, mixed>>
* @return list<array<array-key, mixed>>
*/
public static function values(Filesystem $filesystem, string $uri): array
{
Expand Down
4 changes: 2 additions & 2 deletions src/core/etl/src/Flow/ETL/DSL/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ function rename_replace(string|array $search, string|array $replace): RenameRepl
}

/**
* @param array<string, string> $renames Map of old_name => new_name
* @param array<array-key, string> $renames Map of old_name => new_name
*/
#[DocumentationDSL(module: Module::CORE, type: DSLType::TRANSFORMER)]
function rename_map(array $renames): RenameMapEntryStrategy
Expand All @@ -584,7 +584,7 @@ function rename_map(array $renames): RenameMapEntryStrategy
}

/**
* @param array<string, mixed> $values
* @param array<array-key, mixed> $values
*/
#[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)]
function row(array $values): Row
Expand Down
4 changes: 2 additions & 2 deletions src/core/etl/src/Flow/ETL/Extractor/FileConstants.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ public function __construct(
) {}

/**
* @param array<string, mixed> $row
* @param array<array-key, mixed> $row
*
* @return array<string, mixed>
* @return array<array-key, mixed>
*/
public function fill(array $row): array
{
Expand Down
8 changes: 4 additions & 4 deletions src/core/etl/src/Flow/ETL/Extractor/PartitionColumns.php
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,11 @@ public function apply(Rows $rows, array $names, PartitionTypes $types = new Part
}

/**
* @param array<string, bool> $names
* @param array<string, mixed> $values partition name => value, for the path this row came from
* @param array<string, mixed> $row
* @param array<array-key, bool> $names
* @param array<array-key, mixed> $values partition name => value, for the path this row came from
* @param array<array-key, mixed> $row
*
* @return array<string, mixed>
* @return array<array-key, mixed>
*/
public function fill(array $row, array $names, array $values): array
{
Expand Down
18 changes: 10 additions & 8 deletions src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@
final class RowMerger
{
/**
* @var array<string, true>
* @var array<array-key, true>
*/
private readonly array $dropLeft;

/**
* @var array<string, true>
* @var array<array-key, true>
*/
private readonly array $dropRight;

/**
* @var array<string, array{array<string, true>, array<string, true>, array<string, string>}>
* @var array<string, array{array<array-key, true>, array<array-key, true>, array<array-key, string>}>
*/
private array $plans = [];

Expand Down Expand Up @@ -89,12 +89,12 @@ public function merge(Row $left, Row $right): Row
}

/**
* @param array<string> $leftNames
* @param array<string> $rightNames
* @param list<array-key> $leftNames
* @param list<array-key> $rightNames
*
* @throws DuplicatedEntriesException
*
* @return array{array<string, true>, array<string, true>, array<string, string>}
* @return array{array<array-key, true>, array<array-key, true>, array<array-key, string>}
*/
private function plan(array $leftNames, array $rightNames): array
{
Expand All @@ -121,9 +121,11 @@ private function plan(array $leftNames, array $rightNames): array
}

$keepRight[$name] = true;
$outputName = $this->prefix === '' ? $name : $this->prefix . $name;

if ($this->prefix !== '') {
if ($this->prefix === '') {
$outputName = $name;
} else {
$outputName = $this->prefix . $name;
$renames[$name] = $outputName;
}

Expand Down
8 changes: 4 additions & 4 deletions src/core/etl/src/Flow/ETL/Row.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
final readonly class Row
{
/**
* @param array<string, mixed> $values storage keyed by name; the Schema defines column order (Rows::schema()->references())
* @param array<array-key, mixed> $values storage keyed by name; the Schema defines column order (Rows::schema()->references())
*/
public function __construct(
private array $values,
Expand Down Expand Up @@ -129,7 +129,7 @@ public function hash(Schema $schema, Algorithm $algorithm = new NativePHPHash())
*/
public function names(): array
{
return array_keys($this->values);
return array_map(strval(...), array_keys($this->values));
}

/**
Expand All @@ -152,7 +152,7 @@ public function toArray(bool $withKeys = true): array
}

/**
* @return array<string, mixed>
* @return array<array-key, mixed>
*/
public function values(): array
{
Expand Down Expand Up @@ -195,7 +195,7 @@ private function conform(Schema $schema, bool $checkValues): self
if ($taken !== count($this->values)) {
foreach ($this->values as $name => $_) {
if (!array_key_exists($name, $definitions)) {
throw ColumnMismatchException::unexpectedColumn($name);
throw ColumnMismatchException::unexpectedColumn((string) $name);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
use Flow\ETL\Row\TypedValueComparator;
use Flow\ETL\Schema;

use function array_keys;

final class NativeComparator implements Comparator
{
public function equals(Row $row, Row $nextRow, Schema $schema): bool
{
if ($row->names() !== $nextRow->names()) {
if (array_keys($row->values()) !== array_keys($nextRow->values())) {
return false;
}

Expand Down
2 changes: 0 additions & 2 deletions src/core/etl/src/Flow/ETL/Row/InferredBatch.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public function of(array $batch): Rows
// RawRowValues::$values is declared array<string, mixed>, but PHP hands a numeric
// column name back as an INT key - a pivot names its columns by their own values.
// definition_from_type() takes Reference|string, so the cast is not redundant at runtime.
// @mago-ignore analysis:redundant-cast
$name = (string) $name;
$definition = definition_from_type($name, $this->typeDetector->detectType($value), $value === null);

Expand All @@ -60,7 +59,6 @@ public function of(array $batch): Rows
/** @var mixed $value */
foreach ($rowValues->values as $name => $value) {
// see above - an int key here would miss the column the schema declares as "5"
// @mago-ignore analysis:redundant-cast
$name = (string) $name;
$definition = $schema->get($name);

Expand Down
2 changes: 1 addition & 1 deletion src/core/etl/src/Flow/ETL/Row/RawRowValues.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
final readonly class RawRowValues
{
/**
* @param array<string, mixed> $values
* @param array<array-key, mixed> $values
* @param array<array-key, Metadata> $metadata
*/
public function __construct(
Expand Down
4 changes: 2 additions & 2 deletions src/core/etl/src/Flow/ETL/Row/RowRenaming.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
final readonly class RowRenaming
{
/**
* @param array<string, string> $renames current_name => new_name
* @param array<array-key, string> $renames current_name => new_name
*/
private function __construct(
private array $renames,
) {}

/**
* @param array<string, string> $renames current_name => new_name
* @param array<array-key, string> $renames current_name => new_name
*/
public static function of(array $renames): self
{
Expand Down
4 changes: 2 additions & 2 deletions src/core/etl/src/Flow/ETL/Rows.php
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public static function conformed(Schema $schema, array $rows): self
}

/**
* @return array{schema: Schema, rows: list<array<string, mixed>>}
* @return array{schema: Schema, rows: list<array<array-key, mixed>>}
*/
public function __serialize(): array
{
Expand Down Expand Up @@ -149,7 +149,7 @@ public function __serialize(): array
}

/**
* @param array{schema: Schema, rows: list<array<string, mixed>>} $data
* @param array{schema: Schema, rows: list<array<array-key, mixed>>} $data
*/
public function __unserialize(array $data): void
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
interface RenameEntryStrategy
{
/**
* @return array<string, string> map of current_name => new_name, empty when nothing is renamed
* @return array<array-key, string> map of current_name => new_name; PHP casts a numeric name to an int key, empty when nothing is renamed
*/
public function renames(Schema $schema): array;
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
final readonly class RenameMapEntryStrategy implements RenameEntryStrategy
{
/**
* @param array<string, string> $renames Map of old_name => new_name
* @param array<array-key, string> $renames Map of old_name => new_name
*/
public function __construct(
private array $renames,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
use Throwable;

use function array_search;
use function is_string;

final readonly class RenameEachEntryTransformer implements Transformer
{
Expand All @@ -40,7 +39,7 @@ public function bind(Schema $input): BoundStep

foreach ($this->strategies as $strategy) {
foreach ($strategy->renames($output) as $from => $to) {
$output = $output->rename($from, $to);
$output = $output->rename((string) $from, $to);
}
}

Expand All @@ -57,13 +56,16 @@ public function transform(Rows $rows, FlowContext $context): Rows

foreach ($this->strategies as $strategy) {
foreach ($strategy->renames($schema) as $from => $to) {
// a numeric entry name arrives as int - PHP casts numeric-string array keys
$from = (string) $from;

$schema = $schema->rename($from, $to);

// strategies chain, so a later one renames what an earlier one produced; the map
// has to stay keyed by the row's original name or the projection lands short
$original = array_search($from, $renames, true);

$renames[is_string($original) ? $original : $from] = $to;
$renames[false === $original ? $from : $original] = $to;
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/core/etl/src/Flow/ETL/Transformer/UnpackedColumns.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ public function of(Schema $base, string $prefix, Schema $declared): Schema
}

/**
* @param array<string, mixed> $values
* @param array<array-key, mixed> $values
* @param array<array-key, mixed> $payload
*
* @return array<string, mixed>
* @return array<array-key, mixed>
*/
public function values(array $values, string $prefix, Schema $declared, array $payload): array
{
Expand Down
6 changes: 3 additions & 3 deletions src/core/etl/src/Flow/Floe/FloeLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,9 @@ private function closeWriters(): void
private function openWriter(DestinationStream $stream, FlowContext $context): FloeWriter
{
$declared =
$this->schema ?? $this->inferredSchema ?? throw new FloeException(
'Floe loader has no schema to write with',
);
$this->schema
?? $this->inferredSchema
?? throw new FloeException('Floe loader has no schema to write with');

// partitionBy() keeps the partition columns in the path rather than the body, so the file
// schema must not declare them either. Every other declared column stays - a batch that does
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ enum_schema('enum', BackedStringEnum::class),
xml_schema('xml'),
),
// PHP casts the numeric column name "100" to an int array key
// @mago-ignore analysis:possibly-invalid-argument
row([
'id' => 1234,
'price' => 123.45,
Expand Down Expand Up @@ -159,7 +158,6 @@ public function extract(FlowContext $context): Generator
string_schema('group'),
),
// PHP casts the numeric column name "100" to an int array key
// @mago-ignore analysis:possibly-invalid-argument
row([
'id' => 1234,
'price' => 123.45,
Expand All @@ -182,7 +180,6 @@ public function extract(FlowContext $context): Generator
string_schema('group'),
),
// PHP casts the numeric column name "100" to an int array key
// @mago-ignore analysis:possibly-invalid-argument
row([
'id' => 1234,
'price' => 123.45,
Expand Down
Loading
Loading