Skip to content

support dlo output object schema #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zach-maddox merged 2 commits into from
Sep 16, 2026
Merged

support dlo output object schema #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f79ab5
support dlo output object schema
zach-maddox Sep 15, 2026
47b47ee
linter fix
zach-maddox Sep 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions src/datacustomcode/deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,15 +516,13 @@ def create_data_transform(
"version": "56.0",
}

# outputDataObjects is only set for DMO-backed transforms. The server requires
# the schema of any DMO created/updated by the transform; DLO transforms use
# an existing materialized table and must not include this field.
if isinstance(data_transform_config.permissions.write, DmoPermission):
if not data_transform_config.dataObjects:
if not data_transform_config.dataObjects:
if isinstance(data_transform_config.permissions.write, DmoPermission):
raise ValueError(
"DMO transforms require 'dataObjects' in config.json describing "
"the schema of each output DMO."
)
else:
definition["outputDataObjects"] = [
_data_object_to_output(obj) for obj in data_transform_config.dataObjects
]
Expand Down
114 changes: 113 additions & 1 deletion tests/test_deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,6 +1018,37 @@ def test_get_config_dmo_with_data_objects(self, mock_file):
assert obj.fields[0].dataType == "text"
assert obj.fields[0].keyQualifierFieldName == "KQ_Id1__c"

@patch(
"builtins.open",
new_callable=mock_open,
read_data=(
'{"sdkVersion": "0.1.14", "entryPoint": "entrypoint.py", '
'"dataspace": "default", '
'"permissions": {"read": {"dlo": ["input_dlo"]}, '
'"write": {"dlo": ["output_dlo"]}}, '
'"dataObjects": [{'
'"name": "output_dlo", '
'"label": "Output DLO", '
'"type": "dataLakeObject", '
'"category": "profile", '
'"fields": [{"name": "Id__c", "label": "Id", '
'"dataType": "text", "isPrimaryKey": true, '
'"keyQualifierFieldName": "KQ_Id1__c"}]'
"}]}"
),
)
def test_get_config_dlo_with_data_objects(self, mock_file):
"""config.json parses the optional dataObjects schema for DLO writes."""
result = get_config("/test/dir")
assert isinstance(result, DataTransformConfig)
assert result.dataObjects is not None
assert len(result.dataObjects) == 1
obj = result.dataObjects[0]
assert obj.name == "output_dlo"
assert obj.category == "profile"
assert obj.fields[0].dataType == "text"
assert obj.fields[0].keyQualifierFieldName == "KQ_Id1__c"


class TestCreateDataTransform:
@patch("datacustomcode.deploy.get_config")
Expand Down Expand Up @@ -1286,12 +1317,93 @@ def test_create_data_transform_dmo_emits_output_data_objects(
}
]

@patch("datacustomcode.deploy.get_config")
@patch("datacustomcode.deploy._make_api_call")
def test_create_data_transform_dlo_emits_output_data_objects(
self, mock_make_api_call, mock_get_config
):
"""DLO transforms include outputDataObjects with transformed field names."""
access_token = AccessTokenResponse(
access_token="test_token", instance_url="https://instance.example.com"
)
metadata = CodeExtensionMetadata(
name="test_package",
version="1.0.0",
description="DLO with schema",
computeType="CPU_M",
codeType="script",
)

data_transform_config = DataTransformConfig(
sdkVersion="0.1.14",
entryPoint="entrypoint.py",
dataspace="default",
permissions=Permissions(
read=DloPermission(dlo=["input_dlo"]),
write=DloPermission(dlo=["output_dlo"]),
),
dataObjects=[
DataObject(
name="output_dlo",
label="Output DLO",
type="dataLakeObject",
category="profile",
fields=[
DataObjectField(
name="Id__c",
label="Id",
dataType="text",
isPrimaryKey=True,
keyQualifierFieldName="KQ_Id1__c",
),
DataObjectField(
name="KQ_Id1__c",
label="Key Qualifier Id",
dataType="text",
isPrimaryKey=False,
keyQualifierFieldName=None,
),
],
)
],
)
mock_make_api_call.return_value = {"id": "transform_id"}

create_data_transform(
"/test/dir", access_token, metadata, data_transform_config
)

request_body = mock_make_api_call.call_args[1]["json"]
assert request_body["definition"]["outputDataObjects"] == [
{
"category": "profile",
"fields": [
{
"isPrimaryKey": True,
"keyQualifierField": "KQ_Id1__c",
"label": "Id",
"name": "Id__c",
"type": "text",
},
{
"isPrimaryKey": False,
"label": "Key Qualifier Id",
"name": "KQ_Id1__c",
"type": "text",
},
],
"label": "Output DLO",
"name": "output_dlo",
"type": "dataLakeObject",
}
]

@patch("datacustomcode.deploy.get_config")
@patch("datacustomcode.deploy._make_api_call")
def test_create_data_transform_dlo_omits_output_data_objects(
self, mock_make_api_call, mock_get_config
):
"""DLO transforms must not include outputDataObjects in the payload."""
"""DLO transforms without dataObjects omit outputDataObjects from the payload"""
access_token = AccessTokenResponse(
access_token="test_token", instance_url="https://instance.example.com"
)
Expand Down
Loading