Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5e7e66f

Browse files
kileyskileys
authored
[BEAM-11129] Add namespace and key to portable display data (#15564)
1 parent af711c5 commit 5e7e66f

5 files changed

Lines changed: 136 additions & 17 deletions

File tree

‎model/pipeline/src/main/proto/beam_runner_api.proto‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1718,7 +1718,14 @@ message LabelledPayload {
17181718
string string_value = 2;
17191719
bool bool_value = 3;
17201720
double double_value = 4;
1721+
int64 int_value = 5;
17211722
}
1723+
1724+
// (Required) The key identifies the actual content of the metadata.
1725+
string key = 6;
1726+
1727+
// (Required) The namespace describes the context that specified the key.
1728+
string namespace = 7;
17221729
}
17231730

17241731
// Static display data associated with a pipeline component. Display data is

‎runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,7 @@ public class DisplayDataTranslation {
4141
}
4242

4343
private static final Map<String, Function<DisplayData.Item, ByteString>>
44-
WELL_KNOWN_URN_TRANSLATORS =
45-
ImmutableMap.of(LABELLED, DisplayDataTranslation::translateStringUtf8);
44+
WELL_KNOWN_URN_TRANSLATORS = ImmutableMap.of(LABELLED, DisplayDataTranslation::translate);
4645

4746
public static List<RunnerApi.DisplayData> toProto(DisplayData displayData) {
4847
ImmutableList.Builder<RunnerApi.DisplayData> builder = ImmutableList.builder();
@@ -54,7 +53,7 @@ public static List<RunnerApi.DisplayData> toProto(DisplayData displayData) {
5453
urn = item.getKey();
5554
} else {
5655
urn = LABELLED;
57-
translator = DisplayDataTranslation::translateStringUtf8;
56+
translator = DisplayDataTranslation::translate;
5857
}
5958
builder.add(
6059
RunnerApi.DisplayData.newBuilder()
@@ -65,13 +64,24 @@ public static List<RunnerApi.DisplayData> toProto(DisplayData displayData) {
6564
return builder.build();
6665
}
6766

68-
private static ByteString translateStringUtf8(DisplayData.Item item) {
69-
String value = String.valueOf(item.getValue() == null ? item.getShortValue() : item.getValue());
67+
private static ByteString translate(DisplayData.Item item) {
7068
String label = item.getLabel() == null ? item.getKey() : item.getLabel();
71-
return RunnerApi.LabelledPayload.newBuilder()
72-
.setLabel(label)
73-
.setStringValue(value)
74-
.build()
75-
.toByteString();
69+
String namespace = item.getNamespace() == null ? "" : item.getNamespace().getName();
70+
RunnerApi.LabelledPayload.Builder builder =
71+
RunnerApi.LabelledPayload.newBuilder()
72+
.setKey(item.getKey())
73+
.setLabel(label)
74+
.setNamespace(namespace);
75+
Object valueObj = item.getValue() == null ? item.getShortValue() : item.getValue();
76+
if (valueObj instanceof Boolean) {
77+
builder.setBoolValue((Boolean) valueObj);
78+
} else if (valueObj instanceof Integer || valueObj instanceof Long) {
79+
builder.setIntValue((Long) valueObj);
80+
} else if (valueObj instanceof Number) {
81+
builder.setDoubleValue(((Number) valueObj).doubleValue());
82+
} else {
83+
builder.setStringValue(String.valueOf(valueObj));
84+
}
85+
return builder.build().toByteString();
7686
}
7787
}

‎runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/DisplayDataTranslationTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ public void testTranslation() {
4242
public void populateDisplayData(DisplayData.Builder builder) {
4343
builder.add(DisplayData.item("foo", "value"));
4444
builder.add(DisplayData.item("foo2", "value2").withLabel("label"));
45+
builder.add(DisplayData.item("foo3", true).withLabel("label"));
46+
builder.add(DisplayData.item("foo4", 123.4).withLabel("label"));
47+
builder.add(DisplayData.item("foo5", 4.321f).withLabel("label"));
48+
builder.add(DisplayData.item("foo6", 321).withLabel("label"));
49+
builder.add(DisplayData.item("foo7", 123L).withLabel("label"));
4550
}
4651
})),
4752
containsInAnyOrder(
@@ -51,6 +56,9 @@ public void populateDisplayData(DisplayData.Builder builder) {
5156
RunnerApi.LabelledPayload.newBuilder()
5257
.setLabel("foo")
5358
.setStringValue("value")
59+
.setKey("foo")
60+
.setNamespace(
61+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
5462
.build()
5563
.toByteString())
5664
.build(),
@@ -60,6 +68,69 @@ public void populateDisplayData(DisplayData.Builder builder) {
6068
RunnerApi.LabelledPayload.newBuilder()
6169
.setLabel("label")
6270
.setStringValue("value2")
71+
.setKey("foo2")
72+
.setNamespace(
73+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
74+
.build()
75+
.toByteString())
76+
.build(),
77+
RunnerApi.DisplayData.newBuilder()
78+
.setUrn(DisplayDataTranslation.LABELLED)
79+
.setPayload(
80+
RunnerApi.LabelledPayload.newBuilder()
81+
.setLabel("label")
82+
.setBoolValue(true)
83+
.setKey("foo3")
84+
.setNamespace(
85+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
86+
.build()
87+
.toByteString())
88+
.build(),
89+
RunnerApi.DisplayData.newBuilder()
90+
.setUrn(DisplayDataTranslation.LABELLED)
91+
.setPayload(
92+
RunnerApi.LabelledPayload.newBuilder()
93+
.setLabel("label")
94+
.setDoubleValue(123.4)
95+
.setKey("foo4")
96+
.setNamespace(
97+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
98+
.build()
99+
.toByteString())
100+
.build(),
101+
RunnerApi.DisplayData.newBuilder()
102+
.setUrn(DisplayDataTranslation.LABELLED)
103+
.setPayload(
104+
RunnerApi.LabelledPayload.newBuilder()
105+
.setLabel("label")
106+
.setDoubleValue(4.321000099182129)
107+
.setKey("foo5")
108+
.setNamespace(
109+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
110+
.build()
111+
.toByteString())
112+
.build(),
113+
RunnerApi.DisplayData.newBuilder()
114+
.setUrn(DisplayDataTranslation.LABELLED)
115+
.setPayload(
116+
RunnerApi.LabelledPayload.newBuilder()
117+
.setLabel("label")
118+
.setIntValue(321)
119+
.setKey("foo6")
120+
.setNamespace(
121+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
122+
.build()
123+
.toByteString())
124+
.build(),
125+
RunnerApi.DisplayData.newBuilder()
126+
.setUrn(DisplayDataTranslation.LABELLED)
127+
.setPayload(
128+
RunnerApi.LabelledPayload.newBuilder()
129+
.setLabel("label")
130+
.setIntValue(123)
131+
.setKey("foo7")
132+
.setNamespace(
133+
"org.apache.beam.runners.core.construction.DisplayDataTranslationTest$1")
63134
.build()
64135
.toByteString())
65136
.build()));

‎sdks/python/apache_beam/pipeline_test.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,7 +1016,7 @@ def display_data(self): # type: () -> dict
10161016
'dd_string_value', label='dd_string_label')
10171017
parent_dd['dd_string_2'] = DisplayDataItem('dd_string_value_2')
10181018
parent_dd['dd_bool'] = DisplayDataItem(False, label='dd_bool_label')
1019-
parent_dd['dd_int'] = DisplayDataItem(1.1, label='dd_int_label')
1019+
parent_dd['dd_double'] = DisplayDataItem(1.1, label='dd_double_label')
10201020
return parent_dd
10211021

10221022
p = beam.Pipeline()
@@ -1037,41 +1037,57 @@ def display_data(self): # type: () -> dict
10371037
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10381038
payload=beam_runner_api_pb2.LabelledPayload(
10391039
label='p_dd_string_label',
1040+
key='p_dd_string',
1041+
namespace='apache_beam.pipeline_test.MyPTransform',
10401042
string_value='p_dd_string_value').SerializeToString()),
10411043
beam_runner_api_pb2.DisplayData(
10421044
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10431045
payload=beam_runner_api_pb2.LabelledPayload(
10441046
label='p_dd_string_2',
1047+
key='p_dd_string_2',
1048+
namespace='apache_beam.pipeline_test.MyPTransform',
10451049
string_value='p_dd_string_value_2').SerializeToString()),
10461050
beam_runner_api_pb2.DisplayData(
10471051
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10481052
payload=beam_runner_api_pb2.LabelledPayload(
10491053
label='p_dd_bool_label',
1054+
key='p_dd_bool',
1055+
namespace='apache_beam.pipeline_test.MyPTransform',
10501056
bool_value=True).SerializeToString()),
10511057
beam_runner_api_pb2.DisplayData(
10521058
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10531059
payload=beam_runner_api_pb2.LabelledPayload(
10541060
label='p_dd_int_label',
1055-
double_value=1).SerializeToString()),
1061+
key='p_dd_int',
1062+
namespace='apache_beam.pipeline_test.MyPTransform',
1063+
int_value=1).SerializeToString()),
10561064
beam_runner_api_pb2.DisplayData(
10571065
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10581066
payload=beam_runner_api_pb2.LabelledPayload(
10591067
label='dd_string_label',
1068+
key='dd_string',
1069+
namespace='apache_beam.pipeline_test.MyPTransform',
10601070
string_value='dd_string_value').SerializeToString()),
10611071
beam_runner_api_pb2.DisplayData(
10621072
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10631073
payload=beam_runner_api_pb2.LabelledPayload(
10641074
label='dd_string_2',
1075+
key='dd_string_2',
1076+
namespace='apache_beam.pipeline_test.MyPTransform',
10651077
string_value='dd_string_value_2').SerializeToString()),
10661078
beam_runner_api_pb2.DisplayData(
10671079
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10681080
payload=beam_runner_api_pb2.LabelledPayload(
10691081
label='dd_bool_label',
1082+
key='dd_bool',
1083+
namespace='apache_beam.pipeline_test.MyPTransform',
10701084
bool_value=False).SerializeToString()),
10711085
beam_runner_api_pb2.DisplayData(
10721086
urn=common_urns.StandardDisplayData.DisplayData.LABELLED.urn,
10731087
payload=beam_runner_api_pb2.LabelledPayload(
1074-
label='dd_int_label',
1088+
label='dd_double_label',
1089+
key='dd_double',
1090+
namespace='apache_beam.pipeline_test.MyPTransform',
10751091
double_value=1.1).SerializeToString()),
10761092
])
10771093

‎sdks/python/apache_beam/transforms/display.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,28 @@ def create_payload(dd):
149149
value = display_data_dict['value']
150150
if isinstance(value, str):
151151
return beam_runner_api_pb2.LabelledPayload(
152-
label=label, string_value=value)
152+
label=label,
153+
string_value=value,
154+
key=display_data_dict['key'],
155+
namespace=display_data_dict.get('namespace', ''))
153156
elif isinstance(value, bool):
154157
return beam_runner_api_pb2.LabelledPayload(
155-
label=label, bool_value=value)
156-
elif isinstance(value, (int, float, complex)):
158+
label=label,
159+
bool_value=value,
160+
key=display_data_dict['key'],
161+
namespace=display_data_dict.get('namespace', ''))
162+
elif isinstance(value, int):
157163
return beam_runner_api_pb2.LabelledPayload(
158-
label=label, double_value=value)
164+
label=label,
165+
int_value=value,
166+
key=display_data_dict['key'],
167+
namespace=display_data_dict.get('namespace', ''))
168+
elif isinstance(value, (float, complex)):
169+
return beam_runner_api_pb2.LabelledPayload(
170+
label=label,
171+
double_value=value,
172+
key=display_data_dict['key'],
173+
namespace=display_data_dict.get('namespace', ''))
159174
else:
160175
raise ValueError(
161176
'Unsupported type %s for value of display data %s' %

0 commit comments

Comments
 (0)