# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pipelines: - pipeline: type: composite transforms: - type: Create config: elements: - {a: "x", b: 1} - {a: "x", b: 2} - {a: "x", b: 3} - {a: "y", b: 10} - type: chain name: Filtering input: Create transforms: - type: Sql config: query: "SELECT * FROM PCOLLECTION WHERE b > 2" - type: AssertEqual config: elements: - {a: "x", b: 3} - {a: "y", b: 10} - type: chain name: Grouping input: Create transforms: - type: Sql config: query: "SELECT a, sum(b) as s FROM PCOLLECTION GROUP BY a" - type: AssertEqual config: elements: - {a: "x", s: 6} - {a: "y", s: 10} - type: chain name: Dialect input: Create transforms: - type: Sql config: query: "SELECT a, cast(b AS VARCHAR) as s FROM PCOLLECTION" - type: AssertEqual config: elements: - {a: "x", s: "1"} - {a: "x", s: "2"} - {a: "x", s: "3"} - {a: "y", s: "10"} - pipeline: type: chain transforms: - type: Create name: CreateSampleData config: elements: - { id: 1, name: "John" } - { id: 2, name: "Jane" } - type: Sql name: sql config: query: > SELECT *, CURRENT_TIMESTAMP AS ingest_timestamp FROM PCOLLECTION - type: PyTransform config: constructor: apache_beam.transforms.util.LogElements # Test calcite_connection_properties option with PostgreSQL functions - pipeline: type: chain transforms: - type: Create name: CreateData config: elements: - {id: 1, text: "hello world"} - {id: 2, text: "foo bar baz"} - type: Sql name: SqlWithPostgresFunction config: query: "SELECT id, INITCAP(text) as title_text FROM PCOLLECTION" - type: AssertEqual config: elements: - {id: 1, title_text: "Hello World"} - {id: 2, title_text: "Foo Bar Baz"} options: calcite_connection_properties: fun: "postgresql"