import java.util.TreeMap

/*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements. See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership. The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* License); you may not use this file except in compliance

* with the License. You may obtain a copy of the License at

*

* http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an AS IS BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

plugins {

base

// Apply one top level rat plugin to perform any required license enforcement analysis

id("org.nosphere.apache.rat") version "0.8.1"

// Enable gradle-based release management

id("net.researchgate.release") version "2.8.1"

id("org.apache.beam.module")

id("org.sonarqube") version "3.0"

}

/*************************************************************************************************/

// Configure the root project

tasks.rat {

// Set input directory to that of the root project instead of the CWD. This

// makes .gitignore rules (added below) work properly.

inputDir.set(project.rootDir)

val exclusions = mutableListOf(

// Ignore files we track but do not distribute

"**/.github/**/*",

"**/.gitkeep",

"gradlew",

"gradlew.bat",

"gradle/wrapper/gradle-wrapper.properties",

"**/package-list",

"**/test.avsc",

"**/logical-types.avsc",

"**/user.avsc",

"**/test/resources/**/*.txt",

"**/test/resources/**/*.csv",

"**/test/**/.placeholder",

// Default eclipse excludes neglect subprojects

// Proto/grpc generated wrappers

"**/apache_beam/portability/api/**/*_pb2*.py",

"**/go/pkg/beam/**/*.pb.go",

"**/mock-apis/**/*.pb.go",

// Ignore go.sum files, which don't permit headers

"**/go.sum",

// Ignore Go test data files

"**/go/data/**",

// VCF test files

"**/apache_beam/testing/data/vcf/*",

// JDBC package config files

"**/META-INF/services/java.sql.Driver",

// Website build files

"**/Gemfile.lock",

"**/Rakefile",

"**/.htaccess",

"website/www/site/assets/css/**/*",

"website/www/site/assets/scss/_bootstrap.scss",

"website/www/site/assets/scss/bootstrap/**/*",

"website/www/site/assets/js/**/*",

"website/www/site/static/images/mascot/*.ai",

"website/www/site/static/js/bootstrap*.js",

"website/www/site/static/js/bootstrap/**/*",

"website/www/site/themes",

"website/www/yarn.lock",

"website/www/package.json",

"website/www/site/static/js/hero/lottie-light.min.js",

"website/www/site/static/js/keen-slider.min.js",

"website/www/site/assets/scss/_keen-slider.scss",

// Release automation files

"release/src/main/scripts/*.txt",

// Ignore ownership files

"ownership/**/*",

"**/OWNERS",

// Ignore CPython LICENSE file

"LICENSE.python",

// Ignore vendored cloudpickle files

"sdks/python/apache_beam/internal/cloudpickle/**",

"LICENCE.cloudpickle",

// Json doesn't support comments.

"**/*.json",

// Katas files

"learning/katas/**/course-info.yaml",

"learning/katas/**/task-info.yaml",

"learning/katas/**/course-remote-info.yaml",

"learning/katas/**/section-remote-info.yaml",

"learning/katas/**/lesson-remote-info.yaml",

"learning/katas/**/task-remote-info.yaml",

"learning/katas/**/*.txt",

// Tour Of Beam learning-content metadata and its samples

"learning/tour-of-beam/**/content-info.yaml",

"learning/tour-of-beam/**/module-info.yaml",

"learning/tour-of-beam/**/group-info.yaml",

"learning/tour-of-beam/**/unit-info.yaml",

"learning/tour-of-beam/backend/samples/**/*.md",

// Tour Of Beam example logs

"learning/tour-of-beam/learning-content/**/*.log",

// Tour Of Beam example txt files

"learning/tour-of-beam/learning-content/**/*.txt",

// Tour Of Beam example csv files

"learning/tour-of-beam/learning-content/**/*.csv",

// Tour Of Beam backend autogenerated Datastore indexes

"learning/tour-of-beam/backend/internal/storage/index.yaml",

// Tour Of Beam backend autogenerated Playground GRPC API stubs and mocks

"learning/tour-of-beam/backend/playground_api/api/v1/api.pb.go",

"learning/tour-of-beam/backend/playground_api/api/v1/api_grpc.pb.go",

"learning/tour-of-beam/backend/playground_api/api/v1/mock.go",

// Playground backend autogenerated GRPC API stubs and mocks

"playground/backend/internal/api/v1/api.pb.go",

"playground/backend/internal/api/v1/api_grpc.pb.go",

// Playground infrastructure autogenerated GRPC API stubs and mocks

"playground/infrastructure/api/v1/api_pb2.py",

"playground/infrastructure/api/v1/api_pb2.pyi",

"playground/infrastructure/api/v1/api_pb2_grpc.py",

// test p8 file for SnowflakeIO

"sdks/java/io/snowflake/src/test/resources/invalid_test_rsa_key.p8",

"sdks/java/io/snowflake/src/test/resources/valid_encrypted_test_rsa_key.p8",

"sdks/java/io/snowflake/src/test/resources/valid_unencrypted_test_rsa_key.p8",

// Mockito extensions

"sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

"sdks/java/io/azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

"sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

// JupyterLab extensions

"sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock",

// Autogenerated apitools clients.

"sdks/python/apache_beam/runners/dataflow/internal/clients/*/**/*.py",

// Sample text file for Java quickstart

"sdks/java/maven-archetypes/examples/sample.txt",

// Ignore Flutter autogenerated files for Playground

"playground/frontend/**/*.g.dart",

"playground/frontend/**/*.g.yaml",

"playground/frontend/**/*.gen.dart",

"playground/frontend/**/*.golden.yaml",

"playground/frontend/**/*.mocks.dart",

"playground/frontend/.metadata",

"playground/frontend/pubspec.lock",

// Ignore Flutter autogenerated files for Playground Components

"playground/frontend/**/*.pb.dart",

"playground/frontend/**/*.pbenum.dart",

"playground/frontend/**/*.pbgrpc.dart",

"playground/frontend/**/*.pbjson.dart",

"playground/frontend/playground_components/.metadata",

"playground/frontend/playground_components/pubspec.lock",

// Ignore Flutter autogenerated files for Tour of Beam

"learning/tour-of-beam/frontend/**/*.g.dart",

"learning/tour-of-beam/frontend/**/*.gen.dart",

"learning/tour-of-beam/frontend/.metadata",

"learning/tour-of-beam/frontend/pubspec.lock",

"learning/tour-of-beam/frontend/lib/firebase_options.dart",

// Ignore .gitkeep file

"**/.gitkeep",

// Ignore Flutter localization .arb files (doesn't support comments)

"playground/frontend/lib/l10n/**/*.arb",

// Ignore LICENSES copied onto containers

"sdks/java/container/license_scripts/manual_licenses",

"sdks/python/container/license_scripts/manual_licenses",

// Ignore autogenrated proto files.

"sdks/typescript/src/apache_beam/proto/**/*.ts",

// Ignore typesciript package management.

"sdks/typescript/package-lock.json",

"sdks/typescript/node_modules/**/*",

// Ignore buf autogenerated files.

"**/buf.lock",

// Ignore poetry autogenerated files.

"**/poetry.lock",

// DuetAI training prompts

"learning/prompts/**/*.md",

// Ignore terraform lock files

"**/.terraform.lock.hcl",

// Ignore pytest cache files

"**/.pytest_cache/**"

)

// Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior

// of the Apache Maven Rat plugin since the Apache Ant Rat plugin doesn't do this

// automatically.

val gitIgnore = project(":").file(".gitignore")

if (gitIgnore.exists()) {

val gitIgnoreExcludes = gitIgnore.readLines().filter { it.isNotEmpty() && !it.startsWith("#") }

exclusions.addAll(gitIgnoreExcludes)

}

verbose.set(true)

failOnError.set(true)

setExcludes(exclusions)

}

tasks.check.get().dependsOn(tasks.rat)

// Define root pre/post commit tasks simplifying what is needed

// to be specified on the commandline when executing locally.

// This indirection also makes Jenkins use the branch of the PR

// for the test definitions.

tasks.register("javaPreCommit") {

// We need to list the model/* builds since sdks/java/core doesn't

// depend on any of the model.

dependsOn(":model:pipeline:build")

dependsOn(":model:job-management:build")

dependsOn(":model:fn-execution:build")

dependsOn(":sdks:java:core:buildNeeded")

// Inline :sdks:java:core:buildDependents so we can carve out pieces at a time

dependsOn(":beam-validate-runner:build")

dependsOn(":examples:java:build")

dependsOn(":examples:java:preCommit")

dependsOn(":examples:java:sql:build")

dependsOn(":examples:java:sql:preCommit")

dependsOn(":examples:java:twitter:build")

dependsOn(":examples:java:twitter:preCommit")

dependsOn(":examples:java:iceberg:build")

dependsOn(":examples:multi-language:build")

dependsOn(":model:fn-execution:build")

dependsOn(":model:job-management:build")

dependsOn(":model:pipeline:build")

dependsOn(":runners:core-java:build")

dependsOn(":runners:direct-java:build")

dependsOn(":runners:direct-java:needsRunnerTests")

dependsOn(":runners:extensions-java:metrics:build")

// lowest supported flink version

var flinkVersions = project.ext.get("allFlinkVersions") as Array<*>

dependsOn(":runners:flink:${flinkVersions[0]}:build")

dependsOn(":runners:flink:${flinkVersions[0]}:job-server:build")

dependsOn(":runners:google-cloud-dataflow-java:build")

dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:build")

dependsOn(":runners:google-cloud-dataflow-java:examples:build")

dependsOn(":runners:google-cloud-dataflow-java:worker:build")

dependsOn(":runners:google-cloud-dataflow-java:worker:windmill:build")

dependsOn(":runners:java-fn-execution:build")

dependsOn(":runners:java-job-service:build")

dependsOn(":runners:jet:build")

dependsOn(":runners:local-java:build")

dependsOn(":runners:portability:java:build")

dependsOn(":runners:prism:java:build")

dependsOn(":runners:samza:build")

dependsOn(":runners:samza:job-server:build")

dependsOn(":runners:spark:3:build")

dependsOn(":runners:spark:3:job-server:build")

dependsOn(":runners:twister2:build")

dependsOn(":sdks:java:build-tools:build")

dependsOn(":sdks:java:container:java11:docker")

dependsOn(":sdks:java:core:build")

dependsOn(":sdks:java:core:jmh:build")

dependsOn(":sdks:java:expansion-service:build")

dependsOn(":sdks:java:expansion-service:app:build")

dependsOn(":sdks:java:extensions:arrow:build")

dependsOn(":sdks:java:extensions:avro:build")

dependsOn(":sdks:java:extensions:combiners:build")

dependsOn(":sdks:java:extensions:euphoria:build")

dependsOn(":sdks:java:extensions:google-cloud-platform-core:build")

dependsOn(":sdks:java:extensions:jackson:build")

dependsOn(":sdks:java:extensions:join-library:build")

dependsOn(":sdks:java:extensions:kryo:build")

dependsOn(":sdks:java:extensions:ml:build")

dependsOn(":sdks:java:extensions:protobuf:build")

dependsOn(":sdks:java:extensions:python:build")

dependsOn(":sdks:java:extensions:sbe:build")

dependsOn(":sdks:java:extensions:schemaio-expansion-service:build")

dependsOn(":sdks:java:extensions:sketching:build")

dependsOn(":sdks:java:extensions:sorter:build")

dependsOn(":sdks:java:extensions:timeseries:build")

dependsOn(":sdks:java:extensions:yaml:build")

dependsOn(":sdks:java:extensions:zetasketch:build")

dependsOn(":sdks:java:harness:build")

dependsOn(":sdks:java:harness:jmh:build")

dependsOn(":sdks:java:io:bigquery-io-perf-tests:build")

dependsOn(":sdks:java:io:common:build")

dependsOn(":sdks:java:io:contextualtextio:build")

dependsOn(":sdks:java:io:expansion-service:build")

dependsOn(":sdks:java:io:file-based-io-tests:build")

dependsOn(":sdks:java:io:kafka:jmh:build")

dependsOn(":sdks:java:io:sparkreceiver:3:build")

dependsOn(":sdks:java:io:synthetic:build")

dependsOn(":sdks:java:io:xml:build")

dependsOn(":sdks:java:javadoc:allJavadoc")

dependsOn(":sdks:java:managed:build")

dependsOn("sdks:java:ml:inference:remote:build")

dependsOn("sdks:java:ml:inference:openai:build")

dependsOn(":sdks:java:testing:expansion-service:build")

dependsOn(":sdks:java:testing:jpms-tests:build")

dependsOn(":sdks:java:testing:junit:build")

dependsOn(":sdks:java:testing:load-tests:build")

dependsOn(":sdks:java:testing:nexmark:build")

dependsOn(":sdks:java:testing:test-utils:build")

dependsOn(":sdks:java:testing:tpcds:build")

dependsOn(":sdks:java:testing:watermarks:build")

dependsOn(":sdks:java:transform-service:build")

dependsOn(":sdks:java:transform-service:app:build")

dependsOn(":sdks:java:transform-service:launcher:build")

}

// a precommit task build multiple IOs (except those splitting into single jobs)

tasks.register("javaioPreCommit") {

dependsOn(":sdks:java:io:amqp:build")

dependsOn(":sdks:java:io:cassandra:build")

dependsOn(":sdks:java:io:csv:build")

dependsOn(":sdks:java:io:cdap:build")

dependsOn(":sdks:java:io:clickhouse:build")

dependsOn(":sdks:java:io:datadog:build")

dependsOn(":sdks:java:io:debezium:expansion-service:build")

dependsOn(":sdks:java:io:debezium:build")

dependsOn(":sdks:java:io:elasticsearch:build")

dependsOn(":sdks:java:io:file-schema-transform:build")

dependsOn(":sdks:java:io:google-ads:build")

dependsOn(":sdks:java:io:hbase:build")

dependsOn(":sdks:java:io:hcatalog:build")

dependsOn(":sdks:java:io:influxdb:build")

dependsOn(":sdks:java:io:jdbc:build")

dependsOn(":sdks:java:io:jms:build")

dependsOn(":sdks:java:io:kafka:build")

dependsOn(":sdks:java:io:kafka:upgrade:build")

dependsOn(":sdks:java:extensions:kafka-factories:build")

dependsOn(":sdks:java:io:kudu:build")

dependsOn(":sdks:java:io:mongodb:build")

dependsOn(":sdks:java:io:mqtt:build")

dependsOn(":sdks:java:io:neo4j:build")

dependsOn(":sdks:java:io:parquet:build")

dependsOn(":sdks:java:io:pulsar:build")

dependsOn(":sdks:java:io:rabbitmq:build")

dependsOn(":sdks:java:io:redis:build")

dependsOn(":sdks:java:io:rrio:build")

dependsOn(":sdks:java:io:singlestore:build")

dependsOn(":sdks:java:io:solr:build")

dependsOn(":sdks:java:io:splunk:build")

dependsOn(":sdks:java:io:thrift:build")

dependsOn(":sdks:java:io:tika:build")

}

// a precommit task testing additional supported flink versions not covered by

// the main Java PreCommit (lowest supported version)

tasks.register("flinkPreCommit") {

var flinkVersions = project.ext.get("allFlinkVersions") as Array<*>

for (version in flinkVersions.slice(1..flinkVersions.size - 1)) {

dependsOn(":runners:flink:${version}:build")

dependsOn(":runners:flink:${version}:job-server:build")

}

}

tasks.register("sqlPreCommit") {

dependsOn(":sdks:java:extensions:sql:preCommit")

dependsOn(":sdks:java:extensions:sql:buildDependents")

dependsOn(":sdks:java:extensions:sql:datacatalog:build")

dependsOn(":sdks:java:extensions:sql:expansion-service:build")

dependsOn(":sdks:java:extensions:sql:hcatalog:build")

dependsOn(":sdks:java:extensions:sql:iceberg:build")

dependsOn(":sdks:java:extensions:sql:jdbc:build")

dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")

dependsOn(":sdks:java:extensions:sql:perf-tests:build")

dependsOn(":sdks:java:extensions:sql:udf-test-provider:build")

dependsOn(":sdks:java:extensions:sql:udf:build")

}

tasks.register("javaPreCommitPortabilityApi") {

dependsOn(":runners:google-cloud-dataflow-java:worker:build")

}

tasks.register("javaPostCommit") {

dependsOn(":sdks:java:extensions:google-cloud-platform-core:postCommit")

dependsOn(":sdks:java:extensions:zetasketch:postCommit")

dependsOn(":sdks:java:extensions:ml:postCommit")

}

tasks.register("javaPostCommitSickbay") {

dependsOn(":runners:samza:validatesRunnerSickbay")

for (version in project.ext.get("allFlinkVersions") as Array<*>) {

dependsOn(":runners:flink:${version}:validatesRunnerSickbay")

}

dependsOn(":runners:spark:3:job-server:validatesRunnerSickbay")

dependsOn(":runners:direct-java:validatesRunnerSickbay")

dependsOn(":runners:portability:java:validatesRunnerSickbay")

}

tasks.register("javaHadoopVersionsTest") {

dependsOn(":sdks:java:io:hadoop-common:hadoopVersionsTest")

dependsOn(":sdks:java:io:hadoop-file-system:hadoopVersionsTest")

dependsOn(":sdks:java:io:hadoop-format:hadoopVersionsTest")

dependsOn(":sdks:java:io:hcatalog:hadoopVersionsTest")

dependsOn(":sdks:java:io:iceberg:hadoopVersionsTest")

dependsOn(":sdks:java:io:parquet:hadoopVersionsTest")

dependsOn(":sdks:java:extensions:sorter:hadoopVersionsTest")

dependsOn(":runners:spark:3:hadoopVersionsTest")

}

tasks.register("javaAvroVersionsTest") {

dependsOn(":sdks:java:extensions:avro:avroVersionsTest")

}

tasks.register("sqlPostCommit") {

dependsOn(":sdks:java:extensions:sql:postCommit")

dependsOn(":sdks:java:extensions:sql:jdbc:postCommit")

dependsOn(":sdks:java:extensions:sql:datacatalog:postCommit")

dependsOn(":sdks:java:extensions:sql:iceberg:integrationTest")

dependsOn(":sdks:java:extensions:sql:hadoopVersionsTest")

}

tasks.register("goPreCommit") {

// Ensure the Precommit builds run after the tests, in order to avoid the

// flake described in BEAM-11918. This is done by splitting them into two

// tasks and using "mustRunAfter" to enforce ordering.

dependsOn(":goPrecommitTest")

dependsOn(":goPrecommitBuild")

}

tasks.register("goPrecommitTest") {

dependsOn(":sdks:go:goTest")

}

tasks.register("goPrecommitBuild") {

mustRunAfter(":goPrecommitTest")

dependsOn(":sdks:go:goBuild")

dependsOn(":sdks:go:examples:goBuild")

dependsOn(":sdks:go:test:goBuild")

// Ensure all container Go boot code builds as well.

dependsOn(":sdks:java:container:goBuild")

dependsOn(":sdks:python:container:goBuild")

dependsOn(":sdks:go:container:goBuild")

}

tasks.register("goPortablePreCommit") {

dependsOn(":sdks:go:test:ulrValidatesRunner")

}

tasks.register("goPrismPreCommit") {

dependsOn(":sdks:go:test:prismValidatesRunner")

}

tasks.register("goPostCommitDataflowARM") {

dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64")

}

tasks.register("goPostCommit") {

dependsOn(":sdks:go:test:dataflowValidatesRunner")

}

tasks.register("playgroundPreCommit") {

dependsOn(":playground:lintProto")

dependsOn(":playground:backend:precommit")

dependsOn(":playground:frontend:precommit")

}

tasks.register("pythonPreCommit") {

dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon")

dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310")

dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311")

dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312")

dependsOn(":sdks:python:test-suites:tox:py313:preCommitPy313")

}

tasks.register("pythonPreCommitIT") {

dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon")

dependsOn(":sdks:python:test-suites:dataflow:preCommitIT")

}

tasks.register("pythonDocsPreCommit") {

dependsOn(":sdks:python:test-suites:tox:pycommon:docs")

}

tasks.register("pythonDockerBuildPreCommit") {

dependsOn(":sdks:python:container:py310:docker")

dependsOn(":sdks:python:container:py311:docker")

dependsOn(":sdks:python:container:py312:docker")

dependsOn(":sdks:python:container:py313:docker")

}

tasks.register("pythonLintPreCommit") {

dependsOn(":sdks:python:test-suites:tox:pycommon:linter")

}

tasks.register("pythonFormatterPreCommit") {

dependsOn("sdks:python:test-suites:tox:pycommon:formatter")

}

tasks.register("formatChanges") {

group = "formatting"

description = "Formats CHANGES.md according to the template structure"

doLast {

val changesFile = file("CHANGES.md")

if (!changesFile.exists()) {

throw GradleException("CHANGES.md file not found")

}

val content = changesFile.readText()

val lines = content.lines().toMutableList()

// Find template end (after --> that follows <!-- Template -->)

var templateStartIndex = -1

var templateEndIndex = -1

for (i in lines.indices) {

if (lines[i].trim() == "<!-- Template -->") {

templateStartIndex = i

} else if (templateStartIndex != -1 && lines[i].trim() == "-->") {

templateEndIndex = i

break

}

}

if (templateEndIndex == -1) {

throw GradleException("Template end marker not found in CHANGES.md")

}

// Process each release section

var i = templateEndIndex + 1

val formattedLines = mutableListOf<String>()

// Keep header and template exactly as-is (lines 0 to templateEndIndex inclusive)

formattedLines.addAll(lines.subList(0, templateEndIndex + 1))

// Always add blank line after template

formattedLines.add("")

while (i < lines.size) {

val line = lines[i]

// Check if this is a release header

if (line.startsWith("# [")) {

formattedLines.add(line)

i++

// Expected sections in order (following template)

val expectedSections = listOf(

"## Beam 3.0.0 Development Highlights",

"## Highlights",

"## I/Os",

"## New Features / Improvements",

"## Breaking Changes",

"## Deprecations",

"## Bugfixes",

"## Security Fixes",

"## Known Issues"

)

val sectionContent = mutableMapOf<String, MutableList<String>>()

var currentSection = ""

// Parse existing sections

while (i < lines.size && !lines[i].startsWith("# [")) {

val currentLine = lines[i]

if (currentLine.startsWith("## ")) {

currentSection = currentLine

if (!sectionContent.containsKey(currentSection)) {

sectionContent[currentSection] = mutableListOf()

}

} else if (currentSection.isNotEmpty()) {

sectionContent[currentSection]!!.add(currentLine)

}

i++

}

// Only add sections that actually exist with content

for (section in expectedSections) {

if (sectionContent.containsKey(section)) {

formattedLines.add("")

formattedLines.add(section)

formattedLines.add("")

// Remove empty lines at start and end

val content = sectionContent[section]!!

while (content.isNotEmpty() && content.first().trim().isEmpty()) {

content.removeAt(0)

}

while (content.isNotEmpty() && content.last().trim().isEmpty()) {

content.removeAt(content.size - 1)

}

// Format content according to template rules

val formattedContent = content.map { line ->

// Convert SDK language references from [Language] to (Language)

line.replace(Regex("\\[([^\\]]*(?:Java|Python|Go|Kotlin|TypeScript|YAML)[^\\]]*)\\]")) { matchResult ->

val languages = matchResult.groupValues[1]

// Only convert if it's clearly a language reference (not a link or other content)

if (languages.matches(Regex(".*(?:Java|Python|Go|Kotlin|TypeScript|YAML).*"))) {

"($languages)"

} else {

matchResult.value

}

}

}

formattedLines.addAll(formattedContent)

}

}

if (i < lines.size) {

formattedLines.add("")

}

} else {

i++

}

}

// Write formatted content back

changesFile.writeText(formattedLines.joinToString("\n"))

println("CHANGES.md has been formatted according to template structure")

}

}

tasks.register("validateChanges") {

group = "verification"

description = "Validates CHANGES.md follows required formatting rules"

doLast {

val changesFile = file("CHANGES.md")

if (!changesFile.exists()) {

throw GradleException("CHANGES.md file not found")

}

val content = changesFile.readText()

val lines = content.lines()

val errors = mutableListOf<String>()

// Find template section boundaries

var templateStartIndex = -1

var templateEndIndex = -1

for (i in lines.indices) {

if (lines[i].trim() == "<!-- Template -->") {

templateStartIndex = i

println("Found template start at line ${i+1}")

} else if (templateStartIndex != -1 && lines[i].trim() == "-->") {

templateEndIndex = i

println("Found template end at line ${i+1}")

break

}

}

if (templateStartIndex == -1 || templateEndIndex == -1) {

throw GradleException("Template section not found in CHANGES.md")

}

println("Template section: lines ${templateStartIndex+1} to ${templateEndIndex+1}")

// Find unreleased section after the template section

var unreleasedSectionStart = -1

for (i in (templateEndIndex + 1) until lines.size) {

if (lines[i].startsWith("# [") && lines[i].contains("Unreleased")) {

unreleasedSectionStart = i

println("Found unreleased section at line ${i+1}: ${lines[i]}")

break

}

}

if (unreleasedSectionStart == -1) {

throw GradleException("Unreleased section not found in CHANGES.md")

}

// Check entries in the unreleased section

var i = unreleasedSectionStart + 1

val items = TreeMap<Int, String>()

var lastline = 0

var item = ""

while (i < lines.size && !lines[i].startsWith("# [")) {

val line = lines[i].trim()

if (line.isEmpty()) {

// skip

} else if (line.startsWith("* ")) {

items.put(lastline, item)

lastline = i

item = line

} else if (line.startsWith("##")) {

items.put(lastline, item)

lastline = i

item = ""

} else {

item += line

}

i++

}

items.put(lastline, item)

println("Starting validation from line ${i+1}")

items.forEach { (i, line) ->

if (line.startsWith("* ")) {

println("Checking line ${i+1}: $line")

// Skip comment lines

if (line.startsWith("* [comment]:")) {

println(" Skipping comment line")

} else {

// Rule 1: Check if language references use parentheses instead of brackets

val languagePattern = "\\[(Java|Python|Go|Kotlin|TypeScript|YAML)(?:/(?:Java|Python|Go|Kotlin|TypeScript|YAML))*\\]"

val languageRegex = Regex(languagePattern)

// Check if there's a language reference in brackets

val matches = languageRegex.findAll(line).toList()

if (matches.isNotEmpty()) {

for (match in matches) {

val matchText = match.value

val matchPosition = match.range.first

println(" Found language reference: $matchText at position $matchPosition")

// Check if this is part of an issue link or URL

val beforeMatch = if (matchPosition > 0) line.substring(0, matchPosition) else ""

val isPartOfLink = beforeMatch.contains("[#") ||

beforeMatch.contains("http") ||

line.contains("CVE-")

println(" Is part of link: $isPartOfLink")

if (!isPartOfLink) {

val error = "Line ${i+1}: Language references should use parentheses () instead of brackets []: $line"

println(" Adding error: $error")

errors.add(error)

}

}

} else {

println(" No bracketed language reference found")

}

// Rule 2: Check if each entry has an issue link

val issueLinkPattern = "\\(\\[#[0-9a-zA-Z]+\\]\\(https://github\\.com/apache/beam/issues/[0-9a-zA-Z]+\\)\\)"

val issueLinkRegex = Regex(issueLinkPattern)

val hasIssueLink = issueLinkRegex.containsMatchIn(line)

println(" Has issue link: $hasIssueLink")

if (!hasIssueLink) {

val error = "Line ${i+1}: Missing or malformed issue link. Each entry should end with ([#X](https://github.com/apache/beam/issues/X)): $line"

println(" Adding error: $error")

errors.add(error)

}

}

}

}

println("Found ${errors.size} errors")

if (errors.isNotEmpty()) {

throw GradleException("CHANGES.md validation failed with the following errors:\n${errors.joinToString("\n")}\n\nYou can run `./gradlew formatChanges` to correct some issues.")

}

println("CHANGES.md validation successful")

}

}

tasks.register("python310PostCommit") {

dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")

dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")

dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")

dependsOn(":sdks:python:test-suites:direct:py310:azureIntegrationTest")

// TODO: https://github.com/apache/beam/issues/22651

// The default container uses Python 3.10. The goal here is to

// duild Docker images for TensorRT tests during run time for python versions

// other than 3.10 and add these tests in other python postcommit suites.

dependsOn(":sdks:python:test-suites:dataflow:py310:inferencePostCommitIT")

}

tasks.register("python311PostCommit") {

dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")

dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")

}

tasks.register("python312PostCommit") {

dependsOn(":sdks:python:test-suites:dataflow:py312:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py312:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py312:hdfsIntegrationTest")

dependsOn(":sdks:python:test-suites:portable:py312:postCommitPy312")

dependsOn(":sdks:python:test-suites:dataflow:py312:inferencePostCommitITPy312")

}

tasks.register("python313PostCommit") {

dependsOn(":sdks:python:test-suites:dataflow:py313:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py313:postCommitIT")

dependsOn(":sdks:python:test-suites:direct:py313:hdfsIntegrationTest")

dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy313")

}

tasks.register("portablePythonPreCommit") {

dependsOn(":sdks:python:test-suites:portable:py310:preCommitPy310")

dependsOn(":sdks:python:test-suites:portable:py313:preCommitPy313")

}

tasks.register("pythonSparkPostCommit") {

dependsOn(":sdks:python:test-suites:portable:py310:sparkValidatesRunner")

dependsOn(":sdks:python:test-suites:portable:py313:sparkValidatesRunner")

}

tasks.register("websitePreCommit") {

dependsOn(":website:preCommit")

}

tasks.register("communityMetricsPreCommit") {

dependsOn(":beam-test-infra-metrics:preCommit")

}

tasks.register("communityMetricsProber") {

dependsOn(":beam-test-infra-metrics:checkProber")

}

tasks.register("javaExamplesDataflowPrecommit") {

dependsOn(":runners:google-cloud-dataflow-java:examples:preCommit")

dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit")

dependsOn(":runners:google-cloud-dataflow-java:examplesJavaRunnerV2PreCommit")

}

tasks.register("whitespacePreCommit") {

// TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version.

dependsOn(":sdks:python:test-suites:tox:py310:archiveFilesToLint")

dependsOn(":sdks:python:test-suites:tox:py310:unpackFilesToLint")

dependsOn(":sdks:python:test-suites:tox:py310:whitespacelint")

}

tasks.register("typescriptPreCommit") {

// TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version.

dependsOn(":sdks:python:test-suites:tox:py310:eslint")

dependsOn(":sdks:python:test-suites:tox:py310:jest")

}

tasks.register("pushAllRunnersDockerImages") {

dependsOn(":runners:spark:3:job-server:container:docker")

for (version in project.ext.get("allFlinkVersions") as Array<*>) {

dependsOn(":runners:flink:${version}:job-server-container:docker")

}

doLast {

if (project.hasProperty("prune-images")) {

exec {

executable("docker")

args("system", "prune", "-a", "--force")

}

}

}

}

tasks.register("pushAllSdkDockerImages") {

// Enforce ordering to allow the prune step to happen between runs.

// This will ensure we don't use up too much space (especially in CI environments)

if (!project.hasProperty("skip-runner-images")) {

mustRunAfter(":pushAllRunnersDockerImages")

}

dependsOn(":sdks:java:container:pushAll")

if (!project.hasProperty("skip-python-images")) {

dependsOn(":sdks:python:container:pushAll")

}

dependsOn(":sdks:go:container:pushAll")

dependsOn(":sdks:typescript:container:pushAll")

doLast {

if (project.hasProperty("prune-images")) {

exec {

executable("docker")

args("system", "prune", "-a", "--force")

}

}

}

}

tasks.register("pushAllXlangDockerImages") {

// Enforce ordering to allow the prune step to happen between runs.

// This will ensure we don't use up too much space (especially in CI environments)

if (!project.hasProperty("skip-sdk-images")) {

mustRunAfter(":pushAllSdkDockerImages")

}

dependsOn(":sdks:java:expansion-service:container:docker")

dependsOn(":sdks:java:transform-service:controller-container:docker")

dependsOn(":sdks:python:expansion-service-container:docker")

doLast {

if (project.hasProperty("prune-images")) {

exec {

executable("docker")

args("system", "prune", "-a", "--force")

}

}

}

}

tasks.register("pushAllDockerImages") {

if (!project.hasProperty("skip-runner-images")) {

dependsOn(":pushAllRunnersDockerImages")

}

if (!project.hasProperty("skip-sdk-images")) {

dependsOn(":pushAllSdkDockerImages")

}

if (!project.hasProperty("skip-xlang-images")) {

dependsOn(":pushAllXlangDockerImages")

}

}

// Use this task to validate the environment set up for Go, Python and Java

tasks.register("checkSetup") {

dependsOn(":sdks:go:examples:wordCount")

dependsOn(":sdks:python:wordCount")

dependsOn(":examples:java:wordCount")

}

// if not disabled make spotlessApply dependency of compileJava and compileTestJava

val disableSpotlessCheck: String by project

val isSpotlessDisabled = (project.hasProperty("disableSpotlessCheck") &&

disableSpotlessCheck == "true") || project.hasProperty("disableSpotlessApply")

if (!isSpotlessDisabled) {

subprojects {

afterEvaluate {

tasks.findByName("spotlessApply")?.let {

listOf("compileJava", "compileTestJava").forEach {

t -> tasks.findByName(t)?.let { f -> f.dependsOn("spotlessApply") }

}

}

}

}

}

// Generates external transform config

project.tasks.register("generateExternalTransformsConfig") {

dependsOn(":sdks:python:generateExternalTransformsConfig")

}

// Generates the Managed IO Beam web page

project.tasks.register("generateManagedIOPage") {

dependsOn(":sdks:python:generateManagedIOPage")

}

// Configure the release plugin to do only local work; the release manager determines what, if

// anything, to push. On failure, the release manager can reset the branch without pushing.

release {

revertOnFail = false

tagTemplate = "v${version}"

// workaround from https://github.com/researchgate/gradle-release/issues/281#issuecomment-466876492

release {

with (propertyMissing("git") as net.researchgate.release.GitAdapter.GitConfig) {

requireBranch = "release-.*|master"

pushToRemote = ""

}

}

}

// Reports linkage errors across multiple Apache Beam artifact ids.

//

// To use (from the root of project):

// ./gradlew -Ppublishing -PjavaLinkageArtifactIds=artifactId1,artifactId2,... :checkJavaLinkage

//

// For example:

// ./gradlew -Ppublishing -PjavaLinkageArtifactIds=beam-sdks-java-core,beam-sdks-java-io-jdbc :checkJavaLinkage

//

// Note that this task publishes artifacts into your local Maven repository.

if (project.hasProperty("javaLinkageArtifactIds")) {

if (!project.hasProperty("publishing")) {

throw GradleException("You can only check linkage of Java artifacts if you specify -Ppublishing on the command line as well.")

}