Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 9422b8f

Browse files
ewbankkitewbankkit
authored
Merge pull request #43988 from hashicorp/f-aws_glue_job.worker_type-additions
r/aws_glue_job: Support additional `worker_type` values
2 parents 3b30f26 + 43c6df5 commit 9422b8f

File tree

4 files changed

+63
-23
lines changed

4 files changed

+63
-23
lines changed

‎.changelog/43988.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
resource/aws_glue_job: Support `G.12X`, `G.16X`, `R.1X`, `R.2X`, `R.4X`, and `R.8X` as valid values for `worker_type`
3+
```

‎internal/service/glue/job.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/hashicorp/terraform-provider-aws/internal/errs"
2222
"github.com/hashicorp/terraform-provider-aws/internal/errs/sdkdiag"
2323
"github.com/hashicorp/terraform-provider-aws/internal/flex"
24+
tfslices "github.com/hashicorp/terraform-provider-aws/internal/slices"
2425
tftags "github.com/hashicorp/terraform-provider-aws/internal/tags"
2526
"github.com/hashicorp/terraform-provider-aws/internal/tfresource"
2627
"github.com/hashicorp/terraform-provider-aws/internal/verify"
@@ -235,11 +236,11 @@ func resourceJob() *schema.Resource {
235236
Optional: true,
236237
},
237238
"worker_type": {
238-
Type: schema.TypeString,
239-
Optional: true,
240-
Computed: true,
241-
ConflictsWith: []string{names.AttrMaxCapacity},
242-
ValidateDiagFunc: enum.Validate[awstypes.WorkerType](),
239+
Type: schema.TypeString,
240+
Optional: true,
241+
Computed: true,
242+
ConflictsWith: []string{names.AttrMaxCapacity},
243+
ValidateFunc: validation.StringInSlice(workerType_Values(), false),
243244
},
244245
},
245246
}
@@ -687,3 +688,7 @@ func flattenSourceControlDetails(sourceControlDetails *awstypes.SourceControlDet
687688

688689
return []map[string]any{m}
689690
}
691+
692+
func workerType_Values() []string {
693+
return tfslices.AppendUnique(enum.Values[awstypes.WorkerType](), "G.12X", "G.16X", "R.1X", "R.2X", "R.4X", "R.8X")
694+
}

‎internal/service/glue/job_test.go‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ import (
1212
awstypes "github.com/aws/aws-sdk-go-v2/service/glue/types"
1313
sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest"
1414
"github.com/hashicorp/terraform-plugin-testing/helper/resource"
15+
"github.com/hashicorp/terraform-plugin-testing/knownvalue"
16+
"github.com/hashicorp/terraform-plugin-testing/plancheck"
17+
"github.com/hashicorp/terraform-plugin-testing/statecheck"
1518
"github.com/hashicorp/terraform-plugin-testing/terraform"
19+
"github.com/hashicorp/terraform-plugin-testing/tfjsonpath"
1620
"github.com/hashicorp/terraform-provider-aws/internal/acctest"
1721
"github.com/hashicorp/terraform-provider-aws/internal/conns"
1822
tfglue "github.com/hashicorp/terraform-provider-aws/internal/service/glue"
@@ -702,36 +706,71 @@ func TestAccGlueJob_workerType(t *testing.T) {
702706
Config: testAccJobConfig_workerType(rName, "Standard"),
703707
Check: resource.ComposeTestCheckFunc(
704708
testAccCheckJobExists(ctx, resourceName, &job),
705-
resource.TestCheckResourceAttr(resourceName, "worker_type", "Standard"),
706709
),
710+
ConfigPlanChecks: resource.ConfigPlanChecks{
711+
PreApply: []plancheck.PlanCheck{
712+
plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionCreate),
713+
},
714+
},
715+
ConfigStateChecks: []statecheck.StateCheck{
716+
statecheck.ExpectKnownValue(resourceName, tfjsonpath.New("worker_type"), knownvalue.StringExact("Standard")),
717+
},
707718
},
708719
{
709720
Config: testAccJobConfig_workerType(rName, "G.1X"),
710721
Check: resource.ComposeTestCheckFunc(
711722
testAccCheckJobExists(ctx, resourceName, &job),
712-
resource.TestCheckResourceAttr(resourceName, "worker_type", "G.1X"),
713723
),
724+
ConfigPlanChecks: resource.ConfigPlanChecks{
725+
PreApply: []plancheck.PlanCheck{
726+
plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate),
727+
},
728+
},
729+
ConfigStateChecks: []statecheck.StateCheck{
730+
statecheck.ExpectKnownValue(resourceName, tfjsonpath.New("worker_type"), knownvalue.StringExact("G.1X")),
731+
},
714732
},
715733
{
716734
Config: testAccJobConfig_workerType(rName, "G.2X"),
717735
Check: resource.ComposeTestCheckFunc(
718736
testAccCheckJobExists(ctx, resourceName, &job),
719-
resource.TestCheckResourceAttr(resourceName, "worker_type", "G.2X"),
720737
),
738+
ConfigPlanChecks: resource.ConfigPlanChecks{
739+
PreApply: []plancheck.PlanCheck{
740+
plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate),
741+
},
742+
},
743+
ConfigStateChecks: []statecheck.StateCheck{
744+
statecheck.ExpectKnownValue(resourceName, tfjsonpath.New("worker_type"), knownvalue.StringExact("G.2X")),
745+
},
721746
},
722747
{
723748
Config: testAccJobConfig_workerType(rName, "G.4X"),
724749
Check: resource.ComposeTestCheckFunc(
725750
testAccCheckJobExists(ctx, resourceName, &job),
726-
resource.TestCheckResourceAttr(resourceName, "worker_type", "G.4X"),
727751
),
752+
ConfigPlanChecks: resource.ConfigPlanChecks{
753+
PreApply: []plancheck.PlanCheck{
754+
plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate),
755+
},
756+
},
757+
ConfigStateChecks: []statecheck.StateCheck{
758+
statecheck.ExpectKnownValue(resourceName, tfjsonpath.New("worker_type"), knownvalue.StringExact("G.4X")),
759+
},
728760
},
729761
{
730-
Config: testAccJobConfig_workerType(rName, "G.8X"),
762+
Config: testAccJobConfig_workerType(rName, "R.1X"),
731763
Check: resource.ComposeTestCheckFunc(
732764
testAccCheckJobExists(ctx, resourceName, &job),
733-
resource.TestCheckResourceAttr(resourceName, "worker_type", "G.8X"),
734765
),
766+
ConfigPlanChecks: resource.ConfigPlanChecks{
767+
PreApply: []plancheck.PlanCheck{
768+
plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate),
769+
},
770+
},
771+
ConfigStateChecks: []statecheck.StateCheck{
772+
statecheck.ExpectKnownValue(resourceName, tfjsonpath.New("worker_type"), knownvalue.StringExact("R.1X")),
773+
},
735774
},
736775
{
737776
ResourceName: resourceName,

‎website/docs/r/glue_job.html.markdown‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -216,36 +216,29 @@ resource "aws_glue_job" "example" {
216216

217217
This resource supports the following arguments:
218218

219-
* `region` - (Optional) Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the [provider configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#aws-configuration-reference).
220219
* `command` - (Required) The command of the job. Defined below.
221220
* `connections` - (Optional) The list of connections used for this job.
222221
* `default_arguments` - (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide.
223-
* `non_overridable_arguments` - (Optional) Non-overridable arguments for this job, specified as name-value pairs.
224222
* `description` - (Optional) Description of the job.
223+
* `execution_class` - (Optional) Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`.
225224
* `execution_property` - (Optional) Execution property of the job. Defined below.
226225
* `glue_version` - (Optional) The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html).
227226
* `job_mode` - (Optional) Describes how a job was created. Valid values are `SCRIPT`, `NOTEBOOK` and `VISUAL`.
228227
* `job_run_queuing_enabled` - (Optional) Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
229-
* `execution_class` - (Optional) Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`.
230228
* `maintenance_window` - (Optional) Specifies the day of the week and hour for the maintenance window for streaming jobs.
231229
* `max_capacity` - (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `number_of_workers` and `worker_type` arguments instead with `glue_version` `2.0` and above.
232230
* `max_retries` - (Optional) The maximum number of times to retry this job if it fails.
233231
* `name` - (Required) The name you assign to this job. It must be unique in your account.
232+
* `non_overridable_arguments` - (Optional) Non-overridable arguments for this job, specified as name-value pairs.
234233
* `notification_property` - (Optional) Notification property of the job. Defined below.
234+
* `number_of_workers` - (Optional) The number of workers of a defined workerType that are allocated when a job runs.
235+
* `region` - (Optional) Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the [provider configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#aws-configuration-reference).
235236
* `role_arn` - (Required) The ARN of the IAM role associated with this job.
236237
* `tags` - (Optional) Key-value map of resource tags. If configured with a provider [`default_tags` configuration block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#default_tags-configuration-block) present, tags with matching keys will overwrite those defined at the provider-level.
237238
* `timeout` - (Optional) The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs.
238239
* `security_configuration` - (Optional) The name of the Security Configuration to be associated with the job.
239240
* `source_control_details` - (Optional) The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below.
240-
* `worker_type` - (Optional) The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
241-
* For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
242-
* For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
243-
* For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
244-
* For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
245-
* For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
246-
* For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
247-
* For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
248-
* `number_of_workers` - (Optional) The number of workers of a defined workerType that are allocated when a job runs.
241+
* `worker_type` - (Optional) The type of predefined worker that is allocated when a job runs. Valid values: `Standard`, `G.1X`, `G.2X`, `G.025X`, `G.4X`, `G.8X`, `G.12X`, `G.16X`, `R.1X`, `R.2X`, `R.4X`, `R.8X`, `Z.2X` (Ray jobs). See the [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/worker-types.html) for details.
249242

250243
### command Argument Reference
251244

0 commit comments

Comments
 (0)