diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0f0647b0..0c1ba969 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:e76136cc48f90aa19ba29cdfbd4002111467e44a1c9d905867d98dafafbd03bb + digest: sha256:5df8b62e8da534f7604daef347698f6701e34b3f61713712a3384ac88fc32088 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..c8f413b0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + # Disable version updates for Maven dependencies + open-pull-requests-limit: 0 + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + # Disable version updates for pip dependencies + open-pull-requests-limit: 0 \ No newline at end of file diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index 924f94ae..2092cc74 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -1,34 +1,6 @@ -gcp-docuploader==0.6.3 -google-crc32c==1.3.0 -googleapis-common-protos==1.56.3 -gcp-releasetool==1.9.1 -cryptography==38.0.3 -cachetools==4.2.4 -cffi==1.15.1 -jeepney==0.7.1 -jinja2==3.0.3 -markupsafe==2.0.1 -keyring==23.4.1 -packaging==21.3 -protobuf==3.19.5 -pyjwt==2.4.0 -pyparsing==3.0.9 -pycparser==2.21 -pyperclip==1.8.2 -python-dateutil==2.8.2 -requests==2.27.1 -certifi==2022.9.24 -importlib-metadata==4.8.3 -zipp==3.6.0 -google_api_core==2.8.2 -google-cloud-storage==2.0.0 -google-resumable-media==2.3.3 -google-cloud-core==2.3.1 -typing-extensions==4.1.1 -urllib3==1.26.12 -zipp==3.6.0 -rsa==4.9 -six==1.16.0 -attrs==22.1.0 -google-auth==2.14.1 -idna==3.4 \ No newline at end of file +gcp-docuploader +gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x +wheel +setuptools +typing-extensions +click<8.1.0 \ No newline at end of file diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 71fcafc7..c80f0a87 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -1,27 +1,21 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # -# pip-compile --generate-hashes requirements.in +# pip-compile --allow-unsafe --generate-hashes requirements.in # attrs==22.1.0 \ --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool cachetools==4.2.4 \ --hash=sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693 \ --hash=sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1 - # via - # -r requirements.in - # google-auth -certifi==2022.9.24 \ - --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ - --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 - # via - # -r requirements.in - # requests + # via google-auth +certifi==2022.12.7 \ + --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ + --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 + # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ @@ -87,9 +81,7 @@ cffi==1.15.1 \ --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 - # via - # -r requirements.in - # cryptography + # via cryptography charset-normalizer==2.0.12 \ --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \ --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df @@ -98,62 +90,56 @@ click==8.0.4 \ --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb # via + # -r requirements.in # gcp-docuploader # gcp-releasetool colorlog==6.7.0 \ --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 # via gcp-docuploader -cryptography==38.0.3 \ - --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ - --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ - --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ - --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ - --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ - --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ - --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ - --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ - --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ - --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ - --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ - --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ - --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ - --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ - --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ - --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ - --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ - --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ - --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ - --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ - --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ - --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ - --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ - --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ - --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ - --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 +cryptography==39.0.1 \ + --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \ + --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \ + --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \ + --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \ + --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \ + --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \ + --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \ + --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \ + --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \ + --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \ + --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \ + --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \ + --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \ + --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \ + --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \ + --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \ + --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \ + --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \ + --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \ + --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \ + --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8 # via - # -r requirements.in # gcp-releasetool # secretstorage -gcp-docuploader==0.6.3 \ - --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ - --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b +gcp-docuploader==0.6.4 \ + --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ + --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.9.1 \ - --hash=sha256:952f4055d5d986b070ae2a71c4410b250000f9cc5a1e26398fcd55a5bbc5a15f \ - --hash=sha256:d0d3c814a97c1a237517e837d8cfa668ced8df4b882452578ecef4a4e79c583b +gcp-releasetool==1.10.5 \ + --hash=sha256:174b7b102d704b254f2a26a3eda2c684fd3543320ec239baf771542a2e58e109 \ + --hash=sha256:e29d29927fe2ca493105a82958c6873bb2b90d503acac56be2c229e74de0eec9 # via -r requirements.in google-api-core==2.8.2 \ --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 # via - # -r requirements.in # google-cloud-core # google-cloud-storage google-auth==2.14.1 \ + --hash=sha256:ccaa901f31ad5cbb562615eb8b664b3dd0bf5404a67618e642307f00613eda4d \ --hash=sha256:f5d8701633bebc12e0deea4df8abd8aff31c28b355360597f7f2ee60f2e4d016 # via - # -r requirements.in # gcp-releasetool # google-api-core # google-cloud-core @@ -161,15 +147,11 @@ google-auth==2.14.1 \ google-cloud-core==2.3.1 \ --hash=sha256:113ba4f492467d5bd442c8d724c1a25ad7384045c3178369038840ecdd19346c \ --hash=sha256:34334359cb04187bdc80ddcf613e462dfd7a3aabbc3fe4d118517ab4b9303d53 - # via - # -r requirements.in - # google-cloud-storage + # via google-cloud-storage google-cloud-storage==2.0.0 \ --hash=sha256:a57a15aead0f9dfbd4381f1bfdbe8bf89818a4bd75bab846cafcefb2db846c47 \ --hash=sha256:ec4be60bb223a3a960f0d01697d849b86d91cad815a84915a32ed3635e93a5e7 - # via - # -r requirements.in - # gcp-docuploader + # via gcp-docuploader google-crc32c==1.3.0 \ --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ @@ -214,52 +196,37 @@ google-crc32c==1.3.0 \ --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 - # via - # -r requirements.in - # google-resumable-media + # via google-resumable-media google-resumable-media==2.3.3 \ --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 - # via - # -r requirements.in - # google-cloud-storage + # via google-cloud-storage googleapis-common-protos==1.56.3 \ --hash=sha256:6f1369b58ed6cf3a4b7054a44ebe8d03b29c309257583a2bbdc064cd1e4a1442 \ --hash=sha256:87955d7b3a73e6e803f2572a33179de23989ebba725e05ea42f24838b792e461 - # via - # -r requirements.in - # google-api-core + # via google-api-core idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 - # via - # -r requirements.in - # requests + # via requests importlib-metadata==4.8.3 \ --hash=sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e \ --hash=sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668 + # via keyring +jeepney==0.8.0 \ + --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ + --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 # via - # -r requirements.in - # keyring -jeepney==0.7.1 \ - --hash=sha256:1b5a0ea5c0e7b166b2f5895b91a08c14de8915afda4407fb5022a195224958ac \ - --hash=sha256:fa9e232dfa0c498bd0b8a3a73b8d8a31978304dcef0515adc859d4e096f96f4f - # via - # -r requirements.in # keyring # secretstorage jinja2==3.0.3 \ --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \ --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool keyring==23.4.1 \ --hash=sha256:17e49fb0d6883c2b4445359434dba95aad84aabb29bbff044ad0ed7100232eca \ --hash=sha256:89cbd74d4683ed164c8082fb38619341097741323b3786905c6dac04d6915a55 - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool markupsafe==2.0.1 \ --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ @@ -330,49 +297,39 @@ markupsafe==2.0.1 \ --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 - # via - # -r requirements.in - # jinja2 + # via jinja2 packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 + # via gcp-releasetool +protobuf==3.20.2 \ + --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ + --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ + --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ + --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ + --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ + --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ + --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ + --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ + --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ + --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ + --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ + --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ + --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ + --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ + --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ + --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ + --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ + --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ + --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ + --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ + --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ + --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 # via - # -r requirements.in - # gcp-releasetool -protobuf==3.19.5 \ - --hash=sha256:1867f93b06a183f87696871bb8d1e99ee71dbb69d468ce1f0cc8bf3d30f982f3 \ - --hash=sha256:3c4160b601220627f7e91154e572baf5e161a9c3f445a8242d536ee3d0b7b17c \ - --hash=sha256:4ee2af7051d3b10c8a4fe6fd1a2c69f201fea36aeee7086cf202a692e1b99ee1 \ - --hash=sha256:5266c36cc0af3bb3dbf44f199d225b33da66a9a5c3bdc2b14865ad10eddf0e37 \ - --hash=sha256:5470f892961af464ae6eaf0f3099e2c1190ae8c7f36f174b89491281341f79ca \ - --hash=sha256:66d14b5b90090353efe75c9fb1bf65ef7267383034688d255b500822e37d5c2f \ - --hash=sha256:67efb5d20618020aa9596e17bfc37ca068c28ec0c1507d9507f73c93d46c9855 \ - --hash=sha256:696e6cfab94cc15a14946f2bf72719dced087d437adbd994fff34f38986628bc \ - --hash=sha256:6a02172b9650f819d01fb8e224fc69b0706458fc1ab4f1c669281243c71c1a5e \ - --hash=sha256:6eca9ae238ba615d702387a2ddea635d535d769994a9968c09a4ca920c487ab9 \ - --hash=sha256:950abd6c00e7b51f87ae8b18a0ce4d69fea217f62f171426e77de5061f6d9850 \ - --hash=sha256:9e1d74032f56ff25f417cfe84c8147047732e5059137ca42efad20cbbd25f5e0 \ - --hash=sha256:9e42b1cf2ecd8a1bd161239e693f22035ba99905ae6d7efeac8a0546c7ec1a27 \ - --hash=sha256:9f957ef53e872d58a0afd3bf6d80d48535d28c99b40e75e6634cbc33ea42fd54 \ - --hash=sha256:a89aa0c042e61e11ade320b802d6db4ee5391d8d973e46d3a48172c1597789f8 \ - --hash=sha256:c0f80876a8ff0ae7064084ed094eb86497bd5a3812e6fc96a05318b92301674e \ - --hash=sha256:c44e3282cff74ad18c7e8a0375f407f69ee50c2116364b44492a196293e08b21 \ - --hash=sha256:d249519ba5ecf5dd6b18150c9b6bcde510b273714b696f3923ff8308fc11ae49 \ - --hash=sha256:d3973a2d58aefc7d1230725c2447ce7f86a71cbc094b86a77c6ee1505ac7cdb1 \ - --hash=sha256:dca2284378a5f2a86ffed35c6ac147d14c48b525eefcd1083e5a9ce28dfa8657 \ - --hash=sha256:e63b0b3c42e51c94add62b010366cd4979cb6d5f06158bcae8faac4c294f91e1 \ - --hash=sha256:f2b599a21c9a32e171ec29a2ac54e03297736c578698e11b099d031f79da114b \ - --hash=sha256:f2bde37667b18c2b5280df83bc799204394a5d2d774e4deaf9de0eb741df6833 \ - --hash=sha256:f4f909f4dde413dec435a44b0894956d55bb928ded7d6e3c726556ca4c796e84 \ - --hash=sha256:f976234e20ab2785f54224bcdafa027674e23663b132fa3ca0caa291a6cfbde7 \ - --hash=sha256:f9cebda093c2f6bfed88f1c17cdade09d4d96096421b344026feee236532d4de - # via - # -r requirements.in # gcp-docuploader # gcp-releasetool # google-api-core # google-cloud-storage - # googleapis-common-protos pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba @@ -386,71 +343,61 @@ pyasn1-modules==0.2.8 \ pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via - # -r requirements.in - # cffi + # via cffi pyjwt==2.4.0 \ --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool pyparsing==3.0.9 \ --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via - # -r requirements.in - # packaging + # via packaging pyperclip==1.8.2 \ --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 - # via - # -r requirements.in - # gcp-releasetool + # via gcp-releasetool requests==2.27.1 \ --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d # via - # -r requirements.in # gcp-releasetool # google-api-core # google-cloud-storage rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 - # via - # -r requirements.in - # google-auth + # via google-auth secretstorage==3.3.3 \ --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 # via keyring +setuptools==67.3.2 \ + --hash=sha256:95f00380ef2ffa41d9bba85d95b27689d923c93dfbafed4aecd7cf988a25e012 \ + --hash=sha256:bb6d8e508de562768f2027902929f8523932fcd1fb784e6d573d2cafac995a48 + # via -r requirements.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -r requirements.in # gcp-docuploader # google-auth # python-dateutil -typing-extensions==4.1.1 \ - --hash=sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42 \ - --hash=sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2 +typing-extensions==4.4.0 \ + --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ + --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in urllib3==1.26.12 \ --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 - # via - # -r requirements.in - # requests + # via requests +wheel==0.38.4 \ + --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ + --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8 + # via -r requirements.in zipp==3.6.0 \ --hash=sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832 \ --hash=sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc - # via - # -r requirements.in - # importlib-metadata + # via importlib-metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e78e3ec..23e8fa24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.1.0](https://github.com/googleapis/java-pubsub-group-kafka-connector/compare/v1.0.0...v1.1.0) (2023-03-31) + + +### Features + +* Add pubsublite sink support for credentials settings ([#251](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/251)) ([7290786](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/7290786e82db354f1d310c599f84150a7ec0ef8b)) +* Add pubsublite.ordering.mode to kafka connector ([#228](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/228)) ([c499c39](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/c499c395cef38f9bb4b52d157bc336bff0644b94)) + + +### Bug Fixes + +* **main:** Typo in README.md for PubSubLiteSourceConnector ([#242](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/242)) ([d881eaf](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/d881eafef9bf5ece2391a75bc3d2cb6208a20ba9)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.4.0 ([#232](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/232)) ([f10f9a6](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/f10f9a6546eb6ea65b61fbbe4538edae81b524ab)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.5.0 ([#244](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/244)) ([435cd98](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/435cd98e3ae81f3d698e174782ac81199bdf756f)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.6.0 ([#250](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/250)) ([888d3d9](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/888d3d95ea01ae1535e6aba47fa71c6cb0f0e7e1)) +* Update dependency org.slf4j:slf4j-api to v2.0.6 ([#230](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/230)) ([9dec71b](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/9dec71b8df9acdd738b4738ee060b13ff602e86b)) +* Update dependency org.slf4j:slf4j-api to v2.0.7 ([#243](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/243)) ([1e1c336](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/1e1c33666ee1e0ef3fd45684054dcf25216004f8)) + ## [1.0.0](https://github.com/googleapis/java-pubsub-group-kafka-connector/compare/v0.1.5...v1.0.0) (2022-11-18) diff --git a/README.md b/README.md index 16e5c64f..9dbdf075 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ locally in standalone mode (single process). - `CloudPubSubSinkConnector` and `CloudPubSubSourceConnector` - Create a pair of Pub/Sub [topic](https://cloud.google.com/pubsub/docs/admin#create_a_topic) and [subscription](https://cloud.google.com/pubsub/docs/create-subscription#pull_subscription) - - `PubSubLiteSinkConnector` and `PubSubLiteSinkConnector` + - `PubSubLiteSinkConnector` and `PubSubLiteSourceConnector` - Create a pair of Pub/Sub Lite [topic](https://cloud.google.com/pubsub/lite/docs/topics#create_a_lite_topic) and [subscription](https://cloud.google.com/pubsub/lite/docs/subscriptions#create_a_lite_subscription). @@ -82,7 +82,7 @@ locally in standalone mode (single process). 1. Open [`pubsub-lite-sink-connector.properties`](/config/pubsub-lite-sink-connector.properties). 2. Update `topics`, `pubsublite.project`, `pubsublite.location` and `pubsublite.topic`. - - `PubSubLiteSinkConnector` + - `PubSubLiteSourceConnector` 1. Open [`pubsub-lite-source-connector.properties`](/config/pubsub-lite-source-connector.properties). 2. Update `kafka.topic`, `pubsublite.project`, `pubsublite.location` and `pubsublite.subscription`. @@ -108,18 +108,18 @@ locally in standalone mode (single process). 2. Follow the instructions in the [Kafka quickstart](https://kafka.apache.org/quickstart) to read the message from your Kafka topic. - - `PubSubLiteSinkConnector` - 1. Follow the instructions in the [Kafka quickstart](https://kafka.apache.org/quickstart) - to publish a message to the Kafka topic. - 2. [Pull](https://cloud.google.com/pubsub/docs/publish-receive-messages-console#pull_the_message_from_the_subscription) - the message from your Pub/Sub Lite subscription. - - `PubSubLiteSinkConnector` 1. [Publish](https://cloud.google.com/pubsub/docs/publish-receive-messages-console#publish_a_message_to_the_topic) a message to your Pub/Sub Lite topic. 2. Follow the instructions in the [Kafka quickstart](https://kafka.apache.org/quickstart) to read the message from your Kafka topic. + - `PubSubLiteSourceConnector` + 1. Follow the instructions in the [Kafka quickstart](https://kafka.apache.org/quickstart) + to publish a message to the Kafka topic. + 2. [Pull](https://cloud.google.com/pubsub/docs/publish-receive-messages-console#pull_the_message_from_the_subscription) + the message from your Pub/Sub Lite subscription. + ### Acquire the connector The connector is available from [Maven Central repository](https://search.maven.org/artifact/com.google.cloud/pubsub-group-kafka-connector). diff --git a/pom.xml b/pom.xml index f75bcbcb..d17aa38f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 pubsub-group-kafka-connector - 1.0.0 + 1.1.0 jar Pub/Sub Group Kafka Connector https://github.com/googleapis/java-pubsub-group-kafka-connector @@ -15,54 +15,73 @@ com.google.cloud google-cloud-shared-config - 1.5.4 + 1.5.5 1.8 1.8 - 3.3.1 - 1.120.25 - 1.8.0 - 1.16.0 - 3.21.9 - 2.19.4 - 2.0.3 + 3.4.0 + + + + com.google.cloud + google-cloud-shared-dependencies + 3.6.0 + pom + import + + + com.google.cloud + libraries-bom + 26.11.0 + pom + import + + + + com.google.api.grpc + grpc-google-cloud-pubsublite-v1 + 1.11.1 + + + + + + com.google.api.grpc + proto-google-cloud-pubsublite-v1 + + 1.11.1 + + + com.google.cloud + google-cloud-pubsublite + + 1.11.1 + + + com.google.cloud + pubsublite-kafka + 1.1.2 + com.google.cloud google-cloud-pubsub - ${pubsub.version} com.google.protobuf protobuf-java - ${protobuf-java.vesion} - - - com.google.auth - google-auth-library-oauth2-http - 1.12.1 org.apache.kafka kafka-clients ${kafka.version} - - com.google.cloud - pubsublite-kafka - 1.0.2 - - - com.google.api.grpc - proto-google-cloud-pubsublite-v1 - ${pubsublite.version} - com.google.flogger google-extensions @@ -71,62 +90,51 @@ com.google.code.findbugs jsr305 - 3.0.2 com.google.api gax - ${gax.version} org.slf4j slf4j-api - ${slf4j.version} + 2.0.7 com.google.api gax-grpc - ${gax.version} com.google.auth google-auth-library-credentials - 1.13.0 com.google.api.grpc proto-google-cloud-pubsub-v1 - 1.102.25 com.google.api api-common - 2.2.2 - - - com.google.cloud - google-cloud-pubsublite - ${pubsublite.version} com.google.guava guava - 31.1-jre org.threeten threetenbp - 1.6.4 com.google.errorprone error_prone_annotations - 2.16 com.google.protobuf protobuf-java-util - ${protobuf-java.vesion} + + + com.google.auth + google-auth-library-oauth2-http @@ -141,25 +149,24 @@ junit junit - 4.13.2 test + 4.13.2 org.mockito mockito-core - 4.9.0 test + 4.11.0 com.google.truth truth - 1.1.3 test + 1.1.3 com.google.cloud google-cloud-core - 2.8.28 test @@ -177,13 +184,11 @@ com.google.api.grpc proto-google-cloud-compute-v1 - ${cloud-compute.version} test com.google.cloud google-cloud-compute - ${cloud-compute.version} test @@ -195,13 +200,6 @@ com.google.cloud google-cloud-storage - 2.15.0 - test - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} test @@ -220,7 +218,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.3.0 + 3.5.0 diff --git a/src/main/java/com/google/pubsub/kafka/common/ConnectorCredentialsProvider.java b/src/main/java/com/google/pubsub/kafka/common/ConnectorCredentialsProvider.java index 4c36a2ad..6fccf7fe 100644 --- a/src/main/java/com/google/pubsub/kafka/common/ConnectorCredentialsProvider.java +++ b/src/main/java/com/google/pubsub/kafka/common/ConnectorCredentialsProvider.java @@ -23,29 +23,58 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; public class ConnectorCredentialsProvider implements CredentialsProvider { + private static final List GCP_SCOPE = + Arrays.asList("https://www.googleapis.com/auth/cloud-platform"); - private static final List CPS_SCOPE = - Arrays.asList("https://www.googleapis.com/auth/pubsub"); + CredentialsProvider impl; - GoogleCredentials credentials; + private ConnectorCredentialsProvider(CredentialsProvider impl) { + this.impl = impl; + } + + public static ConnectorCredentialsProvider fromConfig(Map config) { + String credentialsPath = config.get(ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG).toString(); + String credentialsJson = config.get(ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG).toString(); + if (!credentialsPath.isEmpty()) { + if (!credentialsJson.isEmpty()) { + throw new IllegalArgumentException( + "May not set both " + + ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG + + " and " + + ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG); + } + return ConnectorCredentialsProvider.fromFile(credentialsPath); + } else if (!credentialsJson.isEmpty()) { + return ConnectorCredentialsProvider.fromJson(credentialsJson); + } else { + return ConnectorCredentialsProvider.fromDefault(); + } + } - public void loadFromFile(String credentialPath) throws IOException { - this.credentials = GoogleCredentials.fromStream(new FileInputStream(credentialPath)); + public static ConnectorCredentialsProvider fromFile(String credentialPath) { + return new ConnectorCredentialsProvider( + () -> + GoogleCredentials.fromStream(new FileInputStream(credentialPath)) + .createScoped(GCP_SCOPE)); } - public void loadJson(String credentialsJson) throws IOException { - ByteArrayInputStream bs = new ByteArrayInputStream(credentialsJson.getBytes()); - this.credentials = credentials = GoogleCredentials.fromStream(bs); + public static ConnectorCredentialsProvider fromJson(String credentialsJson) { + return new ConnectorCredentialsProvider( + () -> + GoogleCredentials.fromStream(new ByteArrayInputStream(credentialsJson.getBytes())) + .createScoped(GCP_SCOPE)); + } + + public static ConnectorCredentialsProvider fromDefault() { + return new ConnectorCredentialsProvider( + () -> GoogleCredentials.getApplicationDefault().createScoped(GCP_SCOPE)); } @Override public Credentials getCredentials() throws IOException { - if (this.credentials == null) { - return GoogleCredentials.getApplicationDefault().createScoped(this.CPS_SCOPE); - } else { - return this.credentials.createScoped(this.CPS_SCOPE); - } + return impl.getCredentials(); } } diff --git a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java index 7b083d44..eb7d681f 100644 --- a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java +++ b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java @@ -233,13 +233,13 @@ public ConfigDef config() { .define( ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG, Type.STRING, - null, + "", Importance.HIGH, "The path to the GCP credentials file") .define( ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG, Type.STRING, - null, + "", Importance.HIGH, "GCP JSON credentials") .define( diff --git a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java index 5c31a9d8..9faa7910 100644 --- a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java +++ b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java @@ -33,7 +33,6 @@ import com.google.pubsub.kafka.sink.CloudPubSubSinkConnector.OrderingKeySource; import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.PubsubMessage; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -136,24 +135,7 @@ public void start(Map props) { orderingKeySource = OrderingKeySource.getEnum( (String) validatedProps.get(CloudPubSubSinkConnector.ORDERING_KEY_SOURCE)); - gcpCredentialsProvider = new ConnectorCredentialsProvider(); - String credentialsPath = - (String) validatedProps.get(ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG); - String credentialsJson = - (String) validatedProps.get(ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG); - if (credentialsPath != null) { - try { - gcpCredentialsProvider.loadFromFile(credentialsPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else if (credentialsJson != null) { - try { - gcpCredentialsProvider.loadJson(credentialsJson); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + gcpCredentialsProvider = ConnectorCredentialsProvider.fromConfig(validatedProps); if (publisher == null) { // Only do this if we did not use the constructor. createPublisher(); diff --git a/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java b/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java index 4fd649c2..3f507d2f 100644 --- a/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java +++ b/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceConnector.java @@ -22,7 +22,6 @@ import com.google.pubsub.kafka.common.ConnectorCredentialsProvider; import com.google.pubsub.kafka.common.ConnectorUtils; import com.google.pubsub.v1.GetSubscriptionRequest; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -138,25 +137,11 @@ public String version() { public void start(Map props) { // Do a validation of configs here too so that we do not pass null objects to // verifySubscription(). - config().parse(props); - String cpsProject = props.get(ConnectorUtils.CPS_PROJECT_CONFIG); - String cpsSubscription = props.get(CPS_SUBSCRIPTION_CONFIG); - String credentialsPath = props.get(ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG); - String credentialsJson = props.get(ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG); - ConnectorCredentialsProvider credentialsProvider = new ConnectorCredentialsProvider(); - if (credentialsPath != null) { - try { - credentialsProvider.loadFromFile(credentialsPath); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else if (credentialsJson != null) { - try { - credentialsProvider.loadJson(credentialsJson); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + Map validated = config().parse(props); + String cpsProject = validated.get(ConnectorUtils.CPS_PROJECT_CONFIG).toString(); + String cpsSubscription = validated.get(CPS_SUBSCRIPTION_CONFIG).toString(); + ConnectorCredentialsProvider credentialsProvider = + ConnectorCredentialsProvider.fromConfig(validated); verifySubscription(cpsProject, cpsSubscription, credentialsProvider); this.props = props; @@ -271,13 +256,13 @@ public ConfigDef config() { .define( ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG, Type.STRING, - null, + "", Importance.HIGH, "The path to the GCP credentials file") .define( ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG, Type.STRING, - null, + "", Importance.HIGH, "GCP JSON credentials") .define( diff --git a/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java b/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java index 7319cea1..39a8f2ab 100644 --- a/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java +++ b/src/main/java/com/google/pubsub/kafka/source/CloudPubSubSourceTask.java @@ -34,7 +34,6 @@ import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.PubsubMessage; import com.google.pubsub.v1.ReceivedMessage; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -114,11 +113,6 @@ public void start(Map props) { useKafkaHeaders = (Boolean) validatedProps.get(CloudPubSubSourceConnector.USE_KAFKA_HEADERS); makeOrderingKeyAttribute = (Boolean) validatedProps.get(CloudPubSubSourceConnector.CPS_MAKE_ORDERING_KEY_ATTRIBUTE); - ConnectorCredentialsProvider gcpCredentialsProvider = new ConnectorCredentialsProvider(); - String gcpCredentialsFilePath = - (String) validatedProps.get(ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG); - String credentialsJson = - (String) validatedProps.get(ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG); boolean useStreamingPull = (Boolean) validatedProps.get(CloudPubSubSourceConnector.CPS_STREAMING_PULL_ENABLED); long streamingPullBytes = @@ -136,19 +130,9 @@ public void start(Map props) { (Long) validatedProps.get( CloudPubSubSourceConnector.CPS_STREAMING_PULL_MAX_MS_PER_ACK_EXTENSION); - if (gcpCredentialsFilePath != null) { - try { - gcpCredentialsProvider.loadFromFile(gcpCredentialsFilePath); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else if (credentialsJson != null) { - try { - gcpCredentialsProvider.loadJson(credentialsJson); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + ConnectorCredentialsProvider gcpCredentialsProvider = + ConnectorCredentialsProvider.fromConfig(validatedProps); + // Only do this if we did not set it through the constructor. if (subscriber == null) { if (useStreamingPull) { diff --git a/src/main/java/com/google/pubsublite/kafka/sink/ConfigDefs.java b/src/main/java/com/google/pubsublite/kafka/sink/ConfigDefs.java index 82dbd9bd..12d59eb9 100644 --- a/src/main/java/com/google/pubsublite/kafka/sink/ConfigDefs.java +++ b/src/main/java/com/google/pubsublite/kafka/sink/ConfigDefs.java @@ -15,6 +15,7 @@ */ package com.google.pubsublite.kafka.sink; +import com.google.pubsub.kafka.common.ConnectorUtils; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; @@ -25,6 +26,7 @@ private ConfigDefs() {} static final String PROJECT_FLAG = "pubsublite.project"; static final String LOCATION_FLAG = "pubsublite.location"; static final String TOPIC_NAME_FLAG = "pubsublite.topic"; + static final String ORDERING_MODE_FLAG = "pubsublite.ordering.mode"; static ConfigDef config() { return new ConfigDef() @@ -42,6 +44,24 @@ static ConfigDef config() { TOPIC_NAME_FLAG, ConfigDef.Type.STRING, Importance.HIGH, - "The name of the topic to which to publish."); + "The name of the topic to which to publish.") + .define( + ORDERING_MODE_FLAG, + ConfigDef.Type.STRING, + OrderingMode.DEFAULT.name(), + Importance.HIGH, + "The ordering mode to use for publishing to Pub/Sub Lite. If set to `KAFKA`, messages will be republished to the same partition index they were read from on the source topic. Note that this means the Pub/Sub Lite topic *must* have the same number of partitions as the source Kafka topic.") + .define( + ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG, + ConfigDef.Type.STRING, + "", + Importance.HIGH, + "The path to the GCP credentials file") + .define( + ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG, + ConfigDef.Type.STRING, + "", + Importance.HIGH, + "GCP JSON credentials"); } } diff --git a/src/main/java/com/google/pubsublite/kafka/sink/Constants.java b/src/main/java/com/google/pubsublite/kafka/sink/Constants.java index abfed7ae..7dc57b5c 100644 --- a/src/main/java/com/google/pubsublite/kafka/sink/Constants.java +++ b/src/main/java/com/google/pubsublite/kafka/sink/Constants.java @@ -24,6 +24,4 @@ private Constants() {} public static final String KAFKA_OFFSET_HEADER = "x-goog-pubsublite-source-kafka-offset"; public static final String KAFKA_EVENT_TIME_TYPE_HEADER = "x-goog-pubsublite-source-kafka-event-time-type"; - public static final String PUBSUBLITE_KAFKA_SINK_CONNECTOR_NAME = - "JAVA_PUBSUBLITE_KAFKA_SINK_CONNECTOR"; } diff --git a/src/main/java/com/google/pubsublite/kafka/sink/KafkaPartitionRoutingPolicy.java b/src/main/java/com/google/pubsublite/kafka/sink/KafkaPartitionRoutingPolicy.java new file mode 100644 index 00000000..b4faa3cb --- /dev/null +++ b/src/main/java/com/google/pubsublite/kafka/sink/KafkaPartitionRoutingPolicy.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.pubsublite.kafka.sink; + +import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical; + +import com.google.api.gax.rpc.StatusCode.Code; +import com.google.cloud.pubsublite.Partition; +import com.google.cloud.pubsublite.internal.CheckedApiException; +import com.google.cloud.pubsublite.internal.RoutingPolicy; +import com.google.cloud.pubsublite.proto.PubSubMessage; + +/** A routing policy that extracts the original kafka partition and routes to that partition. */ +class KafkaPartitionRoutingPolicy implements RoutingPolicy { + private final long numPartitions; + + KafkaPartitionRoutingPolicy(long numPartitions) { + this.numPartitions = numPartitions; + } + + @Override + public Partition route(PubSubMessage message) throws CheckedApiException { + Partition partition = getPartition(message); + if (partition.value() >= numPartitions) { + throw new CheckedApiException( + "Kafka topic has more partitions than Pub/Sub Lite topic. OrderingMode.KAFKA cannot be used.", + Code.FAILED_PRECONDITION); + } + return partition; + } + + private Partition getPartition(PubSubMessage message) throws CheckedApiException { + try { + return Partition.of( + Long.parseLong( + message + .getAttributesOrThrow(Constants.KAFKA_PARTITION_HEADER) + .getValues(0) + .toStringUtf8())); + } catch (Throwable t) { + throw toCanonical(t); + } + } +} diff --git a/src/main/java/com/google/pubsublite/kafka/sink/OrderingMode.java b/src/main/java/com/google/pubsublite/kafka/sink/OrderingMode.java new file mode 100644 index 00000000..2f24627d --- /dev/null +++ b/src/main/java/com/google/pubsublite/kafka/sink/OrderingMode.java @@ -0,0 +1,23 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.pubsublite.kafka.sink; + +public enum OrderingMode { + /* Order based on the standard Pub/Sub Lite logic. */ + DEFAULT, + /* Send messages to the same partition index they were from in Kafka. */ + KAFKA +} diff --git a/src/main/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTask.java b/src/main/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTask.java index 328b303a..9dd5ab95 100644 --- a/src/main/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTask.java +++ b/src/main/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTask.java @@ -18,9 +18,10 @@ import static com.google.pubsublite.kafka.sink.Schemas.encodeToBytes; import com.google.api.core.ApiService.State; -import com.google.cloud.pubsublite.Message; import com.google.cloud.pubsublite.MessageMetadata; import com.google.cloud.pubsublite.internal.Publisher; +import com.google.cloud.pubsublite.proto.AttributeValues; +import com.google.cloud.pubsublite.proto.PubSubMessage; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableListMultimap; import com.google.protobuf.ByteString; @@ -75,7 +76,7 @@ public void put(Collection collection) { } } for (SinkRecord record : collection) { - Message.Builder message = Message.builder(); + PubSubMessage.Builder message = PubSubMessage.newBuilder(); if (record.key() != null) { message.setKey(encodeToBytes(record.keySchema(), record.key())); } @@ -89,6 +90,7 @@ public void put(Collection collection) { header -> attributes.put( header.key(), Schemas.encodeToBytes(header.schema(), header.value()))); + if (record.topic() != null) { attributes.put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(record.topic())); } @@ -106,7 +108,13 @@ public void put(Collection collection) { ByteString.copyFromUtf8(record.timestampType().name)); message.setEventTime(Timestamps.fromMillis(record.timestamp())); } - message.setAttributes(attributes.build()); + attributes + .build() + .asMap() + .forEach( + (key, values) -> + message.putAttributes( + key, AttributeValues.newBuilder().addAllValues(values).build())); publisher.publish(message.build()); } } diff --git a/src/main/java/com/google/pubsublite/kafka/sink/PublisherFactoryImpl.java b/src/main/java/com/google/pubsublite/kafka/sink/PublisherFactoryImpl.java index b6923287..09c62b21 100644 --- a/src/main/java/com/google/pubsublite/kafka/sink/PublisherFactoryImpl.java +++ b/src/main/java/com/google/pubsublite/kafka/sink/PublisherFactoryImpl.java @@ -18,11 +18,12 @@ import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical; import static com.google.cloud.pubsublite.internal.wire.ServiceClients.addDefaultSettings; import static com.google.cloud.pubsublite.internal.wire.ServiceClients.getCallContext; -import static com.google.pubsublite.kafka.sink.Constants.PUBSUBLITE_KAFKA_SINK_CONNECTOR_NAME; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; -import com.google.cloud.pubsublite.CloudZone; +import com.google.cloud.pubsublite.AdminClient; +import com.google.cloud.pubsublite.AdminClientSettings; +import com.google.cloud.pubsublite.CloudRegionOrZone; import com.google.cloud.pubsublite.MessageMetadata; import com.google.cloud.pubsublite.Partition; import com.google.cloud.pubsublite.ProjectPath; @@ -30,27 +31,30 @@ import com.google.cloud.pubsublite.TopicPath; import com.google.cloud.pubsublite.cloudpubsub.PublisherSettings; import com.google.cloud.pubsublite.internal.Publisher; +import com.google.cloud.pubsublite.internal.wire.PartitionCountWatchingPublisherSettings; import com.google.cloud.pubsublite.internal.wire.PartitionPublisherFactory; import com.google.cloud.pubsublite.internal.wire.PubsubContext; import com.google.cloud.pubsublite.internal.wire.PubsubContext.Framework; import com.google.cloud.pubsublite.internal.wire.RoutingMetadata; -import com.google.cloud.pubsublite.internal.wire.RoutingPublisherBuilder; import com.google.cloud.pubsublite.internal.wire.SinglePartitionPublisherBuilder; +import com.google.cloud.pubsublite.v1.AdminServiceClient; +import com.google.cloud.pubsublite.v1.AdminServiceSettings; import com.google.cloud.pubsublite.v1.PublisherServiceClient; import com.google.cloud.pubsublite.v1.PublisherServiceSettings; +import com.google.pubsub.kafka.common.ConnectorCredentialsProvider; +import java.io.IOException; import java.util.Map; import java.util.Optional; -import org.apache.kafka.common.config.ConfigValue; class PublisherFactoryImpl implements PublisherFactory { private static final Framework FRAMEWORK = Framework.of("KAFKA_CONNECT"); - private PartitionPublisherFactory getPartitionPublisherFactory(TopicPath topic) { + private PartitionPublisherFactory getPartitionPublisherFactory( + TopicPath topic, ConnectorCredentialsProvider credentialsProvider) { return new PartitionPublisherFactory() { - - protected Optional publisherServiceClient = Optional.empty(); + private Optional publisherServiceClient = Optional.empty(); private synchronized PublisherServiceClient getServiceClient() throws ApiException { if (publisherServiceClient.isPresent()) return publisherServiceClient.get(); @@ -61,9 +65,7 @@ private synchronized PublisherServiceClient getServiceClient() throws ApiExcepti addDefaultSettings( topic.location().extractRegion(), PublisherServiceSettings.newBuilder() - .setCredentialsProvider( - PublisherServiceSettings.defaultCredentialsProviderBuilder() - .build())))); + .setCredentialsProvider(credentialsProvider)))); return publisherServiceClient.get(); } catch (Throwable t) { throw toCanonical(t).underlying; @@ -82,8 +84,7 @@ public Publisher newPublisher(Partition partition) throws ApiEx responseStream -> { ApiCallContext context = getCallContext( - PubsubContext.of(Framework.of(PUBSUBLITE_KAFKA_SINK_CONNECTOR_NAME)), - RoutingMetadata.of(topic, partition)); + PubsubContext.of(FRAMEWORK), RoutingMetadata.of(topic, partition)); return client.publishCallable().splitCall(responseStream, context); }); return singlePartitionBuilder.build(); @@ -96,18 +97,41 @@ public void close() {} @Override public Publisher newPublisher(Map params) { - Map config = ConfigDefs.config().validateAll(params); - RoutingPublisherBuilder.Builder builder = RoutingPublisherBuilder.newBuilder(); + Map config = ConfigDefs.config().parse(params); + ConnectorCredentialsProvider credentialsProvider = + ConnectorCredentialsProvider.fromConfig(config); + CloudRegionOrZone location = + CloudRegionOrZone.parse(config.get(ConfigDefs.LOCATION_FLAG).toString()); + PartitionCountWatchingPublisherSettings.Builder builder = + PartitionCountWatchingPublisherSettings.newBuilder(); TopicPath topic = TopicPath.newBuilder() .setProject( - ProjectPath.parse("projects/" + config.get(ConfigDefs.PROJECT_FLAG).value()) - .project()) - .setLocation(CloudZone.parse(config.get(ConfigDefs.LOCATION_FLAG).value().toString())) - .setName(TopicName.of(config.get(ConfigDefs.TOPIC_NAME_FLAG).value().toString())) + ProjectPath.parse("projects/" + config.get(ConfigDefs.PROJECT_FLAG)).project()) + .setLocation(location) + .setName(TopicName.of(config.get(ConfigDefs.TOPIC_NAME_FLAG).toString())) .build(); builder.setTopic(topic); - builder.setPublisherFactory(getPartitionPublisherFactory(topic)); - return builder.build(); + builder.setPublisherFactory(getPartitionPublisherFactory(topic, credentialsProvider)); + try { + builder.setAdminClient( + AdminClient.create( + AdminClientSettings.newBuilder() + .setRegion(location.extractRegion()) + .setServiceClient( + AdminServiceClient.create( + addDefaultSettings( + location.extractRegion(), + AdminServiceSettings.newBuilder() + .setCredentialsProvider(credentialsProvider)))) + .build())); + } catch (IOException e) { + throw new IllegalStateException(e); + } + if (OrderingMode.valueOf(config.get(ConfigDefs.ORDERING_MODE_FLAG).toString()) + == OrderingMode.KAFKA) { + builder.setRoutingPolicyFactory(KafkaPartitionRoutingPolicy::new); + } + return builder.build().instantiate(); } } diff --git a/src/main/java/com/google/pubsublite/kafka/source/ConfigDefs.java b/src/main/java/com/google/pubsublite/kafka/source/ConfigDefs.java index 2dbf1d3a..36942662 100644 --- a/src/main/java/com/google/pubsublite/kafka/source/ConfigDefs.java +++ b/src/main/java/com/google/pubsublite/kafka/source/ConfigDefs.java @@ -15,6 +15,7 @@ */ package com.google.pubsublite.kafka.source; +import com.google.pubsub.kafka.common.ConnectorUtils; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; @@ -63,6 +64,18 @@ static ConfigDef config() { ConfigDef.Type.LONG, 20_000_000, Importance.MEDIUM, - "The number of outstanding bytes per-partition allowed. Set to 20MB by default."); + "The number of outstanding bytes per-partition allowed. Set to 20MB by default.") + .define( + ConnectorUtils.GCP_CREDENTIALS_FILE_PATH_CONFIG, + ConfigDef.Type.STRING, + "", + Importance.HIGH, + "The path to the GCP credentials file") + .define( + ConnectorUtils.GCP_CREDENTIALS_JSON_CONFIG, + ConfigDef.Type.STRING, + "", + Importance.HIGH, + "GCP JSON credentials"); } } diff --git a/src/test/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTaskTest.java b/src/test/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTaskTest.java index d64d5582..dc77d179 100644 --- a/src/test/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTaskTest.java +++ b/src/test/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTaskTest.java @@ -23,12 +23,12 @@ import static org.mockito.Mockito.verify; import static org.mockito.MockitoAnnotations.initMocks; -import com.google.cloud.pubsublite.Message; import com.google.cloud.pubsublite.MessageMetadata; import com.google.cloud.pubsublite.internal.Publisher; import com.google.cloud.pubsublite.internal.testing.FakeApiService; +import com.google.cloud.pubsublite.proto.AttributeValues; +import com.google.cloud.pubsublite.proto.PubSubMessage; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import com.google.protobuf.util.Timestamps; @@ -74,31 +74,21 @@ public class PubSubLiteSinkTaskTest { Schema.BYTES_SCHEMA, KAFKA_MESSAGE2.toByteArray(), -1); - private static final Message SAMPLE_MESSAGE_1 = - Message.builder() + private static final PubSubMessage SAMPLE_MESSAGE_1 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) - .setAttributes( - ImmutableListMultimap.builder() - .put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(KAFKA_TOPIC)) - .put( - Constants.KAFKA_PARTITION_HEADER, - ByteString.copyFromUtf8(Integer.toString(0))) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8(Integer.toString(-1))) - .build()) + .putAttributes(Constants.KAFKA_TOPIC_HEADER, single(KAFKA_TOPIC)) + .putAttributes(Constants.KAFKA_PARTITION_HEADER, single("0")) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("-1")) .build(); - private static final Message SAMPLE_MESSAGE_2 = - Message.builder() + private static final PubSubMessage SAMPLE_MESSAGE_2 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY2)) .setData(KAFKA_MESSAGE2) - .setAttributes( - ImmutableListMultimap.builder() - .put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(KAFKA_TOPIC)) - .put( - Constants.KAFKA_PARTITION_HEADER, - ByteString.copyFromUtf8(Integer.toString(0))) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8(Integer.toString(-1))) - .build()) + .putAttributes(Constants.KAFKA_TOPIC_HEADER, single(KAFKA_TOPIC)) + .putAttributes(Constants.KAFKA_PARTITION_HEADER, single("0")) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("-1")) .build(); private PubSubLiteSinkTask task; @@ -108,6 +98,10 @@ abstract static class FakePublisher extends FakeApiService private @Spy FakePublisher publisher; + private static AttributeValues single(String value) { + return AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8(value)).build(); + } + @Before public void setup() { initMocks(this); @@ -174,7 +168,7 @@ public void testPutWithNullValues() { null, -1)); task.put(records); - Message expectedResult = SAMPLE_MESSAGE_1.toBuilder().setData(ByteString.EMPTY).build(); + PubSubMessage expectedResult = SAMPLE_MESSAGE_1.toBuilder().setData(ByteString.EMPTY).build(); verify(publisher).publish(expectedResult); } @@ -185,7 +179,7 @@ public void testPutWithNullMessage() { records.add( new SinkRecord(KAFKA_TOPIC, 0, Schema.STRING_SCHEMA, null, Schema.BYTES_SCHEMA, null, -1)); task.put(records); - Message expectedResult = + PubSubMessage expectedResult = SAMPLE_MESSAGE_1.toBuilder().setKey(ByteString.EMPTY).setData(ByteString.EMPTY).build(); verify(publisher).publish(expectedResult); } @@ -259,48 +253,35 @@ public void testKafkaMetadata() { null, TimestampType.CREATE_TIME); task.put(ImmutableList.of(record1, record2, record3)); - ImmutableListMultimap attributesBase = - ImmutableListMultimap.builder() - .put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(KAFKA_TOPIC)) - .put(Constants.KAFKA_PARTITION_HEADER, ByteString.copyFromUtf8(Integer.toString(4))) + ImmutableMap attributesBase = + ImmutableMap.builder() + .put(Constants.KAFKA_TOPIC_HEADER, single(KAFKA_TOPIC)) + .put(Constants.KAFKA_PARTITION_HEADER, single(Integer.toString(4))) .build(); - Message message1 = - Message.builder() + PubSubMessage message1 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) .setEventTime(Timestamps.fromMillis(50000)) - .setAttributes( - ImmutableListMultimap.builder() - .putAll(attributesBase) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8("1000")) - .put( - Constants.KAFKA_EVENT_TIME_TYPE_HEADER, - ByteString.copyFromUtf8("CreateTime")) - .build()) + .putAllAttributes(attributesBase) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("1000")) + .putAttributes(Constants.KAFKA_EVENT_TIME_TYPE_HEADER, single("CreateTime")) .build(); - Message message2 = - Message.builder() + PubSubMessage message2 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) .setEventTime(Timestamps.fromMillis(50001)) - .setAttributes( - ImmutableListMultimap.builder() - .putAll(attributesBase) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8("1001")) - .put( - Constants.KAFKA_EVENT_TIME_TYPE_HEADER, - ByteString.copyFromUtf8("LogAppendTime")) - .build()) + .putAllAttributes(attributesBase) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("1001")) + .putAttributes(Constants.KAFKA_EVENT_TIME_TYPE_HEADER, single("LogAppendTime")) .build(); - Message message3 = - Message.builder() + PubSubMessage message3 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) - .setAttributes( - ImmutableListMultimap.builder() - .putAll(attributesBase) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8("1002")) - .build()) + .putAllAttributes(attributesBase) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("1002")) .build(); InOrder order = inOrder(publisher); order.verify(publisher).publish(message1); @@ -336,40 +317,30 @@ public void testKafkaHeaders() { TimestampType.LOG_APPEND_TIME); record2.headers().addString("yourHeader", "yourValue"); task.put(ImmutableList.of(record1, record2)); - ImmutableListMultimap attributesBase = - ImmutableListMultimap.builder() - .put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(KAFKA_TOPIC)) - .put(Constants.KAFKA_PARTITION_HEADER, ByteString.copyFromUtf8(Integer.toString(4))) + ImmutableMap attributesBase = + ImmutableMap.builder() + .put(Constants.KAFKA_TOPIC_HEADER, single(KAFKA_TOPIC)) + .put(Constants.KAFKA_PARTITION_HEADER, single(Integer.toString(4))) .build(); - Message message1 = - Message.builder() + PubSubMessage message1 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) .setEventTime(Timestamps.fromMillis(50000)) - .setAttributes( - ImmutableListMultimap.builder() - .putAll(attributesBase) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8("1000")) - .put( - Constants.KAFKA_EVENT_TIME_TYPE_HEADER, - ByteString.copyFromUtf8("CreateTime")) - .put("myHeader", ByteString.copyFromUtf8("myValue")) - .build()) + .putAllAttributes(attributesBase) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("1000")) + .putAttributes(Constants.KAFKA_EVENT_TIME_TYPE_HEADER, single("CreateTime")) + .putAttributes("myHeader", single("myValue")) .build(); - Message message2 = - Message.builder() + PubSubMessage message2 = + PubSubMessage.newBuilder() .setKey(ByteString.copyFromUtf8(KAFKA_MESSAGE_KEY1)) .setData(KAFKA_MESSAGE1) .setEventTime(Timestamps.fromMillis(50001)) - .setAttributes( - ImmutableListMultimap.builder() - .putAll(attributesBase) - .put(Constants.KAFKA_OFFSET_HEADER, ByteString.copyFromUtf8("1001")) - .put( - Constants.KAFKA_EVENT_TIME_TYPE_HEADER, - ByteString.copyFromUtf8("LogAppendTime")) - .put("yourHeader", ByteString.copyFromUtf8("yourValue")) - .build()) + .putAllAttributes(attributesBase) + .putAttributes(Constants.KAFKA_OFFSET_HEADER, single("1001")) + .putAttributes(Constants.KAFKA_EVENT_TIME_TYPE_HEADER, single("LogAppendTime")) + .putAttributes("yourHeader", single("yourValue")) .build(); InOrder order = inOrder(publisher); order.verify(publisher).publish(message1); diff --git a/src/test/java/it/StandaloneIT.java b/src/test/java/it/StandaloneIT.java index 1c054a04..6de9723f 100644 --- a/src/test/java/it/StandaloneIT.java +++ b/src/test/java/it/StandaloneIT.java @@ -17,6 +17,7 @@ package it; import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.MINUTES; import static junit.framework.TestCase.assertNotNull; import com.google.api.core.ApiFuture; @@ -159,9 +160,8 @@ public class StandaloneIT extends Base { private static final String instanceName = "kafka-it-" + runId; private static final String instanceTemplateName = "kafka-it-template-" + runId; - private static AtomicBoolean initialized = new AtomicBoolean(false); - private static Boolean cpsMessageReceived = false; - private static Boolean pslMessageReceived = false; + private static AtomicBoolean cpsMessageReceived = new AtomicBoolean(false); + private static AtomicBoolean pslMessageReceived = new AtomicBoolean(false); private static Instance gceKafkaInstance; private static String kafkaInstanceIpAddress; @@ -283,7 +283,7 @@ protected static void setupPslResources() throws Exception { .setTopic(pslSinkTopicPath.toString()) .build(); pslSinkSubscription = pslAdminClient.createSubscription(pslSinkSubscription).get(); - log.atInfo().log("Created PSL sink subscription: " + pslSinkSubscriptionPath.toString()); + log.atInfo().log("Created PSL sink subscription: " + pslSinkSubscriptionPath); Topic.Builder sourceTopicBuilder = Topic.newBuilder() @@ -318,7 +318,7 @@ protected static void setupPslResources() throws Exception { .setTopic(pslSourceTopicPath.toString()) .build(); pslSourceSubscription = pslAdminClient.createSubscription(pslSourceSubscription).get(); - log.atInfo().log("Created PSL source subscription: " + pslSinkSubscriptionPath.toString()); + log.atInfo().log("Created PSL source subscription: " + pslSinkSubscriptionPath); } } @@ -404,12 +404,12 @@ public Void apply(Runnable runnable) { } try (InstancesClient instancesClient = InstancesClient.create()) { - instancesClient.deleteAsync(projectId, location, instanceName).get(3, TimeUnit.MINUTES); + instancesClient.deleteAsync(projectId, location, instanceName).get(3, MINUTES); } log.atInfo().log("Deleted Compute Engine instance."); try (InstanceTemplatesClient instanceTemplatesClient = InstanceTemplatesClient.create()) { - instanceTemplatesClient.deleteAsync(projectId, instanceTemplateName).get(3, TimeUnit.MINUTES); + instanceTemplatesClient.deleteAsync(projectId, instanceTemplateName).get(3, MINUTES); } log.atInfo().log("Deleted Compute Engine instance template."); } @@ -434,7 +434,7 @@ public void testCpsSinkConnector() throws Exception { .forEach( (metricName, metric) -> { if (metricName.name() == "record-send-total") { - log.atInfo().log("record-send-total: " + metric.metricValue().toString()); + log.atInfo().log("record-send-total: " + metric.metricValue()); } }); kafkaProducer.close(); @@ -452,7 +452,7 @@ public void testCpsSinkConnector() throws Exception { assertThat(message.getData().toStringUtf8()).isEqualTo("value0"); assertThat(message.getAttributesMap().get(ConnectorUtils.CPS_MESSAGE_KEY_ATTRIBUTE)) .isEqualTo("key0"); - this.cpsMessageReceived = true; + this.cpsMessageReceived.set(true); consumer.ack(); }; @@ -465,7 +465,7 @@ public void testCpsSinkConnector() throws Exception { // Shut down the subscriber after 30s. Stop receiving messages. subscriber.stopAsync(); } - assertThat(this.cpsMessageReceived).isTrue(); + assertThat(this.cpsMessageReceived.get()).isTrue(); } @Test(timeout = 5 * 60 * 1000L) @@ -573,7 +573,7 @@ public void testPslSinkConnector() throws Exception { .forEach( (metricName, metric) -> { if (metricName.name() == "record-send-total") { - log.atInfo().log("record-send-total: " + metric.metricValue().toString()); + log.atInfo().log("record-send-total: " + metric.metricValue()); } }); kafkaProducer.close(); @@ -588,8 +588,7 @@ public void testPslSinkConnector() throws Exception { log.atInfo().log("Received message: " + message); assertThat(message.getData().toStringUtf8()).isEqualTo("value0"); assertThat(message.getOrderingKey()).isEqualTo("key0"); - this.pslMessageReceived = true; - log.atInfo().log("this.pslMessageReceived: " + this.pslMessageReceived); + this.pslMessageReceived.set(true); consumer.ack(); }; @@ -606,12 +605,12 @@ public void testPslSinkConnector() throws Exception { .build()); try { subscriber.startAsync().awaitRunning(); - subscriber.awaitTerminated(3, TimeUnit.MINUTES); + subscriber.awaitTerminated(3, MINUTES); } catch (TimeoutException timeoutException) { // Shut down the subscriber after 3 minutes. Stop receiving messages. subscriber.stopAsync(); } - assertThat(this.pslMessageReceived).isTrue(); + assertThat(this.pslMessageReceived.get()).isTrue(); } @Test(timeout = 5 * 60 * 1000L) @@ -634,30 +633,8 @@ public void testPslSourceConnector() throws Exception { PubsubMessage msg0 = PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("msg0")).build(); ApiFuture publishFuture = publisher.publish(msg0); - ApiFutures.addCallback( - publishFuture, - new ApiFutureCallback() { - - @Override - public void onFailure(Throwable throwable) { - if (throwable instanceof ApiException) { - ApiException apiException = ((ApiException) throwable); - // details on the API exception - log.atInfo().log(apiException.fillInStackTrace().toString()); - } - Assert.fail("Error publishing message : " + msg0); - } - - @Override - public void onSuccess(String messageId) { - // Once published, returns server-assigned message ids (unique within the topic) - log.atInfo().log("Published message ID: " + messageId); - } - }, - MoreExecutors.directExecutor()); - - // Sleep for 1min. - Thread.sleep(60 * 1000); + log.atInfo().log("Published message ID: " + publishFuture.get(1, MINUTES)); + publisher.stopAsync().awaitTerminated(); // Consume from Kafka connect. Properties consumer_props = new Properties(); diff --git a/versions.txt b/versions.txt index d2fe15f2..af274e56 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -pubsub-group-kafka-connector:1.0.0:1.0.0 +pubsub-group-kafka-connector:1.1.0:1.1.0