diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index d304652e1c..f569205572 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:6f431774e11cc46619cf093fd1481193c4024031073697fa18f0099b943aab88 -# created: 2023-12-01T19:50:20.444857406Z + digest: sha256:81f75d962cd28b7ad10740a643b8069b8fa0357cb495b782eef8560bb7a8fd65 +# created: 2023-12-05T19:16:19.735195992Z diff --git a/.kokoro/conformance.sh b/.kokoro/conformance.sh index a5d22f1a9a..0229a03a70 100755 --- a/.kokoro/conformance.sh +++ b/.kokoro/conformance.sh @@ -60,7 +60,9 @@ do configFlag="--enable_features_all" else echo "Testing the client with default settings for optional features..." - configFlag="" + # skipping routing cookie and retry info tests. When the feature is disabled, these + # tests are expected to fail + configFlag="-skip _Retry_WithRoutingCookie\|_Retry_WithRetryInfo" fi pushd . diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index c5c11bbe79..445c5c1f09 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -170,30 +170,30 @@ colorlog==6.7.0 \ --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 # via gcp-docuploader -cryptography==41.0.2 \ - --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ - --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ - --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ - --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ - --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ - --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ - --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ - --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ - --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ - --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ - --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ - --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ - --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ - --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ - --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ - --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ - --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ - --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ - --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ - --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ - --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ - --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ - --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 +cryptography==41.0.6 \ + --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \ + --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \ + --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \ + --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \ + --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \ + --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \ + --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \ + --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \ + --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \ + --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \ + --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \ + --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \ + --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \ + --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \ + --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \ + --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \ + --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \ + --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \ + --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \ + --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \ + --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \ + --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \ + --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae # via # gcp-releasetool # secretstorage diff --git a/CHANGELOG.md b/CHANGELOG.md index e03e2c9321..3c6f5c0ccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [2.31.0](https://github.com/googleapis/java-bigtable/compare/v2.30.0...v2.31.0) (2024-01-12) + + +### Features + +* Add a flag to add / remove routing cookie from callable chain ([#2032](https://github.com/googleapis/java-bigtable/issues/2032)) ([201e631](https://github.com/googleapis/java-bigtable/commit/201e631f893b1edacdd5760c1d180b212dc9e38a)) +* Adding feature flags for routing cookie and retry info ([#2031](https://github.com/googleapis/java-bigtable/issues/2031)) ([08c5bf1](https://github.com/googleapis/java-bigtable/commit/08c5bf1fd76258387135c8c3abe75f13bcdcc1f6)) +* Count row merging errors as internal errors ([#2045](https://github.com/googleapis/java-bigtable/issues/2045)) ([fc7845b](https://github.com/googleapis/java-bigtable/commit/fc7845bd4cefca05bccc4dc3a9f727fd20f5adf6)) +* Enable feature flag when setting is enabled ([#2043](https://github.com/googleapis/java-bigtable/issues/2043)) ([e0d90db](https://github.com/googleapis/java-bigtable/commit/e0d90db67b3ea52d833f7d6bcd78e3f7e91ff301)) +* Handle retry info so client respect the delay server sets ([#2026](https://github.com/googleapis/java-bigtable/issues/2026)) ([f1b7fc7](https://github.com/googleapis/java-bigtable/commit/f1b7fc79ad3fd9006e430e48430331b360bb22e3)) + + +### Bug Fixes + +* **deps:** Update the Java code generator (gapic-generator-java) to 2.31.0 ([#2044](https://github.com/googleapis/java-bigtable/issues/2044)) ([d9042a5](https://github.com/googleapis/java-bigtable/commit/d9042a567f284424efb4af69f757883c9781dce3)) +* Fix RetryInfo algorithm and tests ([#2041](https://github.com/googleapis/java-bigtable/issues/2041)) ([dad7517](https://github.com/googleapis/java-bigtable/commit/dad751736112323c578b3c90d9587fc182105747)) + + +### Dependencies + +* Update dependency com.google.cloud:gapic-libraries-bom to v1.27.0 ([#2030](https://github.com/googleapis/java-bigtable/issues/2030)) ([a492d02](https://github.com/googleapis/java-bigtable/commit/a492d02bdc52cb81d8804a4d7cd363b5807bdd47)) +* Update dependency com.google.truth.extensions:truth-proto-extension to v1.2.0 ([#2035](https://github.com/googleapis/java-bigtable/issues/2035)) ([46e1e03](https://github.com/googleapis/java-bigtable/commit/46e1e0335f9969fa1b60acdf17e9b8abbc312ca2)) + ## [2.30.0](https://github.com/googleapis/java-bigtable/compare/v2.29.1...v2.30.0) (2023-12-05) diff --git a/README.md b/README.md index 5caf555e94..a1a00577d2 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-bigtable - 2.29.1 + 2.30.0 ``` @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.27.0') +implementation platform('com.google.cloud:libraries-bom:26.29.0') implementation 'com.google.cloud:google-cloud-bigtable' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigtable:2.29.1' +implementation 'com.google.cloud:google-cloud-bigtable:2.30.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.29.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.30.0" ``` @@ -609,7 +609,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigtable/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigtable.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigtable/2.29.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigtable/2.30.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 7512e99da6..1288d2edf6 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom com.google.cloud google-cloud-shared-config - 1.6.1 + 1.7.1 @@ -63,42 +63,42 @@ com.google.cloud google-cloud-bigtable - 2.30.0 + 2.31.0 com.google.cloud google-cloud-bigtable-emulator - 0.167.0 + 0.168.0 com.google.cloud google-cloud-bigtable-emulator-core - 0.167.0 + 0.168.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 com.google.cloud google-cloud-bigtable-stats - 2.30.0 + 2.31.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index b0733f362d..980ea7ed25 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -7,13 +7,13 @@ com.google.cloud google-cloud-shared-config - 1.6.1 + 1.7.1 com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom @@ -66,14 +66,14 @@ com.google.cloud gapic-libraries-bom - 1.25.0 + 1.27.0 pom import com.google.cloud google-cloud-shared-dependencies - 3.20.0 + 3.21.0 pom import diff --git a/google-cloud-bigtable-emulator-core/pom.xml b/google-cloud-bigtable-emulator-core/pom.xml index b00a569bc0..52e348d8fd 100644 --- a/google-cloud-bigtable-emulator-core/pom.xml +++ b/google-cloud-bigtable-emulator-core/pom.xml @@ -7,11 +7,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.30.0 + 2.31.0 google-cloud-bigtable-emulator-core - 0.167.0 + 0.168.0 A Java wrapper for the Cloud Bigtable emulator. diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 63241aa35d..972ca8905c 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.167.0 + 0.168.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -81,14 +81,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigtable-emulator-core - 0.167.0 + 0.168.0 diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index b5e46be9d3..7a58836626 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -5,7 +5,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 4.0.0 @@ -13,7 +13,7 @@ through Stackdriver. Built-in metrics will be implemented with shaded OpenCensus so it won't interfere with customer's application metrics. --> google-cloud-bigtable-stats - 2.30.0 + 2.31.0 Experimental project to shade OpenCensus dependencies. @@ -21,7 +21,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import diff --git a/google-cloud-bigtable/clirr-ignored-differences.xml b/google-cloud-bigtable/clirr-ignored-differences.xml index 4bb4684c38..f6411f9ee3 100644 --- a/google-cloud-bigtable/clirr-ignored-differences.xml +++ b/google-cloud-bigtable/clirr-ignored-differences.xml @@ -34,6 +34,12 @@ com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub * + + + 7002 + com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub + * + 8001 @@ -150,4 +156,28 @@ 8001 com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerBatchedUnaryCallable + + + 6001 + com/google/cloud/bigtable/gaxx/retrying/ApiResultRetryAlgorithm + * + + + + 7004 + com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable + * + + + + 7004 + com/google/cloud/bigtable/data/v2/models/MutateRowsException + * + + + + 7009 + com/google/cloud/bigtable/data/v2/models/MutateRowsException + * + diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 57c976d877..cc39950662 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 2.30.0 + 2.31.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,11 +12,11 @@ com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 - 2.30.0 + 2.31.0 google-cloud-bigtable @@ -38,8 +38,8 @@ batch-bigtable.googleapis.com:443 - 1.59.1 - 3.25.1 + 1.60.1 + 3.25.2 @@ -47,14 +47,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import @@ -161,7 +161,10 @@ grpc-alts runtime - + + org.checkerframework + checker-qual + com.google.http-client google-http-client diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java index 60fb95c75c..0e807e45fc 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java @@ -20,6 +20,6 @@ @InternalApi("For internal use only") public final class Version { // {x-version-update-start:google-cloud-bigtable:current} - public static String VERSION = "2.30.0"; + public static String VERSION = "2.31.0"; // {x-version-update-end} } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClient.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClient.java index 5271632246..ea42627935 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClient.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClient.java @@ -1396,6 +1396,7 @@ public final Table modifyColumnFamilies( * ModifyColumnFamiliesRequest.newBuilder() * .setName(TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]").toString()) * .addAllModifications(new ArrayList()) + * .setIgnoreWarnings(true) * .build(); * Table response = baseBigtableTableAdminClient.modifyColumnFamilies(request); * } @@ -1428,6 +1429,7 @@ public final Table modifyColumnFamilies(ModifyColumnFamiliesRequest request) { * ModifyColumnFamiliesRequest.newBuilder() * .setName(TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]").toString()) * .addAllModifications(new ArrayList()) + * .setIgnoreWarnings(true) * .build(); * ApiFuture future = * baseBigtableTableAdminClient.modifyColumnFamiliesCallable().futureCall(request); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java index a917ad2cc5..f4262827c5 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java @@ -456,6 +456,12 @@ public BigtableInstanceAdminStub createStub() throws IOException { "Transport not supported: %s", getTransportChannelProvider().getTransportName())); } + /** Returns the default service name. */ + @Override + public String getServiceName() { + return "bigtableadmin"; + } + /** Returns a builder for the default ExecutorProvider for this service. */ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuilder() { return InstantiatingExecutorProvider.newBuilder(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java index 7abc6f4723..7a1049b626 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java @@ -544,6 +544,12 @@ public BigtableTableAdminStub createStub() throws IOException { "Transport not supported: %s", getTransportChannelProvider().getTransportName())); } + /** Returns the default service name. */ + @Override + public String getServiceName() { + return "bigtableadmin"; + } + /** Returns a builder for the default ExecutorProvider for this service. */ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuilder() { return InstantiatingExecutorProvider.newBuilder(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java index 968ebaef26..f84a5dd098 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java @@ -25,6 +25,7 @@ import com.google.api.gax.batching.Batcher; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.rpc.ApiExceptions; +import com.google.api.gax.rpc.ClientContext; import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.ServerStream; import com.google.api.gax.rpc.ServerStreamingCallable; @@ -166,6 +167,18 @@ public static BigtableDataClient create(BigtableDataSettings settings) throws IO return new BigtableDataClient(stub); } + /** + * Constructs an instance of BigtableDataClient with the provided client context. This is used by + * {@link BigtableDataClientFactory} and the client context will not be closed unless {@link + * BigtableDataClientFactory#close()} is called. + */ + static BigtableDataClient createWithClientContext( + BigtableDataSettings settings, ClientContext context) throws IOException { + EnhancedBigtableStub stub = + EnhancedBigtableStub.createWithClientContext(settings.getStubSettings(), context); + return new BigtableDataClient(stub); + } + @InternalApi("Visible for testing") BigtableDataClient(EnhancedBigtableStub stub) { this.stub = stub; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java index d4561ab4df..c35500a189 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java @@ -17,13 +17,8 @@ import com.google.api.core.BetaApi; import com.google.api.gax.core.BackgroundResource; -import com.google.api.gax.core.FixedCredentialsProvider; -import com.google.api.gax.core.FixedExecutorProvider; import com.google.api.gax.rpc.ClientContext; -import com.google.api.gax.rpc.FixedHeaderProvider; -import com.google.api.gax.rpc.FixedTransportChannelProvider; -import com.google.api.gax.rpc.FixedWatchdogProvider; -import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; +import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; import java.io.IOException; import javax.annotation.Nonnull; @@ -78,7 +73,8 @@ public final class BigtableDataClientFactory implements AutoCloseable { */ public static BigtableDataClientFactory create(BigtableDataSettings defaultSettings) throws IOException { - ClientContext sharedClientContext = ClientContext.create(defaultSettings.getStubSettings()); + ClientContext sharedClientContext = + EnhancedBigtableStub.createClientContext(defaultSettings.getStubSettings()); return new BigtableDataClientFactory(sharedClientContext, defaultSettings); } @@ -110,12 +106,16 @@ public void close() throws Exception { * release all resources, first close all of the created clients and then this factory instance. */ public BigtableDataClient createDefault() { - BigtableDataSettings.Builder settingsBuilder = defaultSettings.toBuilder(); - patchStubSettings(settingsBuilder.stubSettings()); - BigtableDataSettings settings = settingsBuilder.build(); - try { - return BigtableDataClient.create(settings); + ClientContext clientContext = + sharedClientContext + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory( + defaultSettings.getStubSettings())) + .build(); + + return BigtableDataClient.createWithClientContext(defaultSettings, clientContext); } catch (IOException e) { // Should never happen because the connection has been established already throw new RuntimeException( @@ -133,12 +133,16 @@ public BigtableDataClient createDefault() { * release all resources, first close all of the created clients and then this factory instance. */ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) throws IOException { - BigtableDataSettings.Builder settingsBuilder = - defaultSettings.toBuilder().setAppProfileId(appProfileId); - - patchStubSettings(settingsBuilder.stubSettings()); + BigtableDataSettings settings = + defaultSettings.toBuilder().setAppProfileId(appProfileId).build(); - return BigtableDataClient.create(settingsBuilder.build()); + ClientContext clientContext = + sharedClientContext + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory(settings.getStubSettings())) + .build(); + return BigtableDataClient.createWithClientContext(settings, clientContext); } /** @@ -152,16 +156,22 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro */ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull String instanceId) throws IOException { - BigtableDataSettings.Builder settingsBuilder = + BigtableDataSettings settings = defaultSettings .toBuilder() .setProjectId(projectId) .setInstanceId(instanceId) - .setDefaultAppProfileId(); + .setDefaultAppProfileId() + .build(); - patchStubSettings(settingsBuilder.stubSettings()); + ClientContext clientContext = + sharedClientContext + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory(settings.getStubSettings())) + .build(); - return BigtableDataClient.create(settingsBuilder.build()); + return BigtableDataClient.createWithClientContext(settings, clientContext); } /** @@ -176,32 +186,19 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull public BigtableDataClient createForInstance( @Nonnull String projectId, @Nonnull String instanceId, @Nonnull String appProfileId) throws IOException { - BigtableDataSettings.Builder settingsBuilder = + BigtableDataSettings settings = defaultSettings .toBuilder() .setProjectId(projectId) .setInstanceId(instanceId) - .setAppProfileId(appProfileId); - - patchStubSettings(settingsBuilder.stubSettings()); - - return BigtableDataClient.create(settingsBuilder.build()); - } - - // Update stub settings to use shared resources in this factory - private void patchStubSettings(EnhancedBigtableStubSettings.Builder stubSettings) { - stubSettings - // Channel refreshing will be configured in the shared ClientContext. Derivative clients - // won't be able to reconfigure the refreshing logic - .setRefreshingChannel(false) - .setTransportChannelProvider( - FixedTransportChannelProvider.create(sharedClientContext.getTransportChannel())) - .setCredentialsProvider( - FixedCredentialsProvider.create(sharedClientContext.getCredentials())) - .setExecutorProvider(FixedExecutorProvider.create(sharedClientContext.getExecutor())) - .setStreamWatchdogProvider( - FixedWatchdogProvider.create(sharedClientContext.getStreamWatchdog())) - .setHeaderProvider(FixedHeaderProvider.create(sharedClientContext.getHeaders())) - .setClock(sharedClientContext.getClock()); + .setAppProfileId(appProfileId) + .build(); + ClientContext clientContext = + sharedClientContext + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory(settings.getStubSettings())) + .build(); + return BigtableDataClient.createWithClientContext(settings, clientContext); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/MutateRowsException.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/MutateRowsException.java index d1c0eda844..4ae0606ab9 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/MutateRowsException.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/MutateRowsException.java @@ -17,6 +17,7 @@ import com.google.api.core.InternalApi; import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.ErrorDetails; import com.google.api.gax.rpc.StatusCode; import com.google.auto.value.AutoValue; import com.google.bigtable.v2.MutateRowsRequest; @@ -53,16 +54,36 @@ public Object getTransportCode() { * applications. */ @InternalApi - public MutateRowsException( + public static MutateRowsException create( @Nullable Throwable rpcError, @Nonnull List failedMutations, boolean retryable) { - super("Some mutations failed to apply", rpcError, LOCAL_STATUS, retryable); + ErrorDetails errorDetails = null; + if (rpcError instanceof ApiException) { + errorDetails = ((ApiException) rpcError).getErrorDetails(); + } + + return new MutateRowsException(rpcError, failedMutations, retryable, errorDetails); + } + + private MutateRowsException( + @Nullable Throwable rpcError, + @Nonnull List failedMutations, + boolean retryable, + @Nullable ErrorDetails errorDetails) { + super(rpcError, LOCAL_STATUS, retryable, errorDetails); Preconditions.checkNotNull(failedMutations); Preconditions.checkArgument(!failedMutations.isEmpty(), "failedMutations can't be empty"); this.failedMutations = failedMutations; } + // TODO: remove this after we add a ctor in gax to pass in a Throwable, a message and error + // details. + @Override + public String getMessage() { + return "Some mutations failed to apply"; + } + /** * Retrieve all of the failed mutations. This list will contain failures for all of the mutations * that have failed across all of the retry attempts so far. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java index dc5953a95a..78796f217c 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java @@ -159,6 +159,12 @@ public BigtableStub createStub() throws IOException { "Transport not supported: %s", getTransportChannelProvider().getTransportName())); } + /** Returns the default service name. */ + @Override + public String getServiceName() { + return "bigtable"; + } + /** Returns a builder for the default ExecutorProvider for this service. */ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuilder() { return InstantiatingExecutorProvider.newBuilder(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolder.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolder.java index 7d7ca6a029..7a153cfd5f 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolder.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolder.java @@ -55,14 +55,14 @@ Metadata injectCookiesInRequestHeaders(Metadata headers) { * COOKIE_KEY_PREFIX to cookies. Values in trailers will override the value set in initial * metadata for the same keys. */ - void extractCookiesFromMetadata(@Nullable Metadata trailers) { - if (trailers == null) { + void extractCookiesFromMetadata(@Nullable Metadata metadata) { + if (metadata == null) { return; } - for (String key : trailers.keys()) { + for (String key : metadata.keys()) { if (key.startsWith(COOKIE_KEY_PREFIX)) { Metadata.Key metadataKey = Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER); - String value = trailers.get(metadataKey); + String value = metadata.get(metadataKey); cookies.put(metadataKey, value); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 705b3027ed..b43b680e1a 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -28,6 +28,7 @@ import com.google.api.gax.grpc.GrpcCallSettings; import com.google.api.gax.grpc.GrpcRawCallableFactory; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetryingExecutorWithContext; @@ -37,7 +38,9 @@ import com.google.api.gax.rpc.RequestParamsExtractor; import com.google.api.gax.rpc.ServerStreamingCallSettings; import com.google.api.gax.rpc.ServerStreamingCallable; +import com.google.api.gax.rpc.UnaryCallSettings; import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.tracing.ApiTracerFactory; import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.api.gax.tracing.SpanName; import com.google.api.gax.tracing.TracedServerStreamingCallable; @@ -107,6 +110,8 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; +import com.google.cloud.bigtable.gaxx.retrying.RetryInfoRetryAlgorithm; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -147,6 +152,8 @@ public class EnhancedBigtableStub implements AutoCloseable { private final EnhancedBigtableStubSettings settings; private final ClientContext clientContext; + + private final boolean closeClientContext; private final RequestContext requestContext; private final FlowController bulkMutationFlowController; private final DynamicFlowControlStats bulkMutationDynamicFlowControlStats; @@ -169,13 +176,20 @@ public class EnhancedBigtableStub implements AutoCloseable { public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings) throws IOException { - settings = finalizeSettings(settings, Tags.getTagger(), Stats.getStatsRecorder()); - return new EnhancedBigtableStub(settings, ClientContext.create(settings)); + settings = settings.toBuilder().setTracerFactory(createBigtableTracerFactory(settings)).build(); + ClientContext clientContext = createClientContext(settings); + + return new EnhancedBigtableStub(settings, clientContext); + } + + public static EnhancedBigtableStub createWithClientContext( + EnhancedBigtableStubSettings settings, ClientContext clientContext) throws IOException { + + return new EnhancedBigtableStub(settings, clientContext, false); } - public static EnhancedBigtableStubSettings finalizeSettings( - EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats) + public static ClientContext createClientContext(EnhancedBigtableStubSettings settings) throws IOException { EnhancedBigtableStubSettings.Builder builder = settings.toBuilder(); @@ -185,11 +199,14 @@ public static EnhancedBigtableStubSettings finalizeSettings( // workaround JWT audience issues patchCredentials(builder); - // patch cookies interceptor - InstantiatingGrpcChannelProvider.Builder transportProvider = null; - if (builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider) { - transportProvider = - ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder(); + InstantiatingGrpcChannelProvider.Builder transportProvider = + builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider + ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder() + : null; + + if (builder.getEnableRoutingCookie() && transportProvider != null) { + // TODO: this also need to be added to BigtableClientFactory + // patch cookies interceptor transportProvider.setInterceptorProvider(() -> ImmutableList.of(new CookiesInterceptor())); } @@ -216,49 +233,53 @@ public static EnhancedBigtableStubSettings finalizeSettings( builder.setTransportChannelProvider(transportProvider.build()); } + return ClientContext.create(builder.build()); + } + + public static ApiTracerFactory createBigtableTracerFactory( + EnhancedBigtableStubSettings settings) { + return createBigtableTracerFactory(settings, Tags.getTagger(), Stats.getStatsRecorder()); + } + + @VisibleForTesting + public static ApiTracerFactory createBigtableTracerFactory( + EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats) { + String projectId = settings.getProjectId(); + String instanceId = settings.getInstanceId(); + String appProfileId = settings.getAppProfileId(); + ImmutableMap attributes = ImmutableMap.builder() - .put(RpcMeasureConstants.BIGTABLE_PROJECT_ID, TagValue.create(settings.getProjectId())) - .put( - RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(settings.getInstanceId())) - .put( - RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, - TagValue.create(settings.getAppProfileId())) + .put(RpcMeasureConstants.BIGTABLE_PROJECT_ID, TagValue.create(projectId)) + .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(instanceId)) + .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId)) .build(); ImmutableMap builtinAttributes = ImmutableMap.builder() - .put("project_id", settings.getProjectId()) - .put("instance", settings.getInstanceId()) - .put("app_profile", settings.getAppProfileId()) + .put("project_id", projectId) + .put("instance", instanceId) + .put("app_profile", appProfileId) .build(); - // Inject Opencensus instrumentation - builder.setTracerFactory( - new CompositeTracerFactory( - ImmutableList.of( - // Add OpenCensus Tracing - new OpencensusTracerFactory( - ImmutableMap.builder() - // Annotate traces with the same tags as metrics - .put( - RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), - settings.getProjectId()) - .put( - RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), - settings.getInstanceId()) - .put( - RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), - settings.getAppProfileId()) - // Also annotate traces with library versions - .put("gax", GaxGrpcProperties.getGaxGrpcVersion()) - .put("grpc", GaxGrpcProperties.getGrpcVersion()) - .put("gapic", Version.VERSION) - .build()), - // Add OpenCensus Metrics - MetricsTracerFactory.create(tagger, stats, attributes), - BuiltinMetricsTracerFactory.create(builtinAttributes), - // Add user configured tracer - settings.getTracerFactory()))); - return builder.build(); + + return new CompositeTracerFactory( + ImmutableList.of( + // Add OpenCensus Tracing + new OpencensusTracerFactory( + ImmutableMap.builder() + // Annotate traces with the same tags as metrics + .put(RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), projectId) + .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), instanceId) + .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), appProfileId) + // Also annotate traces with library versions + .put("gax", GaxGrpcProperties.getGaxGrpcVersion()) + .put("grpc", GaxGrpcProperties.getGrpcVersion()) + .put("gapic", Version.VERSION) + .build()), + // Add OpenCensus Metrics + MetricsTracerFactory.create(tagger, stats, attributes), + BuiltinMetricsTracerFactory.create(builtinAttributes), + // Add user configured tracer + settings.getTracerFactory())); } private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings) @@ -297,8 +318,16 @@ private static void patchCredentials(EnhancedBigtableStubSettings.Builder settin } public EnhancedBigtableStub(EnhancedBigtableStubSettings settings, ClientContext clientContext) { + this(settings, clientContext, true); + } + + public EnhancedBigtableStub( + EnhancedBigtableStubSettings settings, + ClientContext clientContext, + boolean closeClientContext) { this.settings = settings; this.clientContext = clientContext; + this.closeClientContext = closeClientContext; this.requestContext = RequestContext.create( settings.getProjectId(), settings.getInstanceId(), settings.getAppProfileId()); @@ -371,11 +400,7 @@ public ServerStreamingCallable createReadRowsCallable( new TracedServerStreamingCallable<>( readRowsUserCallable, clientContext.getTracerFactory(), span); - // CookieHolder needs to be injected to the CallOptions outside of retries, otherwise retry - // attempts won't see a CookieHolder. - ServerStreamingCallable withCookie = new CookiesServerStreamingCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -411,9 +436,7 @@ public UnaryCallable createReadRowCallable(RowAdapter new TracedUnaryCallable<>( firstRow, clientContext.getTracerFactory(), getSpanName("ReadRow")); - UnaryCallable withCookie = new CookiesUnaryCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -485,7 +508,7 @@ public Map extract(ReadRowsRequest readRowsRequest) { new ReadRowsRetryCompletedCallable<>(withBigtableTracer); ServerStreamingCallable retrying2 = - Callables.retrying(retrying1, innerSettings, clientContext); + withRetries(retrying1, innerSettings); return new FilterMarkerRowsCallable<>(retrying2, rowAdapter); } @@ -568,7 +591,7 @@ public Map extract( new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable> retryable = - Callables.retrying(withBigtableTracer, settings.sampleRowKeysSettings(), clientContext); + withRetries(withBigtableTracer, settings.sampleRowKeysSettings()); return createUserFacingUnaryCallable( methodName, new SampleRowKeysCallable(retryable, requestContext)); @@ -607,7 +630,7 @@ public Map extract(MutateRowRequest mutateRowRequest) { new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying(withBigtableTracer, settings.mutateRowSettings(), clientContext); + withRetries(withBigtableTracer, settings.mutateRowSettings()); return createUserFacingUnaryCallable( methodName, new MutateRowCallable(retrying, requestContext)); @@ -631,11 +654,17 @@ public Map extract(MutateRowRequest mutateRowRequest) { private UnaryCallable createBulkMutateRowsCallable() { UnaryCallable baseCallable = createMutateRowsBaseCallable(); + UnaryCallable withCookie = baseCallable; + + if (settings.getEnableRoutingCookie()) { + withCookie = new CookiesUnaryCallable<>(baseCallable); + } + UnaryCallable flowControlCallable = null; if (settings.bulkMutateRowsSettings().isLatencyBasedThrottlingEnabled()) { flowControlCallable = new DynamicFlowControlCallable( - baseCallable, + withCookie, bulkMutationFlowController, bulkMutationDynamicFlowControlStats, settings.bulkMutateRowsSettings().getTargetRpcLatencyMs(), @@ -643,7 +672,7 @@ private UnaryCallable createBulkMutateRowsCallable() { } UnaryCallable userFacing = new BulkMutateRowsUserFacingCallable( - flowControlCallable != null ? flowControlCallable : baseCallable, requestContext); + flowControlCallable != null ? flowControlCallable : withCookie, requestContext); SpanName spanName = getSpanName("MutateRows"); @@ -654,9 +683,7 @@ private UnaryCallable createBulkMutateRowsCallable() { new TracedUnaryCallable<>( tracedBatcherUnaryCallable, clientContext.getTracerFactory(), spanName); - UnaryCallable withCookie = new CookiesUnaryCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -760,11 +787,19 @@ public Map extract(MutateRowsRequest mutateRowsRequest) { ServerStreamingCallable withBigtableTracer = new BigtableTracerStreamingCallable<>(convertException); + BasicResultRetryAlgorithm resultRetryAlgorithm; + if (settings.getEnableRetryInfo()) { + resultRetryAlgorithm = new RetryInfoRetryAlgorithm<>(); + } else { + resultRetryAlgorithm = new ApiResultRetryAlgorithm<>(); + } + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>( - new ApiResultRetryAlgorithm(), + resultRetryAlgorithm, new ExponentialRetryAlgorithm( settings.bulkMutateRowsSettings().getRetrySettings(), clientContext.getClock())); + RetryingExecutorWithContext retryingExecutor = new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); @@ -772,7 +807,8 @@ public Map extract(MutateRowsRequest mutateRowsRequest) { clientContext.getDefaultCallContext(), withBigtableTracer, retryingExecutor, - settings.bulkMutateRowsSettings().getRetryableCodes()); + settings.bulkMutateRowsSettings().getRetryableCodes(), + retryAlgorithm); } /** @@ -810,7 +846,7 @@ public Map extract( new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying(withBigtableTracer, settings.checkAndMutateRowSettings(), clientContext); + withRetries(withBigtableTracer, settings.checkAndMutateRowSettings()); return createUserFacingUnaryCallable( methodName, new CheckAndMutateRowCallable(retrying, requestContext)); @@ -851,8 +887,7 @@ public Map extract(ReadModifyWriteRowRequest request) { new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying( - withBigtableTracer, settings.readModifyWriteRowSettings(), clientContext); + withRetries(withBigtableTracer, settings.readModifyWriteRowSettings()); return createUserFacingUnaryCallable( methodName, new ReadModifyWriteRowCallable(retrying, requestContext)); @@ -932,16 +967,13 @@ public Map extract( new BigtableTracerStreamingCallable<>(watched); ServerStreamingCallable retrying = - Callables.retrying(withBigtableTracer, innerSettings, clientContext); + withRetries(withBigtableTracer, innerSettings); SpanName span = getSpanName("GenerateInitialChangeStreamPartitions"); ServerStreamingCallable traced = new TracedServerStreamingCallable<>(retrying, clientContext.getTracerFactory(), span); - ServerStreamingCallable withCookie = - new CookiesServerStreamingCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -1010,7 +1042,7 @@ public Map extract( new BigtableTracerStreamingCallable<>(watched); ServerStreamingCallable readChangeStreamCallable = - Callables.retrying(withBigtableTracer, innerSettings, clientContext); + withRetries(withBigtableTracer, innerSettings); ServerStreamingCallable readChangeStreamUserCallable = @@ -1021,10 +1053,7 @@ public Map extract( new TracedServerStreamingCallable<>( readChangeStreamUserCallable, clientContext.getTracerFactory(), span); - ServerStreamingCallable withCookie = - new CookiesServerStreamingCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -1037,11 +1066,7 @@ private UnaryCallable createUserFacin UnaryCallable traced = new TracedUnaryCallable<>(inner, clientContext.getTracerFactory(), getSpanName(methodName)); - // CookieHolder needs to be injected to the CallOptions outside of retries, otherwise retry - // attempts won't see a CookieHolder. - UnaryCallable withCookie = new CookiesUnaryCallable<>(traced); - - return withCookie.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } private UnaryCallable createPingAndWarmCallable() { @@ -1062,6 +1087,40 @@ public Map extract(PingAndWarmRequest request) { Collections.emptySet()); return pingAndWarm.withDefaultCallContext(clientContext.getDefaultCallContext()); } + + private UnaryCallable withRetries( + UnaryCallable innerCallable, UnaryCallSettings unaryCallSettings) { + UnaryCallable retrying; + if (settings.getEnableRetryInfo()) { + retrying = + com.google.cloud.bigtable.gaxx.retrying.Callables.retrying( + innerCallable, unaryCallSettings, clientContext); + } else { + retrying = Callables.retrying(innerCallable, unaryCallSettings, clientContext); + } + if (settings.getEnableRoutingCookie()) { + return new CookiesUnaryCallable<>(retrying); + } + return retrying; + } + + private ServerStreamingCallable withRetries( + ServerStreamingCallable innerCallable, + ServerStreamingCallSettings serverStreamingCallSettings) { + + ServerStreamingCallable retrying; + if (settings.getEnableRetryInfo()) { + retrying = + com.google.cloud.bigtable.gaxx.retrying.Callables.retrying( + innerCallable, serverStreamingCallSettings, clientContext); + } else { + retrying = Callables.retrying(innerCallable, serverStreamingCallSettings, clientContext); + } + if (settings.getEnableRoutingCookie()) { + return new CookiesServerStreamingCallable<>(retrying); + } + return retrying; + } // // @@ -1130,11 +1189,13 @@ private SpanName getSpanName(String methodName) { @Override public void close() { - for (BackgroundResource backgroundResource : clientContext.getBackgroundResources()) { - try { - backgroundResource.close(); - } catch (Exception e) { - throw new IllegalStateException("Failed to close resource", e); + if (closeClientContext) { + for (BackgroundResource backgroundResource : clientContext.getBackgroundResources()) { + try { + backgroundResource.close(); + } catch (Exception e) { + throw new IllegalStateException("Failed to close resource", e); + } } } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index cffd9c85df..44e4752cd5 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -15,13 +15,13 @@ */ package com.google.cloud.bigtable.data.v2.stub; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.api.gax.batching.BatchingCallSettings; import com.google.api.gax.batching.BatchingSettings; import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.batching.FlowController; import com.google.api.gax.batching.FlowController.LimitExceededBehavior; -import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.grpc.ChannelPoolSettings; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; @@ -32,7 +32,6 @@ import com.google.api.gax.rpc.StubSettings; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; -import com.google.auth.Credentials; import com.google.bigtable.v2.FeatureFlags; import com.google.bigtable.v2.PingAndWarmRequest; import com.google.cloud.bigtable.Version; @@ -211,6 +210,8 @@ public class EnhancedBigtableStubSettings extends StubSettings primedTableIds; private final Map jwtAudienceMapping; + private final boolean enableRoutingCookie; + private final boolean enableRetryInfo; private final ServerStreamingCallSettings readRowsSettings; private final UnaryCallSettings readRowSettings; @@ -252,6 +253,8 @@ private EnhancedBigtableStubSettings(Builder builder) { isRefreshingChannel = builder.isRefreshingChannel; primedTableIds = builder.primedTableIds; jwtAudienceMapping = builder.jwtAudienceMapping; + enableRoutingCookie = builder.enableRoutingCookie; + enableRetryInfo = builder.enableRetryInfo; // Per method settings. readRowsSettings = builder.readRowsSettings.build(); @@ -313,6 +316,24 @@ public Map getJwtAudienceMapping() { return jwtAudienceMapping; } + /** + * Gets if routing cookie is enabled. If true, client will retry a request with extra metadata + * server sent back. + */ + @BetaApi("Routing cookie is not currently stable and may change in the future") + public boolean getEnableRoutingCookie() { + return enableRoutingCookie; + } + + /** + * Gets if RetryInfo is enabled. If true, client bases retry decision and back off time on server + * returned RetryInfo value. Otherwise, client uses {@link RetrySettings}. + */ + @BetaApi("RetryInfo is not currently stable and may change in the future") + public boolean getEnableRetryInfo() { + return enableRetryInfo; + } + /** Returns a builder for the default ChannelProvider for this service. */ public static InstantiatingGrpcChannelProvider.Builder defaultGrpcTransportProviderBuilder() { return BigtableStubSettings.defaultGrpcTransportProviderBuilder() @@ -595,6 +616,8 @@ public static class Builder extends StubSettings.Builder primedTableIds; private Map jwtAudienceMapping; + private boolean enableRoutingCookie; + private boolean enableRetryInfo; private final ServerStreamingCallSettings.Builder readRowsSettings; private final UnaryCallSettings.Builder readRowSettings; @@ -627,6 +650,8 @@ private Builder() { primedTableIds = ImmutableList.of(); jwtAudienceMapping = DEFAULT_JWT_AUDIENCE_MAPPING; setCredentialsProvider(defaultCredentialsProviderBuilder().build()); + this.enableRoutingCookie = true; + this.enableRetryInfo = true; // Defaults provider BigtableStubSettings.Builder baseDefaults = BigtableStubSettings.newBuilder(); @@ -745,6 +770,8 @@ private Builder(EnhancedBigtableStubSettings settings) { isRefreshingChannel = settings.isRefreshingChannel; primedTableIds = settings.primedTableIds; jwtAudienceMapping = settings.jwtAudienceMapping; + enableRoutingCookie = settings.enableRoutingCookie; + enableRetryInfo = settings.enableRetryInfo; // Per method settings. readRowsSettings = settings.readRowsSettings.toBuilder(); @@ -893,6 +920,44 @@ public Map getJwtAudienceMapping() { return jwtAudienceMapping; } + /** + * Sets if routing cookie is enabled. If true, client will retry a request with extra metadata + * server sent back. + */ + @BetaApi("Routing cookie is not currently stable and may change in the future") + public Builder setEnableRoutingCookie(boolean enableRoutingCookie) { + this.enableRoutingCookie = enableRoutingCookie; + return this; + } + + /** + * Gets if routing cookie is enabled. If true, client will retry a request with extra metadata + * server sent back. + */ + @BetaApi("Routing cookie is not currently stable and may change in the future") + public boolean getEnableRoutingCookie() { + return enableRoutingCookie; + } + + /** + * Sets if RetryInfo is enabled. If true, client bases retry decision and back off time on + * server returned RetryInfo value. Otherwise, client uses {@link RetrySettings}. + */ + @BetaApi("RetryInfo is not currently stable and may change in the future") + public Builder setEnableRetryInfo(boolean enableRetryInfo) { + this.enableRetryInfo = enableRetryInfo; + return this; + } + + /** + * Gets if RetryInfo is enabled. If true, client bases retry decision and back off time on + * server returned RetryInfo value. Otherwise, client uses {@link RetrySettings}. + */ + @BetaApi("RetryInfo is not currently stable and may change in the future") + public boolean getEnableRetryInfo() { + return enableRetryInfo; + } + /** Returns the builder for the settings used for calls to readRows. */ public ServerStreamingCallSettings.Builder readRowsSettings() { return readRowsSettings; @@ -956,26 +1021,6 @@ public UnaryCallSettings.Builder pingAndWarmSettings() public EnhancedBigtableStubSettings build() { Preconditions.checkState(projectId != null, "Project id must be set"); Preconditions.checkState(instanceId != null, "Instance id must be set"); - if (isRefreshingChannel) { - Preconditions.checkArgument( - getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider, - "refreshingChannel only works with InstantiatingGrpcChannelProviders"); - InstantiatingGrpcChannelProvider.Builder channelProviderBuilder = - ((InstantiatingGrpcChannelProvider) getTransportChannelProvider()).toBuilder(); - Credentials credentials = null; - if (getCredentialsProvider() != null) { - try { - credentials = getCredentialsProvider().getCredentials(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - // Use shared credentials - this.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); - channelProviderBuilder.setChannelPrimer( - BigtableChannelPrimer.create(credentials, projectId, instanceId, appProfileId)); - this.setTransportChannelProvider(channelProviderBuilder.build()); - } if (this.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled()) { // only set mutate rows feature flag when this feature is enabled @@ -983,6 +1028,9 @@ public EnhancedBigtableStubSettings build() { featureFlags.setMutateRowsRateLimit2(true); } + featureFlags.setRoutingCookie(this.getEnableRoutingCookie()); + featureFlags.setRetryInfo(this.getEnableRetryInfo()); + // Serialize the web64 encode the bigtable feature flags ByteArrayOutputStream boas = new ByteArrayOutputStream(); try { @@ -1019,6 +1067,8 @@ public String toString() { .add("isRefreshingChannel", isRefreshingChannel) .add("primedTableIds", primedTableIds) .add("jwtAudienceMapping", jwtAudienceMapping) + .add("enableRoutingCookie", enableRoutingCookie) + .add("enableRetryInfo", enableRetryInfo) .add("readRowsSettings", readRowsSettings) .add("readRowSettings", readRowSettings) .add("sampleRowKeysSettings", sampleRowKeysSettings) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java index b049219a95..155ea43211 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java @@ -19,7 +19,9 @@ import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.ApiExceptionFactory; @@ -110,6 +112,8 @@ public Object getTransportCode() { @Nullable private List originalIndexes; @Nonnull private final Set retryableCodes; @Nullable private final List permanentFailures; + @Nonnull private final RetryAlgorithm retryAlgorithm; + @Nonnull private TimedAttemptSettings attemptSettings; // Parent controller private RetryingFuture externalFuture; @@ -137,11 +141,14 @@ public List apply(Throwable throwable) { @Nonnull UnaryCallable> innerCallable, @Nonnull MutateRowsRequest originalRequest, @Nonnull ApiCallContext callContext, - @Nonnull Set retryableCodes) { + @Nonnull Set retryableCodes, + @Nonnull RetryAlgorithm retryAlgorithm) { this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable"); this.currentRequest = Preconditions.checkNotNull(originalRequest, "currentRequest"); this.callContext = Preconditions.checkNotNull(callContext, "callContext"); this.retryableCodes = Preconditions.checkNotNull(retryableCodes, "retryableCodes"); + this.retryAlgorithm = retryAlgorithm; + this.attemptSettings = retryAlgorithm.createFirstAttempt(); permanentFailures = Lists.newArrayList(); } @@ -229,13 +236,15 @@ private void handleAttemptError(Throwable rpcError) { Builder builder = lastRequest.toBuilder().clearEntries(); List newOriginalIndexes = Lists.newArrayList(); + attemptSettings = retryAlgorithm.createNextAttempt(null, entryError, null, attemptSettings); + for (int i = 0; i < currentRequest.getEntriesCount(); i++) { int origIndex = getOriginalIndex(i); FailedMutation failedMutation = FailedMutation.create(origIndex, entryError); allFailures.add(failedMutation); - if (!failedMutation.getError().isRetryable()) { + if (!retryAlgorithm.shouldRetry(null, failedMutation.getError(), null, attemptSettings)) { permanentFailures.add(failedMutation); } else { // Schedule the mutation entry for the next RPC by adding it to the request builder and @@ -248,7 +257,7 @@ private void handleAttemptError(Throwable rpcError) { currentRequest = builder.build(); originalIndexes = newOriginalIndexes; - throw new MutateRowsException(rpcError, allFailures.build(), entryError.isRetryable()); + throw MutateRowsException.create(rpcError, allFailures.build(), builder.getEntriesCount() > 0); } /** @@ -256,7 +265,7 @@ private void handleAttemptError(Throwable rpcError) { * transient failures are found, their corresponding mutations are scheduled for the next RPC. The * caller is notified of both new found errors and pre-existing permanent errors in the thrown * {@link MutateRowsException}. If no errors exist, then the attempt future is successfully - * completed. + * completed. We don't currently handle RetryInfo on entry level failures. */ private void handleAttemptSuccess(List responses) { List allFailures = Lists.newArrayList(permanentFailures); @@ -317,7 +326,7 @@ private void handleAttemptSuccess(List responses) { if (!allFailures.isEmpty()) { boolean isRetryable = builder.getEntriesCount() > 0; - throw new MutateRowsException(null, allFailures, isRetryable); + throw MutateRowsException.create(null, allFailures, isRetryable); } } @@ -352,10 +361,10 @@ private static ApiException createSyntheticErrorForRpcFailure(Throwable overallR ApiException requestApiException = (ApiException) overallRequestError; return ApiExceptionFactory.createException( - "Didn't receive a result for this mutation entry", overallRequestError, requestApiException.getStatusCode(), - requestApiException.isRetryable()); + requestApiException.isRetryable(), + requestApiException.getErrorDetails()); } return ApiExceptionFactory.createException( diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable.java index ff0daf78bb..8ad1db258d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable.java @@ -16,6 +16,7 @@ package com.google.cloud.bigtable.data.v2.stub.mutaterows; import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetryingExecutorWithContext; import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.rpc.ApiCallContext; @@ -44,23 +45,26 @@ public class MutateRowsRetryingCallable extends UnaryCallable callable; private final RetryingExecutorWithContext executor; private final ImmutableSet retryCodes; + private final RetryAlgorithm retryAlgorithm; public MutateRowsRetryingCallable( @Nonnull ApiCallContext callContextPrototype, @Nonnull ServerStreamingCallable callable, @Nonnull RetryingExecutorWithContext executor, - @Nonnull Set retryCodes) { + @Nonnull Set retryCodes, + @Nonnull RetryAlgorithm retryAlgorithm) { this.callContextPrototype = Preconditions.checkNotNull(callContextPrototype); this.callable = Preconditions.checkNotNull(callable); this.executor = Preconditions.checkNotNull(executor); this.retryCodes = ImmutableSet.copyOf(retryCodes); + this.retryAlgorithm = retryAlgorithm; } @Override public RetryingFuture futureCall(MutateRowsRequest request, ApiCallContext inputContext) { ApiCallContext context = callContextPrototype.nullToSelf(inputContext); MutateRowsAttemptCallable retryCallable = - new MutateRowsAttemptCallable(callable.all(), request, context, retryCodes); + new MutateRowsAttemptCallable(callable.all(), request, context, retryCodes, retryAlgorithm); RetryingFuture retryingFuture = executor.createFuture(retryCallable, context); retryCallable.setExternalFuture(retryingFuture); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/StateMachine.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/StateMachine.java index 6791679829..64ac3e29e2 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/StateMachine.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/StateMachine.java @@ -15,6 +15,8 @@ */ package com.google.cloud.bigtable.data.v2.stub.readrows; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.InternalException; import com.google.bigtable.v2.ReadRowsResponse.CellChunk; import com.google.cloud.bigtable.data.v2.internal.ByteStringComparator; import com.google.cloud.bigtable.data.v2.models.RowAdapter.RowBuilder; @@ -22,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.EvictingQueue; import com.google.protobuf.ByteString; +import io.grpc.Status; import java.util.List; /** @@ -252,6 +255,21 @@ State handleChunk(CellChunk chunk) { new State() { @Override State handleLastScannedRow(ByteString rowKey) { + if (lastCompleteRowKey != null) { + int cmp = ByteStringComparator.INSTANCE.compare(lastCompleteRowKey, rowKey); + String direction = "increasing"; + if (reversed) { + cmp *= -1; + direction = "decreasing"; + } + + validate( + cmp < 0, + "AWAITING_NEW_ROW: last scanned key must be strictly " + + direction + + ". New last scanned key=" + + rowKey); + } completeRow = adapter.createScanMarkerRow(rowKey); lastCompleteRowKey = rowKey; return AWAITING_ROW_CONSUME; @@ -259,6 +277,10 @@ State handleLastScannedRow(ByteString rowKey) { @Override State handleChunk(CellChunk chunk) { + // Make sure to populate the rowKey before validations so that validation failures include + // the new key + rowKey = chunk.getRowKey(); + validate(!chunk.getResetRow(), "AWAITING_NEW_ROW: can't reset"); validate(!chunk.getRowKey().isEmpty(), "AWAITING_NEW_ROW: rowKey missing"); validate(chunk.hasFamilyName(), "AWAITING_NEW_ROW: family missing"); @@ -468,9 +490,9 @@ private void validate(boolean condition, String message) { } } - static class InvalidInputException extends RuntimeException { + static class InvalidInputException extends InternalException { InvalidInputException(String message) { - super(message); + super(message, null, GrpcStatusCode.of(Status.Code.INTERNAL), false); } } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ApiResultRetryAlgorithm.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ApiResultRetryAlgorithm.java index c7f3d18b62..d71a044235 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ApiResultRetryAlgorithm.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ApiResultRetryAlgorithm.java @@ -16,36 +16,38 @@ package com.google.cloud.bigtable.gaxx.retrying; import com.google.api.core.InternalApi; -import com.google.api.gax.retrying.ResultRetryAlgorithm; -import com.google.api.gax.retrying.TimedAttemptSettings; +import com.google.api.gax.retrying.BasicResultRetryAlgorithm; +import com.google.api.gax.retrying.RetryingContext; import com.google.api.gax.rpc.ApiException; -import com.google.api.gax.rpc.DeadlineExceededException; -import org.threeten.bp.Duration; /** For internal use, public for technical reasons. */ @InternalApi -public class ApiResultRetryAlgorithm implements ResultRetryAlgorithm { - // Duration to sleep on if the error is DEADLINE_EXCEEDED. - public static final Duration DEADLINE_SLEEP_DURATION = Duration.ofMillis(1); +public class ApiResultRetryAlgorithm extends BasicResultRetryAlgorithm { + /** Returns true if previousThrowable is an {@link ApiException} that is retryable. */ @Override - public TimedAttemptSettings createNextAttempt( - Throwable prevThrowable, ResponseT prevResponse, TimedAttemptSettings prevSettings) { - if (prevThrowable != null && prevThrowable instanceof DeadlineExceededException) { - return TimedAttemptSettings.newBuilder() - .setGlobalSettings(prevSettings.getGlobalSettings()) - .setRetryDelay(prevSettings.getRetryDelay()) - .setRpcTimeout(prevSettings.getRpcTimeout()) - .setRandomizedRetryDelay(DEADLINE_SLEEP_DURATION) - .setAttemptCount(prevSettings.getAttemptCount() + 1) - .setFirstAttemptStartTimeNanos(prevSettings.getFirstAttemptStartTimeNanos()) - .build(); - } - return null; + public boolean shouldRetry(Throwable previousThrowable, ResponseT previousResponse) { + return (previousThrowable instanceof ApiException) + && ((ApiException) previousThrowable).isRetryable(); } + /** + * If {@link RetryingContext#getRetryableCodes()} is not null: Returns true if the status code of + * previousThrowable is in the list of retryable code of the {@link RetryingContext}. + * + *

Otherwise it returns the result of {@link #shouldRetry(Throwable, Object)}. + */ @Override - public boolean shouldRetry(Throwable prevThrowable, ResponseT prevResponse) { - return (prevThrowable instanceof ApiException) && ((ApiException) prevThrowable).isRetryable(); + public boolean shouldRetry( + RetryingContext context, Throwable previousThrowable, ResponseT previousResponse) { + if (context.getRetryableCodes() != null) { + // Ignore the isRetryable() value of the throwable if the RetryingContext has a specific list + // of codes that should be retried. + return (previousThrowable instanceof ApiException) + && context + .getRetryableCodes() + .contains(((ApiException) previousThrowable).getStatusCode().getCode()); + } + return shouldRetry(previousThrowable, previousResponse); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/AttemptCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/AttemptCallable.java new file mode 100644 index 0000000000..3599e1e4df --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/AttemptCallable.java @@ -0,0 +1,84 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.NonCancellableFuture; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.common.base.Preconditions; +import java.util.concurrent.Callable; +import org.threeten.bp.Duration; + +// TODO: remove this once ApiResultRetryAlgorithm is added to gax. +/** + * A callable representing an attempt to make an RPC call. This class is used from {@link + * RetryingCallable}. + * + * @param request type + * @param response type + */ +@InternalApi +public class AttemptCallable implements Callable { + private final UnaryCallable callable; + private final RequestT request; + private final ApiCallContext originalCallContext; + + private volatile RetryingFuture externalFuture; + + AttemptCallable( + UnaryCallable callable, RequestT request, ApiCallContext callContext) { + this.callable = Preconditions.checkNotNull(callable); + this.request = Preconditions.checkNotNull(request); + this.originalCallContext = Preconditions.checkNotNull(callContext); + } + + public void setExternalFuture(RetryingFuture externalFuture) { + this.externalFuture = Preconditions.checkNotNull(externalFuture); + } + + @Override + public ResponseT call() { + ApiCallContext callContext = originalCallContext; + + try { + // Set the RPC timeout if the caller did not provide their own. + Duration rpcTimeout = externalFuture.getAttemptSettings().getRpcTimeout(); + if (!rpcTimeout.isZero() && callContext.getTimeout() == null) { + callContext = callContext.withTimeout(rpcTimeout); + } + + externalFuture.setAttemptFuture(new NonCancellableFuture()); + if (externalFuture.isDone()) { + return null; + } + + callContext + .getTracer() + .attemptStarted(request, externalFuture.getAttemptSettings().getOverallAttemptCount()); + + ApiFuture internalFuture = callable.futureCall(request, callContext); + externalFuture.setAttemptFuture(internalFuture); + } catch (Throwable e) { + externalFuture.setAttemptFuture(ApiFutures.immediateFailedFuture(e)); + } + + return null; + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/Callables.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/Callables.java new file mode 100644 index 0000000000..a78e7643b0 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/Callables.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.ExponentialRetryAlgorithm; +import com.google.api.gax.retrying.RetryAlgorithm; +import com.google.api.gax.retrying.ScheduledRetryingExecutor; +import com.google.api.gax.retrying.StreamingRetryAlgorithm; +import com.google.api.gax.rpc.ClientContext; +import com.google.api.gax.rpc.ServerStreamingCallSettings; +import com.google.api.gax.rpc.ServerStreamingCallable; +import com.google.api.gax.rpc.UnaryCallSettings; +import com.google.api.gax.rpc.UnaryCallable; + +// TODO: remove this once ApiResultRetryAlgorithm is added to gax. +/** + * Class with utility methods to create callable objects using provided settings. + * + *

The callable objects wrap a given direct callable with features like retry and exception + * translation. + */ +@InternalApi +public class Callables { + + private Callables() {} + + public static UnaryCallable retrying( + UnaryCallable innerCallable, + UnaryCallSettings callSettings, + ClientContext clientContext) { + + UnaryCallSettings settings = callSettings; + + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new RetryInfoRetryAlgorithm<>(), + new ExponentialRetryAlgorithm(settings.getRetrySettings(), clientContext.getClock())); + ScheduledRetryingExecutor executor = + new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); + + return new RetryingCallable<>(clientContext.getDefaultCallContext(), innerCallable, executor); + } + + public static ServerStreamingCallable retrying( + ServerStreamingCallable innerCallable, + ServerStreamingCallSettings callSettings, + ClientContext clientContext) { + + ServerStreamingCallSettings settings = callSettings; + + StreamingRetryAlgorithm retryAlgorithm = + new StreamingRetryAlgorithm<>( + new RetryInfoRetryAlgorithm<>(), + new ExponentialRetryAlgorithm(settings.getRetrySettings(), clientContext.getClock())); + + ScheduledRetryingExecutor retryingExecutor = + new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); + + return new RetryingServerStreamingCallable<>( + innerCallable, retryingExecutor, settings.getResumptionStrategy()); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryInfoRetryAlgorithm.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryInfoRetryAlgorithm.java new file mode 100644 index 0000000000..085b48bbb5 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryInfoRetryAlgorithm.java @@ -0,0 +1,102 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.BasicResultRetryAlgorithm; +import com.google.api.gax.retrying.RetryingContext; +import com.google.api.gax.retrying.TimedAttemptSettings; +import com.google.api.gax.rpc.ApiException; +import com.google.protobuf.util.Durations; +import com.google.rpc.RetryInfo; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.threeten.bp.Duration; + +// TODO move this algorithm to gax +/** + * This retry algorithm checks the metadata of an exception for additional error details. If the + * metadata has a RetryInfo field, use the retry delay to set the wait time between attempts. + */ +@InternalApi +public class RetryInfoRetryAlgorithm extends BasicResultRetryAlgorithm { + + @Override + public TimedAttemptSettings createNextAttempt( + Throwable prevThrowable, ResponseT prevResponse, TimedAttemptSettings prevSettings) { + Duration retryDelay = extractRetryDelay(prevThrowable); + if (retryDelay != null) { + return prevSettings + .toBuilder() + .setRandomizedRetryDelay(retryDelay) + .setAttemptCount(prevSettings.getAttemptCount() + 1) + .setOverallAttemptCount(prevSettings.getAttemptCount() + 1) + .build(); + } + return null; + } + + /** Returns true if previousThrowable is an {@link ApiException} that is retryable. */ + @Override + public boolean shouldRetry(Throwable previousThrowable, ResponseT previousResponse) { + return shouldRetry(null, previousThrowable, previousResponse); + } + + /** + * If {@link RetryingContext#getRetryableCodes()} is not null: Returns true if the status code of + * previousThrowable is in the list of retryable code of the {@link RetryingContext}. + * + *

Otherwise it returns the result of {@link #shouldRetry(Throwable, Object)}. + */ + @Override + public boolean shouldRetry( + @Nullable RetryingContext context, Throwable previousThrowable, ResponseT previousResponse) { + if (extractRetryDelay(previousThrowable) != null) { + // First check if server wants us to retry + return true; + } + if (context != null && context.getRetryableCodes() != null) { + // Ignore the isRetryable() value of the throwable if the RetryingContext has a specific list + // of codes that should be retried. + return ((previousThrowable instanceof ApiException) + && context + .getRetryableCodes() + .contains(((ApiException) previousThrowable).getStatusCode().getCode())); + } + // Server didn't have retry information and there's no retry context, use the local status + // code config. + return previousThrowable instanceof ApiException + && ((ApiException) previousThrowable).isRetryable(); + } + + @Nullable + static Duration extractRetryDelay(@Nullable Throwable throwable) { + if (throwable == null) { + return null; + } + if (!(throwable instanceof ApiException)) { + return null; + } + ApiException exception = (ApiException) throwable; + if (exception.getErrorDetails() == null) { + return null; + } + if (exception.getErrorDetails().getRetryInfo() == null) { + return null; + } + RetryInfo retryInfo = exception.getErrorDetails().getRetryInfo(); + return Duration.ofMillis(Durations.toMillis(retryInfo.getRetryDelay())); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingCallable.java new file mode 100644 index 0000000000..d78bf08322 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingCallable.java @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.RetryingExecutorWithContext; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.common.base.Preconditions; + +// TODO: remove this once ApiResultRetryAlgorithm is added to gax. +/** + * A UnaryCallable that will keep issuing calls to an inner callable until it succeeds or times out. + */ +@InternalApi +public class RetryingCallable extends UnaryCallable { + private final ApiCallContext callContextPrototype; + private final UnaryCallable callable; + private final RetryingExecutorWithContext executor; + + public RetryingCallable( + ApiCallContext callContextPrototype, + UnaryCallable callable, + RetryingExecutorWithContext executor) { + this.callContextPrototype = (ApiCallContext) Preconditions.checkNotNull(callContextPrototype); + this.callable = (UnaryCallable) Preconditions.checkNotNull(callable); + this.executor = (RetryingExecutorWithContext) Preconditions.checkNotNull(executor); + } + + public RetryingFuture futureCall(RequestT request, ApiCallContext inputContext) { + ApiCallContext context = this.callContextPrototype.nullToSelf(inputContext); + AttemptCallable retryCallable = + new AttemptCallable(this.callable, request, context); + RetryingFuture retryingFuture = + this.executor.createFuture(retryCallable, inputContext); + retryCallable.setExternalFuture(retryingFuture); + retryCallable.call(); + return retryingFuture; + } + + public String toString() { + return String.format("retrying(%s)", this.callable); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingServerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingServerStreamingCallable.java new file mode 100644 index 0000000000..504cf4f2b7 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/RetryingServerStreamingCallable.java @@ -0,0 +1,99 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.core.InternalApi; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.ScheduledRetryingExecutor; +import com.google.api.gax.retrying.ServerStreamingAttemptException; +import com.google.api.gax.retrying.StreamResumptionStrategy; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.ResponseObserver; +import com.google.api.gax.rpc.ServerStreamingCallable; + +// TODO: remove this once ApiResultRetryAlgorithm is added to gax. +/** + * A ServerStreamingCallable that implements resumable retries. + * + *

Wraps a request, a {@link ResponseObserver} and an inner {@link ServerStreamingCallable} and + * coordinates retries between them. When the inner callable throws an error, this class will + * schedule retries using the configured {@link ScheduledRetryingExecutor}. + * + *

Streams can be resumed using a {@link StreamResumptionStrategy}. The {@link + * StreamResumptionStrategy} is notified of incoming responses and is expected to track the progress + * of the stream. Upon receiving an error, the {@link StreamResumptionStrategy} is asked to modify + * the original request to resume the stream. + */ +@InternalApi +public final class RetryingServerStreamingCallable + extends ServerStreamingCallable { + + private final ServerStreamingCallable innerCallable; + private final ScheduledRetryingExecutor executor; + private final StreamResumptionStrategy resumptionStrategyPrototype; + + public RetryingServerStreamingCallable( + ServerStreamingCallable innerCallable, + ScheduledRetryingExecutor executor, + StreamResumptionStrategy resumptionStrategyPrototype) { + this.innerCallable = innerCallable; + this.executor = executor; + this.resumptionStrategyPrototype = resumptionStrategyPrototype; + } + + @Override + public void call( + RequestT request, + final ResponseObserver responseObserver, + ApiCallContext context) { + + ServerStreamingAttemptCallable attemptCallable = + new ServerStreamingAttemptCallable<>( + innerCallable, + resumptionStrategyPrototype.createNew(), + request, + context, + responseObserver); + + RetryingFuture retryingFuture = executor.createFuture(attemptCallable, context); + attemptCallable.setExternalFuture(retryingFuture); + attemptCallable.start(); + + // Bridge the future result back to the external responseObserver + ApiFutures.addCallback( + retryingFuture, + new ApiFutureCallback() { + @Override + public void onFailure(Throwable throwable) { + // Make sure to unwrap the underlying ApiException + if (throwable instanceof ServerStreamingAttemptException) { + throwable = throwable.getCause(); + } + responseObserver.onError(throwable); + } + + @Override + public void onSuccess(Void ignored) { + responseObserver.onComplete(); + } + }, + directExecutor()); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ServerStreamingAttemptCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ServerStreamingAttemptCallable.java new file mode 100644 index 0000000000..793cf2e91c --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/retrying/ServerStreamingAttemptCallable.java @@ -0,0 +1,366 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.gaxx.retrying; + +import com.google.api.core.InternalApi; +import com.google.api.core.SettableApiFuture; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.ServerStreamingAttemptException; +import com.google.api.gax.retrying.StreamResumptionStrategy; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.ResponseObserver; +import com.google.api.gax.rpc.ServerStreamingCallable; +import com.google.api.gax.rpc.StateCheckingResponseObserver; +import com.google.api.gax.rpc.StreamController; +import com.google.common.base.Preconditions; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; + +// TODO: remove this once ApiResultRetryAlgorithm is added to gax. +/** + * A callable that generates Server Streaming attempts. At any one time, it is responsible for at + * most a single outstanding attempt. During an attempt, it proxies all incoming message to the + * outer {@link ResponseObserver} and the {@link StreamResumptionStrategy}. Once the attempt + * completes, the external {@link RetryingFuture} future is notified. If the {@link RetryingFuture} + * decides to retry the attempt, it will invoke {@link #call()}. + * + *

The lifecycle of this class is: + * + *

    + *
  1. The caller instantiates this class. + *
  2. The caller sets the {@link RetryingFuture} via {@link #setExternalFuture(RetryingFuture)}. + * The {@link RetryingFuture} will be responsible for scheduling future attempts. + *
  3. The caller calls {@link #start()}. This notifies the outer {@link ResponseObserver} that + * call is about to start. + *
  4. The outer {@link ResponseObserver} configures inbound flow control via the {@link + * StreamController} that it received in {@link ResponseObserver#onStart(StreamController)}. + *
  5. The attempt call is sent via the inner/upstream {@link ServerStreamingCallable}. + *
  6. A future representing the end state of the inner attempt is passed to the outer {@link + * RetryingFuture}. + *
  7. All messages received from the inner {@link ServerStreamingCallable} are recorded by the + * {@link StreamResumptionStrategy}. + *
  8. All messages received from the inner {@link ServerStreamingCallable} are forwarded to the + * outer {@link ResponseObserver}. + *
  9. Upon attempt completion (either success or failure) are communicated to the outer {@link + * RetryingFuture}. + *
  10. If the {@link RetryingFuture} decides to resume the RPC, it will invoke {@link #call()}, + * which will consult the {@link StreamResumptionStrategy} for the resuming request and + * restart the process at step 5. + *
  11. Once the {@link RetryingFuture} decides to stop the retry loop, it will notify the outer + * {@link ResponseObserver}. + *
+ * + *

This class is meant to be used as middleware between an outer {@link ResponseObserver} and an + * inner {@link ServerStreamingCallable}. As such it follows the general threading model of {@link + * ServerStreamingCallable}s: + * + *

    + *
  • {@code onStart} must be called in the same thread that invoked {@code call()} + *
  • The outer {@link ResponseObserver} can call {@code request()} and {@code cancel()} on this + * class' {@link StreamController} from any thread + *
  • The inner callable will serialize calls to {@code onResponse()}, {@code onError()} and + * {@code onComplete} + *
+ * + *

With this model in mind, this class only needs to synchronize access data that is shared + * between: the outer {@link ResponseObserver} (via this class' {@link StreamController}) and the + * inner {@link ServerStreamingCallable}: pendingRequests, cancellationCause and the current + * innerController. + * + * @param request type + * @param response type + */ +@InternalApi +public final class ServerStreamingAttemptCallable implements Callable { + private final Object lock = new Object(); + + private final ServerStreamingCallable innerCallable; + private final StreamResumptionStrategy resumptionStrategy; + private final RequestT initialRequest; + private ApiCallContext context; + private final ResponseObserver outerObserver; + + // Start state + private boolean autoFlowControl = true; + private boolean isStarted; + + // Outer state + private Throwable cancellationCause; + + private int pendingRequests; + + private RetryingFuture outerRetryingFuture; + + // Internal retry state + private int numAttempts; + + private StreamController innerController; + + private boolean seenSuccessSinceLastError; + private SettableApiFuture innerAttemptFuture; + + public ServerStreamingAttemptCallable( + ServerStreamingCallable innerCallable, + StreamResumptionStrategy resumptionStrategy, + RequestT initialRequest, + ApiCallContext context, + ResponseObserver outerObserver) { + this.innerCallable = innerCallable; + this.resumptionStrategy = resumptionStrategy; + this.initialRequest = initialRequest; + this.context = context; + this.outerObserver = outerObserver; + } + + /** Sets controlling {@link RetryingFuture}. Must be called be before {@link #start()}. */ + void setExternalFuture(RetryingFuture retryingFuture) { + Preconditions.checkState(!isStarted, "Can't change the RetryingFuture once the call has start"); + Preconditions.checkNotNull(retryingFuture, "RetryingFuture can't be null"); + + this.outerRetryingFuture = retryingFuture; + } + + /** + * Starts the initial call. The call is attempted on the caller's thread. Further call attempts + * will be scheduled by the {@link RetryingFuture}. + */ + public void start() { + Preconditions.checkState(!isStarted, "Already started"); + + // Initialize the outer observer + outerObserver.onStart( + new StreamController() { + @Override + public void disableAutoInboundFlowControl() { + Preconditions.checkState( + !isStarted, "Can't disable auto flow control once the stream is started"); + autoFlowControl = false; + } + + @Override + public void request(int count) { + onRequest(count); + } + + @Override + public void cancel() { + onCancel(); + } + }); + + if (autoFlowControl) { + synchronized (lock) { + pendingRequests = Integer.MAX_VALUE; + } + } + isStarted = true; + + // Call the inner callable + call(); + } + + /** + * Sends the actual RPC. The request being sent will first be transformed by the {@link + * StreamResumptionStrategy}. + * + *

This method expects to be called by one thread at a time. Furthermore, it expects that the + * current RPC finished before the next time it's called. + */ + @Override + public Void call() { + Preconditions.checkState(isStarted, "Must be started first"); + + RequestT request = + (++numAttempts == 1) ? initialRequest : resumptionStrategy.getResumeRequest(initialRequest); + + // Should never happen. onAttemptError will check if ResumptionStrategy can create a resume + // request, + // which the RetryingFuture/StreamResumptionStrategy should respect. + Preconditions.checkState(request != null, "ResumptionStrategy returned a null request."); + + innerAttemptFuture = SettableApiFuture.create(); + seenSuccessSinceLastError = false; + + ApiCallContext attemptContext = context; + + if (!outerRetryingFuture.getAttemptSettings().getRpcTimeout().isZero() + && attemptContext.getTimeout() == null) { + attemptContext = + attemptContext.withTimeout(outerRetryingFuture.getAttemptSettings().getRpcTimeout()); + } + + attemptContext + .getTracer() + .attemptStarted(request, outerRetryingFuture.getAttemptSettings().getOverallAttemptCount()); + + innerCallable.call( + request, + new StateCheckingResponseObserver() { + @Override + public void onStartImpl(StreamController controller) { + onAttemptStart(controller); + } + + @Override + public void onResponseImpl(ResponseT response) { + onAttemptResponse(response); + } + + @Override + public void onErrorImpl(Throwable t) { + onAttemptError(t); + } + + @Override + public void onCompleteImpl() { + onAttemptComplete(); + } + }, + attemptContext); + + outerRetryingFuture.setAttemptFuture(innerAttemptFuture); + + return null; + } + + /** + * Called by the inner {@link ServerStreamingCallable} when the call is about to start. This will + * transfer unfinished state from the previous attempt. + * + * @see ResponseObserver#onStart(StreamController) + */ + private void onAttemptStart(StreamController controller) { + if (!autoFlowControl) { + controller.disableAutoInboundFlowControl(); + } + + Throwable localCancellationCause; + int numToRequest = 0; + + synchronized (lock) { + innerController = controller; + + localCancellationCause = this.cancellationCause; + + if (!autoFlowControl) { + numToRequest = pendingRequests; + } + } + + if (localCancellationCause != null) { + controller.cancel(); + } else if (numToRequest > 0) { + controller.request(numToRequest); + } + } + + /** + * Called when the outer {@link ResponseObserver} wants to prematurely cancel the stream. + * + * @see StreamController#cancel() + */ + private void onCancel() { + StreamController localInnerController; + + synchronized (lock) { + if (cancellationCause != null) { + return; + } + // NOTE: BasicRetryingFuture will replace j.u.c.CancellationExceptions with it's own, + // which will not have the current stacktrace, so a special wrapper has be used here. + cancellationCause = + new ServerStreamingAttemptException( + new CancellationException("User cancelled stream"), + resumptionStrategy.canResume(), + seenSuccessSinceLastError); + localInnerController = innerController; + } + + if (localInnerController != null) { + localInnerController.cancel(); + } + } + + /** + * Called when the outer {@link ResponseObserver} is ready for more data. + * + * @see StreamController#request(int) + */ + private void onRequest(int count) { + Preconditions.checkState(!autoFlowControl, "Automatic flow control is enabled"); + Preconditions.checkArgument(count > 0, "Count must be > 0"); + + final StreamController localInnerController; + + synchronized (lock) { + int maxInc = Integer.MAX_VALUE - pendingRequests; + count = Math.min(maxInc, count); + + pendingRequests += count; + localInnerController = this.innerController; + } + + // Note: there is a race condition here where the count might go to the previous attempt's + // StreamController after it failed. But it doesn't matter, because the controller will just + // ignore it and the current controller will pick it up onStart. + if (localInnerController != null) { + localInnerController.request(count); + } + } + + /** Called when the inner callable has responses to deliver. */ + private void onAttemptResponse(ResponseT message) { + if (!autoFlowControl) { + synchronized (lock) { + pendingRequests--; + } + } + // Update local state to allow for future resume. + seenSuccessSinceLastError = true; + message = resumptionStrategy.processResponse(message); + // Notify the outer observer. + outerObserver.onResponse(message); + } + + /** + * Called when the current RPC fails. The error will be bubbled up to the outer {@link + * RetryingFuture} via the {@link #innerAttemptFuture}. + */ + private void onAttemptError(Throwable throwable) { + Throwable localCancellationCause; + synchronized (lock) { + localCancellationCause = cancellationCause; + } + + if (localCancellationCause != null) { + // Take special care to preserve the cancellation's stack trace. + innerAttemptFuture.setException(localCancellationCause); + } else { + // Wrap the original exception and provide more context for StreamingRetryAlgorithm. + innerAttemptFuture.setException( + new ServerStreamingAttemptException( + throwable, resumptionStrategy.canResume(), seenSuccessSinceLastError)); + } + } + + /** + * Called when the current RPC successfully completes. Notifies the outer {@link RetryingFuture} + * via {@link #innerAttemptFuture}. + */ + private void onAttemptComplete() { + innerAttemptFuture.set(null); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/functional/ReadRowsTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/functional/ReadRowsTest.java new file mode 100644 index 0000000000..1a74eb5aa8 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/functional/ReadRowsTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.functional; + +import com.google.api.gax.rpc.InternalException; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.FakeServiceBuilder; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.StringValue; +import io.grpc.Server; +import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ReadRowsTest { + private FakeService service; + private Server server; + + @Before + public void setUp() throws Exception { + service = new FakeService(); + server = FakeServiceBuilder.create(service).start(); + } + + @After + public void tearDown() throws Exception { + server.shutdown(); + } + + @Test + public void rowMergingErrorsUseInternalStatus() throws Exception { + BigtableDataSettings settings = + BigtableDataSettings.newBuilderForEmulator(server.getPort()) + .setProjectId("fake-project") + .setInstanceId("fake-instance") + .build(); + + service.readRowsResponses.add( + ReadRowsResponse.newBuilder() + .addChunks( + ReadRowsResponse.CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("z")) + .setFamilyName(StringValue.newBuilder().setValue("f")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("q")).build()) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("v")) + .setCommitRow(true)) + .addChunks( + ReadRowsResponse.CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("a")) + .setFamilyName(StringValue.newBuilder().setValue("f")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("q")).build()) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("v")) + .setCommitRow(true)) + .build()); + + try (BigtableDataClient client = BigtableDataClient.create(settings)) { + Assert.assertThrows( + InternalException.class, + () -> { + for (Row ignored : client.readRows(Query.create("fake-table"))) {} + }); + } + } + + static class FakeService extends BigtableGrpc.BigtableImplBase { + private List readRowsResponses = + Collections.synchronizedList(new ArrayList<>()); + + @Override + public void readRows( + ReadRowsRequest request, StreamObserver responseObserver) { + for (ReadRowsResponse r : readRowsResponses) { + responseObserver.onNext(r); + } + responseObserver.onCompleted(); + } + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 7b58e14f7c..6578dbad24 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -18,13 +18,17 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.TruthJUnit.assume; +import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.core.SettableApiFuture; +import com.google.api.gax.batching.Batcher; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.StreamController; import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.models.BulkMutation; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange; @@ -38,8 +42,17 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ManagedChannelBuilder; +import io.grpc.MethodDescriptor; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -48,6 +61,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -308,6 +322,93 @@ public void reversed() { .inOrder(); } + @Test + @Ignore("Test taking too long to run, ignore for now") + public void reversedWithForcedResumption() throws IOException, InterruptedException { + assume() + .withMessage("reverse scans are not supported in the emulator") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + + BigtableDataClient client = testEnvRule.env().getDataClient(); + String tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); + String uniqueKey = prefix + "-rev-queries2"; + + // Add enough rows that ensures resumption logic is forced + Random random; + List expectedResults; + try (Batcher batcher = client.newBulkMutationBatcher(tableId)) { + + byte[] valueBytes = new byte[1024]; + random = new Random(); + + expectedResults = new ArrayList<>(); + + for (int i = 0; i < 2 * 1024; i++) { + ByteString key = ByteString.copyFromUtf8(String.format("%s-%05d", uniqueKey, i)); + ByteString qualifier = ByteString.copyFromUtf8("q"); + long timestamp = System.currentTimeMillis() * 1000; + random.nextBytes(valueBytes); + ByteString value = ByteString.copyFrom(valueBytes); + + batcher.add(RowMutationEntry.create(key).setCell(familyId, qualifier, timestamp, value)); + expectedResults.add( + Row.create( + key, + ImmutableList.of( + RowCell.create(familyId, qualifier, timestamp, ImmutableList.of(), value)))); + } + } + Collections.reverse(expectedResults); + + BigtableDataSettings.Builder settingsBuilder = + testEnvRule.env().getDataClientSettings().toBuilder(); + + settingsBuilder.stubSettings().readRowsSettings().retrySettings().setMaxAttempts(100); + + InstantiatingGrpcChannelProvider.Builder transport = + ((InstantiatingGrpcChannelProvider) + settingsBuilder.stubSettings().getTransportChannelProvider()) + .toBuilder(); + ApiFunction oldConfigurator = + transport.getChannelConfigurator(); + + // Randomly camp the deadline to force a timeout to force a retry + transport.setChannelConfigurator( + (ManagedChannelBuilder c) -> { + if (oldConfigurator != null) { + c = oldConfigurator.apply(c); + } + return c.intercept( + new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + if (method.getBareMethodName().equals("ReadRows")) { + callOptions = + callOptions.withDeadlineAfter(random.nextInt(200), TimeUnit.MILLISECONDS); + } + + return next.newCall(method, callOptions); + } + }); + }); + settingsBuilder.stubSettings().setTransportChannelProvider(transport.build()); + + try (BigtableDataClient patchedClient = BigtableDataClient.create(settingsBuilder.build())) { + for (int i = 0; i < 10; i++) { + List actualResults = new ArrayList<>(); + for (Row row : + patchedClient.readRows(Query.create(tableId).prefix(uniqueKey).reversed(true))) { + actualResults.add(row); + Thread.sleep(1); + } + assertThat(actualResults).containsExactlyElementsIn(expectedResults).inOrder(); + } + } + } + @Test public void readSingleNonexistentAsyncCallback() throws Exception { ApiFuture future = diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolderTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolderTest.java index 5dac053523..edf0b87fd9 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolderTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/CookiesHolderTest.java @@ -37,10 +37,13 @@ import com.google.bigtable.v2.ReadRowsResponse; import com.google.bigtable.v2.SampleRowKeysRequest; import com.google.bigtable.v2.SampleRowKeysResponse; +import com.google.bigtable.v2.StreamContinuationToken; import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataClientFactory; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.FakeServiceBuilder; import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord; import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; import com.google.cloud.bigtable.data.v2.models.Mutation; import com.google.cloud.bigtable.data.v2.models.Query; @@ -58,6 +61,7 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; +import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -77,12 +81,18 @@ public class CookiesHolderTest { Metadata.Key.of("x-goog-cbt-cookie-routing", Metadata.ASCII_STRING_MARSHALLER); private static final Metadata.Key ROUTING_COOKIE_2 = Metadata.Key.of("x-goog-cbt-cookie-random", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key ROUTING_COOKIE_HEADER = + Metadata.Key.of("x-goog-cbt-cookie-header", Metadata.ASCII_STRING_MARSHALLER); private static final Metadata.Key BAD_KEY = Metadata.Key.of("x-goog-cbt-not-cookie", Metadata.ASCII_STRING_MARSHALLER); + + private static final String testHeaderCookie = "header-cookie"; private static final String testCookie = "test-routing-cookie"; + private static final String routingCookie1Header = "should-be-overridden"; private Server server; private final FakeService fakeService = new FakeService(); + private BigtableDataSettings.Builder settings; private BigtableDataClient client; private final List serverMetadata = new ArrayList<>(); @@ -101,7 +111,16 @@ public ServerCall.Listener interceptCall( if (metadata.containsKey(ROUTING_COOKIE_1)) { methods.add(serverCall.getMethodDescriptor().getBareMethodName()); } - return serverCallHandler.startCall(serverCall, metadata); + return serverCallHandler.startCall( + new ForwardingServerCall.SimpleForwardingServerCall(serverCall) { + @Override + public void sendHeaders(Metadata responseHeaders) { + responseHeaders.put(ROUTING_COOKIE_HEADER, testHeaderCookie); + responseHeaders.put(ROUTING_COOKIE_1, routingCookie1Header); + super.sendHeaders(responseHeaders); + } + }, + metadata); } }; @@ -138,6 +157,8 @@ public ServerCall.Listener interceptCall( .build()) .setRetryableCodes(StatusCode.Code.UNAVAILABLE); + this.settings = settings; + client = BigtableDataClient.create(settings.build()); } @@ -161,7 +182,13 @@ public void testReadRows() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); assertThat(lastMetadata) - .containsAtLeast(ROUTING_COOKIE_1.name(), "readRows", ROUTING_COOKIE_2.name(), testCookie); + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "readRows", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); serverMetadata.clear(); @@ -177,7 +204,13 @@ public void testReadRow() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); assertThat(lastMetadata) - .containsAtLeast(ROUTING_COOKIE_1.name(), "readRows", ROUTING_COOKIE_2.name(), testCookie); + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "readRows", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); serverMetadata.clear(); @@ -196,7 +229,12 @@ public void testMutateRows() { assertThat(lastMetadata) .containsAtLeast( - ROUTING_COOKIE_1.name(), "mutateRows", ROUTING_COOKIE_2.name(), testCookie); + ROUTING_COOKIE_1.name(), + "mutateRows", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); serverMetadata.clear(); @@ -212,7 +250,13 @@ public void testMutateRow() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); assertThat(lastMetadata) - .containsAtLeast(ROUTING_COOKIE_1.name(), "mutateRow", ROUTING_COOKIE_2.name(), testCookie); + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "mutateRow", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); serverMetadata.clear(); @@ -230,7 +274,58 @@ public void testSampleRowKeys() { assertThat(lastMetadata) .containsAtLeast( - ROUTING_COOKIE_1.name(), "sampleRowKeys", ROUTING_COOKIE_2.name(), testCookie); + ROUTING_COOKIE_1.name(), + "sampleRowKeys", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); + assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); + + serverMetadata.clear(); + } + + @Test + public void testReadChangeStream() { + for (ChangeStreamRecord record : + client.readChangeStream(ReadChangeStreamQuery.create("table"))) {} + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + + assertThat(lastMetadata) + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "readChangeStream", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); + assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); + + serverMetadata.clear(); + } + + @Test + public void testGenerateInitialChangeStreamPartition() { + client.generateInitialChangeStreamPartitions("table").iterator().hasNext(); + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + + // generateInitialChangeStreamPartition uses SimpleStreamResumptionStrategy which means + // it can't resume from the middle of the stream. So we are not able to send a header + // for error responses. + assertThat(lastMetadata) + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "generateInitialChangeStreamPartitions", + ROUTING_COOKIE_2.name(), + testCookie); assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); serverMetadata.clear(); @@ -247,7 +342,9 @@ public void testNoCookieSucceedReadRows() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); - assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_1.name(), ROUTING_COOKIE_2.name()); + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name()); + // Should contain initial metadata + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); serverMetadata.clear(); } @@ -263,8 +360,8 @@ public void testNoCookieSucceedReadRow() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); - assertThat(lastMetadata) - .doesNotContainKeys(ROUTING_COOKIE_1.name(), ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); serverMetadata.clear(); } @@ -282,8 +379,8 @@ public void testNoCookieSucceedMutateRows() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); - assertThat(lastMetadata) - .doesNotContainKeys(ROUTING_COOKIE_1.name(), ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); serverMetadata.clear(); } @@ -299,8 +396,8 @@ public void testNoCookieSucceedMutateRow() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); - assertThat(lastMetadata) - .doesNotContainKeys(ROUTING_COOKIE_1.name(), ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); serverMetadata.clear(); } @@ -316,6 +413,43 @@ public void testNoCookieSucceedSampleRowKeys() { Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); + + serverMetadata.clear(); + } + + @Test + public void testNoCookieSucceedReadChangeStream() { + fakeService.returnCookie = false; + + for (ChangeStreamRecord record : + client.readChangeStream(ReadChangeStreamQuery.create("table"))) {} + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + + assertThat(lastMetadata).doesNotContainKeys(ROUTING_COOKIE_2.name(), BAD_KEY.name()); + assertThat(lastMetadata).containsAtLeast(ROUTING_COOKIE_1.name(), routingCookie1Header); + + serverMetadata.clear(); + + serverMetadata.clear(); + } + + @Test + public void testNoCookieSucceedGenerateInitialChangeStreamParition() { + fakeService.returnCookie = false; + + client.generateInitialChangeStreamPartitions("table").iterator().hasNext(); + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + assertThat(lastMetadata) .doesNotContainKeys(ROUTING_COOKIE_1.name(), ROUTING_COOKIE_2.name(), BAD_KEY.name()); @@ -379,7 +513,7 @@ public void sendHeaders(Metadata headers) { } @Test - public void testAllMethodsAreCalled() throws InterruptedException { + public void testAllMethodsAreCalled() { // This test ensures that all methods respect the retry cookie except for the ones that are // explicitly added to the methods list. It requires that any newly method is exercised in this // test. This is enforced by introspecting grpc method descriptors. @@ -409,7 +543,8 @@ public void testAllMethodsAreCalled() throws InterruptedException { client.generateInitialChangeStreamPartitions("fake-table").iterator().hasNext(); fakeService.count.set(0); - client.readChangeStream(ReadChangeStreamQuery.create("fake-table")).iterator().hasNext(); + for (ChangeStreamRecord record : + client.readChangeStream(ReadChangeStreamQuery.create("fake-table"))) {} Set expected = BigtableGrpc.getServiceDescriptor().getMethods().stream() @@ -422,6 +557,103 @@ public void testAllMethodsAreCalled() throws InterruptedException { assertThat(methods).containsExactlyElementsIn(expected); } + @Test + public void testCookieSetWithBigtableClientFactory() throws Exception { + try (BigtableDataClientFactory factory = BigtableDataClientFactory.create(settings.build())) { + BigtableDataClient client1 = factory.createDefault(); + BigtableDataClient client2 = factory.createForAppProfile("app-profile"); + + client1.readRows(Query.create("fake-table")).iterator().hasNext(); + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + Metadata lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + + assertThat(lastMetadata) + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "readRows", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); + assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); + + // Reset fake service status + fakeService.count.set(0); + serverMetadata.clear(); + + client2.readRows(Query.create("fake-table")).iterator().hasNext(); + + assertThat(fakeService.count.get()).isGreaterThan(1); + assertThat(serverMetadata).hasSize(fakeService.count.get()); + + lastMetadata = serverMetadata.get(fakeService.count.get() - 1); + + assertThat(lastMetadata) + .containsAtLeast( + ROUTING_COOKIE_1.name(), + "readRows", + ROUTING_COOKIE_2.name(), + testCookie, + ROUTING_COOKIE_HEADER.name(), + testHeaderCookie); + assertThat(lastMetadata).doesNotContainKeys(BAD_KEY.name()); + + serverMetadata.clear(); + } + } + + @Test + public void testDisableRoutingCookie() throws IOException { + // This test disables routing cookie in the client settings and ensures that none of the routing + // cookie + // is added. + settings.stubSettings().setEnableRoutingCookie(false); + try (BigtableDataClient client = BigtableDataClient.create(settings.build())) { + client.readRows(Query.create("fake-table")).iterator().hasNext(); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.mutateRow(RowMutation.create("fake-table", "key").setCell("cf", "q", "v")); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("key").setCell("cf", "q", "v"))); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.sampleRowKeys("fake-table"); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.checkAndMutateRow( + ConditionalRowMutation.create("fake-table", "key") + .then(Mutation.create().setCell("cf", "q", "v"))); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.readModifyWriteRow( + ReadModifyWriteRow.create("fake-table", "key").append("cf", "q", "v")); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + client.generateInitialChangeStreamPartitions("fake-table").iterator().hasNext(); + assertThat(fakeService.count.get()).isEqualTo(2); + fakeService.count.set(0); + + for (ChangeStreamRecord record : + client.readChangeStream(ReadChangeStreamQuery.create("fake-table"))) {} + + assertThat(fakeService.count.get()).isEqualTo(2); + + assertThat(methods).isEmpty(); + } + } + static class FakeService extends BigtableGrpc.BigtableImplBase { private boolean returnCookie = true; @@ -448,6 +680,7 @@ public void mutateRow( if (count.getAndIncrement() < 1) { Metadata trailers = new Metadata(); maybePopulateCookie(trailers, "mutateRow"); + responseObserver.onNext(MutateRowResponse.getDefaultInstance()); StatusRuntimeException exception = new StatusRuntimeException(Status.UNAVAILABLE, trailers); responseObserver.onError(exception); return; @@ -462,6 +695,7 @@ public void mutateRows( if (count.getAndIncrement() < 1) { Metadata trailers = new Metadata(); maybePopulateCookie(trailers, "mutateRows"); + responseObserver.onNext(MutateRowsResponse.getDefaultInstance()); StatusRuntimeException exception = new StatusRuntimeException(Status.UNAVAILABLE, trailers); responseObserver.onError(exception); return; @@ -479,6 +713,7 @@ public void sampleRowKeys( if (count.getAndIncrement() < 1) { Metadata trailers = new Metadata(); maybePopulateCookie(trailers, "sampleRowKeys"); + responseObserver.onNext(SampleRowKeysResponse.getDefaultInstance()); StatusRuntimeException exception = new StatusRuntimeException(Status.UNAVAILABLE, trailers); responseObserver.onError(exception); return; @@ -524,6 +759,14 @@ public void readChangeStream( if (count.getAndIncrement() < 1) { Metadata trailers = new Metadata(); maybePopulateCookie(trailers, "readChangeStream"); + responseObserver.onNext( + ReadChangeStreamResponse.newBuilder() + .setHeartbeat( + ReadChangeStreamResponse.Heartbeat.newBuilder() + .setContinuationToken( + StreamContinuationToken.newBuilder().setToken("a").build()) + .build()) + .build()); StatusRuntimeException exception = new StatusRuntimeException(Status.UNAVAILABLE, trailers); responseObserver.onError(exception); return; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettingsTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettingsTest.java index fbd6442e0c..a57d42f6f1 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettingsTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettingsTest.java @@ -20,7 +20,6 @@ import com.google.api.gax.batching.BatchingSettings; import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ServerStreamingCallSettings; @@ -77,6 +76,8 @@ public void settingsAreNotLostTest() { CredentialsProvider credentialsProvider = Mockito.mock(CredentialsProvider.class); WatchdogProvider watchdogProvider = Mockito.mock(WatchdogProvider.class); Duration watchdogInterval = Duration.ofSeconds(12); + boolean enableRoutingCookie = false; + boolean enableRetryInfo = false; EnhancedBigtableStubSettings.Builder builder = EnhancedBigtableStubSettings.newBuilder() @@ -87,7 +88,9 @@ public void settingsAreNotLostTest() { .setEndpoint(endpoint) .setCredentialsProvider(credentialsProvider) .setStreamWatchdogProvider(watchdogProvider) - .setStreamWatchdogCheckInterval(watchdogInterval); + .setStreamWatchdogCheckInterval(watchdogInterval) + .setEnableRoutingCookie(enableRoutingCookie) + .setEnableRetryInfo(enableRetryInfo); verifyBuilder( builder, @@ -98,7 +101,9 @@ public void settingsAreNotLostTest() { endpoint, credentialsProvider, watchdogProvider, - watchdogInterval); + watchdogInterval, + enableRoutingCookie, + enableRetryInfo); verifySettings( builder.build(), projectId, @@ -108,7 +113,9 @@ public void settingsAreNotLostTest() { endpoint, credentialsProvider, watchdogProvider, - watchdogInterval); + watchdogInterval, + enableRoutingCookie, + enableRetryInfo); verifyBuilder( builder.build().toBuilder(), projectId, @@ -118,7 +125,9 @@ public void settingsAreNotLostTest() { endpoint, credentialsProvider, watchdogProvider, - watchdogInterval); + watchdogInterval, + enableRoutingCookie, + enableRetryInfo); } private void verifyBuilder( @@ -130,7 +139,9 @@ private void verifyBuilder( String endpoint, CredentialsProvider credentialsProvider, WatchdogProvider watchdogProvider, - Duration watchdogInterval) { + Duration watchdogInterval, + boolean enableRoutingCookie, + boolean enableRetryInfo) { assertThat(builder.getProjectId()).isEqualTo(projectId); assertThat(builder.getInstanceId()).isEqualTo(instanceId); assertThat(builder.getAppProfileId()).isEqualTo(appProfileId); @@ -139,6 +150,8 @@ private void verifyBuilder( assertThat(builder.getCredentialsProvider()).isEqualTo(credentialsProvider); assertThat(builder.getStreamWatchdogProvider()).isSameInstanceAs(watchdogProvider); assertThat(builder.getStreamWatchdogCheckInterval()).isEqualTo(watchdogInterval); + assertThat(builder.getEnableRoutingCookie()).isEqualTo(enableRoutingCookie); + assertThat(builder.getEnableRetryInfo()).isEqualTo(enableRetryInfo); } private void verifySettings( @@ -150,7 +163,9 @@ private void verifySettings( String endpoint, CredentialsProvider credentialsProvider, WatchdogProvider watchdogProvider, - Duration watchdogInterval) { + Duration watchdogInterval, + boolean enableRoutingCookie, + boolean enableRetryInfo) { assertThat(settings.getProjectId()).isEqualTo(projectId); assertThat(settings.getInstanceId()).isEqualTo(instanceId); assertThat(settings.getAppProfileId()).isEqualTo(appProfileId); @@ -159,6 +174,8 @@ private void verifySettings( assertThat(settings.getCredentialsProvider()).isEqualTo(credentialsProvider); assertThat(settings.getStreamWatchdogProvider()).isSameInstanceAs(watchdogProvider); assertThat(settings.getStreamWatchdogCheckInterval()).isEqualTo(watchdogInterval); + assertThat(settings.getEnableRoutingCookie()).isEqualTo(enableRoutingCookie); + assertThat(settings.getEnableRetryInfo()).isEqualTo(enableRetryInfo); } @Test @@ -781,6 +798,72 @@ public void isRefreshingChannelFalseValueTest() { assertThat(builder.build().toBuilder().isRefreshingChannel()).isFalse(); } + @Test + public void routingCookieIsEnabled() throws IOException { + String dummyProjectId = "my-project"; + String dummyInstanceId = "my-instance"; + CredentialsProvider credentialsProvider = Mockito.mock(CredentialsProvider.class); + Mockito.when(credentialsProvider.getCredentials()).thenReturn(new FakeCredentials()); + EnhancedBigtableStubSettings.Builder builder = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(dummyProjectId) + .setInstanceId(dummyInstanceId) + .setCredentialsProvider(credentialsProvider); + + assertThat(builder.getEnableRoutingCookie()).isTrue(); + assertThat(builder.build().getEnableRoutingCookie()).isTrue(); + assertThat(builder.build().toBuilder().getEnableRoutingCookie()).isTrue(); + } + + public void enableRetryInfoDefaultValueTest() throws IOException { + String dummyProjectId = "my-project"; + String dummyInstanceId = "my-instance"; + CredentialsProvider credentialsProvider = Mockito.mock(CredentialsProvider.class); + Mockito.when(credentialsProvider.getCredentials()).thenReturn(new FakeCredentials()); + EnhancedBigtableStubSettings.Builder builder = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(dummyProjectId) + .setInstanceId(dummyInstanceId) + .setCredentialsProvider(credentialsProvider); + assertThat(builder.getEnableRetryInfo()).isTrue(); + assertThat(builder.build().getEnableRetryInfo()).isTrue(); + assertThat(builder.build().toBuilder().getEnableRetryInfo()).isTrue(); + } + + @Test + public void routingCookieFalseValueSet() throws IOException { + String dummyProjectId = "my-project"; + String dummyInstanceId = "my-instance"; + CredentialsProvider credentialsProvider = Mockito.mock(CredentialsProvider.class); + Mockito.when(credentialsProvider.getCredentials()).thenReturn(new FakeCredentials()); + EnhancedBigtableStubSettings.Builder builder = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(dummyProjectId) + .setInstanceId(dummyInstanceId) + .setEnableRoutingCookie(false) + .setCredentialsProvider(credentialsProvider); + assertThat(builder.getEnableRoutingCookie()).isFalse(); + assertThat(builder.build().getEnableRoutingCookie()).isFalse(); + assertThat(builder.build().toBuilder().getEnableRoutingCookie()).isFalse(); + } + + @Test + public void enableRetryInfoFalseValueTest() throws IOException { + String dummyProjectId = "my-project"; + String dummyInstanceId = "my-instance"; + CredentialsProvider credentialsProvider = Mockito.mock(CredentialsProvider.class); + Mockito.when(credentialsProvider.getCredentials()).thenReturn(new FakeCredentials()); + EnhancedBigtableStubSettings.Builder builder = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(dummyProjectId) + .setInstanceId(dummyInstanceId) + .setEnableRetryInfo(false) + .setCredentialsProvider(credentialsProvider); + assertThat(builder.getEnableRetryInfo()).isFalse(); + assertThat(builder.build().getEnableRetryInfo()).isFalse(); + assertThat(builder.build().toBuilder().getEnableRetryInfo()).isFalse(); + } + static final String[] SETTINGS_LIST = { "projectId", "instanceId", @@ -788,6 +871,8 @@ public void isRefreshingChannelFalseValueTest() { "isRefreshingChannel", "primedTableIds", "jwtAudienceMapping", + "enableRoutingCookie", + "enableRetryInfo", "readRowsSettings", "readRowSettings", "sampleRowKeysSettings", @@ -872,16 +957,6 @@ public void refreshingChannelSetFixedCredentialProvider() throws Exception { .setRefreshingChannel(true) .setCredentialsProvider(credentialsProvider); assertThat(builder.isRefreshingChannel()).isTrue(); - // Verify that isRefreshing setting is not lost and stubSettings will always return the same - // credential - EnhancedBigtableStubSettings stubSettings = builder.build(); - assertThat(stubSettings.isRefreshingChannel()).isTrue(); - assertThat(stubSettings.getCredentialsProvider()).isInstanceOf(FixedCredentialsProvider.class); - assertThat(stubSettings.getCredentialsProvider().getCredentials()) - .isEqualTo(expectedCredentials); - assertThat(stubSettings.toBuilder().isRefreshingChannel()).isTrue(); - assertThat(stubSettings.toBuilder().getCredentialsProvider().getCredentials()) - .isEqualTo(expectedCredentials); } private static class FakeCredentials extends Credentials { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java index e36eb1a8a9..eacf145bcb 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java @@ -549,6 +549,7 @@ public void testBulkMutationFlowControlFeatureFlagIsNotSet() throws Exception { FeatureFlags featureFlags = FeatureFlags.parseFrom(decodedFlags); assertThat(featureFlags.getMutateRowsRateLimit()).isFalse(); assertThat(featureFlags.getMutateRowsRateLimit2()).isFalse(); + stub.close(); } @Test diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/RetryInfoTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/RetryInfoTest.java new file mode 100644 index 0000000000..ba61ee5350 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/RetryInfoTest.java @@ -0,0 +1,767 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.ErrorDetails; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.api.gax.rpc.InternalException; +import com.google.api.gax.rpc.UnavailableException; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.CheckAndMutateRowRequest; +import com.google.bigtable.v2.CheckAndMutateRowResponse; +import com.google.bigtable.v2.GenerateInitialChangeStreamPartitionsRequest; +import com.google.bigtable.v2.GenerateInitialChangeStreamPartitionsResponse; +import com.google.bigtable.v2.MutateRowRequest; +import com.google.bigtable.v2.MutateRowResponse; +import com.google.bigtable.v2.MutateRowsRequest; +import com.google.bigtable.v2.MutateRowsResponse; +import com.google.bigtable.v2.ReadChangeStreamRequest; +import com.google.bigtable.v2.ReadChangeStreamResponse; +import com.google.bigtable.v2.ReadModifyWriteRowRequest; +import com.google.bigtable.v2.ReadModifyWriteRowResponse; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.SampleRowKeysRequest; +import com.google.bigtable.v2.SampleRowKeysResponse; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.MutateRowsException; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.ReadChangeStreamQuery; +import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Queues; +import com.google.protobuf.Any; +import com.google.rpc.RetryInfo; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcServerRule; +import java.io.IOException; +import java.time.Duration; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class RetryInfoTest { + + @Rule public GrpcServerRule serverRule = new GrpcServerRule(); + + private static final Metadata.Key ERROR_DETAILS_KEY = + Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER); + + private FakeBigtableService service; + private BigtableDataClient client; + private BigtableDataSettings.Builder settings; + + private AtomicInteger attemptCounter = new AtomicInteger(); + private com.google.protobuf.Duration delay = + com.google.protobuf.Duration.newBuilder().setSeconds(2).setNanos(0).build(); + + @Before + public void setUp() throws IOException { + service = new FakeBigtableService(); + serverRule.getServiceRegistry().addService(service); + + settings = + BigtableDataSettings.newBuilder() + .setProjectId("fake-project") + .setInstanceId("fake-instance") + .setCredentialsProvider(NoCredentialsProvider.create()); + + settings + .stubSettings() + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(serverRule.getChannel()))) + // channel priming doesn't work with FixedTransportChannelProvider. Disable it for the test + .setRefreshingChannel(false) + .build(); + + this.client = BigtableDataClient.create(settings.build()); + } + + @Test + public void testReadRow() { + verifyRetryInfoIsUsed(() -> client.readRow("table", "row"), true); + } + + @Test + public void testReadRowNonRetryableErrorWithRetryInfo() { + verifyRetryInfoIsUsed(() -> client.readRow("table", "row"), false); + } + + @Test + public void testReadRowDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled(() -> newClient.readRow("table", "row")); + } + } + + @Test + public void testReadRowServerNotReturningRetryInfo() { + verifyNoRetryInfo(() -> client.readRow("table", "row"), true); + } + + @Test + public void testReadRowServerNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo(() -> newClient.readRow("table", "row"), true); + } + } + + @Test + public void testReadRows() { + verifyRetryInfoIsUsed(() -> client.readRows(Query.create("table")).iterator().hasNext(), true); + } + + @Test + public void testReadRowsNonRetraybleErrorWithRetryInfo() { + verifyRetryInfoIsUsed(() -> client.readRows(Query.create("table")).iterator().hasNext(), false); + } + + @Test + public void testReadRowsDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled( + () -> newClient.readRows(Query.create("table")).iterator().hasNext()); + } + } + + @Test + public void testReadRowsServerNotReturningRetryInfo() { + verifyNoRetryInfo(() -> client.readRows(Query.create("table")).iterator().hasNext(), true); + } + + @Test + public void testReadRowsServerNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo(() -> newClient.readRows(Query.create("table")).iterator().hasNext(), true); + } + } + + @Test + public void testMutateRows() { + verifyRetryInfoIsUsed( + () -> + client.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("row-key-1").setCell("cf", "q", "v"))), + true); + } + + @Test + public void testMutateRowsNonRetryableErrorWithRetryInfo() { + verifyRetryInfoIsUsed( + () -> + client.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("row-key-1").setCell("cf", "q", "v"))), + false); + } + + @Test + public void testMutateRowsDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled( + () -> + newClient.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("row-key-1").setCell("cf", "q", "v")))); + } + } + + @Test + public void testMutateRowsServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> + client.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("row-key-1").setCell("cf", "q", "v"))), + true); + } + + @Test + public void testMutateRowsServerNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> + newClient.bulkMutateRows( + BulkMutation.create("fake-table") + .add(RowMutationEntry.create("row-key-1").setCell("cf", "q", "v"))), + true); + } + } + + @Test + public void testMutateRow() { + verifyRetryInfoIsUsed( + () -> client.mutateRow(RowMutation.create("table", "key").setCell("cf", "q", "v")), true); + } + + @Test + public void testMutateRowNonRetryableErrorWithRetryInfo() { + verifyRetryInfoIsUsed( + () -> client.mutateRow(RowMutation.create("table", "key").setCell("cf", "q", "v")), false); + } + + @Test + public void testMutateRowDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + + verifyRetryInfoCanBeDisabled( + () -> newClient.mutateRow(RowMutation.create("table", "key").setCell("cf", "q", "v"))); + } + } + + @Test + public void testMutateRowServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> client.mutateRow(RowMutation.create("table", "key").setCell("cf", "q", "v")), true); + } + + @Test + public void testMutateRowServerNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> newClient.mutateRow(RowMutation.create("table", "key").setCell("cf", "q", "v")), + true); + } + } + + @Test + public void testSampleRowKeys() { + verifyRetryInfoIsUsed(() -> client.sampleRowKeys("table"), true); + } + + @Test + public void testSampleRowKeysNonRetryableErrorWithRetryInfo() { + verifyRetryInfoIsUsed(() -> client.sampleRowKeys("table"), false); + } + + @Test + public void testSampleRowKeysDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled(() -> newClient.sampleRowKeys("table")); + } + } + + @Test + public void testSampleRowKeysServerNotReturningRetryInfo() { + verifyNoRetryInfo(() -> client.sampleRowKeys("table"), true); + } + + @Test + public void testSampleRowKeysServerNotReturningRetryInfoClientDisabledHandling() + throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo(() -> newClient.sampleRowKeys("table"), true); + } + } + + @Test + public void testCheckAndMutateRow() { + verifyRetryInfoIsUsed( + () -> + client.checkAndMutateRow( + ConditionalRowMutation.create("table", "key") + .condition(Filters.FILTERS.value().regex("old-value")) + .then(Mutation.create().setCell("cf", "q", "v"))), + true); + } + + @Test + public void testCheckAndMutateDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient client = BigtableDataClient.create(settings.build())) { + ApiException exception = enqueueNonRetryableExceptionWithDelay(delay); + try { + client.checkAndMutateRow( + ConditionalRowMutation.create("table", "key") + .condition(Filters.FILTERS.value().regex("old-value")) + .then(Mutation.create().setCell("cf", "q", "v"))); + } catch (ApiException e) { + assertThat(e.getStatusCode()).isEqualTo(exception.getStatusCode()); + } + assertThat(attemptCounter.get()).isEqualTo(1); + } + } + + @Test + public void testCheckAndMutateServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> + client.checkAndMutateRow( + ConditionalRowMutation.create("table", "key") + .condition(Filters.FILTERS.value().regex("old-value")) + .then(Mutation.create().setCell("cf", "q", "v"))), + false); + } + + @Test + public void testCheckAndMutateServerNotReturningRetryInfoClientDisabledHandling() + throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> + newClient.checkAndMutateRow( + ConditionalRowMutation.create("table", "key") + .condition(Filters.FILTERS.value().regex("old-value")) + .then(Mutation.create().setCell("cf", "q", "v"))), + false); + } + } + + @Test + public void testReadModifyWrite() { + verifyRetryInfoIsUsed( + () -> + client.readModifyWriteRow( + ReadModifyWriteRow.create("table", "row").append("cf", "q", "v")), + true); + } + + @Test + public void testReadModifyWriteDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient client = BigtableDataClient.create(settings.build())) { + ApiException exception = enqueueNonRetryableExceptionWithDelay(delay); + try { + client.readModifyWriteRow(ReadModifyWriteRow.create("table", "row").append("cf", "q", "v")); + } catch (ApiException e) { + assertThat(e.getStatusCode()).isEqualTo(exception.getStatusCode()); + } + assertThat(attemptCounter.get()).isEqualTo(1); + } + } + + @Test + public void testReadModifyWriteServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> + client.readModifyWriteRow( + ReadModifyWriteRow.create("table", "row").append("cf", "q", "v")), + false); + } + + @Test + public void testReadModifyWriteNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> + newClient.readModifyWriteRow( + ReadModifyWriteRow.create("table", "row").append("cf", "q", "v")), + false); + } + } + + @Test + public void testReadChangeStream() { + verifyRetryInfoIsUsed( + () -> client.readChangeStream(ReadChangeStreamQuery.create("table")).iterator().hasNext(), + true); + } + + @Test + public void testReadChangeStreamNonRetryableErrorWithRetryInfo() { + verifyRetryInfoIsUsed( + () -> client.readChangeStream(ReadChangeStreamQuery.create("table")).iterator().hasNext(), + false); + } + + @Test + public void testReadChangeStreamDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled( + () -> + newClient + .readChangeStream(ReadChangeStreamQuery.create("table")) + .iterator() + .hasNext()); + } + } + + @Test + public void testReadChangeStreamServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> client.readChangeStream(ReadChangeStreamQuery.create("table")).iterator().hasNext(), + true); + } + + @Test + public void testReadChangeStreamNotReturningRetryInfoClientDisabledHandling() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> + newClient + .readChangeStream(ReadChangeStreamQuery.create("table")) + .iterator() + .hasNext(), + true); + } + } + + @Test + public void testGenerateInitialChangeStreamPartition() { + verifyRetryInfoIsUsed( + () -> client.generateInitialChangeStreamPartitions("table").iterator().hasNext(), true); + } + + @Test + public void testGenerateInitialChangeStreamPartitionNonRetryableError() { + verifyRetryInfoIsUsed( + () -> client.generateInitialChangeStreamPartitions("table").iterator().hasNext(), false); + } + + @Test + public void testGenerateInitialChangeStreamPartitionDisableRetryInfo() throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyRetryInfoCanBeDisabled( + () -> newClient.generateInitialChangeStreamPartitions("table").iterator().hasNext()); + } + } + + @Test + public void testGenerateInitialChangeStreamServerNotReturningRetryInfo() { + verifyNoRetryInfo( + () -> client.generateInitialChangeStreamPartitions("table").iterator().hasNext(), true); + } + + @Test + public void testGenerateInitialChangeStreamServerNotReturningRetryInfoClientDisabledHandling() + throws IOException { + settings.stubSettings().setEnableRetryInfo(false); + + try (BigtableDataClient newClient = BigtableDataClient.create(settings.build())) { + verifyNoRetryInfo( + () -> newClient.generateInitialChangeStreamPartitions("table").iterator().hasNext(), + true); + } + } + + // Test the case where server returns retry info and client enables handling of retry info + private void verifyRetryInfoIsUsed(Runnable runnable, boolean retryableError) { + if (retryableError) { + enqueueRetryableExceptionWithDelay(delay); + } else { + enqueueNonRetryableExceptionWithDelay(delay); + } + Stopwatch stopwatch = Stopwatch.createStarted(); + runnable.run(); + stopwatch.stop(); + + assertThat(attemptCounter.get()).isEqualTo(2); + assertThat(stopwatch.elapsed()).isAtLeast(Duration.ofSeconds(delay.getSeconds())); + } + + // Test the case where server returns retry info but client disabled handling of retry info + private void verifyRetryInfoCanBeDisabled(Runnable runnable) { + enqueueRetryableExceptionWithDelay(delay); + Stopwatch stopwatch = Stopwatch.createStarted(); + runnable.run(); + stopwatch.stop(); + + assertThat(attemptCounter.get()).isEqualTo(2); + assertThat(stopwatch.elapsed()).isLessThan(Duration.ofSeconds(delay.getSeconds())); + + attemptCounter.set(0); + ApiException expectedApiException = enqueueNonRetryableExceptionWithDelay(delay); + ApiException actualException = + assertThrows("non retryable operations should fail", ApiException.class, runnable::run); + if (actualException instanceof MutateRowsException) { + assertThat( + ((MutateRowsException) actualException) + .getFailedMutations() + .get(0) + .getError() + .getStatusCode()) + .isEqualTo(expectedApiException.getStatusCode()); + } else { + assertThat(actualException.getStatusCode()).isEqualTo(expectedApiException.getStatusCode()); + } + assertThat(attemptCounter.get()).isEqualTo(1); + } + + // Test the case where server does not return retry info + private void verifyNoRetryInfo(Runnable runnable, boolean operationRetryable) { + enqueueRetryableExceptionNoRetryInfo(); + + if (!operationRetryable) { + assertThrows("non retryable operation should fail", ApiException.class, runnable::run); + assertThat(attemptCounter.get()).isEqualTo(1); + } else { + Stopwatch stopwatch = Stopwatch.createStarted(); + runnable.run(); + stopwatch.stop(); + + assertThat(attemptCounter.get()).isEqualTo(2); + assertThat(stopwatch.elapsed()).isLessThan(Duration.ofSeconds(delay.getSeconds())); + } + + attemptCounter.set(0); + + ApiException expectedApiException = enqueueNonRetryableExceptionNoRetryInfo(); + + ApiException actualApiException = + assertThrows("non retryable error should fail", ApiException.class, runnable::run); + if (actualApiException instanceof MutateRowsException) { + assertThat( + ((MutateRowsException) actualApiException) + .getFailedMutations() + .get(0) + .getError() + .getStatusCode()) + .isEqualTo(expectedApiException.getStatusCode()); + } else { + assertThat(actualApiException.getStatusCode()) + .isEqualTo(expectedApiException.getStatusCode()); + } + + assertThat(attemptCounter.get()).isEqualTo(1); + } + + private void enqueueRetryableExceptionWithDelay(com.google.protobuf.Duration delay) { + Metadata trailers = new Metadata(); + RetryInfo retryInfo = RetryInfo.newBuilder().setRetryDelay(delay).build(); + ErrorDetails errorDetails = + ErrorDetails.builder().setRawErrorMessages(ImmutableList.of(Any.pack(retryInfo))).build(); + byte[] status = + com.google.rpc.Status.newBuilder().addDetails(Any.pack(retryInfo)).build().toByteArray(); + trailers.put(ERROR_DETAILS_KEY, status); + + ApiException exception = + new UnavailableException( + new StatusRuntimeException(Status.UNAVAILABLE, trailers), + GrpcStatusCode.of(Status.Code.UNAVAILABLE), + true, + errorDetails); + + service.expectations.add(exception); + } + + private ApiException enqueueNonRetryableExceptionWithDelay(com.google.protobuf.Duration delay) { + Metadata trailers = new Metadata(); + RetryInfo retryInfo = RetryInfo.newBuilder().setRetryDelay(delay).build(); + ErrorDetails errorDetails = + ErrorDetails.builder().setRawErrorMessages(ImmutableList.of(Any.pack(retryInfo))).build(); + byte[] status = + com.google.rpc.Status.newBuilder().addDetails(Any.pack(retryInfo)).build().toByteArray(); + trailers.put(ERROR_DETAILS_KEY, status); + + ApiException exception = + new InternalException( + new StatusRuntimeException(Status.INTERNAL, trailers), + GrpcStatusCode.of(Status.Code.INTERNAL), + false, + errorDetails); + + service.expectations.add(exception); + + return exception; + } + + private void enqueueRetryableExceptionNoRetryInfo() { + ApiException exception = + new UnavailableException( + new StatusRuntimeException(Status.UNAVAILABLE), + GrpcStatusCode.of(Status.Code.UNAVAILABLE), + true); + service.expectations.add(exception); + } + + private ApiException enqueueNonRetryableExceptionNoRetryInfo() { + ApiException exception = + new InternalException( + new StatusRuntimeException(Status.INTERNAL), + GrpcStatusCode.of(Status.Code.INTERNAL), + false); + + service.expectations.add(exception); + + return exception; + } + + private class FakeBigtableService extends BigtableGrpc.BigtableImplBase { + Queue expectations = Queues.newArrayDeque(); + + @Override + public void readRows( + ReadRowsRequest request, StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(ReadRowsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void mutateRow( + MutateRowRequest request, StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(MutateRowResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void mutateRows( + MutateRowsRequest request, StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + MutateRowsResponse.Builder builder = MutateRowsResponse.newBuilder(); + for (int i = 0; i < request.getEntriesCount(); i++) { + builder.addEntriesBuilder().setIndex(i); + } + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void sampleRowKeys( + SampleRowKeysRequest request, StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(SampleRowKeysResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void checkAndMutateRow( + CheckAndMutateRowRequest request, + StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(CheckAndMutateRowResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void readModifyWriteRow( + ReadModifyWriteRowRequest request, + StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(ReadModifyWriteRowResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void generateInitialChangeStreamPartitions( + GenerateInitialChangeStreamPartitionsRequest request, + StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext(GenerateInitialChangeStreamPartitionsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + + @Override + public void readChangeStream( + ReadChangeStreamRequest request, + StreamObserver responseObserver) { + attemptCounter.incrementAndGet(); + if (expectations.isEmpty()) { + responseObserver.onNext( + ReadChangeStreamResponse.newBuilder() + .setCloseStream(ReadChangeStreamResponse.CloseStream.getDefaultInstance()) + .build()); + responseObserver.onCompleted(); + } else { + Exception expectedRpc = expectations.poll(); + responseObserver.onError(expectedRpc); + } + } + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java index d8e3402b84..527e41e046 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java @@ -126,8 +126,13 @@ public void sendHeaders(Metadata headers) { .setAppProfileId(APP_PROFILE_ID) .build(); EnhancedBigtableStubSettings stubSettings = - EnhancedBigtableStub.finalizeSettings( - settings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder()); + settings + .getStubSettings() + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory( + settings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder())) + .build(); attempts = stubSettings.readRowsSettings().getRetrySettings().getMaxAttempts(); stub = new EnhancedBigtableStub(stubSettings, ClientContext.create(stubSettings)); @@ -142,8 +147,15 @@ public void sendHeaders(Metadata headers) { .setAppProfileId(APP_PROFILE_ID) .build(); EnhancedBigtableStubSettings noHeaderStubSettings = - EnhancedBigtableStub.finalizeSettings( - noHeaderSettings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder()); + noHeaderSettings + .getStubSettings() + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory( + noHeaderSettings.getStubSettings(), + Tags.getTagger(), + localStats.getStatsRecorder())) + .build(); noHeaderStub = new EnhancedBigtableStub(noHeaderStubSettings, ClientContext.create(noHeaderStubSettings)); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java index da989b65dc..2894568f27 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java @@ -121,8 +121,13 @@ public void setUp() throws Exception { .setAppProfileId(APP_PROFILE_ID) .build(); EnhancedBigtableStubSettings stubSettings = - EnhancedBigtableStub.finalizeSettings( - settings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder()); + settings + .getStubSettings() + .toBuilder() + .setTracerFactory( + EnhancedBigtableStub.createBigtableTracerFactory( + settings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder())) + .build(); stub = new EnhancedBigtableStub(stubSettings, ClientContext.create(stubSettings)); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java index 358ff01cde..e5d12ccaeb 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java @@ -16,16 +16,19 @@ package com.google.cloud.bigtable.data.v2.stub.mutaterows; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.any; import com.google.api.core.AbstractApiFuture; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.UnaryCallable; import com.google.api.gax.rpc.UnavailableException; @@ -47,6 +50,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.Mockito; import org.threeten.bp.Duration; @RunWith(JUnit4.class) @@ -64,6 +68,8 @@ public class MutateRowsAttemptCallableTest { private Set retryCodes; private ApiCallContext callContext; private MockRetryingFuture parentFuture; + private final RetryAlgorithm mockRetryAlgorithm = + Mockito.mock(RetryAlgorithm.class); @Before public void setUp() { @@ -71,6 +77,12 @@ public void setUp() { retryCodes = ImmutableSet.of(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); callContext = GrpcCallContext.createDefault(); parentFuture = new MockRetryingFuture(); + Mockito.when(mockRetryAlgorithm.shouldRetry(any(), any(), any(), any())) + .thenAnswer( + input -> { + Throwable throwable = input.getArgument(1); + return ((ApiException) throwable).isRetryable(); + }); } @Test @@ -84,7 +96,8 @@ public void singleEntrySuccessTest() throws Exception { .build()); MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); @@ -107,7 +120,8 @@ public void missingEntry() { .build()); MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); @@ -140,7 +154,8 @@ public void testNoRpcTimeout() { .build()); MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); @@ -172,7 +187,8 @@ public void mixedTest() { .build()); MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); // Make the only call @@ -230,7 +246,8 @@ public void nextAttemptTest() { .build()); MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); // Make the first call @@ -295,7 +312,8 @@ public ApiFuture> futureCall( // Make the call MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); @@ -347,7 +365,8 @@ public ApiFuture> futureCall( // Make the call MutateRowsAttemptCallable attemptCallable = - new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); + new MutateRowsAttemptCallable( + innerCallable, request, callContext, retryCodes, mockRetryAlgorithm); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptorTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptorTest.java index 81d5c67396..237444ba84 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptorTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsBatchingDescriptorTest.java @@ -138,7 +138,7 @@ public void splitExceptionWithFailedMutationsTest() { // Threw an exception at 1st and 3rd entry MutateRowsException serverError = - new MutateRowsException( + MutateRowsException.create( null, ImmutableList.of( MutateRowsException.FailedMutation.create( diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 2c9af872ff..0ecbd8a400 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index 5f0ad3cb46..3b233c9223 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import diff --git a/pom.xml b/pom.xml index d1ceadc642..f38a4edf88 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 2.30.0 + 2.31.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.6.1 + 1.7.1 @@ -153,27 +153,27 @@ com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 com.google.cloud google-cloud-bigtable - 2.30.0 + 2.31.0 @@ -184,12 +184,12 @@ com.google.truth truth - 1.1.5 + 1.2.0 com.google.truth.extensions truth-proto-extension - 1.1.5 + 1.2.0 test diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 60044ab20a..01ad910df2 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.30.0 + 2.31.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import diff --git a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminProto.java b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminProto.java index 3c575dd878..31db57ad7a 100644 --- a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminProto.java +++ b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminProto.java @@ -250,224 +250,225 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + "\203\001\n\025UndeleteTableMetadata\022\014\n\004name\030\001 \001(\t\022" + ".\n\nstart_time\030\002 \001(\0132\032.google.protobuf.Ti" + "mestamp\022,\n\010end_time\030\003 \001(\0132\032.google.proto" - + "buf.Timestamp\"\341\002\n\033ModifyColumnFamiliesRe" + + "buf.Timestamp\"\200\003\n\033ModifyColumnFamiliesRe" + "quest\0229\n\004name\030\001 \001(\tB+\342A\001\002\372A$\n\"bigtablead" + "min.googleapis.com/Table\022_\n\rmodification" + "s\030\002 \003(\0132B.google.bigtable.admin.v2.Modif" + "yColumnFamiliesRequest.ModificationB\004\342A\001" - + "\002\032\245\001\n\014Modification\022\n\n\002id\030\001 \001(\t\0228\n\006create" - + "\030\002 \001(\0132&.google.bigtable.admin.v2.Column" - + "FamilyH\000\0228\n\006update\030\003 \001(\0132&.google.bigtab" - + "le.admin.v2.ColumnFamilyH\000\022\016\n\004drop\030\004 \001(\010" - + "H\000B\005\n\003mod\"\\\n\037GenerateConsistencyTokenReq" - + "uest\0229\n\004name\030\001 \001(\tB+\342A\001\002\372A$\n\"bigtableadm" - + "in.googleapis.com/Table\"=\n GenerateConsi" - + "stencyTokenResponse\022\031\n\021consistency_token" - + "\030\001 \001(\t\"u\n\027CheckConsistencyRequest\0229\n\004nam" - + "e\030\001 \001(\tB+\342A\001\002\372A$\n\"bigtableadmin.googleap" - + "is.com/Table\022\037\n\021consistency_token\030\002 \001(\tB" - + "\004\342A\001\002\".\n\030CheckConsistencyResponse\022\022\n\ncon" - + "sistent\030\001 \001(\010\"\351\001\n\024SnapshotTableRequest\0229" - + "\n\004name\030\001 \001(\tB+\342A\001\002\372A$\n\"bigtableadmin.goo" - + "gleapis.com/Table\022>\n\007cluster\030\002 \001(\tB-\342A\001\002" - + "\372A&\n$bigtableadmin.googleapis.com/Cluste" - + "r\022\031\n\013snapshot_id\030\003 \001(\tB\004\342A\001\002\022&\n\003ttl\030\004 \001(" - + "\0132\031.google.protobuf.Duration\022\023\n\013descript" - + "ion\030\005 \001(\t\"R\n\022GetSnapshotRequest\022<\n\004name\030" - + "\001 \001(\tB.\342A\001\002\372A\'\n%bigtableadmin.googleapis" - + ".com/Snapshot\"|\n\024ListSnapshotsRequest\022=\n" - + "\006parent\030\001 \001(\tB-\342A\001\002\372A&\n$bigtableadmin.go" - + "ogleapis.com/Cluster\022\021\n\tpage_size\030\002 \001(\005\022" - + "\022\n\npage_token\030\003 \001(\t\"g\n\025ListSnapshotsResp" - + "onse\0225\n\tsnapshots\030\001 \003(\0132\".google.bigtabl" - + "e.admin.v2.Snapshot\022\027\n\017next_page_token\030\002" - + " \001(\t\"U\n\025DeleteSnapshotRequest\022<\n\004name\030\001 " - + "\001(\tB.\342A\001\002\372A\'\n%bigtableadmin.googleapis.c" - + "om/Snapshot\"\304\001\n\025SnapshotTableMetadata\022H\n" - + "\020original_request\030\001 \001(\0132..google.bigtabl" - + "e.admin.v2.SnapshotTableRequest\0220\n\014reque" - + "st_time\030\002 \001(\0132\032.google.protobuf.Timestam" - + "p\022/\n\013finish_time\030\003 \001(\0132\032.google.protobuf" - + ".Timestamp\"\330\001\n\037CreateTableFromSnapshotMe" - + "tadata\022R\n\020original_request\030\001 \001(\01328.googl" - + "e.bigtable.admin.v2.CreateTableFromSnaps" - + "hotRequest\0220\n\014request_time\030\002 \001(\0132\032.googl" - + "e.protobuf.Timestamp\022/\n\013finish_time\030\003 \001(" - + "\0132\032.google.protobuf.Timestamp\"\245\001\n\023Create" - + "BackupRequest\022=\n\006parent\030\001 \001(\tB-\342A\001\002\372A&\n$" - + "bigtableadmin.googleapis.com/Cluster\022\027\n\t" - + "backup_id\030\002 \001(\tB\004\342A\001\002\0226\n\006backup\030\003 \001(\0132 ." - + "google.bigtable.admin.v2.BackupB\004\342A\001\002\"\230\001" - + "\n\024CreateBackupMetadata\022\014\n\004name\030\001 \001(\t\022\024\n\014" - + "source_table\030\002 \001(\t\022.\n\nstart_time\030\003 \001(\0132\032" - + ".google.protobuf.Timestamp\022,\n\010end_time\030\004" - + " \001(\0132\032.google.protobuf.Timestamp\"\204\001\n\023Upd" - + "ateBackupRequest\0226\n\006backup\030\001 \001(\0132 .googl" - + "e.bigtable.admin.v2.BackupB\004\342A\001\002\0225\n\013upda" - + "te_mask\030\002 \001(\0132\032.google.protobuf.FieldMas" - + "kB\004\342A\001\002\"N\n\020GetBackupRequest\022:\n\004name\030\001 \001(" - + "\tB,\342A\001\002\372A%\n#bigtableadmin.googleapis.com" - + "/Backup\"Q\n\023DeleteBackupRequest\022:\n\004name\030\001" - + " \001(\tB,\342A\001\002\372A%\n#bigtableadmin.googleapis." - + "com/Backup\"\234\001\n\022ListBackupsRequest\022=\n\006par" - + "ent\030\001 \001(\tB-\342A\001\002\372A&\n$bigtableadmin.google" - + "apis.com/Cluster\022\016\n\006filter\030\002 \001(\t\022\020\n\010orde" - + "r_by\030\003 \001(\t\022\021\n\tpage_size\030\004 \001(\005\022\022\n\npage_to" - + "ken\030\005 \001(\t\"a\n\023ListBackupsResponse\0221\n\007back" - + "ups\030\001 \003(\0132 .google.bigtable.admin.v2.Bac" - + "kup\022\027\n\017next_page_token\030\002 \001(\t\"\347\001\n\021CopyBac" - + "kupRequest\022=\n\006parent\030\001 \001(\tB-\342A\001\002\372A&\n$big" - + "tableadmin.googleapis.com/Cluster\022\027\n\tbac" - + "kup_id\030\002 \001(\tB\004\342A\001\002\022C\n\rsource_backup\030\003 \001(" - + "\tB,\342A\001\002\372A%\n#bigtableadmin.googleapis.com" - + "/Backup\0225\n\013expire_time\030\004 \001(\0132\032.google.pr" - + "otobuf.TimestampB\004\342A\001\002\"\315\001\n\022CopyBackupMet" - + "adata\0226\n\004name\030\001 \001(\tB(\372A%\n#bigtableadmin." - + "googleapis.com/Backup\022@\n\022source_backup_i" - + "nfo\030\002 \001(\0132$.google.bigtable.admin.v2.Bac" - + "kupInfo\022=\n\010progress\030\003 \001(\0132+.google.bigta" - + "ble.admin.v2.OperationProgress2\242*\n\022Bigta" - + "bleTableAdmin\022\253\001\n\013CreateTable\022,.google.b" - + "igtable.admin.v2.CreateTableRequest\032\037.go" - + "ogle.bigtable.admin.v2.Table\"M\332A\025parent," - + "table_id,table\202\323\344\223\002/\"*/v2/{parent=projec" - + "ts/*/instances/*}/tables:\001*\022\212\002\n\027CreateTa" - + "bleFromSnapshot\0228.google.bigtable.admin." - + "v2.CreateTableFromSnapshotRequest\032\035.goog" - + "le.longrunning.Operation\"\225\001\312A(\n\005Table\022\037C" - + "reateTableFromSnapshotMetadata\332A\037parent," - + "table_id,source_snapshot\202\323\344\223\002B\"=/v2/{par" - + "ent=projects/*/instances/*}/tables:creat" - + "eFromSnapshot:\001*\022\244\001\n\nListTables\022+.google" - + ".bigtable.admin.v2.ListTablesRequest\032,.g" - + "oogle.bigtable.admin.v2.ListTablesRespon" - + "se\";\332A\006parent\202\323\344\223\002,\022*/v2/{parent=project" - + "s/*/instances/*}/tables\022\221\001\n\010GetTable\022).g" - + "oogle.bigtable.admin.v2.GetTableRequest\032" - + "\037.google.bigtable.admin.v2.Table\"9\332A\004nam" - + "e\202\323\344\223\002,\022*/v2/{name=projects/*/instances/" - + "*/tables/*}\022\316\001\n\013UpdateTable\022,.google.big" - + "table.admin.v2.UpdateTableRequest\032\035.goog" - + "le.longrunning.Operation\"r\312A\034\n\005Table\022\023Up" - + "dateTableMetadata\332A\021table,update_mask\202\323\344" - + "\223\002920/v2/{table.name=projects/*/instance" - + "s/*/tables/*}:\005table\022\216\001\n\013DeleteTable\022,.g" - + "oogle.bigtable.admin.v2.DeleteTableReque" - + "st\032\026.google.protobuf.Empty\"9\332A\004name\202\323\344\223\002" - + ",**/v2/{name=projects/*/instances/*/tabl" - + "es/*}\022\306\001\n\rUndeleteTable\022..google.bigtabl" - + "e.admin.v2.UndeleteTableRequest\032\035.google" - + ".longrunning.Operation\"f\312A\036\n\005Table\022\025Unde" - + "leteTableMetadata\332A\004name\202\323\344\223\0028\"3/v2/{nam" - + "e=projects/*/instances/*/tables/*}:undel" - + "ete:\001*\022\317\001\n\024ModifyColumnFamilies\0225.google" - + ".bigtable.admin.v2.ModifyColumnFamiliesR" - + "equest\032\037.google.bigtable.admin.v2.Table\"" - + "_\332A\022name,modifications\202\323\344\223\002D\"?/v2/{name=" - + "projects/*/instances/*/tables/*}:modifyC" - + "olumnFamilies:\001*\022\231\001\n\014DropRowRange\022-.goog" - + "le.bigtable.admin.v2.DropRowRangeRequest" - + "\032\026.google.protobuf.Empty\"B\202\323\344\223\002<\"7/v2/{n" - + "ame=projects/*/instances/*/tables/*}:dro" - + "pRowRange:\001*\022\350\001\n\030GenerateConsistencyToke" - + "n\0229.google.bigtable.admin.v2.GenerateCon" - + "sistencyTokenRequest\032:.google.bigtable.a" - + "dmin.v2.GenerateConsistencyTokenResponse" - + "\"U\332A\004name\202\323\344\223\002H\"C/v2/{name=projects/*/in" - + "stances/*/tables/*}:generateConsistencyT" - + "oken:\001*\022\332\001\n\020CheckConsistency\0221.google.bi" - + "gtable.admin.v2.CheckConsistencyRequest\032" - + "2.google.bigtable.admin.v2.CheckConsiste" - + "ncyResponse\"_\332A\026name,consistency_token\202\323" - + "\344\223\002@\";/v2/{name=projects/*/instances/*/t" - + "ables/*}:checkConsistency:\001*\022\352\001\n\rSnapsho" - + "tTable\022..google.bigtable.admin.v2.Snapsh" - + "otTableRequest\032\035.google.longrunning.Oper" - + "ation\"\211\001\312A!\n\010Snapshot\022\025SnapshotTableMeta" - + "data\332A$name,cluster,snapshot_id,descript" - + "ion\202\323\344\223\0028\"3/v2/{name=projects/*/instance" - + "s/*/tables/*}:snapshot:\001*\022\250\001\n\013GetSnapsho" - + "t\022,.google.bigtable.admin.v2.GetSnapshot" - + "Request\032\".google.bigtable.admin.v2.Snaps" - + "hot\"G\332A\004name\202\323\344\223\002:\0228/v2/{name=projects/*" - + "/instances/*/clusters/*/snapshots/*}\022\273\001\n" - + "\rListSnapshots\022..google.bigtable.admin.v" - + "2.ListSnapshotsRequest\032/.google.bigtable" - + ".admin.v2.ListSnapshotsResponse\"I\332A\006pare" - + "nt\202\323\344\223\002:\0228/v2/{parent=projects/*/instanc" - + "es/*/clusters/*}/snapshots\022\242\001\n\016DeleteSna" - + "pshot\022/.google.bigtable.admin.v2.DeleteS" - + "napshotRequest\032\026.google.protobuf.Empty\"G" - + "\332A\004name\202\323\344\223\002:*8/v2/{name=projects/*/inst" - + "ances/*/clusters/*/snapshots/*}\022\340\001\n\014Crea" - + "teBackup\022-.google.bigtable.admin.v2.Crea" - + "teBackupRequest\032\035.google.longrunning.Ope" - + "ration\"\201\001\312A\036\n\006Backup\022\024CreateBackupMetada" - + "ta\332A\027parent,backup_id,backup\202\323\344\223\002@\"6/v2/" - + "{parent=projects/*/instances/*/clusters/" - + "*}/backups:\006backup\022\240\001\n\tGetBackup\022*.googl" - + "e.bigtable.admin.v2.GetBackupRequest\032 .g" - + "oogle.bigtable.admin.v2.Backup\"E\332A\004name\202" - + "\323\344\223\0028\0226/v2/{name=projects/*/instances/*/" - + "clusters/*/backups/*}\022\303\001\n\014UpdateBackup\022-" - + ".google.bigtable.admin.v2.UpdateBackupRe" - + "quest\032 .google.bigtable.admin.v2.Backup\"" - + "b\332A\022backup,update_mask\202\323\344\223\002G2=/v2/{backu" - + "p.name=projects/*/instances/*/clusters/*" - + "/backups/*}:\006backup\022\234\001\n\014DeleteBackup\022-.g" - + "oogle.bigtable.admin.v2.DeleteBackupRequ" - + "est\032\026.google.protobuf.Empty\"E\332A\004name\202\323\344\223" - + "\0028*6/v2/{name=projects/*/instances/*/clu" - + "sters/*/backups/*}\022\263\001\n\013ListBackups\022,.goo" - + "gle.bigtable.admin.v2.ListBackupsRequest" - + "\032-.google.bigtable.admin.v2.ListBackupsR" - + "esponse\"G\332A\006parent\202\323\344\223\0028\0226/v2/{parent=pr" - + "ojects/*/instances/*/clusters/*}/backups" - + "\022\273\001\n\014RestoreTable\022-.google.bigtable.admi" - + "n.v2.RestoreTableRequest\032\035.google.longru" - + "nning.Operation\"]\312A\035\n\005Table\022\024RestoreTabl" - + "eMetadata\202\323\344\223\0027\"2/v2/{parent=projects/*/" - + "instances/*}/tables:restore:\001*\022\355\001\n\nCopyB" - + "ackup\022+.google.bigtable.admin.v2.CopyBac" - + "kupRequest\032\035.google.longrunning.Operatio" - + "n\"\222\001\312A\034\n\006Backup\022\022CopyBackupMetadata\332A*pa" - + "rent,backup_id,source_backup,expire_time" - + "\202\323\344\223\002@\";/v2/{parent=projects/*/instances" - + "/*/clusters/*}/backups:copy:\001*\022\354\001\n\014GetIa" - + "mPolicy\022\".google.iam.v1.GetIamPolicyRequ" - + "est\032\025.google.iam.v1.Policy\"\240\001\332A\010resource" - + "\202\323\344\223\002\216\001\";/v2/{resource=projects/*/instan" - + "ces/*/tables/*}:getIamPolicy:\001*ZL\"G/v2/{" - + "resource=projects/*/instances/*/clusters" - + "/*/backups/*}:getIamPolicy:\001*\022\363\001\n\014SetIam" - + "Policy\022\".google.iam.v1.SetIamPolicyReque" - + "st\032\025.google.iam.v1.Policy\"\247\001\332A\017resource," - + "policy\202\323\344\223\002\216\001\";/v2/{resource=projects/*/" - + "instances/*/tables/*}:setIamPolicy:\001*ZL\"" - + "G/v2/{resource=projects/*/instances/*/cl" - + "usters/*/backups/*}:setIamPolicy:\001*\022\244\002\n\022" - + "TestIamPermissions\022(.google.iam.v1.TestI" - + "amPermissionsRequest\032).google.iam.v1.Tes" - + "tIamPermissionsResponse\"\270\001\332A\024resource,pe" - + "rmissions\202\323\344\223\002\232\001\"A/v2/{resource=projects" - + "/*/instances/*/tables/*}:testIamPermissi" - + "ons:\001*ZR\"M/v2/{resource=projects/*/insta" - + "nces/*/clusters/*/backups/*}:testIamPerm" - + "issions:\001*\032\336\002\312A\034bigtableadmin.googleapis" - + ".com\322A\273\002https://www.googleapis.com/auth/" - + "bigtable.admin,https://www.googleapis.co" - + "m/auth/bigtable.admin.table,https://www." - + "googleapis.com/auth/cloud-bigtable.admin" - + ",https://www.googleapis.com/auth/cloud-b" - + "igtable.admin.table,https://www.googleap" - + "is.com/auth/cloud-platform,https://www.g" - + "oogleapis.com/auth/cloud-platform.read-o" - + "nlyB\337\001\n\034com.google.bigtable.admin.v2B\027Bi" - + "gtableTableAdminProtoP\001Z=google.golang.o" - + "rg/genproto/googleapis/bigtable/admin/v2" - + ";admin\252\002\036Google.Cloud.Bigtable.Admin.V2\312" - + "\002\036Google\\Cloud\\Bigtable\\Admin\\V2\352\002\"Googl" - + "e::Cloud::Bigtable::Admin::V2b\006proto3" + + "\002\022\035\n\017ignore_warnings\030\003 \001(\010B\004\342A\001\001\032\245\001\n\014Mod" + + "ification\022\n\n\002id\030\001 \001(\t\0228\n\006create\030\002 \001(\0132&." + + "google.bigtable.admin.v2.ColumnFamilyH\000\022" + + "8\n\006update\030\003 \001(\0132&.google.bigtable.admin." + + "v2.ColumnFamilyH\000\022\016\n\004drop\030\004 \001(\010H\000B\005\n\003mod" + + "\"\\\n\037GenerateConsistencyTokenRequest\0229\n\004n" + + "ame\030\001 \001(\tB+\342A\001\002\372A$\n\"bigtableadmin.google" + + "apis.com/Table\"=\n GenerateConsistencyTok" + + "enResponse\022\031\n\021consistency_token\030\001 \001(\t\"u\n" + + "\027CheckConsistencyRequest\0229\n\004name\030\001 \001(\tB+" + + "\342A\001\002\372A$\n\"bigtableadmin.googleapis.com/Ta" + + "ble\022\037\n\021consistency_token\030\002 \001(\tB\004\342A\001\002\".\n\030" + + "CheckConsistencyResponse\022\022\n\nconsistent\030\001" + + " \001(\010\"\351\001\n\024SnapshotTableRequest\0229\n\004name\030\001 " + + "\001(\tB+\342A\001\002\372A$\n\"bigtableadmin.googleapis.c" + + "om/Table\022>\n\007cluster\030\002 \001(\tB-\342A\001\002\372A&\n$bigt" + + "ableadmin.googleapis.com/Cluster\022\031\n\013snap" + + "shot_id\030\003 \001(\tB\004\342A\001\002\022&\n\003ttl\030\004 \001(\0132\031.googl" + + "e.protobuf.Duration\022\023\n\013description\030\005 \001(\t" + + "\"R\n\022GetSnapshotRequest\022<\n\004name\030\001 \001(\tB.\342A" + + "\001\002\372A\'\n%bigtableadmin.googleapis.com/Snap" + + "shot\"|\n\024ListSnapshotsRequest\022=\n\006parent\030\001" + + " \001(\tB-\342A\001\002\372A&\n$bigtableadmin.googleapis." + + "com/Cluster\022\021\n\tpage_size\030\002 \001(\005\022\022\n\npage_t" + + "oken\030\003 \001(\t\"g\n\025ListSnapshotsResponse\0225\n\ts" + + "napshots\030\001 \003(\0132\".google.bigtable.admin.v" + + "2.Snapshot\022\027\n\017next_page_token\030\002 \001(\t\"U\n\025D" + + "eleteSnapshotRequest\022<\n\004name\030\001 \001(\tB.\342A\001\002" + + "\372A\'\n%bigtableadmin.googleapis.com/Snapsh" + + "ot\"\304\001\n\025SnapshotTableMetadata\022H\n\020original" + + "_request\030\001 \001(\0132..google.bigtable.admin.v" + + "2.SnapshotTableRequest\0220\n\014request_time\030\002" + + " \001(\0132\032.google.protobuf.Timestamp\022/\n\013fini" + + "sh_time\030\003 \001(\0132\032.google.protobuf.Timestam" + + "p\"\330\001\n\037CreateTableFromSnapshotMetadata\022R\n" + + "\020original_request\030\001 \001(\01328.google.bigtabl" + + "e.admin.v2.CreateTableFromSnapshotReques" + + "t\0220\n\014request_time\030\002 \001(\0132\032.google.protobu" + + "f.Timestamp\022/\n\013finish_time\030\003 \001(\0132\032.googl" + + "e.protobuf.Timestamp\"\245\001\n\023CreateBackupReq" + + "uest\022=\n\006parent\030\001 \001(\tB-\342A\001\002\372A&\n$bigtablea" + + "dmin.googleapis.com/Cluster\022\027\n\tbackup_id" + + "\030\002 \001(\tB\004\342A\001\002\0226\n\006backup\030\003 \001(\0132 .google.bi" + + "gtable.admin.v2.BackupB\004\342A\001\002\"\230\001\n\024CreateB" + + "ackupMetadata\022\014\n\004name\030\001 \001(\t\022\024\n\014source_ta" + + "ble\030\002 \001(\t\022.\n\nstart_time\030\003 \001(\0132\032.google.p" + + "rotobuf.Timestamp\022,\n\010end_time\030\004 \001(\0132\032.go" + + "ogle.protobuf.Timestamp\"\204\001\n\023UpdateBackup" + + "Request\0226\n\006backup\030\001 \001(\0132 .google.bigtabl" + + "e.admin.v2.BackupB\004\342A\001\002\0225\n\013update_mask\030\002" + + " \001(\0132\032.google.protobuf.FieldMaskB\004\342A\001\002\"N" + + "\n\020GetBackupRequest\022:\n\004name\030\001 \001(\tB,\342A\001\002\372A" + + "%\n#bigtableadmin.googleapis.com/Backup\"Q" + + "\n\023DeleteBackupRequest\022:\n\004name\030\001 \001(\tB,\342A\001" + + "\002\372A%\n#bigtableadmin.googleapis.com/Backu" + + "p\"\234\001\n\022ListBackupsRequest\022=\n\006parent\030\001 \001(\t" + + "B-\342A\001\002\372A&\n$bigtableadmin.googleapis.com/" + + "Cluster\022\016\n\006filter\030\002 \001(\t\022\020\n\010order_by\030\003 \001(" + + "\t\022\021\n\tpage_size\030\004 \001(\005\022\022\n\npage_token\030\005 \001(\t" + + "\"a\n\023ListBackupsResponse\0221\n\007backups\030\001 \003(\013" + + "2 .google.bigtable.admin.v2.Backup\022\027\n\017ne" + + "xt_page_token\030\002 \001(\t\"\347\001\n\021CopyBackupReques" + + "t\022=\n\006parent\030\001 \001(\tB-\342A\001\002\372A&\n$bigtableadmi" + + "n.googleapis.com/Cluster\022\027\n\tbackup_id\030\002 " + + "\001(\tB\004\342A\001\002\022C\n\rsource_backup\030\003 \001(\tB,\342A\001\002\372A" + + "%\n#bigtableadmin.googleapis.com/Backup\0225" + + "\n\013expire_time\030\004 \001(\0132\032.google.protobuf.Ti" + + "mestampB\004\342A\001\002\"\315\001\n\022CopyBackupMetadata\0226\n\004" + + "name\030\001 \001(\tB(\372A%\n#bigtableadmin.googleapi" + + "s.com/Backup\022@\n\022source_backup_info\030\002 \001(\013" + + "2$.google.bigtable.admin.v2.BackupInfo\022=" + + "\n\010progress\030\003 \001(\0132+.google.bigtable.admin" + + ".v2.OperationProgress2\242*\n\022BigtableTableA" + + "dmin\022\253\001\n\013CreateTable\022,.google.bigtable.a" + + "dmin.v2.CreateTableRequest\032\037.google.bigt" + + "able.admin.v2.Table\"M\332A\025parent,table_id," + + "table\202\323\344\223\002/\"*/v2/{parent=projects/*/inst" + + "ances/*}/tables:\001*\022\212\002\n\027CreateTableFromSn" + + "apshot\0228.google.bigtable.admin.v2.Create" + + "TableFromSnapshotRequest\032\035.google.longru" + + "nning.Operation\"\225\001\312A(\n\005Table\022\037CreateTabl" + + "eFromSnapshotMetadata\332A\037parent,table_id," + + "source_snapshot\202\323\344\223\002B\"=/v2/{parent=proje" + + "cts/*/instances/*}/tables:createFromSnap" + + "shot:\001*\022\244\001\n\nListTables\022+.google.bigtable" + + ".admin.v2.ListTablesRequest\032,.google.big" + + "table.admin.v2.ListTablesResponse\";\332A\006pa" + + "rent\202\323\344\223\002,\022*/v2/{parent=projects/*/insta" + + "nces/*}/tables\022\221\001\n\010GetTable\022).google.big" + + "table.admin.v2.GetTableRequest\032\037.google." + + "bigtable.admin.v2.Table\"9\332A\004name\202\323\344\223\002,\022*" + + "/v2/{name=projects/*/instances/*/tables/" + + "*}\022\316\001\n\013UpdateTable\022,.google.bigtable.adm" + + "in.v2.UpdateTableRequest\032\035.google.longru" + + "nning.Operation\"r\312A\034\n\005Table\022\023UpdateTable" + + "Metadata\332A\021table,update_mask\202\323\344\223\002920/v2/" + + "{table.name=projects/*/instances/*/table" + + "s/*}:\005table\022\216\001\n\013DeleteTable\022,.google.big" + + "table.admin.v2.DeleteTableRequest\032\026.goog" + + "le.protobuf.Empty\"9\332A\004name\202\323\344\223\002,**/v2/{n" + + "ame=projects/*/instances/*/tables/*}\022\306\001\n" + + "\rUndeleteTable\022..google.bigtable.admin.v" + + "2.UndeleteTableRequest\032\035.google.longrunn" + + "ing.Operation\"f\312A\036\n\005Table\022\025UndeleteTable" + + "Metadata\332A\004name\202\323\344\223\0028\"3/v2/{name=project" + + "s/*/instances/*/tables/*}:undelete:\001*\022\317\001" + + "\n\024ModifyColumnFamilies\0225.google.bigtable" + + ".admin.v2.ModifyColumnFamiliesRequest\032\037." + + "google.bigtable.admin.v2.Table\"_\332A\022name," + + "modifications\202\323\344\223\002D\"?/v2/{name=projects/" + + "*/instances/*/tables/*}:modifyColumnFami" + + "lies:\001*\022\231\001\n\014DropRowRange\022-.google.bigtab" + + "le.admin.v2.DropRowRangeRequest\032\026.google" + + ".protobuf.Empty\"B\202\323\344\223\002<\"7/v2/{name=proje" + + "cts/*/instances/*/tables/*}:dropRowRange" + + ":\001*\022\350\001\n\030GenerateConsistencyToken\0229.googl" + + "e.bigtable.admin.v2.GenerateConsistencyT" + + "okenRequest\032:.google.bigtable.admin.v2.G" + + "enerateConsistencyTokenResponse\"U\332A\004name" + + "\202\323\344\223\002H\"C/v2/{name=projects/*/instances/*" + + "/tables/*}:generateConsistencyToken:\001*\022\332" + + "\001\n\020CheckConsistency\0221.google.bigtable.ad" + + "min.v2.CheckConsistencyRequest\0322.google." + + "bigtable.admin.v2.CheckConsistencyRespon" + + "se\"_\332A\026name,consistency_token\202\323\344\223\002@\";/v2" + + "/{name=projects/*/instances/*/tables/*}:" + + "checkConsistency:\001*\022\352\001\n\rSnapshotTable\022.." + + "google.bigtable.admin.v2.SnapshotTableRe" + + "quest\032\035.google.longrunning.Operation\"\211\001\312" + + "A!\n\010Snapshot\022\025SnapshotTableMetadata\332A$na" + + "me,cluster,snapshot_id,description\202\323\344\223\0028" + + "\"3/v2/{name=projects/*/instances/*/table" + + "s/*}:snapshot:\001*\022\250\001\n\013GetSnapshot\022,.googl" + + "e.bigtable.admin.v2.GetSnapshotRequest\032\"" + + ".google.bigtable.admin.v2.Snapshot\"G\332A\004n" + + "ame\202\323\344\223\002:\0228/v2/{name=projects/*/instance" + + "s/*/clusters/*/snapshots/*}\022\273\001\n\rListSnap" + + "shots\022..google.bigtable.admin.v2.ListSna" + + "pshotsRequest\032/.google.bigtable.admin.v2" + + ".ListSnapshotsResponse\"I\332A\006parent\202\323\344\223\002:\022" + + "8/v2/{parent=projects/*/instances/*/clus" + + "ters/*}/snapshots\022\242\001\n\016DeleteSnapshot\022/.g" + + "oogle.bigtable.admin.v2.DeleteSnapshotRe" + + "quest\032\026.google.protobuf.Empty\"G\332A\004name\202\323" + + "\344\223\002:*8/v2/{name=projects/*/instances/*/c" + + "lusters/*/snapshots/*}\022\340\001\n\014CreateBackup\022" + + "-.google.bigtable.admin.v2.CreateBackupR" + + "equest\032\035.google.longrunning.Operation\"\201\001" + + "\312A\036\n\006Backup\022\024CreateBackupMetadata\332A\027pare" + + "nt,backup_id,backup\202\323\344\223\002@\"6/v2/{parent=p" + + "rojects/*/instances/*/clusters/*}/backup" + + "s:\006backup\022\240\001\n\tGetBackup\022*.google.bigtabl" + + "e.admin.v2.GetBackupRequest\032 .google.big" + + "table.admin.v2.Backup\"E\332A\004name\202\323\344\223\0028\0226/v" + + "2/{name=projects/*/instances/*/clusters/" + + "*/backups/*}\022\303\001\n\014UpdateBackup\022-.google.b" + + "igtable.admin.v2.UpdateBackupRequest\032 .g" + + "oogle.bigtable.admin.v2.Backup\"b\332A\022backu" + + "p,update_mask\202\323\344\223\002G2=/v2/{backup.name=pr" + + "ojects/*/instances/*/clusters/*/backups/" + + "*}:\006backup\022\234\001\n\014DeleteBackup\022-.google.big" + + "table.admin.v2.DeleteBackupRequest\032\026.goo" + + "gle.protobuf.Empty\"E\332A\004name\202\323\344\223\0028*6/v2/{" + + "name=projects/*/instances/*/clusters/*/b" + + "ackups/*}\022\263\001\n\013ListBackups\022,.google.bigta" + + "ble.admin.v2.ListBackupsRequest\032-.google" + + ".bigtable.admin.v2.ListBackupsResponse\"G" + + "\332A\006parent\202\323\344\223\0028\0226/v2/{parent=projects/*/" + + "instances/*/clusters/*}/backups\022\273\001\n\014Rest" + + "oreTable\022-.google.bigtable.admin.v2.Rest" + + "oreTableRequest\032\035.google.longrunning.Ope" + + "ration\"]\312A\035\n\005Table\022\024RestoreTableMetadata" + + "\202\323\344\223\0027\"2/v2/{parent=projects/*/instances" + + "/*}/tables:restore:\001*\022\355\001\n\nCopyBackup\022+.g" + + "oogle.bigtable.admin.v2.CopyBackupReques" + + "t\032\035.google.longrunning.Operation\"\222\001\312A\034\n\006" + + "Backup\022\022CopyBackupMetadata\332A*parent,back" + + "up_id,source_backup,expire_time\202\323\344\223\002@\";/" + + "v2/{parent=projects/*/instances/*/cluste" + + "rs/*}/backups:copy:\001*\022\354\001\n\014GetIamPolicy\022\"" + + ".google.iam.v1.GetIamPolicyRequest\032\025.goo" + + "gle.iam.v1.Policy\"\240\001\332A\010resource\202\323\344\223\002\216\001\";" + + "/v2/{resource=projects/*/instances/*/tab" + + "les/*}:getIamPolicy:\001*ZL\"G/v2/{resource=" + + "projects/*/instances/*/clusters/*/backup" + + "s/*}:getIamPolicy:\001*\022\363\001\n\014SetIamPolicy\022\"." + + "google.iam.v1.SetIamPolicyRequest\032\025.goog" + + "le.iam.v1.Policy\"\247\001\332A\017resource,policy\202\323\344" + + "\223\002\216\001\";/v2/{resource=projects/*/instances" + + "/*/tables/*}:setIamPolicy:\001*ZL\"G/v2/{res" + + "ource=projects/*/instances/*/clusters/*/" + + "backups/*}:setIamPolicy:\001*\022\244\002\n\022TestIamPe" + + "rmissions\022(.google.iam.v1.TestIamPermiss" + + "ionsRequest\032).google.iam.v1.TestIamPermi" + + "ssionsResponse\"\270\001\332A\024resource,permissions" + + "\202\323\344\223\002\232\001\"A/v2/{resource=projects/*/instan" + + "ces/*/tables/*}:testIamPermissions:\001*ZR\"" + + "M/v2/{resource=projects/*/instances/*/cl" + + "usters/*/backups/*}:testIamPermissions:\001" + + "*\032\336\002\312A\034bigtableadmin.googleapis.com\322A\273\002h" + + "ttps://www.googleapis.com/auth/bigtable." + + "admin,https://www.googleapis.com/auth/bi" + + "gtable.admin.table,https://www.googleapi" + + "s.com/auth/cloud-bigtable.admin,https://" + + "www.googleapis.com/auth/cloud-bigtable.a" + + "dmin.table,https://www.googleapis.com/au" + + "th/cloud-platform,https://www.googleapis" + + ".com/auth/cloud-platform.read-onlyB\337\001\n\034c" + + "om.google.bigtable.admin.v2B\027BigtableTab" + + "leAdminProtoP\001Z=google.golang.org/genpro" + + "to/googleapis/bigtable/admin/v2;admin\252\002\036" + + "Google.Cloud.Bigtable.Admin.V2\312\002\036Google\\" + + "Cloud\\Bigtable\\Admin\\V2\352\002\"Google::Cloud:" + + ":Bigtable::Admin::V2b\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( @@ -620,7 +621,7 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_google_bigtable_admin_v2_ModifyColumnFamiliesRequest_descriptor, new java.lang.String[] { - "Name", "Modifications", + "Name", "Modifications", "IgnoreWarnings", }); internal_static_google_bigtable_admin_v2_ModifyColumnFamiliesRequest_Modification_descriptor = internal_static_google_bigtable_admin_v2_ModifyColumnFamiliesRequest_descriptor diff --git a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequest.java b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequest.java index 5eaa481fee..6c0d53debd 100644 --- a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequest.java +++ b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequest.java @@ -1819,6 +1819,24 @@ public com.google.bigtable.admin.v2.ModifyColumnFamiliesRequest.Modification get return modifications_.get(index); } + public static final int IGNORE_WARNINGS_FIELD_NUMBER = 3; + private boolean ignoreWarnings_ = false; + /** + * + * + *

+   * Optional. If true, ignore safety checks when modifying the column families.
+   * 
+ * + * bool ignore_warnings = 3 [(.google.api.field_behavior) = OPTIONAL]; + * + * @return The ignoreWarnings. + */ + @java.lang.Override + public boolean getIgnoreWarnings() { + return ignoreWarnings_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override @@ -1839,6 +1857,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io for (int i = 0; i < modifications_.size(); i++) { output.writeMessage(2, modifications_.get(i)); } + if (ignoreWarnings_ != false) { + output.writeBool(3, ignoreWarnings_); + } getUnknownFields().writeTo(output); } @@ -1854,6 +1875,9 @@ public int getSerializedSize() { for (int i = 0; i < modifications_.size(); i++) { size += com.google.protobuf.CodedOutputStream.computeMessageSize(2, modifications_.get(i)); } + if (ignoreWarnings_ != false) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(3, ignoreWarnings_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -1872,6 +1896,7 @@ public boolean equals(final java.lang.Object obj) { if (!getName().equals(other.getName())) return false; if (!getModificationsList().equals(other.getModificationsList())) return false; + if (getIgnoreWarnings() != other.getIgnoreWarnings()) return false; if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -1889,6 +1914,8 @@ public int hashCode() { hash = (37 * hash) + MODIFICATIONS_FIELD_NUMBER; hash = (53 * hash) + getModificationsList().hashCode(); } + hash = (37 * hash) + IGNORE_WARNINGS_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getIgnoreWarnings()); hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2038,6 +2065,7 @@ public Builder clear() { modificationsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000002); + ignoreWarnings_ = false; return this; } @@ -2091,6 +2119,9 @@ private void buildPartial0(com.google.bigtable.admin.v2.ModifyColumnFamiliesRequ if (((from_bitField0_ & 0x00000001) != 0)) { result.name_ = name_; } + if (((from_bitField0_ & 0x00000004) != 0)) { + result.ignoreWarnings_ = ignoreWarnings_; + } } @java.lang.Override @@ -2171,6 +2202,9 @@ public Builder mergeFrom(com.google.bigtable.admin.v2.ModifyColumnFamiliesReques } } } + if (other.getIgnoreWarnings() != false) { + setIgnoreWarnings(other.getIgnoreWarnings()); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -2218,6 +2252,12 @@ public Builder mergeFrom( } break; } // case 18 + case 24: + { + ignoreWarnings_ = input.readBool(); + bitField0_ |= 0x00000004; + break; + } // case 24 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { @@ -2833,6 +2873,59 @@ public Builder removeModifications(int index) { return modificationsBuilder_; } + private boolean ignoreWarnings_; + /** + * + * + *
+     * Optional. If true, ignore safety checks when modifying the column families.
+     * 
+ * + * bool ignore_warnings = 3 [(.google.api.field_behavior) = OPTIONAL]; + * + * @return The ignoreWarnings. + */ + @java.lang.Override + public boolean getIgnoreWarnings() { + return ignoreWarnings_; + } + /** + * + * + *
+     * Optional. If true, ignore safety checks when modifying the column families.
+     * 
+ * + * bool ignore_warnings = 3 [(.google.api.field_behavior) = OPTIONAL]; + * + * @param value The ignoreWarnings to set. + * @return This builder for chaining. + */ + public Builder setIgnoreWarnings(boolean value) { + + ignoreWarnings_ = value; + bitField0_ |= 0x00000004; + onChanged(); + return this; + } + /** + * + * + *
+     * Optional. If true, ignore safety checks when modifying the column families.
+     * 
+ * + * bool ignore_warnings = 3 [(.google.api.field_behavior) = OPTIONAL]; + * + * @return This builder for chaining. + */ + public Builder clearIgnoreWarnings() { + bitField0_ = (bitField0_ & ~0x00000004); + ignoreWarnings_ = false; + onChanged(); + return this; + } + @java.lang.Override public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { return super.setUnknownFields(unknownFields); diff --git a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequestOrBuilder.java b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequestOrBuilder.java index e386edffea..382407649d 100644 --- a/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequestOrBuilder.java +++ b/proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequestOrBuilder.java @@ -135,4 +135,17 @@ public interface ModifyColumnFamiliesRequestOrBuilder */ com.google.bigtable.admin.v2.ModifyColumnFamiliesRequest.ModificationOrBuilder getModificationsOrBuilder(int index); + + /** + * + * + *
+   * Optional. If true, ignore safety checks when modifying the column families.
+   * 
+ * + * bool ignore_warnings = 3 [(.google.api.field_behavior) = OPTIONAL]; + * + * @return The ignoreWarnings. + */ + boolean getIgnoreWarnings(); } diff --git a/proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_table_admin.proto b/proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_table_admin.proto index 8c516abe7a..62cd7d6555 100644 --- a/proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_table_admin.proto +++ b/proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_table_admin.proto @@ -734,6 +734,9 @@ message ModifyColumnFamiliesRequest { // family, for example). repeated Modification modifications = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. If true, ignore safety checks when modifying the column families. + bool ignore_warnings = 3 [(google.api.field_behavior) = OPTIONAL]; } // Request message for diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index b59ed7b835..9949e24f5f 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.30.0 + 2.31.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.30.0 + 2.31.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.30.0 + 2.31.0 pom import com.google.cloud google-cloud-bigtable-bom - 2.30.0 + 2.31.0 pom import diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlags.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlags.java index 953adab411..ed3a97f9a8 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlags.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlags.java @@ -145,6 +145,44 @@ public boolean getLastScannedRowResponses() { return lastScannedRowResponses_; } + public static final int ROUTING_COOKIE_FIELD_NUMBER = 6; + private boolean routingCookie_ = false; + /** + * + * + *
+   * Notify the server that the client supports using encoded routing cookie
+   * strings to retry requests with.
+   * 
+ * + * bool routing_cookie = 6; + * + * @return The routingCookie. + */ + @java.lang.Override + public boolean getRoutingCookie() { + return routingCookie_; + } + + public static final int RETRY_INFO_FIELD_NUMBER = 7; + private boolean retryInfo_ = false; + /** + * + * + *
+   * Notify the server that the client supports using retry info back off
+   * durations to retry requests with.
+   * 
+ * + * bool retry_info = 7; + * + * @return The retryInfo. + */ + @java.lang.Override + public boolean getRetryInfo() { + return retryInfo_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override @@ -171,6 +209,12 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io if (mutateRowsRateLimit2_ != false) { output.writeBool(5, mutateRowsRateLimit2_); } + if (routingCookie_ != false) { + output.writeBool(6, routingCookie_); + } + if (retryInfo_ != false) { + output.writeBool(7, retryInfo_); + } getUnknownFields().writeTo(output); } @@ -192,6 +236,12 @@ public int getSerializedSize() { if (mutateRowsRateLimit2_ != false) { size += com.google.protobuf.CodedOutputStream.computeBoolSize(5, mutateRowsRateLimit2_); } + if (routingCookie_ != false) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(6, routingCookie_); + } + if (retryInfo_ != false) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(7, retryInfo_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -211,6 +261,8 @@ public boolean equals(final java.lang.Object obj) { if (getMutateRowsRateLimit() != other.getMutateRowsRateLimit()) return false; if (getMutateRowsRateLimit2() != other.getMutateRowsRateLimit2()) return false; if (getLastScannedRowResponses() != other.getLastScannedRowResponses()) return false; + if (getRoutingCookie() != other.getRoutingCookie()) return false; + if (getRetryInfo() != other.getRetryInfo()) return false; if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -230,6 +282,10 @@ public int hashCode() { hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getMutateRowsRateLimit2()); hash = (37 * hash) + LAST_SCANNED_ROW_RESPONSES_FIELD_NUMBER; hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getLastScannedRowResponses()); + hash = (37 * hash) + ROUTING_COOKIE_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getRoutingCookie()); + hash = (37 * hash) + RETRY_INFO_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getRetryInfo()); hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -379,6 +435,8 @@ public Builder clear() { mutateRowsRateLimit_ = false; mutateRowsRateLimit2_ = false; lastScannedRowResponses_ = false; + routingCookie_ = false; + retryInfo_ = false; return this; } @@ -426,6 +484,12 @@ private void buildPartial0(com.google.bigtable.v2.FeatureFlags result) { if (((from_bitField0_ & 0x00000008) != 0)) { result.lastScannedRowResponses_ = lastScannedRowResponses_; } + if (((from_bitField0_ & 0x00000010) != 0)) { + result.routingCookie_ = routingCookie_; + } + if (((from_bitField0_ & 0x00000020) != 0)) { + result.retryInfo_ = retryInfo_; + } } @java.lang.Override @@ -485,6 +549,12 @@ public Builder mergeFrom(com.google.bigtable.v2.FeatureFlags other) { if (other.getLastScannedRowResponses() != false) { setLastScannedRowResponses(other.getLastScannedRowResponses()); } + if (other.getRoutingCookie() != false) { + setRoutingCookie(other.getRoutingCookie()); + } + if (other.getRetryInfo() != false) { + setRetryInfo(other.getRetryInfo()); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -535,6 +605,18 @@ public Builder mergeFrom( bitField0_ |= 0x00000004; break; } // case 40 + case 48: + { + routingCookie_ = input.readBool(); + bitField0_ |= 0x00000010; + break; + } // case 48 + case 56: + { + retryInfo_ = input.readBool(); + bitField0_ |= 0x00000020; + break; + } // case 56 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { @@ -784,6 +866,118 @@ public Builder clearLastScannedRowResponses() { return this; } + private boolean routingCookie_; + /** + * + * + *
+     * Notify the server that the client supports using encoded routing cookie
+     * strings to retry requests with.
+     * 
+ * + * bool routing_cookie = 6; + * + * @return The routingCookie. + */ + @java.lang.Override + public boolean getRoutingCookie() { + return routingCookie_; + } + /** + * + * + *
+     * Notify the server that the client supports using encoded routing cookie
+     * strings to retry requests with.
+     * 
+ * + * bool routing_cookie = 6; + * + * @param value The routingCookie to set. + * @return This builder for chaining. + */ + public Builder setRoutingCookie(boolean value) { + + routingCookie_ = value; + bitField0_ |= 0x00000010; + onChanged(); + return this; + } + /** + * + * + *
+     * Notify the server that the client supports using encoded routing cookie
+     * strings to retry requests with.
+     * 
+ * + * bool routing_cookie = 6; + * + * @return This builder for chaining. + */ + public Builder clearRoutingCookie() { + bitField0_ = (bitField0_ & ~0x00000010); + routingCookie_ = false; + onChanged(); + return this; + } + + private boolean retryInfo_; + /** + * + * + *
+     * Notify the server that the client supports using retry info back off
+     * durations to retry requests with.
+     * 
+ * + * bool retry_info = 7; + * + * @return The retryInfo. + */ + @java.lang.Override + public boolean getRetryInfo() { + return retryInfo_; + } + /** + * + * + *
+     * Notify the server that the client supports using retry info back off
+     * durations to retry requests with.
+     * 
+ * + * bool retry_info = 7; + * + * @param value The retryInfo to set. + * @return This builder for chaining. + */ + public Builder setRetryInfo(boolean value) { + + retryInfo_ = value; + bitField0_ |= 0x00000020; + onChanged(); + return this; + } + /** + * + * + *
+     * Notify the server that the client supports using retry info back off
+     * durations to retry requests with.
+     * 
+ * + * bool retry_info = 7; + * + * @return This builder for chaining. + */ + public Builder clearRetryInfo() { + bitField0_ = (bitField0_ & ~0x00000020); + retryInfo_ = false; + onChanged(); + return this; + } + @java.lang.Override public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { return super.setUnknownFields(unknownFields); diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsOrBuilder.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsOrBuilder.java index 0696b9d05a..40dc1a2ad9 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsOrBuilder.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsOrBuilder.java @@ -80,4 +80,32 @@ public interface FeatureFlagsOrBuilder * @return The lastScannedRowResponses. */ boolean getLastScannedRowResponses(); + + /** + * + * + *
+   * Notify the server that the client supports using encoded routing cookie
+   * strings to retry requests with.
+   * 
+ * + * bool routing_cookie = 6; + * + * @return The routingCookie. + */ + boolean getRoutingCookie(); + + /** + * + * + *
+   * Notify the server that the client supports using retry info back off
+   * durations to retry requests with.
+   * 
+ * + * bool retry_info = 7; + * + * @return The retryInfo. + */ + boolean getRetryInfo(); } diff --git a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsProto.java b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsProto.java index 91b2c97e8c..2072fb2852 100644 --- a/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsProto.java +++ b/proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FeatureFlagsProto.java @@ -41,16 +41,17 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { static { java.lang.String[] descriptorData = { "\n&google/bigtable/v2/feature_flags.proto" - + "\022\022google.bigtable.v2\"\212\001\n\014FeatureFlags\022\025\n" + + "\022\022google.bigtable.v2\"\266\001\n\014FeatureFlags\022\025\n" + "\rreverse_scans\030\001 \001(\010\022\036\n\026mutate_rows_rate" + "_limit\030\003 \001(\010\022\037\n\027mutate_rows_rate_limit2\030" + "\005 \001(\010\022\"\n\032last_scanned_row_responses\030\004 \001(" - + "\010B\275\001\n\026com.google.bigtable.v2B\021FeatureFla" - + "gsProtoP\001Z:google.golang.org/genproto/go" - + "ogleapis/bigtable/v2;bigtable\252\002\030Google.C" - + "loud.Bigtable.V2\312\002\030Google\\Cloud\\Bigtable" - + "\\V2\352\002\033Google::Cloud::Bigtable::V2b\006proto" - + "3" + + "\010\022\026\n\016routing_cookie\030\006 \001(\010\022\022\n\nretry_info\030" + + "\007 \001(\010B\275\001\n\026com.google.bigtable.v2B\021Featur" + + "eFlagsProtoP\001Z:google.golang.org/genprot" + + "o/googleapis/bigtable/v2;bigtable\252\002\030Goog" + + "le.Cloud.Bigtable.V2\312\002\030Google\\Cloud\\Bigt" + + "able\\V2\352\002\033Google::Cloud::Bigtable::V2b\006p" + + "roto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( @@ -65,6 +66,8 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { "MutateRowsRateLimit", "MutateRowsRateLimit2", "LastScannedRowResponses", + "RoutingCookie", + "RetryInfo", }); } diff --git a/proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/feature_flags.proto b/proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/feature_flags.proto index 942646c6b5..d3128c5c67 100644 --- a/proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/feature_flags.proto +++ b/proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/feature_flags.proto @@ -50,4 +50,12 @@ message FeatureFlags { // Notify the server that the client supports the last_scanned_row field // in ReadRowsResponse for long-running scans. bool last_scanned_row_responses = 4; + + // Notify the server that the client supports using encoded routing cookie + // strings to retry requests with. + bool routing_cookie = 6; + + // Notify the server that the client supports using retry info back off + // durations to retry requests with. + bool retry_info = 7; } diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index e7462e744d..6209a9e2ff 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 2.29.1 + 2.30.0 @@ -42,7 +42,7 @@ com.google.truth truth - 1.1.5 + 1.2.0 test diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 377134cef1..2350b0d0f4 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -52,7 +52,7 @@ com.google.truth truth - 1.1.5 + 1.2.0 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 7bd6ab705d..b7fad804aa 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 2.30.0 + 2.31.0 @@ -41,7 +41,7 @@ com.google.truth truth - 1.1.5 + 1.2.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index b5e05baa96..388cd638b5 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -53,7 +53,7 @@ com.google.truth truth - 1.1.5 + 1.2.0 test diff --git a/test-proxy/pom.xml b/test-proxy/pom.xml index c3c1549277..3990f657ef 100644 --- a/test-proxy/pom.xml +++ b/test-proxy/pom.xml @@ -12,11 +12,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.30.0 + 2.31.0 - 2.30.0 + 2.31.0 diff --git a/test-proxy/src/main/java/com/google/cloud/bigtable/testproxy/CbtTestProxy.java b/test-proxy/src/main/java/com/google/cloud/bigtable/testproxy/CbtTestProxy.java index 2ebb609388..6e563d4df0 100644 --- a/test-proxy/src/main/java/com/google/cloud/bigtable/testproxy/CbtTestProxy.java +++ b/test-proxy/src/main/java/com/google/cloud/bigtable/testproxy/CbtTestProxy.java @@ -208,6 +208,9 @@ public synchronized void createClient( .setInstanceId(request.getInstanceId()) .setAppProfileId(request.getAppProfileId()); + settingsBuilder.stubSettings().setEnableRoutingCookie(false); + settingsBuilder.stubSettings().setEnableRetryInfo(false); + if (request.hasPerOperationTimeout()) { Duration newTimeout = Duration.ofMillis(Durations.toMillis(request.getPerOperationTimeout())); settingsBuilder = overrideTimeoutSetting(newTimeout, settingsBuilder); @@ -231,6 +234,8 @@ public synchronized void createClient( } } settingsBuilder.stubSettings().bulkMutateRowsSettings().setServerInitiatedFlowControl(true); + settingsBuilder.stubSettings().setEnableRoutingCookie(true); + settingsBuilder.stubSettings().setEnableRetryInfo(true); } // Create and store CbtClient for later use diff --git a/versions.txt b/versions.txt index d46263f551..d652a91293 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:2.30.0:2.30.0 -grpc-google-cloud-bigtable-admin-v2:2.30.0:2.30.0 -grpc-google-cloud-bigtable-v2:2.30.0:2.30.0 -proto-google-cloud-bigtable-admin-v2:2.30.0:2.30.0 -proto-google-cloud-bigtable-v2:2.30.0:2.30.0 -google-cloud-bigtable-emulator:0.167.0:0.167.0 -google-cloud-bigtable-emulator-core:0.167.0:0.167.0 +google-cloud-bigtable:2.31.0:2.31.0 +grpc-google-cloud-bigtable-admin-v2:2.31.0:2.31.0 +grpc-google-cloud-bigtable-v2:2.31.0:2.31.0 +proto-google-cloud-bigtable-admin-v2:2.31.0:2.31.0 +proto-google-cloud-bigtable-v2:2.31.0:2.31.0 +google-cloud-bigtable-emulator:0.168.0:0.168.0 +google-cloud-bigtable-emulator-core:0.168.0:0.168.0