diff --git a/CHANGELOG.md b/CHANGELOG.md index c8507d308d..30c39f5ecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.2.0](https://www.github.com/googleapis/java-bigquery/compare/v2.1.13...v2.2.0) (2021-10-01) + + +### Features + +* add support for AvroOptions ([#1630](https://www.github.com/googleapis/java-bigquery/issues/1630)) ([10c1961](https://www.github.com/googleapis/java-bigquery/commit/10c1961f53ab6ba1b71ead9c51a369bf14389c49)) + ### [2.1.13](https://www.github.com/googleapis/java-bigquery/compare/v2.1.12...v2.1.13) (2021-09-29) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 414b622825..56645dd933 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.1.13 + 2.2.0 diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml new file mode 100644 index 0000000000..5dba6ab7a7 --- /dev/null +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -0,0 +1,10 @@ + + + + + 7006 + com/google/cloud/bigquery/FormatOptions + com.google.cloud.bigquery.FormatOptions avro() + com.google.cloud.bigquery.AvroOptions + + diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 5a05701f2f..b88bb19a57 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.1.13 + 2.2.0 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.1.13 + 2.2.0 google-cloud-bigquery diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AvroOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AvroOptions.java new file mode 100644 index 0000000000..dd5964f2db --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AvroOptions.java @@ -0,0 +1,112 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +/** + * Google BigQuery options for AVRO format. This class wraps some properties of AVRO files used by + * BigQuery to parse external data. + */ +public final class AvroOptions extends FormatOptions { + + private static final long serialVersionUID = 2293570529308612712L; + + private final Boolean useAvroLogicalTypes; + + public static final class Builder { + + private Boolean useAvroLogicalTypes; + + private Builder() {} + + private Builder(AvroOptions avroOptions) { + this.useAvroLogicalTypes = avroOptions.useAvroLogicalTypes; + } + + /** + * [Optional] Sets whether BigQuery should interpret logical types as the corresponding BigQuery + * data type (for example, TIMESTAMP), instead of using the raw type (for example, INTEGER). + */ + public Builder setUseAvroLogicalTypes(boolean useAvroLogicalTypes) { + this.useAvroLogicalTypes = useAvroLogicalTypes; + return this; + } + + /** Creates a {@code AvroOptions} object. */ + public AvroOptions build() { + return new AvroOptions(this); + } + } + + private AvroOptions(Builder builder) { + super(FormatOptions.AVRO); + this.useAvroLogicalTypes = builder.useAvroLogicalTypes; + } + + /** + * Returns whether BigQuery should interpret logical types as the corresponding BigQuery data type + * (for example, TIMESTAMP), instead of using the raw type (for example, INTEGER). + */ + public Boolean useAvroLogicalTypes() { + return useAvroLogicalTypes; + } + + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", getType()) + .add("useAvroLogicalTypes", useAvroLogicalTypes) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(getType(), useAvroLogicalTypes); + } + + @Override + public boolean equals(Object obj) { + return obj == this + || obj instanceof AvroOptions && Objects.equals(toPb(), ((AvroOptions) obj).toPb()); + } + + com.google.api.services.bigquery.model.AvroOptions toPb() { + com.google.api.services.bigquery.model.AvroOptions avroOptions = + new com.google.api.services.bigquery.model.AvroOptions(); + avroOptions.setUseAvroLogicalTypes(useAvroLogicalTypes); + return avroOptions; + } + + /** Returns a builder for a AvroOptions object. */ + public static AvroOptions.Builder newBuilder() { + return new AvroOptions.Builder(); + } + + static AvroOptions fromPb(com.google.api.services.bigquery.model.AvroOptions avroOptions) { + Builder builder = newBuilder(); + if (avroOptions.getUseAvroLogicalTypes() != null) { + builder.setUseAvroLogicalTypes(avroOptions.getUseAvroLogicalTypes()); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 2b68a7ff22..6ca64a8d78 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -300,6 +300,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getDecimalTargetTypes() != null) { externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes()); } + if (getFormatOptions() != null && FormatOptions.AVRO.equals(getFormatOptions().getType())) { + externalConfigurationPb.setAvroOptions(((AvroOptions) getFormatOptions()).toPb()); + } if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) { externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb()); } @@ -459,6 +462,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setConnectionId(externalDataConfiguration.getConnectionId()); } builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); + if (externalDataConfiguration.getAvroOptions() != null) { + builder.setFormatOptions(AvroOptions.fromPb(externalDataConfiguration.getAvroOptions())); + } if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } @@ -508,6 +514,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getIgnoreUnknownValues() != null) { builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); } + if (externalDataConfiguration.getAvroOptions() != null) { + builder.setFormatOptions(AvroOptions.fromPb(externalDataConfiguration.getAvroOptions())); + } if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java index 92a57fc8f0..c4f4111752 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java @@ -91,8 +91,8 @@ public static FormatOptions datastoreBackup() { } /** Default options for AVRO format. */ - public static FormatOptions avro() { - return new FormatOptions(AVRO); + public static AvroOptions avro() { + return AvroOptions.newBuilder().build(); } /** Default options for BIGTABLE format. */ @@ -120,6 +120,8 @@ public static FormatOptions of(String format) { checkArgument(!isNullOrEmpty(format), "Provided format is null or empty"); if (format.equals(CSV)) { return csv(); + } else if (format.equals(AVRO)) { + return avro(); } else if (format.equals(DATASTORE_BACKUP)) { return datastoreBackup(); } else if (format.equals(GOOGLE_SHEETS)) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AvroOptionsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AvroOptionsTest.java new file mode 100644 index 0000000000..f40660fd7b --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AvroOptionsTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class AvroOptionsTest { + + private static final Boolean USE_AVRO_LOGICAL_TYPES = true; + private static final AvroOptions AVRO_OPTIONS = + AvroOptions.newBuilder().setUseAvroLogicalTypes(USE_AVRO_LOGICAL_TYPES).build(); + + @Test + public void testToBuilder() { + compareAvroOptions(AVRO_OPTIONS, AVRO_OPTIONS.toBuilder().build()); + AvroOptions avroOptions = AVRO_OPTIONS.toBuilder().setUseAvroLogicalTypes(false).build(); + assertEquals(false, avroOptions.useAvroLogicalTypes()); + avroOptions = avroOptions.toBuilder().setUseAvroLogicalTypes(true).build(); + compareAvroOptions(AVRO_OPTIONS, avroOptions); + } + + @Test + public void testBuilder() { + assertEquals(FormatOptions.AVRO, AVRO_OPTIONS.getType()); + assertEquals(USE_AVRO_LOGICAL_TYPES, AVRO_OPTIONS.useAvroLogicalTypes()); + } + + @Test + public void testToAndFromPb() { + compareAvroOptions(AVRO_OPTIONS, AvroOptions.fromPb(AVRO_OPTIONS.toPb())); + AvroOptions avroOptions = + AvroOptions.newBuilder().setUseAvroLogicalTypes(USE_AVRO_LOGICAL_TYPES).build(); + compareAvroOptions(avroOptions, AvroOptions.fromPb(avroOptions.toPb())); + } + + private void compareAvroOptions(AvroOptions expected, AvroOptions value) { + assertEquals(expected, value); + assertEquals(expected.useAvroLogicalTypes(), value.useAvroLogicalTypes()); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index 38dcd27142..b7b9531e55 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -50,6 +50,7 @@ public class ExternalTableDefinitionTest { private static final String COMPRESSION = "GZIP"; private static final String CONNECTION_ID = "123456789"; private static final Boolean AUTODETECT = true; + private static final AvroOptions AVRO_OPTIONS = AvroOptions.newBuilder().build(); private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build(); private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS = HivePartitioningOptions.newBuilder() @@ -67,6 +68,9 @@ public class ExternalTableDefinitionTest { .setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS) .build(); + private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO = + ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, AVRO_OPTIONS).build(); + @Test public void testToBuilder() { compareExternalTableDefinition( @@ -109,6 +113,7 @@ public void testBuilder() { assertEquals(TableDefinition.Type.EXTERNAL, EXTERNAL_TABLE_DEFINITION.getType()); assertEquals(COMPRESSION, EXTERNAL_TABLE_DEFINITION.getCompression()); assertEquals(CONNECTION_ID, EXTERNAL_TABLE_DEFINITION.getConnectionId()); + assertEquals(AVRO_OPTIONS, EXTERNAL_TABLE_DEFINITION_AVRO.getFormatOptions()); assertEquals(CSV_OPTIONS, EXTERNAL_TABLE_DEFINITION.getFormatOptions()); assertEquals(IGNORE_UNKNOWN_VALUES, EXTERNAL_TABLE_DEFINITION.ignoreUnknownValues()); assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index a2f164f8af..deed2f11b4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -56,7 +56,7 @@ public class LoadJobConfigurationTest { ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); private static final Boolean AUTODETECT = true; - private static final Boolean USERAVROLOGICALTYPES = true; + private static final Boolean USE_AVRO_LOGICAL_TYPES = true; private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); @@ -128,7 +128,7 @@ public class LoadJobConfigurationTest { .setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION) .setTimePartitioning(TIME_PARTITIONING) .setClustering(CLUSTERING) - .setUseAvroLogicalTypes(USERAVROLOGICALTYPES) + .setUseAvroLogicalTypes(USE_AVRO_LOGICAL_TYPES) .setLabels(LABELS) .setJobTimeoutMs(TIMEOUT) .setRangePartitioning(RANGE_PARTITIONING) diff --git a/pom.xml b/pom.xml index 4d28453688..b46a240f80 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.1.13 + 2.2.0 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -77,7 +77,7 @@ com.google.cloud google-cloud-bigquery - 2.1.13 + 2.2.0 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 7145c55743..367196a0d5 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.1.12 + 2.1.13 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 8f574ae87a..b43dea32e0 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.1.13 + 2.2.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 06629d9976..7ddea439d8 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -47,7 +47,7 @@ com.google.cloud libraries-bom - 23.0.0 + 23.1.0 pom import diff --git a/versions.txt b/versions.txt index 1646b28f87..bc8d5d4cce 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.1.13:2.1.13 \ No newline at end of file +google-cloud-bigquery:2.2.0:2.2.0 \ No newline at end of file