8000 Bigquery: Expose labels for LoadJobConfiguration. by pmakani · Pull Request #6387 · googleapis/google-cloud-java · GitHub
[go: up one dir, main page]
Skip to content

Bigquery: Expose labels for LoadJobConfiguration. #6387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pmakani merged 2 commits into from
Oct 2, 2019
Merged

Bigquery: Expose labels for LoadJobConfiguration. #6387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
29445c2
expose labels
Sep 30, 2019
85c089b
modified test case
Sep 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions ...s/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Ints;
import java.util.List;
import java.util.Map;
import java.util.Objects;

/**
Expand All @@ -50,6 +51,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
private final TimePartitioning timePartitioning;
private final Clustering clustering;
private final Boolean useAvroLogicalTypes;
private final Map<String, String> labels;

public static final class Builder extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
implements LoadConfiguration.Builder {
Expand All @@ -70,6 +72,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
private TimePartitioning timePartitioning;
private Clustering clustering;
private Boolean useAvroLogicalTypes;
private Map<String, String> labels;

private Builder() {
super(Type.LOAD);
Expand All @@ -93,6 +96,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
this.timePartitioning = loadConfiguration.timePartitioning;
this.clustering = loadConfiguration.clustering;
this.useAvroLogicalTypes = loadConfiguration.useAvroLogicalTypes;
this.labels = loadConfiguration.labels;
}

private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
Expand Down Expand Up @@ -166,6 +170,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
loadConfigurationPb.getDestinationEncryptionConfiguration())
.build();
}
if (configurationPb.getLabels() != null) {
this.labels = configurationPb.getLabels();
}
}

@Override
Expand Down Expand Up @@ -263,6 +270,20 @@ public Builder setSchemaUpdateOptions(List<JobInfo.SchemaUpdateOption> schemaUpd
return this;
}

/**
* The labels associated with this job. You can use these to organize and group your jobs. Label
* keys and values can be no longer than 63 characters, can only contain lowercase letters,
* numeric characters, underscores and dashes. International characters are allowed. Label
* values are optional. Label keys must start with a letter and each label in the list must have
* a different key.
*
* @param labels labels or {@code null} for none
*/
public Builder setLabels(Map<String, String> labels) {
this.labels = labels;
return this;
}

@Override
public LoadJobConfiguration build() {
return new LoadJobConfiguration(this);
Expand All @@ -286,6 +307,7 @@ private LoadJobConfiguration(Builder builder) {
this.timePartitioning = builder.timePartitioning;
this.clustering = builder.clustering;
this.useAvroLogicalTypes = builder.useAvroLogicalTypes;
this.labels = builder.labels;
}

@Override
Expand Down Expand Up @@ -378,6 +400,11 @@ public List<JobInfo.SchemaUpdateOption> getSchemaUpdateOptions() {
return schemaUpdateOptions;
}

/** Returns the labels associated with this job */
public Map<String, String> getLabels() {
return labels;
}

@Override
public Builder toBuilder() {
return new Builder(this);
Expand All @@ -400,7 +427,8 @@ ToStringHelper toStringHelper() {
.add("autodetect", autodetect)
.add("timePartitioning", timePartitioning)
.add("clustering", clustering)
.add("useAvroLogicalTypes", useAvroLogicalTypes);
.add("useAvroLogicalTypes", useAvroLogicalTypes)
.add("labels", labels);
}

@Override
Expand All @@ -425,6 +453,8 @@ LoadJobConfiguration setProjectId(String projectId) {
@Override
com.google.api.services.bigquery.model.JobConfiguration toPb() {
JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad();
com.google.api.services.bigquery.model.JobConfiguration jobConfiguration =
new com.google.api.services.bigquery.model.JobConfiguration();
loadConfigurationPb.setDestinationTable(destinationTable.toPb());
if (createDisposition != null) {
loadConfigurationPb.setCreateDisposition(createDisposition.toString());
Expand Down Expand Up @@ -482,8 +512,11 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
loadConfigurationPb.setClustering(clustering.toPb());
}
loadConfigurationPb.setUseAvroLogicalTypes(useAvroLogicalTypes);
return new com.google.api.services.bigquery.model.JobConfiguration()
.setLoad(loadConfigurationPb);
if (labels != null) {
jobConfiguration.setLabels(labels);
}
jobConfiguration.setLoad(loadConfigurationPb);
return jobConfiguration;
}

/**
Expand Down
8 changes: 8 additions & 0 deletions ...ogle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
import com.google.cloud.bigquery.TimePartitioning.Type;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import org.junit.Test;

public class LoadJobConfigurationTest {
Expand Down Expand Up @@ -58,6 +60,8 @@ public class LoadJobConfigurationTest {
private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY);
private static final Clustering CLUSTERING =
Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build();
private static final Map<String, String> LABELS =
ImmutableMap.of("test-job-name", "test-load-job");
private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV =
LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
.setCreateDisposition(CREATE_DISPOSITION)
Expand All @@ -71,6 +75,7 @@ public class LoadJobConfigurationTest {
.setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION)
.setTimePartitioning(TIME_PARTITIONING)
.setClustering(CLUSTERING)
.setLabels(LABELS)
.build();
private static final DatastoreBackupOptions BACKUP_OPTIONS =
DatastoreBackupOptions.newBuilder()
Expand All @@ -86,6 +91,7 @@ public class LoadJobConfigurationTest {
.setSchema(TABLE_SCHEMA)
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
.setAutodetect(AUTODETECT)
.setLabels(LABELS)
.build();
private static final LoadJobConfiguration LOAD_CONFIGURATION_AVRO =
LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
Expand All @@ -101,6 +107,7 @@ public class LoadJobConfigurationTest {
.setTimePartitioning(TIME_PARTITIONING)
.setClustering(CLUSTERING)
.setUseAvroLogicalTypes(USERAVROLOGICALTYPES)
.setLabels(LABELS)
.build();

@Test
Expand Down Expand Up @@ -216,5 +223,6 @@ private void compareLoadJobConfiguration(
assertEquals(expected.getTimePartitioning(), value.getTimePartitioning());
assertEquals(expected.getClustering(), value.getClustering());
assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes());
assertEquals(expected.getLabels(), value.getLabels());
}
}
8 changes: 8 additions & 0 deletions ...ents/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ public class ITBigQueryTest {
public static void beforeClass() throws InterruptedException, TimeoutException {
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
Map<String, String> labels = ImmutableMap.of("test-job-name", "test-load-job");
bigquery = bigqueryHelper.getOptions().getService();
storage = storageHelper.getOptions().getService();
storage.create(BucketInfo.of(BUCKET));
Expand All @@ -302,10 +303,13 @@ public static void beforeClass() throws InterruptedException, TimeoutException {
TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json())
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
.setSchema(TABLE_SCHEMA)
.setLabels(labels)
.build();
Job job = bigquery.create(JobInfo.of(configuration));
job = job.waitFor();
assertNull(job.getStatus().getError());
LoadJobConfiguration loadJobConfiguration = job.getConfiguration();
assertEquals(labels, loadJobConfiguration.getLabels());
}

@AfterClass
Expand Down Expand Up @@ -1533,13 +1537,17 @@ public void testQueryJobWithDryRun() throws InterruptedException, TimeoutExcepti
public void testExtractJob() throws InterruptedException, TimeoutException {
String tableName = "test_export_job_table";
TableId destinationTable = TableId.of(DATASET, tableName);
Map<String, String> labels = ImmutableMap.of("test-job-name", "test-load-extract-job");
LoadJobConfiguration configuration =
LoadJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE)
.setSchema(SIMPLE_SCHEMA)
.setLabels(labels)
.build();
Job remoteLoadJob = bigquery.create(JobInfo.of(configuration));
remoteLoadJob = remoteLoadJob.waitFor();
assertNull(remoteLoadJob.getStatus().getError());
LoadJobConfiguration loadJobConfiguration = remoteLoadJob.getConfiguration();
assertEquals(labels, loadJobConfiguration.getLabels());

ExtractJobConfiguration extractConfiguration =
ExtractJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE)
Expand Down
0