From d3ffde390cbef7854ab3b1aedf4cb7b40ed81e42 Mon Sep 17 00:00:00 2001 From: peihe Date: Tue, 13 Oct 2015 11:06:06 -0700 Subject: [PATCH 1/2] Fix a BigQuery output issue in DirectPipelineRunner getOrCreateTable() was called for each output element, and it introduces unnecessary BigQuery APIs call and duplicated messages. ----Release Notes---- [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=105328508 --- .../com/google/cloud/dataflow/sdk/io/BigQueryIO.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java b/sdk/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java index 3dbccd3b03..ccb474aaa6 100644 --- a/sdk/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java +++ b/sdk/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java @@ -1353,9 +1353,6 @@ private static void evaluateWriteHelper( if (ref.getProjectId() == null) { ref.setProjectId(options.getProject()); } - LOG.info("Writing to BigQuery table {}", toTableSpec(ref)); - inserter.getOrCreateTable( - ref, transform.writeDisposition, transform.createDisposition, transform.schema); List rows = getOrCreateMapListValue(tableRows, ref); rows.add(windowedValue.getValue()); @@ -1363,6 +1360,12 @@ private static void evaluateWriteHelper( } for (TableReference ref : tableRows.keySet()) { + LOG.info("Writing to BigQuery table {}", toTableSpec(ref)); + // {@link BigQueryTableInserter#getOrCreateTable} validates {@link CreateDisposition} + // and {@link WriteDisposition}. + // For each {@link TableReference}, it can only be called before rows are written. + inserter.getOrCreateTable( + ref, transform.writeDisposition, transform.createDisposition, transform.schema); inserter.insertAll(ref, tableRows.get(ref)); } } catch (IOException e) { From eac7a85b04ac0731c0cde1a8b790b19205783a1b Mon Sep 17 00:00:00 2001 From: davor Date: Mon, 5 Oct 2015 15:08:21 -0700 Subject: [PATCH 2/2] Version management Declare version 1.2.0. Prepare for version 1.2.1. ----Release Notes---- [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=104700491 --- examples/pom.xml | 2 +- maven-archetypes/examples/pom.xml | 2 +- maven-archetypes/starter/pom.xml | 2 +- pom.xml | 2 +- sdk/pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index e59b855819..5dc1b250c2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -22,7 +22,7 @@ com.google.cloud.dataflow google-cloud-dataflow-java-sdk-parent - 1.1.1-SNAPSHOT + 1.2.1-SNAPSHOT com.google.cloud.dataflow diff --git a/maven-archetypes/examples/pom.xml b/maven-archetypes/examples/pom.xml index cdb3744d89..8b0d5e2c0f 100644 --- a/maven-archetypes/examples/pom.xml +++ b/maven-archetypes/examples/pom.xml @@ -22,7 +22,7 @@ com.google.cloud.dataflow google-cloud-dataflow-java-sdk-parent - 1.1.1-SNAPSHOT + 1.2.1-SNAPSHOT ../../pom.xml diff --git a/maven-archetypes/starter/pom.xml b/maven-archetypes/starter/pom.xml index cdbb5137ea..0298889abf 100644 --- a/maven-archetypes/starter/pom.xml +++ b/maven-archetypes/starter/pom.xml @@ -22,7 +22,7 @@ com.google.cloud.dataflow google-cloud-dataflow-java-sdk-parent - 1.1.1-SNAPSHOT + 1.2.1-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 6b56dc45a2..a936936f25 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ http://cloud.google.com/dataflow 2013 - 1.1.1-SNAPSHOT + 1.2.1-SNAPSHOT diff --git a/sdk/pom.xml b/sdk/pom.xml index 4c07023d67..da58421744 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -22,7 +22,7 @@ com.google.cloud.dataflow google-cloud-dataflow-java-sdk-parent - 1.1.1-SNAPSHOT + 1.2.1-SNAPSHOT com.google.cloud.dataflow