8000 Update Bigtable+Beam hello world examples and move them into java-doc… · georg78sf/java-docs-samples@b435d4d · GitHub
[go: up one dir, main page]
Skip to content

Commit b435d4d

Browse files
billyjacobsonbillyjacobson
authored
Update Bigtable+Beam hello world examples and move them into java-docs-samples (GoogleCloudPlatform#2503)
Existing samples are here: https://github.com/GoogleCloudPlatform/cloud-bigtable-examples/tree/master/java/dataflow-connector-examples/src/main/java/com/google/cloud/bigtable/dataflow/example
1 parent e308272 commit b435d4d

File tree

4 files changed

+392
-0
lines changed

4 files changed

+392
-0
lines changed

‎bigtable/beam/helloworld/pom.xml

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright 2020 Google LLC
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
22+
<groupId>com.example.bigtable</groupId>
23+
<artifactId>helloworld</artifactId>
24+
<version>1.0-SNAPSHOT</version>
25+
26+
<properties>
27+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
28+
<maven.compiler.source>1.8</maven.compiler.source>
29+
<maven.compiler.target>1.8</maven.compiler.target>
30+
<apache_beam.version>2.19.0</apache_beam.version>
31+
</properties>
32+
33+
<dependencies>
34+
<!-- [START bigtable_beam_connector_dependency] -->
35+
<dependency>
36+
<groupId>com.google.cloud.bigtable</groupId>
37+
<artifactId>bigtable-hbase-beam</artifactId>
38+
<version>1.14.0</version>
39+
</dependency>
40+
<!-- [END bigtable_beam_connector_dependency] -->
41+
42+
<dependency>
43+
<groupId>org.apache.beam</groupId>
44+
<artifactId>beam-runners-direct-java</artifactId>
45+
<version>${apache_beam.version}</version>
46+
</dependency>
47+
<dependency>
48+
<groupId>org.apache.beam</groupId>
49+
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
50+
<version>${apache_beam.version}</version>
51+
</dependency>
52+
<dependency>
53+
<groupId>org.apache.beam</groupId>
54+
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
55+
<version>${apache_beam.version}</version>
56+
</dependency>
57+
58+
<dependency>
59+
<groupId>junit</groupId>
60+
<artifactId>junit</artifactId>
61+
<version>4.13</version>
62+
<scope>test</scope>
63+
</dependency>
64+
<dependency>
65+
<groupId>com.google.truth</groupId>
66+
<artifactId>truth</artifactId>
67+
<version>1.0</version>
68+
<scope>test</scope>
69+
</dependency>
70+
</dependencies>
71+
</project>

‎bigtable/beam/helloworld/src/main/java/HelloWorldRead.java

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
// [START bigtable_beam_helloworld_read]
17+
import com.google.cloud.bigtable.beam.CloudBigtableIO;
18+
import com.google.cloud.bigtable.beam.CloudBigtableScanConfiguration;
19+
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
20+
import org.apache.beam.sdk.Pipeline;
21+
import org.apache.beam.sdk.io.Read;
22+
import org.apache.beam.sdk.options.Default;
23+
import org.apache.beam.sdk.options.Description;
24+
import org.apache.beam.sdk.options.PipelineOptionsFactory;
25+
import org.apache.beam.sdk.transforms.DoFn;
26+
import org.apache.beam.sdk.transforms.ParDo;
27+
import org.apache.hadoop.hbase.client.Result;
28+
import org.apache.hadoop.hbase.client.Scan;
29+
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
30+
import org.apache.hadoop.hbase.util.Bytes;
31+
32+
public class HelloWorldRead {
33+
public static void main(String[] args) {
34+
BigtableOptions options =
35+
PipelineOptionsFactory.fromArgs(args).withValidation().as(BigtableOptions.class);
36+
Pipeline p = Pipeline.create(options);
37+
38+
Scan scan = new Scan();
39+
scan.setCacheBlocks(false);
40+
scan.setFilter(new FirstKeyOnlyFilter());
41+
42+
CloudBigtableScanConfiguration config =
43+
new CloudBigtableScanConfiguration.Builder()
44+
.withProjectId(options.getBigtableProjectId())
45+
.withInstanceId(options.getBigtableInstanceId())
46+
.withTableId(options.getBigtableTableId())
47+
.withScan(scan)
48+
.build();
49+
50+
// [START bigtable_beam_helloworld_read_transforms]
51+
p.apply(Read.from(CloudBigtableIO.read(config)))
52+
.apply(
53+
ParDo.of(
54+
new DoFn<Result, Void>() {
55+
@ProcessElement
56+
public void processElement(@Element Result row, OutputReceiver<Void> out) {
57+
System.out.println(Bytes.toString(row.getRow()));
58+
}
59+
}));
60+
// [END bigtable_beam_helloworld_read_transforms]
61+
62+
p.run().waitUntilFinish();
63+
}
64+
65+
public interface BigtableOptions extends DataflowPipelineOptions {
66+
@Description("The Bigtable project ID, this can be different than your Dataflow project")
67+
@Default.String("bigtable-project")
68+
String getBigtableProjectId();
69+
70+
void setBigtableProjectId(String bigtableProjectId);
71+
72+
@Description("The Bigtable instance ID")
73+
@Default.String("bigtable-instance")
74+
String getBigtableInstanceId();
75+
76+
void setBigtableInstanceId(String bigtableInstanceId);
77+
78+
@Description("The Bigtable table ID in the instance.")
79+
@Default.String("bigtable-table")
80+
String getBigtableTableId();
81+
82+
void setBigtableTableId(String bigtableTableId);
83+
}
84+
}
85+
// [END bigtable_beam_helloworld_read]
86+

‎bigtable/beam/helloworld/src/main/java/HelloWorldWrite.java

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
// [START bigtable_beam_helloworld_write]
17+
import com.google.cloud.bigtable.beam.CloudBigtableIO;
18+
import com.google.cloud.bigtable.beam.CloudBigtableTableConfiguration;
19+
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
20+
import org.apache.beam.sdk.Pipeline;
21+
import org.apache.beam.sdk.options.Default;
22+
import org.apache.beam.sdk.options.Description;
23+
import org.apache.beam.sdk.options.PipelineOptionsFactory;
24+
import org.apache.beam.sdk.transforms.Create;
25+
import org.apache.beam.sdk.transforms.DoFn;
26+
import org.apache.beam.sdk.transforms.ParDo;
27+
import org.apache.hadoop.hbase.client.Mutation;
28+
import org.apache.hadoop.hbase.client.Put;
29+
import org.apache.hadoop.hbase.util.Bytes;
30+
31+
public class HelloWorldWrite {
32+
public static void main(String[] args) {
33+
// [START bigtable_beam_helloworld_create_pipeline]
34+
BigtableOptions options =
35+
PipelineOptionsFactory.fromArgs(args).withValidation().as(BigtableOptions.class);
36+
Pipeline p = Pipeline.create(options);
37+
// [END bigtable_beam_helloworld_create_pipeline]
38+
39+
// [START bigtable_beam_helloworld_write_config]
40+
CloudBigtableTableConfiguration bigtableTableConfig =
41+
new CloudBigtableTableConfiguration.Builder()
42+
.withProjectId(options.getBigtableProjectId())
43+
.withInstanceId(options.getBigtableInstanceId())
44+
.withTableId(options.getBigtableTableId())
45+
.build();
46+
// [END bigtable_beam_helloworld_write_config]
47+
48+
// [START bigtable_beam_helloworld_write_transforms]
49+
p.apply(Create.of("phone#4c410523#20190501", "phone#4c410523#20190502"))
50+
.apply(
51+
ParDo.of(
52+
new DoFn<String, Mutation>() {
53+
@ProcessElement
54+
public void processElement(@Element String rowkey, OutputReceiver<Mutation> out) {
55+
long timestamp = System.currentTimeMillis();
56+
Put row = new Put(Bytes.toBytes(rowkey));
57+
58+
row.addColumn(
59+
Bytes.toBytes("stats_summary"),
60+
Bytes.toBytes("os_build"),
61+
timestamp,
62+
Bytes.toBytes("android"));
63+
out.output(row);
64+
}
65+
}))
66+
.apply(CloudBigtableIO.writeToTable(bigtableTableConfig));
67+
// [END bigtable_beam_helloworld_write_transforms]
68+
69+
p.run().waitUntilFinish();
70+
}
71+
72+
// [START bigtable_beam_helloworld_options]
73+
public interface BigtableOptions extends DataflowPipelineOptions {
74+
@Description("The Bigtable project ID, this can be different than your Dataflow project")
75+
@Default.String("bigtable-project")
76+
String getBigtableProjectId();
77+
78+
void setBigtableProjectId(String bigtableProjectId);
79+
80+
@Description("The Bigtable instance ID")
81+
@Default.String("bigtable-instance")
82+
String getBigtableInstanceId();
83+
84+
void setBigtableInstanceId(String bigtableInstanceId);
85+
86+
@Description("The Bigtable table ID in the instance.")
87+
@Default.String("bigtable-table")
88+
String getBigtableTableId();
89+
90+
void setBigtableTableId(String bigtableTableId);
91+
}
92+
// [END bigtable_beam_helloworld_options]
93+
}
94+
// [END bigtable_beam_helloworld_write]

‎bigtable/beam/helloworld/src/test/java/HelloWorldTest.java

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import static com.google.common.truth.Truth.assertThat;
18+
import static org.junit.Assert.assertNotNull;
19+
20+
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
21+
import java.io.ByteArrayOutputStream;
22+
import java.io.IOException;
23+
import java.io.PrintStream;
24+
import java.util.UUID;
25+
import org.apache.hadoop.hbase.HColumnDescriptor;
26+
import org.apache.hadoop.hbase.HTableDescriptor;
27+
import org.apache.hadoop.hbase.TableName;
28+
import org.apache.hadoop.hbase.client.Admin;
29+
import org.apache.hadoop.hbase.client.Connection;
30+
import org.apache.hadoop.hbase.client.Put;
31+
import org.apache.hadoop.hbase.client.Result;
32+
import org.apache.hadoop.hbase.client.ResultScanner;
33+
import org.apache.hadoop.hbase.client.Scan;
34+
import org.apache.hadoop.hbase.client.Table;
35+
import org.apache.hadoop.hbase.util.Bytes;
36+
import org.junit.AfterClass;
37+
import org.junit.Before;
38+
import org.junit.BeforeClass;
39+
import org.junit.Test;
40+
41+
public class HelloWorldTest {
42+
private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE";
43+
private static final String TABLE_ID =
44+
"mobile-time-series-" + UUID.randomUUID().toString().substring(0, 20);
45+
private static final String COLUMN_FAMILY_NAME = "stats_summary";
46+
47+
private static String projectId;
48+
private static String instanceId;
49+
private ByteArrayOutputStream bout;
50+
51+
private static String requireEnv(String varName) {
52+
String value = System.getenv(varName);
53+
assertNotNull(
54+
String.format("Environment variable '%s' is required to perform these tests.", varName),
55+
value);
56+
return value;
57+
}
58+
59+
@BeforeClass
60+
public static void beforeClass() {
61+
projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
62+
instanceId = requireEnv(INSTANCE_ENV);
63+
try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
64+
Admin admin = connection.getAdmin();
65+
HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
66+
descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
67+
admin.createTable(descriptor);
68+
69+
Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));
70+
71+
String rowKey = "phone#4c410523#20190401";
72+
Put put = new Put(Bytes.toBytes(rowKey));
73+
74+
put.addColumn(
75+
Bytes.toBytes(COLUMN_FAMILY_NAME), Bytes.toBytes("os_name"), Bytes.toBytes("android"));
76+
table.put(put);
77+
78+
} catch (Exception e) {
79+
System.out.println("Error during beforeClass: \n" + e.toString());
80+
}
81+
}
82+
83+
@Before
84+
public void setupStream() {
85+
bout = new ByteArrayOutputStream();
86+
System.setOut(new PrintStream(bout));
87+
}
88+
89+
@AfterClass
90+
public static void afterClass() {
91+
try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
92+
Admin admin = connection.getAdmin();
93+
Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));
94+
admin.disableTable(table.getName());
95+
admin.deleteTable(table.getName());
96+
} catch (Exception e) {
97+
System.out.println("Error during afterClass: \n" + e.toString());
98+
}
99+
}
100+
101+
@Test
102+
public void testWrite() {
103+
HelloWorldWrite.main(
104+
new String[] {
105+
"--bigtableProjectId=" + projectId,
106+
"--bigtableInstanceId=" + instanceId,
107+
"--bigtableTableId=" + TABLE_ID,
108+
"--runner=DirectRunner"
109+
});
110+
111+
long count = 0;
112+
try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
113+
Table table = connection.getTable(TableName.valueOf(TABLE_ID));
114+
Scan scan = new Scan();
115+
116+
ResultScanner rows = table.getScanner(scan);
117+
118+
for (Result row : rows) {
119+
count++;
120+
}
121+
} catch (IOException e) {
122+
System.out.println(
123+
"Unable to initialize service client, as a network error occurred: \n" + e.toString());
124+
}
125+
assertThat(count).isGreaterThan(0);
126+
}
127+
128+
@Test
129+
public void testRead() {
130+
HelloWorldRead.main(
131+
new String[] {
132+
"--bigtableProjectId=" + projectId,
133+
"--bigtableInstanceId=" + instanceId,
134+
"--bigtableTableId=" + TABLE_ID,
135+
"--runner=DirectRunner"
136+
});
137+
138+
String output = bout.toString();
139+
assertThat(output).contains("phone#");
140+
}
141+
}

0 commit comments

Comments
 (0)
0